├── LICENSE ├── README.md ├── backend-2dbs.py ├── crontab.sh ├── index.css ├── index.html ├── index.js ├── index_with_escape.js ├── migrate-split.py ├── old-and-experiments ├── backend-1db-regex.py ├── backend-2dbs-regex.py ├── migrate-index.py ├── migrate-or-update.py └── migrate.py ├── rarbg16.png ├── rarbg32.png ├── restart.sh ├── screenshot-night-theme.png ├── screenshot.png ├── uvicorn_logging.conf ├── watchdog-and-restart.py └── watchdog.py /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Roman Davydov 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # 🧲 rarbg-dump-search 2 | 3 | 4 | 5 | ![](https://img.shields.io/github/license/rdavydov/rarbg-dump-search?style=for-the-badge&logo=github&color=purple&logoColor=thistle) 6 | ![](https://img.shields.io/github/stars/rdavydov/rarbg-dump-search?style=for-the-badge&logo=github&color=darkblue&logoColor=aquamarine) 7 | ![](https://img.shields.io/github/forks/rdavydov/rarbg-dump-search?style=for-the-badge&logo=github&color=darkblue&logoColor=aquamarine) 8 | ![](https://img.shields.io/github/watchers/rdavydov/rarbg-dump-search?style=for-the-badge&logo=github&color=darkblue&logoColor=aquamarine) 9 | ![](https://img.shields.io/github/last-commit/rdavydov/rarbg-dump-search?style=for-the-badge&logo=github&color=darkgreen&logoColor=lightgreen) 10 | 11 |
12 | 13 | - Made a DB out of "everything.txt" from 14 | - Made a back-end with FastAPI that talks to the DB and serves requests 15 | - Made a front-end with day/night theme to search the DB via the back-end 16 | - Deployed the front-end to the GitHub Pages 17 | 18 | *Dump is fully migrated into 2 DBs, total number of magnets is **3.468.029**.* 19 | 20 |
21 | 22 | Please support the development: 23 | | | 24 | |-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| 25 | |Buy Me A Coffee (rdavydov)| 26 | 27 | or you can donate crypto: 28 | 29 | | | | 30 | |-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------| 31 | |Donate DOGE | `DAKzncwKkpfPCm1xVU7u2pConpXwX7HS3D` *(DOGE)*| 32 | 33 |
34 | 35 | Day theme: 36 | 37 | ![Day theme](https://raw.githubusercontent.com/rdavydov/rarbg-dump-search/main/screenshot.png) 38 | 39 | Night theme: 40 | 41 | ![Night theme](https://raw.githubusercontent.com/rdavydov/rarbg-dump-search/main/screenshot-night-theme.png) 42 | -------------------------------------------------------------------------------- /backend-2dbs.py: -------------------------------------------------------------------------------- 1 | # run command: TZ='Europe/Moscow' uvicorn backend-2dbs:app --host 0.0.0.0 --port 31337 --log-config uvicorn_logging.conf 2 | 3 | from fastapi import FastAPI 4 | from fastapi.middleware.cors import CORSMiddleware 5 | from fastapi.middleware.httpsredirect import HTTPSRedirectMiddleware 6 | from pymongo import MongoClient 7 | from dotenv import load_dotenv 8 | import os 9 | import re 10 | 11 | load_dotenv() # Load environment variables from .env file 12 | 13 | app = FastAPI() 14 | client_1 = MongoClient(os.getenv("MONGODB_CONNECTION_STRING_1")) 15 | db_1 = client_1[os.getenv("DB_NAME_1")] 16 | collection_1 = db_1[os.getenv("COLLECTION_NAME_1")] 17 | 18 | client_2 = MongoClient(os.getenv("MONGODB_CONNECTION_STRING_2")) 19 | db_2 = client_2[os.getenv("DB_NAME_2")] 20 | collection_2 = db_2[os.getenv("COLLECTION_NAME_2")] 21 | 22 | app.add_middleware( 23 | CORSMiddleware, 24 | # allow_origins=["*"], 25 | allow_origins=[ 26 | "https://rdavydov.github.io", 27 | "https://rar-bg.vercel.app", 28 | ], 29 | allow_credentials=True, 30 | allow_methods=["*"], 31 | allow_headers=["*"], 32 | ) 33 | app.add_middleware(HTTPSRedirectMiddleware) 34 | 35 | 36 | @app.get("/") 37 | async def search(q: str): 38 | keywords = q.split() 39 | regex_patterns = [re.escape(keyword) for keyword in keywords] 40 | 41 | regex_queries = [ 42 | {"magnet_link": {"$regex": pattern, "$options": "i"}} for pattern in regex_patterns 43 | ] 44 | 45 | search_pipeline = [ 46 | {"$match": {"$and": regex_queries}}, 47 | {"$project": {"magnet_link": 1}} 48 | ] 49 | 50 | results_1 = collection_1.aggregate(search_pipeline) 51 | results_2 = collection_2.aggregate(search_pipeline) 52 | 53 | all_results = [] 54 | for result in results_1: 55 | all_results.append(result["magnet_link"]) 56 | for result in results_2: 57 | all_results.append(result["magnet_link"]) 58 | 59 | return {"results": all_results} 60 | -------------------------------------------------------------------------------- /crontab.sh: -------------------------------------------------------------------------------- 1 | service cron start 2 | crontab -l > mycron 3 | echo "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" >> mycron # not always needed, depends on your system 4 | 5 | # The 2>&1 at the end of the command redirects the standard error (stderr) output to the same location as the standard output (stdout). 6 | # With the >> redirection operator used in the command (>> /path/to/uvicorn.log), the output will be appended to the existing log file each time the command is executed. 7 | # In this expression, */12 in the second field means "every 12 hours". 8 | # The pkill -f command sends a signal to the process to terminate it. 9 | # By default, pkill sends the SIGTERM signal, which allows the process to perform a graceful shutdown by handling the signal and cleaning up resources before exiting. 10 | 11 | echo "0 */12 * * * /root/rarbg-dump/restart.sh" >> mycron 12 | 13 | crontab mycron 14 | rm mycron -------------------------------------------------------------------------------- /index.css: -------------------------------------------------------------------------------- 1 | * { 2 | font-family: 'Trebuchet MS', 'Lucida Sans Unicode', 'Lucida Grande', 'Lucida Sans', Arial, sans-serif; 3 | transition: all 0.3s ease; 4 | } 5 | 6 | body { 7 | margin: 0; 8 | padding: 0; 9 | min-height: 100vh; 10 | display: flex; 11 | flex-direction: column; 12 | } 13 | 14 | h1, h2 { 15 | text-align: center; 16 | } 17 | 18 | .search-bar { 19 | display: flex; 20 | align-items: center; 21 | background: linear-gradient(to right, #2e52bd, #1623b0); 22 | padding: 20px; 23 | z-index: 1; 24 | top: 0; 25 | position: sticky; 26 | max-width: 960px; 27 | margin: 0 auto; 28 | border-radius: 5px; 29 | width: 100%; 30 | } 31 | 32 | #search-input { 33 | margin-right: 10px; 34 | padding: 8px; 35 | flex: 1; 36 | border: none; 37 | border-radius: 5px; 38 | background-color: var(--bg-color); 39 | color: var(--text-color); 40 | font-size: 1em; 41 | } 42 | 43 | #search-button { 44 | padding: 8px 16px; 45 | background-color: var(--bg-color); 46 | color: var(--text-color); 47 | border: none; 48 | cursor: pointer; 49 | border-radius: 5px; 50 | } 51 | 52 | #search-button:hover { 53 | filter: invert(1); 54 | } 55 | 56 | #search-button:active { 57 | background-color: red; 58 | } 59 | 60 | #results { 61 | list-style: none; 62 | padding: 0; 63 | margin: 0 auto; 64 | max-width: 960px; 65 | overflow-wrap: break-word; 66 | width: 100%; 67 | } 68 | 69 | #results li { 70 | align-items: center; 71 | justify-content: space-between; 72 | padding: 10px; 73 | text-align: left; 74 | } 75 | 76 | #results li + li { 77 | border-top: 1px solid #1623b0; 78 | } 79 | 80 | #results li:nth-child(odd) { 81 | background-color: var(--odd-color); 82 | } 83 | 84 | #results li:hover { 85 | background-color: var(--hover-color); 86 | } 87 | 88 | #results li a:active { 89 | color: red; 90 | } 91 | 92 | .magnet-emoji { 93 | font-size: small; 94 | margin-left: 5px; 95 | } 96 | 97 | /* Day theme styles */ 98 | body.theme-day { 99 | --bg-color: #f2f2f2; 100 | --text-color: #333; 101 | --odd-color: #d2ecff; 102 | --hover-color: #aafffe; 103 | background-color: var(--bg-color); 104 | color: var(--text-color); 105 | } 106 | 107 | /* Night theme styles */ 108 | body.theme-night { 109 | --bg-color: #050840; 110 | --text-color: #fff; 111 | --odd-color: #1623b0; 112 | --hover-color: #2e52bd; 113 | background-color: var(--bg-color); 114 | color: var(--text-color); 115 | } 116 | 117 | .switch { 118 | position: relative; 119 | display: inline-block; 120 | width: 40px; 121 | height: 20px; 122 | margin-left: 10px; 123 | } 124 | 125 | .switch input { 126 | opacity: 0; 127 | width: 0; 128 | height: 0; 129 | } 130 | 131 | .switch:hover { 132 | filter: invert(1); 133 | } 134 | 135 | .slider { 136 | position: absolute; 137 | cursor: pointer; 138 | top: 0; 139 | left: 0; 140 | right: 0; 141 | bottom: 0; 142 | background-color: #ccc; 143 | -webkit-transition: .4s; 144 | transition: .4s; 145 | border-radius: 20px; 146 | } 147 | 148 | .slider:before { 149 | position: absolute; 150 | content: ""; 151 | height: 16px; 152 | width: 16px; 153 | left: 2px; 154 | bottom: 2px; 155 | background-color: #fff; 156 | -webkit-transition: .4s; 157 | transition: .4s; 158 | border-radius: 50%; 159 | } 160 | 161 | input:checked + .slider { 162 | background-color: #007bff; 163 | } 164 | 165 | input:focus + .slider { 166 | box-shadow: 0 0 1px #007bff; 167 | } 168 | 169 | input:checked + .slider:before { 170 | -webkit-transform: translateX(20px); 171 | -ms-transform: translateX(20px); 172 | transform: translateX(20px); 173 | } 174 | 175 | a { 176 | text-decoration: none; 177 | color: var(--text-color); 178 | } 179 | 180 | footer { 181 | text-align: center; 182 | font-size: small; 183 | margin: 0; 184 | padding: 0; 185 | position: sticky; 186 | bottom: 0; 187 | background-color: var(--bg-color); 188 | color: var(--text-color); 189 | } 190 | 191 | footer a { 192 | display: inline-block; 193 | } 194 | 195 | footer a { 196 | /* border-bottom: 1px solid green; */ 197 | text-decoration: underline; 198 | } 199 | 200 | footer a:hover { 201 | text-decoration: none; 202 | } 203 | 204 | footer img { 205 | transition-duration: 2s; 206 | transition-property: transform; 207 | margin-right: 0.3em; 208 | } 209 | 210 | footer img:hover, 211 | footer a:hover img { 212 | transform: rotate(360deg); 213 | -webkit-transform: rotate(360deg); 214 | } 215 | 216 | footer sup:hover { 217 | font-size: small; 218 | } 219 | 220 | footer p { 221 | margin: 5px; 222 | } 223 | 224 | #donate { 225 | -webkit-animation: glow 1s ease-in-out infinite alternate; 226 | -moz-animation: glow 1s ease-in-out infinite alternate; 227 | animation: glow 1s ease-in-out infinite alternate; 228 | } 229 | 230 | @-webkit-keyframes glow { 231 | from { 232 | text-shadow: 0 0 5px #fff, 0 0 10px #fff, 0 0 20px #00ffff, 0 0 30px #00ffff, 0 0 40px #00ffff, 0 0 50px #00ffff, 0 0 60px #00ffff; 233 | } 234 | to { 235 | text-shadow: 0 0 10px #fff, 0 0 20px #00ffdd, 0 0 30px #00ffdd, 0 0 40px #00ffdd, 0 0 50px #00ffdd, 0 0 60px #00ffdd, 0 0 70px #00ffdd; 236 | } 237 | } 238 | -------------------------------------------------------------------------------- /index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | RARBG Dump Search 7 | 8 | 9 | 10 | 11 | 12 | 13 |

RARBG Dump Search

14 | 22 |

Results will be here

23 | 24 | 32 | 33 | 34 | 65 | 66 | 67 | 68 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | document.addEventListener('DOMContentLoaded', function () { 2 | const themeToggle = document.getElementById('theme-switch'); 3 | const body = document.body; 4 | 5 | // Check if the theme preference exists in localStorage 6 | const savedTheme = localStorage.getItem('theme'); 7 | if (savedTheme) { 8 | // Apply the saved theme preference 9 | body.classList.add(savedTheme); 10 | themeToggle.checked = savedTheme === 'theme-night'; 11 | } 12 | 13 | themeToggle.addEventListener('change', function () { 14 | if (this.checked) { 15 | body.classList.remove('theme-day'); 16 | body.classList.add('theme-night'); 17 | // Save the theme preference to localStorage 18 | localStorage.setItem('theme', 'theme-night'); 19 | } else { 20 | body.classList.remove('theme-night'); 21 | body.classList.add('theme-day'); 22 | // Save the theme preference to localStorage 23 | localStorage.setItem('theme', 'theme-day'); 24 | } 25 | }); 26 | 27 | const searchInput = document.getElementById('search-input'); 28 | const searchButton = document.getElementById('search-button'); 29 | const resultsList = document.getElementById('results'); 30 | const resultsHeader = document.getElementById('results-header'); 31 | 32 | searchButton.addEventListener('click', function () { 33 | const query = searchInput.value.trim().replace(/\s/g, ' '); 34 | 35 | if (query) { 36 | fuzzySearch(query); 37 | } else displayResults(0); 38 | }); 39 | 40 | searchInput.addEventListener('keyup', function (event) { 41 | const query = event.target.value.trim().replace(/\s/g, ' '); 42 | if (event.key === 'Enter') 43 | if (query) { 44 | fuzzySearch(query); 45 | } else displayResults(0); 46 | }); 47 | 48 | function fuzzySearch(query) { 49 | resultsList.innerHTML = '

Searching...

'; 50 | resultsHeader.textContent = 'Results'; 51 | 52 | const url = decodeURL('aHR0cHM6Ly9yYXJiZy5tb29vLmNvbS8/cT0'); 53 | fetch(url + query) 54 | .then(response => response.json()) 55 | .then(data => { 56 | const results = data.results; 57 | displayResults(results); 58 | }) 59 | .catch(error => { 60 | resultsList.innerHTML = `

Error searching: ${error}

`; 61 | console.error('Error searching:', error); 62 | }); 63 | } 64 | 65 | function displayResults(results) { 66 | resultsList.innerHTML = ''; 67 | 68 | if (results.length === 0 || !results) { 69 | resultsList.innerHTML = '

No results found

'; 70 | } else { 71 | resultsHeader.textContent = `Results: ${results.length}`; 72 | 73 | results.forEach(result => { 74 | const magnetLink = document.createElement('a'); 75 | magnetLink.href = result; 76 | magnetLink.onclick = function () { 77 | this.style.color = "gray"; 78 | } 79 | 80 | const dnStartIndex = result.indexOf('&dn=') + 4; 81 | const dnEndIndex = result.indexOf('&', dnStartIndex); 82 | const dn = dnEndIndex !== -1 ? result.substring(dnStartIndex, dnEndIndex) : result.substring(dnStartIndex); 83 | 84 | const li = document.createElement('li'); 85 | li.appendChild(magnetLink); 86 | resultsList.appendChild(li); 87 | 88 | const magnetEmoji = document.createElement('span'); 89 | magnetEmoji.textContent = '🧲'; 90 | magnetEmoji.className = 'magnet-emoji'; 91 | 92 | magnetLink.appendChild(document.createTextNode(dn)); 93 | magnetLink.appendChild(magnetEmoji); 94 | }); 95 | } 96 | } 97 | 98 | function decodeURL(encodedURL) { 99 | const decodedString = atob(encodedURL); 100 | return decodedString; 101 | } 102 | }); 103 | -------------------------------------------------------------------------------- /index_with_escape.js: -------------------------------------------------------------------------------- 1 | document.addEventListener('DOMContentLoaded', function () { 2 | const themeToggle = document.getElementById('theme-switch'); 3 | const body = document.body; 4 | 5 | // Check if the theme preference exists in localStorage 6 | const savedTheme = localStorage.getItem('theme'); 7 | if (savedTheme) { 8 | // Apply the saved theme preference 9 | body.classList.add(savedTheme); 10 | themeToggle.checked = savedTheme === 'theme-night'; 11 | } 12 | 13 | themeToggle.addEventListener('change', function () { 14 | if (this.checked) { 15 | body.classList.remove('theme-day'); 16 | body.classList.add('theme-night'); 17 | // Save the theme preference to localStorage 18 | localStorage.setItem('theme', 'theme-night'); 19 | } else { 20 | body.classList.remove('theme-night'); 21 | body.classList.add('theme-day'); 22 | // Save the theme preference to localStorage 23 | localStorage.setItem('theme', 'theme-day'); 24 | } 25 | }); 26 | 27 | const searchInput = document.getElementById('search-input'); 28 | const searchButton = document.getElementById('search-button'); 29 | const resultsList = document.getElementById('results'); 30 | const resultsHeader = document.getElementById('results-header'); 31 | 32 | searchButton.addEventListener('click', function () { 33 | const query = searchInput.value.trim().replace(/\s/g, ' '); 34 | 35 | if (query) { 36 | fuzzySearch(query); 37 | } else displayResults(0); 38 | }); 39 | 40 | searchInput.addEventListener('keyup', function (event) { 41 | const query = event.target.value.trim().replace(/\s/g, ' '); 42 | if (event.key === 'Enter') 43 | if (query) { 44 | fuzzySearch(query); 45 | } else displayResults(0); 46 | }); 47 | 48 | function escapeSpecialCharacters(string) { 49 | // List of special characters to escape 50 | const specialCharacters = /[.*+?^${}()|[\]\\]/g; 51 | return string.replace(specialCharacters, '\\$&'); 52 | } 53 | 54 | function fuzzySearch(query) { 55 | resultsList.innerHTML = '

Searching...

'; 56 | resultsHeader.textContent = 'Results'; 57 | 58 | const url = decodeURL('aHR0cHM6Ly9yYXJiZy1kdW1wLnJ1bi5nb29ybS5pby8/cT0'); 59 | fetch(url + escapeSpecialCharacters(query), { 60 | mode: 'cors', 61 | headers: { 62 | 'Origin': 'https://rarbg.coomer.party' // Replace with the actual origin of your website 63 | } 64 | }) 65 | .then(response => response.json()) 66 | .then(data => { 67 | const results = data.results; 68 | displayResults(results); 69 | }) 70 | .catch(error => { 71 | resultsList.innerHTML = `

Error searching: ${error}

`; 72 | console.error('Error searching:', error); 73 | }); 74 | } 75 | 76 | function displayResults(results) { 77 | resultsList.innerHTML = ''; 78 | 79 | if (results.length === 0 || !results) { 80 | resultsList.innerHTML = '

No results found

'; 81 | } else { 82 | resultsHeader.textContent = `Results: ${results.length}`; 83 | 84 | results.forEach(result => { 85 | const magnetLink = document.createElement('a'); 86 | magnetLink.href = result; 87 | magnetLink.onclick = function () { 88 | this.style.color = "gray"; 89 | } 90 | 91 | const dnStartIndex = result.indexOf('&dn=') + 4; 92 | const dnEndIndex = result.indexOf('&', dnStartIndex); 93 | const dn = dnEndIndex !== -1 ? result.substring(dnStartIndex, dnEndIndex) : result.substring(dnStartIndex); 94 | 95 | const li = document.createElement('li'); 96 | li.appendChild(magnetLink); 97 | resultsList.appendChild(li); 98 | 99 | const magnetEmoji = document.createElement('span'); 100 | magnetEmoji.textContent = '🧲'; 101 | magnetEmoji.className = 'magnet-emoji'; 102 | 103 | magnetLink.appendChild(document.createTextNode(dn)); 104 | magnetLink.appendChild(magnetEmoji); 105 | }); 106 | } 107 | } 108 | 109 | function decodeURL(encodedURL) { 110 | const decodedString = atob(encodedURL); 111 | return decodedString; 112 | } 113 | }); 114 | -------------------------------------------------------------------------------- /migrate-split.py: -------------------------------------------------------------------------------- 1 | # fast migration script to MongoDB Atlas that splits the magnets into 2 dbs 2 | # good for new empty db filling 3 | # speed will be around 5K doc/s 4 | 5 | import pymongo 6 | from dotenv import load_dotenv 7 | from tqdm import tqdm 8 | import os 9 | 10 | load_dotenv() # Load environment variables from .env file 11 | 12 | # MongoDB Atlas connection details for the first account 13 | connection_string_1 = os.getenv("MONGODB_CONNECTION_STRING_1") 14 | db_name_1 = os.getenv("DB_NAME_1") 15 | collection_name_1 = os.getenv("COLLECTION_NAME_1") 16 | 17 | # MongoDB Atlas connection details for the second account 18 | connection_string_2 = os.getenv("MONGODB_CONNECTION_STRING_2") 19 | db_name_2 = os.getenv("DB_NAME_2") 20 | collection_name_2 = os.getenv("COLLECTION_NAME_2") 21 | 22 | # Connect to the first MongoDB Atlas account 23 | client_1 = pymongo.MongoClient(connection_string_1) 24 | db_1 = client_1[db_name_1] 25 | collection_1 = db_1[collection_name_1] 26 | 27 | # Connect to the second MongoDB Atlas account 28 | client_2 = pymongo.MongoClient(connection_string_2) 29 | db_2 = client_2[db_name_2] 30 | collection_2 = db_2[collection_name_2] 31 | 32 | # Open the "everything.txt" file and migrate data 33 | batch_size = 10000 # Number of documents to insert in each batch 34 | batch_1 = [] 35 | batch_2 = [] 36 | current_batch = 1 # Counter variable for the current batch 37 | # Get the total number of documents 38 | total_documents = sum(1 for _ in open("everything.txt")) 39 | 40 | with tqdm(total=total_documents, unit="doc") as pbar: 41 | pbar.set_postfix( 42 | inserted=0, 43 | remaining=total_documents, 44 | total=total_documents, 45 | refresh=False, 46 | ) 47 | 48 | with open("everything.txt", "r") as file: 49 | for line in file: 50 | magnet_link = line.strip() 51 | 52 | # Create a document and add it to the appropriate batch 53 | document = {"magnet_link": magnet_link} 54 | if current_batch == 1: 55 | batch_1.append(document) 56 | current_batch = 2 57 | else: 58 | batch_2.append(document) 59 | current_batch = 1 60 | 61 | # Insert the batches when they reach the specified size 62 | if len(batch_1) == batch_size: 63 | collection_1.insert_many(batch_1) 64 | batch_1 = [] # Reset the batch 65 | if len(batch_2) == batch_size: 66 | collection_2.insert_many(batch_2) 67 | batch_2 = [] # Reset the batch 68 | 69 | pbar.update(1) # Update the progress bar 70 | pbar.set_postfix( 71 | inserted=pbar.n % batch_size, 72 | remaining=total_documents - pbar.n, 73 | total=total_documents, 74 | refresh=False, 75 | ) 76 | 77 | # Insert the remaining documents in the last batches 78 | if batch_1: 79 | collection_1.insert_many(batch_1) 80 | pbar.update(len(batch_1)) 81 | pbar.set_postfix( 82 | inserted=pbar.n % batch_size, 83 | remaining=total_documents - pbar.n, 84 | total=total_documents, 85 | refresh=False, 86 | ) 87 | if batch_2: 88 | collection_2.insert_many(batch_2) 89 | pbar.update(len(batch_2)) 90 | pbar.set_postfix( 91 | inserted=pbar.n % batch_size, 92 | remaining=total_documents - pbar.n, 93 | total=total_documents, 94 | refresh=False, 95 | ) 96 | 97 | # Close the MongoDB connections 98 | client_1.close() 99 | client_2.close() 100 | -------------------------------------------------------------------------------- /old-and-experiments/backend-1db-regex.py: -------------------------------------------------------------------------------- 1 | # run command: TZ='Europe/Moscow' uvicorn backend-2dbs:app --host 0.0.0.0 --port 31337 --log-config uvicorn_logging.conf 2 | 3 | from fastapi import FastAPI 4 | from fastapi.middleware.cors import CORSMiddleware 5 | from pymongo import MongoClient 6 | from dotenv import load_dotenv 7 | import os 8 | import re 9 | 10 | load_dotenv() # Load environment variables from .env file 11 | 12 | app = FastAPI() 13 | client = MongoClient(os.getenv("MONGODB_CONNECTION_STRING")) 14 | db = client[os.getenv("DB_NAME")] 15 | collection = db[os.getenv("COLLECTION_NAME")] 16 | 17 | app.add_middleware(CORSMiddleware, 18 | allow_origins=["*"], 19 | allow_credentials=True, 20 | allow_methods=["*"], 21 | allow_headers=["*"]) 22 | 23 | 24 | @app.get("/search") 25 | async def search(query: str): 26 | keywords = query.split() 27 | regex_pattern = ".*".join(re.escape(keyword) for keyword in keywords) 28 | results = collection.find( 29 | {"magnet_link": {"$regex": regex_pattern, "$options": "i"}} 30 | ) 31 | return {"results": [result["magnet_link"] for result in results]} 32 | -------------------------------------------------------------------------------- /old-and-experiments/backend-2dbs-regex.py: -------------------------------------------------------------------------------- 1 | # run command: TZ='Europe/Moscow' uvicorn backend-2dbs:app --host 0.0.0.0 --port 31337 --log-config uvicorn_logging.conf 2 | 3 | from fastapi import FastAPI 4 | from fastapi.middleware.cors import CORSMiddleware 5 | from pymongo import MongoClient 6 | from dotenv import load_dotenv 7 | import os 8 | import re 9 | 10 | load_dotenv() # Load environment variables from .env file 11 | 12 | app = FastAPI() 13 | client_1 = MongoClient(os.getenv("MONGODB_CONNECTION_STRING_1")) 14 | db_1 = client_1[os.getenv("DB_NAME_1")] 15 | collection_1 = db_1[os.getenv("COLLECTION_NAME_1")] 16 | 17 | client_2 = MongoClient(os.getenv("MONGODB_CONNECTION_STRING_2")) 18 | db_2 = client_2[os.getenv("DB_NAME_2")] 19 | collection_2 = db_2[os.getenv("COLLECTION_NAME_2")] 20 | 21 | app.add_middleware(CORSMiddleware, 22 | allow_origins=["*"], 23 | allow_credentials=True, 24 | allow_methods=["*"], 25 | allow_headers=["*"]) 26 | 27 | 28 | @app.get("/search") 29 | async def search(query: str): 30 | keywords = query.split() 31 | regex_pattern = ".*".join(re.escape(keyword) for keyword in keywords) 32 | 33 | results_1 = collection_1.find( 34 | {"magnet_link": {"$regex": regex_pattern, "$options": "i"}} 35 | ) 36 | results_2 = collection_2.find( 37 | {"magnet_link": {"$regex": regex_pattern, "$options": "i"}} 38 | ) 39 | 40 | all_results = [] 41 | for result in results_1: 42 | all_results.append(result["magnet_link"]) 43 | for result in results_2: 44 | all_results.append(result["magnet_link"]) 45 | 46 | return {"results": all_results} 47 | -------------------------------------------------------------------------------- /old-and-experiments/migrate-index.py: -------------------------------------------------------------------------------- 1 | # Fast migration script to MongoDB Atlas that splits magnets into 2 DBs. 2 | # This script is meant to be run on new empty MongoDB Atlas instances for the first time DBs filling. 3 | # It includes the text indexing approach using MongoDB's text search feature. 4 | # This script creates text indexes on the magnet_link field for both collections before migrating the data. 5 | # This will enable faster text search on the indexed field in MongoDB. 6 | 7 | # Speed will be around 5K doc/s. 8 | 9 | import pymongo 10 | from dotenv import load_dotenv 11 | from tqdm import tqdm 12 | import os 13 | 14 | load_dotenv() # Load environment variables from .env file 15 | 16 | # MongoDB Atlas connection details for the first account 17 | connection_string_1 = os.getenv("MONGODB_CONNECTION_STRING_1") 18 | db_name_1 = os.getenv("DB_NAME_1") 19 | collection_name_1 = os.getenv("COLLECTION_NAME_1") 20 | 21 | # MongoDB Atlas connection details for the second account 22 | connection_string_2 = os.getenv("MONGODB_CONNECTION_STRING_2") 23 | db_name_2 = os.getenv("DB_NAME_2") 24 | collection_name_2 = os.getenv("COLLECTION_NAME_2") 25 | 26 | # Connect to the first MongoDB Atlas account 27 | client_1 = pymongo.MongoClient(connection_string_1) 28 | db_1 = client_1[db_name_1] 29 | collection_1 = db_1[collection_name_1] 30 | 31 | # Connect to the second MongoDB Atlas account 32 | client_2 = pymongo.MongoClient(connection_string_2) 33 | db_2 = client_2[db_name_2] 34 | collection_2 = db_2[collection_name_2] 35 | 36 | # Create text indexes on the "magnet_link" field 37 | collection_1.create_index([("magnet_link", "text")]) 38 | collection_2.create_index([("magnet_link", "text")]) 39 | 40 | # Open the "everything.txt" file and migrate data 41 | batch_size = 10000 # Number of documents to insert in each batch 42 | batch_1 = [] 43 | batch_2 = [] 44 | current_batch = 1 # Counter variable for the current batch 45 | # Get the total number of documents 46 | total_documents = sum(1 for _ in open("everything.txt")) 47 | 48 | with tqdm(total=total_documents, unit="doc") as pbar: 49 | pbar.set_postfix( 50 | inserted=0, 51 | remaining=total_documents, 52 | total=total_documents, 53 | refresh=False, 54 | ) 55 | 56 | with open("everything.txt", "r") as file: 57 | for line in file: 58 | magnet_link = line.strip() 59 | 60 | # Create a document and add it to the appropriate batch 61 | document = {"magnet_link": magnet_link} 62 | if current_batch == 1: 63 | batch_1.append(document) 64 | current_batch = 2 65 | else: 66 | batch_2.append(document) 67 | current_batch = 1 68 | 69 | # Insert the batches when they reach the specified size 70 | if len(batch_1) == batch_size: 71 | collection_1.insert_many(batch_1) 72 | batch_1 = [] # Reset the batch 73 | if len(batch_2) == batch_size: 74 | collection_2.insert_many(batch_2) 75 | batch_2 = [] # Reset the batch 76 | 77 | pbar.update(1) # Update the progress bar 78 | pbar.set_postfix( 79 | inserted=pbar.n % batch_size, 80 | remaining=total_documents - pbar.n, 81 | total=total_documents, 82 | refresh=False, 83 | ) 84 | 85 | # Insert the remaining documents in the last batches 86 | if batch_1: 87 | collection_1.insert_many(batch_1) 88 | pbar.update(len(batch_1)) 89 | pbar.set_postfix( 90 | inserted=pbar.n % batch_size, 91 | remaining=total_documents - pbar.n, 92 | total=total_documents, 93 | refresh=False, 94 | ) 95 | if batch_2: 96 | collection_2.insert_many(batch_2) 97 | pbar.update(len(batch_2)) 98 | pbar.set_postfix( 99 | inserted=pbar.n % batch_size, 100 | remaining=total_documents - pbar.n, 101 | total=total_documents, 102 | refresh=False, 103 | ) 104 | 105 | # Close the MongoDB connections 106 | client_1.close() 107 | client_2.close() 108 | -------------------------------------------------------------------------------- /old-and-experiments/migrate-or-update.py: -------------------------------------------------------------------------------- 1 | # very slow because of the overhead of the MongoDB connection 2 | # speed will be around 1.78 lines/s 3 | 4 | import pymongo 5 | from dotenv import load_dotenv 6 | import os 7 | from tqdm import tqdm 8 | 9 | load_dotenv() # Load environment variables from .env file 10 | 11 | # MongoDB Atlas connection details 12 | connection_string = os.getenv("MONGODB_CONNECTION_STRING") 13 | db_name = os.getenv("DB_NAME") 14 | collection_name = os.getenv("COLLECTION_NAME") 15 | 16 | # Connect to MongoDB Atlas 17 | client = pymongo.MongoClient(connection_string) 18 | db = client[db_name] 19 | collection = db[collection_name] 20 | 21 | # Open the "everything.txt" file and migrate data 22 | with open("everything.txt", "r") as file: 23 | batch_size = 10000 # Number of documents to process in each batch 24 | total_count = 0 25 | skipped_count = 0 26 | progress_bar = tqdm(total=sum(1 for _ in file), unit='lines') 27 | 28 | file.seek(0) # Reset the file pointer to the beginning 29 | 30 | for line in file: 31 | magnet_link = line.strip() 32 | 33 | # Check if the magnet link already exists in the collection 34 | if collection.count_documents({"magnet_link": magnet_link}) > 0: 35 | skipped_count += 1 36 | progress_bar.update(1) 37 | continue 38 | 39 | # Create a document and insert into MongoDB 40 | document = {"magnet_link": magnet_link} 41 | collection.insert_one(document) 42 | 43 | total_count += 1 44 | progress_bar.update(1) 45 | if total_count % batch_size == 0: 46 | tqdm.write( 47 | f"Processed: {total_count} | Skipped: {skipped_count} | Remaining: {progress_bar.total - progress_bar.n}") 48 | 49 | progress_bar.close() 50 | 51 | print("Migration complete!") 52 | print(f"Total documents: {total_count}") 53 | print(f"Skipped documents (already existing): {skipped_count}") 54 | 55 | # Close the MongoDB connection 56 | client.close() 57 | -------------------------------------------------------------------------------- /old-and-experiments/migrate.py: -------------------------------------------------------------------------------- 1 | # fast migration script to MongoDB Atlas because of the batches 2 | # good for new empty db filling 3 | # speed will be around 1076.34 doc/s 4 | 5 | import pymongo 6 | from dotenv import load_dotenv 7 | from tqdm import tqdm 8 | import os 9 | 10 | load_dotenv() # Load environment variables from .env file 11 | 12 | # MongoDB Atlas connection details 13 | connection_string = os.getenv("MONGODB_CONNECTION_STRING") 14 | db_name = os.getenv("DB_NAME") 15 | collection_name = os.getenv("COLLECTION_NAME") 16 | 17 | # Connect to MongoDB Atlas 18 | client = pymongo.MongoClient(connection_string) 19 | db = client[db_name] 20 | collection = db[collection_name] 21 | 22 | # Open the "everything.txt" file and migrate data 23 | batch_size = 10000 # Number of documents to insert in each batch 24 | batch = [] 25 | # Get the total number of documents 26 | total_documents = sum(1 for _ in open("everything.txt")) 27 | 28 | with tqdm(total=total_documents, ncols=80, unit="doc") as pbar: 29 | pbar.set_postfix( 30 | inserted=0, 31 | remaining=total_documents, 32 | total=total_documents, 33 | refresh=False, 34 | ) 35 | 36 | with open("everything.txt", "r") as file: 37 | for line in file: 38 | magnet_link = line.strip() 39 | 40 | # Create a document and add it to the batch 41 | document = {"magnet_link": magnet_link} 42 | batch.append(document) 43 | 44 | # Insert the batch when it reaches the specified size 45 | if len(batch) == batch_size: 46 | collection.insert_many(batch) 47 | batch = [] # Reset the batch 48 | 49 | pbar.update(1) # Update the progress bar 50 | pbar.set_postfix( 51 | inserted=pbar.n % batch_size, 52 | remaining=total_documents - pbar.n, 53 | total=total_documents, 54 | refresh=False, 55 | ) 56 | 57 | # Insert the remaining documents in the last batch 58 | if batch: 59 | collection.insert_many(batch) 60 | pbar.update(len(batch)) 61 | pbar.set_postfix( 62 | inserted=pbar.n % batch_size, 63 | remaining=0, 64 | total=total_documents, 65 | refresh=False, 66 | ) 67 | 68 | # Close the MongoDB connection 69 | client.close() 70 | -------------------------------------------------------------------------------- /rarbg16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rdavydov/rarbg-dump-search/3b52ccb301727eeace2c0969abbe5e28e80cf97e/rarbg16.png -------------------------------------------------------------------------------- /rarbg32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rdavydov/rarbg-dump-search/3b52ccb301727eeace2c0969abbe5e28e80cf97e/rarbg32.png -------------------------------------------------------------------------------- /restart.sh: -------------------------------------------------------------------------------- 1 | # pkill -f "uvicorn backend-2dbs:app" 2 | # sleep 2 3 | # uvicorn backend-2dbs:app --host 0.0.0.0 --port 31337 --app-dir /root/rarbg-dump 4 | 5 | cd /root/rarbg-dump 6 | 7 | tmux kill-ses -t uvicorn 8 | sleep 3 9 | 10 | tmux new -s uvicorn -d 11 | tmux send-keys 'TZ='Europe/Moscow' uvicorn backend-2dbs:app --host 0.0.0.0 --port 31337 --log-config uvicorn_logging.conf' C-m -------------------------------------------------------------------------------- /screenshot-night-theme.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rdavydov/rarbg-dump-search/3b52ccb301727eeace2c0969abbe5e28e80cf97e/screenshot-night-theme.png -------------------------------------------------------------------------------- /screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rdavydov/rarbg-dump-search/3b52ccb301727eeace2c0969abbe5e28e80cf97e/screenshot.png -------------------------------------------------------------------------------- /uvicorn_logging.conf: -------------------------------------------------------------------------------- 1 | [loggers] 2 | keys=root 3 | 4 | [handlers] 5 | keys=consoleHandler,fileHandler 6 | 7 | [formatters] 8 | keys=consoleFormatter,fileFormatter 9 | 10 | [logger_root] 11 | level=INFO 12 | handlers=consoleHandler,fileHandler 13 | 14 | [handler_consoleHandler] 15 | class=StreamHandler 16 | formatter=consoleFormatter 17 | args=(sys.stdout,) 18 | 19 | [handler_fileHandler] 20 | class=FileHandler 21 | formatter=fileFormatter 22 | args=('uvicorn.log', 'a') 23 | 24 | [formatter_consoleFormatter] 25 | format=%(asctime)s - %(levelname)s - "%(message)s" 26 | datefmt=%Y-%m-%d %H:%M:%S 27 | 28 | [formatter_fileFormatter] 29 | format=%(asctime)s - %(levelname)s - "%(message)s" 30 | datefmt=%Y-%m-%d %H:%M:%S 31 | -------------------------------------------------------------------------------- /watchdog-and-restart.py: -------------------------------------------------------------------------------- 1 | # run command: TZ='Europe/Moscow' python watchdog-and-restart.py 2 | 3 | import subprocess 4 | import time 5 | import logging 6 | from datetime import datetime 7 | 8 | 9 | def setup_logger(): 10 | logger = logging.getLogger("watchdog") 11 | logger.setLevel(logging.INFO) 12 | 13 | # Create a formatter with timezone information 14 | formatter = logging.Formatter( 15 | fmt="%(asctime)s - %(levelname)s - %(message)s", 16 | datefmt="%Y-%m-%d %H:%M:%S %Z%z", 17 | ) 18 | 19 | # Create a StreamHandler for logging to the console 20 | stream_handler = logging.StreamHandler() 21 | stream_handler.setLevel(logging.INFO) 22 | stream_handler.setFormatter(formatter) 23 | logger.addHandler(stream_handler) 24 | 25 | # Create a FileHandler for logging to a file 26 | file_handler = logging.FileHandler("watchdog-and-restart.log") 27 | file_handler.setLevel(logging.INFO) 28 | file_handler.setFormatter(formatter) 29 | logger.addHandler(file_handler) 30 | 31 | return logger 32 | 33 | 34 | def run_uvicorn(logger): 35 | command = [ 36 | "uvicorn", 37 | "backend-2dbs:app", 38 | "--host", 39 | "0.0.0.0", 40 | "--port", 41 | "31337", 42 | "--log-config", 43 | "uvicorn_logging.conf" 44 | ] 45 | subprocess.run(command) 46 | 47 | 48 | def restart_uvicorn(logger): 49 | logger.info("Restarting Uvicorn process...") 50 | process = subprocess.Popen( 51 | ["pgrep", "-f", "uvicorn backend-2dbs:app"], stdout=subprocess.PIPE) 52 | output, _ = process.communicate() 53 | if process.returncode == 0: 54 | process_ids = output.decode().strip().split("\n") 55 | for process_id in process_ids: 56 | subprocess.run(["kill", process_id]) 57 | run_uvicorn(logger) 58 | 59 | 60 | def watchdog(): 61 | logger = setup_logger() 62 | while True: 63 | current_time = datetime.now() 64 | # restarts uvicorn every 4 hours 65 | if current_time.hour % 4 == 0 and current_time.minute == 0: 66 | restart_uvicorn(logger) 67 | else: 68 | process = subprocess.Popen( 69 | ["pgrep", "-f", "uvicorn backend-2dbs:app"], stdout=subprocess.PIPE) 70 | # Since we are only interested in whether the command executed successfully or not, and not the actual output, 71 | # we assign the output stream to output and ignore the error stream by using the _ variable. By convention, 72 | # the _ variable is often used as a throwaway variable to indicate that a value 73 | # is intentionally being ignored or not used in the code. 74 | output, _ = process.communicate() 75 | if process.returncode != 0: 76 | logger.info("Uvicorn process is not running. Restarting...") 77 | run_uvicorn(logger) 78 | time.sleep(10) # sleep for 10 seconds 79 | 80 | 81 | if __name__ == "__main__": 82 | watchdog() 83 | -------------------------------------------------------------------------------- /watchdog.py: -------------------------------------------------------------------------------- 1 | # run command: TZ='Europe/Moscow' python watchdog.py 2 | 3 | import subprocess 4 | import time 5 | import logging 6 | 7 | 8 | def setup_logger(): 9 | logger = logging.getLogger("watchdog") 10 | logger.setLevel(logging.INFO) 11 | 12 | # Create a formatter with timezone information 13 | formatter = logging.Formatter( 14 | fmt="%(asctime)s - %(levelname)s - %(message)s", 15 | datefmt="%Y-%m-%d %H:%M:%S %Z%z", 16 | ) 17 | 18 | # Create a StreamHandler for logging to the console 19 | stream_handler = logging.StreamHandler() 20 | stream_handler.setLevel(logging.INFO) 21 | stream_handler.setFormatter(formatter) 22 | logger.addHandler(stream_handler) 23 | 24 | # Create a FileHandler for logging to a file 25 | file_handler = logging.FileHandler("watchdog.log") 26 | file_handler.setLevel(logging.INFO) 27 | file_handler.setFormatter(formatter) 28 | logger.addHandler(file_handler) 29 | 30 | return logger 31 | 32 | 33 | def run_uvicorn(logger): 34 | command = [ 35 | "uvicorn", 36 | "backend-2dbs:app", 37 | "--host", 38 | "0.0.0.0", 39 | "--port", 40 | "31337", 41 | "--log-config", 42 | "uvicorn_logging.conf" 43 | ] 44 | subprocess.run(command) 45 | 46 | 47 | def watchdog(): 48 | logger = setup_logger() 49 | while True: 50 | process = subprocess.Popen( 51 | ["pgrep", "-f", "uvicorn backend-2dbs:app"], stdout=subprocess.PIPE) 52 | # Since we are only interested in whether the command executed successfully or not, and not the actual output, 53 | # we assign the output stream to output and ignore the error stream by using the _ variable. By convention, 54 | # the _ variable is often used as a throwaway variable to indicate that a value 55 | # is intentionally being ignored or not used in the code. 56 | output, _ = process.communicate() 57 | if process.returncode != 0: 58 | logger.info("Uvicorn process is not running. Restarting...") 59 | run_uvicorn(logger) 60 | time.sleep(10) # sleep for 10 seconds 61 | 62 | 63 | if __name__ == "__main__": 64 | watchdog() 65 | --------------------------------------------------------------------------------