├── .file_contents_output └── latest_file_contents.txt ├── .gitignore ├── .repo_summary └── consolidated_summary.txt ├── Dockerfile ├── README.md ├── backup-data.sh ├── clean-data.py ├── count.py ├── docker-build.sh ├── docker-compose.yaml ├── docker-run.sh ├── fetch.py ├── fetch_editors-picks.py ├── fetch_indicators.py ├── fetch_more.py ├── fetch_scripts.py ├── fetch_strategies.py └── requirements.txt /.file_contents_output/latest_file_contents.txt: -------------------------------------------------------------------------------- 1 | 2 | ======================================== 3 | File: /home/hasnocool/Github/active/tradingview-script-downloader/count.py 4 | ======================================== 5 | #!/usr/bin/python 6 | 7 | import os 8 | 9 | def get_directory_size(directory): 10 | total_size = 0 11 | for dirpath, _, filenames in os.walk(directory): 12 | for filename in filenames: 13 | file_path = os.path.join(dirpath, filename) 14 | total_size += os.path.getsize(file_path) 15 | return total_size 16 | 17 | def format_size(size): 18 | # Adapted from https://stackoverflow.com/a/1094933 19 | units = ["B", "KB", "MB", "GB", "TB"] 20 | index = 0 21 | while size >= 1024 and index < len(units) - 1: 22 | size /= 1024 23 | index += 1 24 | return f"{size:.2f} {units[index]}" 25 | 26 | # Specify the directory path you want to count files in 27 | directory_path = "PineScripts" 28 | 29 | file_count = sum([len(files) for _, _, files in os.walk(directory_path)]) 30 | directory_size = get_directory_size(directory_path) 31 | 32 | print(f"The number of files in '{directory_path}' is: {file_count}") 33 | print(f"The total size of the directory is: {format_size(directory_size)}") 34 | 35 | ======================================== 36 | File: /home/hasnocool/Github/active/tradingview-script-downloader/fetch_more.py 37 | ======================================== 38 | #!/usr/bin/python 39 | 40 | import os 41 | import requests 42 | from bs4 import BeautifulSoup 43 | from selenium import webdriver 44 | from selenium.webdriver.common.by import By 45 | from selenium.common.exceptions import NoSuchElementException 46 | from selenium.webdriver.firefox.options import Options 47 | import time 48 | import concurrent.futures 49 | 50 | # Set the path to the geckodriver executable 51 | geckodriver_path = os.path.join(os.getcwd(), 'geckodriver') 52 | 53 | # Configure Firefox options for headless browsing 54 | firefox_options = Options() 55 | firefox_options.add_argument("--headless") 56 | 57 | # Launch Firefox WebDriver with headless options 58 | driver = webdriver.Firefox(executable_path=geckodriver_path, options=firefox_options) 59 | 60 | base_url = 'https://www.tradingview.com/scripts/' 61 | 62 | def download_page(page): 63 | url = base_url + f'page-{page}/?script_type=strategies&script_access=open&sort=month_popular&route_range=1' 64 | response = requests.get(url) 65 | soup = BeautifulSoup(response.content, 'html.parser') 66 | 67 | script_items = soup.find_all('div', class_='tv-feed__item') 68 | if not script_items: 69 | return # Return if there are no more script items on the page 70 | 71 | for item in script_items: 72 | if item.get('data-widget-type') == 'idea': 73 | script_data = item.get('data-widget-data') 74 | script_name = item.find('a', class_='tv-widget-idea__title').text.strip() 75 | script_author = item.find('span', class_='tv-card-user-info__name').text.strip() 76 | script_description = item.find('p', class_='tv-widget-idea__description-row').text.strip() 77 | print('Script Name:', script_name) 78 | print('Author:', script_author) 79 | print('Description:', script_description) 80 | print('---') 81 | 82 | # Extract PineScript code 83 | script_url = item.find('a', class_='tv-widget-idea__title').get('href') 84 | script_full_url = 'https://www.tradingview.com' + script_url 85 | 86 | # Open the script page with Firefox 87 | driver.get(script_full_url) 88 | time.sleep(1) # Add a delay to allow the page to fully load 89 | 90 | try: 91 | # Find and expand the collapse buttons 92 | collapse_buttons = driver.find_elements(By.CLASS_NAME, "collapseBtn") 93 | for button in collapse_buttons: 94 | driver.execute_script("arguments[0].click();", button) 95 | time.sleep(0.5) # Add a delay after expanding each collapse button 96 | 97 | # Get the PineScript code element 98 | script_code_element = driver.find_element(By.CLASS_NAME, "tv-chart-view__script-wrap") 99 | 100 | # Get the PineScript code text using innerText property 101 | script_code = script_code_element.get_property("innerText").strip() 102 | 103 | # Create directory if it doesn't exist 104 | directory = 'PineScripts' 105 | os.makedirs(directory, exist_ok=True) 106 | 107 | # Replace invalid characters in script name 108 | invalid_characters = '/\\?%*:|"<>' 109 | for char in invalid_characters: 110 | script_name = script_name.replace(char, '-') 111 | 112 | # Save PineScript to a file with .pine extension 113 | filename = os.path.join(directory, f"{script_name}.pine") 114 | with open(filename, 'w') as file: 115 | file.write(f"Script Name: {script_name}\n") 116 | file.write(f"Author: {script_author}\n") 117 | file.write(f"Description: {script_description}\n") 118 | file.write("PineScript code:\n") 119 | file.write(script_code) 120 | 121 | print(f"Saved PineScript code to {filename}\n") 122 | 123 | except NoSuchElementException: 124 | print(f"Element not found. Skipping script: {script_name}") 125 | 126 | # Number of pages to download concurrently 127 | num_pages = 3 128 | 129 | with concurrent.futures.ThreadPoolExecutor() as executor: 130 | page_range = range(1, num_pages + 1) 131 | executor.map(download_page, page_range) 132 | 133 | # Close the WebDriver 134 | driver.quit() 135 | 136 | ======================================== 137 | File: /home/hasnocool/Github/active/tradingview-script-downloader/clean-data.py 138 | ======================================== 139 | #!/usr/bin/python 140 | 141 | import os 142 | import re 143 | 144 | # Function to clean the PineScript 145 | def clean_pinescript(script): 146 | lines = script.split('\n') 147 | 148 | # Remove numbered lines dynamically 149 | lines = [line for line in lines if not re.match(r"^\d+\s*$", line.strip())] 150 | 151 | # Find the index of "Copy code" line 152 | copy_code_index = -1 153 | for i, line in enumerate(lines): 154 | if line.strip() == 'Copy code': 155 | copy_code_index = i 156 | break 157 | 158 | # Remove the last line if it contains the word "Expand" followed by parentheses and a number 159 | if re.match(r"Expand \(\d+ lines\)", lines[-1].strip()): 160 | lines = lines[:-1] 161 | 162 | # Add comments to the lines preceding "Copy code" 163 | comment_lines = [] 164 | for i in range(copy_code_index): 165 | line = lines[i].strip() 166 | if line: 167 | comment_lines.append(f"// {line}") 168 | lines = comment_lines + lines[copy_code_index + 1:] 169 | 170 | # Remove unnecessary lines and format PineScript code properly 171 | cleaned_script = '\n'.join(lines).strip() 172 | cleaned_script = cleaned_script.replace('\n\n', '\n') 173 | cleaned_script = cleaned_script.replace('=', ' = ') 174 | cleaned_script = cleaned_script.replace(':', ': ') 175 | cleaned_script = cleaned_script.replace(' ', ' ') 176 | return cleaned_script 177 | 178 | # Specify the folder paths 179 | input_folder = 'PineScripts' 180 | output_folder = 'PineScripts_Cleaned' 181 | 182 | # Create the output folder if it doesn't exist 183 | os.makedirs(output_folder, exist_ok=True) 184 | 185 | # Iterate through the files in the input folder 186 | for file_name in os.listdir(input_folder): 187 | if file_name.endswith('.pine'): 188 | input_file_path = os.path.join(input_folder, file_name) 189 | output_file_path = os.path.join(output_folder, file_name) 190 | 191 | with open(input_file_path, 'r') as file: 192 | script = file.read() 193 | 194 | cleaned_script = clean_pinescript(script) 195 | 196 | with open(output_file_path, 'w') as cleaned_file: 197 | cleaned_file.write(cleaned_script) 198 | 199 | print(f"Cleaned script saved to: {output_file_path}") 200 | 201 | ======================================== 202 | File: /home/hasnocool/Github/active/tradingview-script-downloader/fetch_editors-picks.py 203 | ======================================== 204 | import os 205 | import asyncio 206 | import aiohttp 207 | from bs4 import BeautifulSoup 208 | from selenium import webdriver 209 | from selenium.webdriver.common.by import By 210 | from selenium.common.exceptions import NoSuchElementException 211 | from selenium.webdriver.firefox.options import Options 212 | import time 213 | 214 | # Set the path to the geckodriver executable 215 | geckodriver_path = os.path.join(os.getcwd(), 'geckodriver') 216 | 217 | # Configure Firefox options for headless browsing 218 | firefox_options = Options() 219 | firefox_options.add_argument("--headless") 220 | 221 | # Launch Firefox WebDriver with headless options 222 | driver = webdriver.Firefox(executable_path=geckodriver_path, options=firefox_options) 223 | 224 | base_url = 'https://www.tradingview.com/scripts/editors-picks/' 225 | 226 | async def download_page(session, page): 227 | url = base_url + f'page-{page}/?script_type=strategies&script_access=open&sort=month_popular&route_range=1' 228 | 229 | async with session.get(url) as response: 230 | content = await response.text() 231 | 232 | soup = BeautifulSoup(content, 'html.parser') 233 | 234 | script_items = soup.find_all('div', class_='tv-feed__item') 235 | if not script_items: 236 | return # Return if there are no more script items on the page 237 | 238 | for item in script_items: 239 | if item.get('data-widget-type') == 'idea': 240 | script_data = item.get('data-widget-data') 241 | script_name = item.find('a', class_='tv-widget-idea__title').text.strip() 242 | script_author = item.find('span', class_='tv-card-user-info__name').text.strip() 243 | script_description = item.find('p', class_='tv-widget-idea__description-row').text.strip() 244 | print('Script Name:', script_name) 245 | print('Author:', script_author) 246 | print('Description:', script_description) 247 | print('---') 248 | 249 | # Replace invalid characters in script name 250 | invalid_characters = '/\\?%*:|"<>' 251 | for char in invalid_characters: 252 | script_name = script_name.replace(char, '-') 253 | 254 | # Create directory if it doesn't exist 255 | directory = 'PineScripts_Editors-Picks' 256 | os.makedirs(directory, exist_ok=True) 257 | 258 | # Check if the file already exists 259 | filename = os.path.join(directory, f"{script_name}.pine") 260 | if os.path.exists(filename): 261 | print(f"Skipping page {page}. File already exists: {filename}\n") 262 | continue 263 | 264 | # Extract PineScript code 265 | script_url = item.find('a', class_='tv-widget-idea__title').get('href') 266 | script_full_url = 'https://www.tradingview.com' + script_url 267 | 268 | # Open the script page with Firefox 269 | driver.get(script_full_url) 270 | time.sleep(2) # Add a delay to allow the page to fully load 271 | 272 | try: 273 | # Find and expand the collapse buttons 274 | collapse_buttons = driver.find_elements(By.CLASS_NAME, "collapseBtn") 275 | for button in collapse_buttons: 276 | driver.execute_script("arguments[0].click();", button) 277 | time.sleep(1) # Add a delay after expanding each collapse button 278 | 279 | # Get the PineScript code element 280 | script_code_element = driver.find_element(By.CLASS_NAME, "tv-chart-view__script-wrap") 281 | 282 | # Get the PineScript code text using innerText property 283 | script_code = script_code_element.get_property("innerText").strip() 284 | 285 | # Save PineScript to a file with .pine extension 286 | with open(filename, 'w') as file: 287 | file.write(f"Script Name: {script_name}\n") 288 | file.write(f"Author: {script_author}\n") 289 | file.write(f"Description: {script_description}\n") 290 | file.write("PineScript code:\n") 291 | file.write(script_code) 292 | 293 | print(f"Saved PineScript code to {filename}\n") 294 | 295 | except NoSuchElementException: 296 | print(f"Element not found. Skipping script: {script_name}") 297 | 298 | async def process_pages(): 299 | async with aiohttp.ClientSession() as session: 300 | page = 1 301 | while True: 302 | await download_page(session, page) 303 | page += 1 304 | 305 | # Number of pages to download concurrently 306 | num_concurrent_pages = 3 307 | 308 | # Run the event loop 309 | loop = asyncio.get_event_loop() 310 | tasks = [process_pages() for _ in range(num_concurrent_pages)] 311 | loop.run_until_complete(asyncio.gather(*tasks)) 312 | 313 | # Close the WebDriver 314 | driver.quit() 315 | 316 | ======================================== 317 | File: /home/hasnocool/Github/active/tradingview-script-downloader/fetch.py 318 | ======================================== 319 | #!/usr/bin/python 320 | 321 | import os 322 | import requests 323 | from bs4 import BeautifulSoup 324 | from selenium import webdriver 325 | from selenium.webdriver.common.by import By 326 | from selenium.common.exceptions import NoSuchElementException 327 | import time 328 | 329 | # Set the path to the geckodriver executable 330 | geckodriver_path = os.path.join(os.getcwd(), 'geckodriver') 331 | 332 | # Launch Firefox WebDriver 333 | driver = webdriver.Firefox(executable_path=geckodriver_path) 334 | 335 | base_url = 'https://www.tradingview.com/scripts/' 336 | page = 1 337 | 338 | while True: 339 | url = base_url + f'page-{page}/?script_type=strategies&script_access=open&sort=month_popular&route_range=1' 340 | response = requests.get(url) 341 | soup = BeautifulSoup(response.content, 'html.parser') 342 | 343 | script_items = soup.find_all('div', class_='tv-feed__item') 344 | if not script_items: 345 | break # Break the loop if there are no more script items on the page 346 | 347 | for item in script_items: 348 | if item.get('data-widget-type') == 'idea': 349 | script_data = item.get('data-widget-data') 350 | script_name = item.find('a', class_='tv-widget-idea__title').text.strip() 351 | script_author = item.find('span', class_='tv-card-user-info__name').text.strip() 352 | script_description = item.find('p', class_='tv-widget-idea__description-row').text.strip() 353 | print('Script Name:', script_name) 354 | print('Author:', script_author) 355 | print('Description:', script_description) 356 | print('---') 357 | 358 | # Extract PineScript code 359 | script_url = item.find('a', class_='tv-widget-idea__title').get('href') 360 | script_full_url = 'https://www.tradingview.com' + script_url 361 | 362 | # Open the script page with Firefox 363 | driver.get(script_full_url) 364 | time.sleep(2) # Add a delay to allow the page to fully load 365 | 366 | try: 367 | # Find and expand the collapse buttons 368 | collapse_buttons = driver.find_elements(By.CLASS_NAME, "collapseBtn") 369 | for button in collapse_buttons: 370 | driver.execute_script("arguments[0].click();", button) 371 | time.sleep(1) # Add a delay after expanding each collapse button 372 | 373 | # Get the PineScript code element 374 | script_code_element = driver.find_element(By.CLASS_NAME, "tv-chart-view__script-wrap") 375 | 376 | # Get the PineScript code text using innerText property 377 | script_code = script_code_element.get_property("innerText").strip() 378 | 379 | # Create directory if it doesn't exist 380 | directory = 'PineScripts' 381 | os.makedirs(directory, exist_ok=True) 382 | 383 | # Replace invalid characters in script name 384 | invalid_characters = '/\\?%*:|"<>' 385 | for char in invalid_characters: 386 | script_name = script_name.replace(char, '-') 387 | 388 | # Save PineScript to a file with .pine extension 389 | filename = os.path.join(directory, f"{script_name}.pine") 390 | with open(filename, 'w') as file: 391 | file.write(f"Script Name: {script_name}\n") 392 | file.write(f"Author: {script_author}\n") 393 | file.write(f"Description: {script_description}\n") 394 | file.write("PineScript code:\n\n") 395 | file.write(script_code) 396 | 397 | print(f"Saved PineScript code to {filename}\n") 398 | 399 | except NoSuchElementException: 400 | print(f"Element not found. Skipping script: {script_name}") 401 | 402 | page += 1 403 | 404 | # Close the WebDriver 405 | driver.quit() 406 | 407 | ======================================== 408 | File: /home/hasnocool/Github/active/tradingview-script-downloader/fetch_scripts.py 409 | ======================================== 410 | import os 411 | import asyncio 412 | import aiohttp 413 | from bs4 import BeautifulSoup 414 | from selenium import webdriver 415 | from selenium.webdriver.common.by import By 416 | from selenium.common.exceptions import NoSuchElementException 417 | from selenium.webdriver.firefox.options import Options 418 | import time 419 | 420 | # Set the path to the geckodriver executable 421 | geckodriver_path = os.path.join(os.getcwd(), 'geckodriver') 422 | 423 | # Configure Firefox options for headless browsing 424 | firefox_options = Options() 425 | firefox_options.add_argument("--headless") 426 | 427 | # Launch Firefox WebDriver with headless options 428 | driver = webdriver.Firefox(executable_path=geckodriver_path, options=firefox_options) 429 | 430 | base_url = 'https://www.tradingview.com/scripts/' 431 | 432 | async def download_page(session, page): 433 | url = base_url + f'page-{page}/?script_type=strategies&script_access=open&sort=month_popular&route_range=1' 434 | 435 | async with session.get(url) as response: 436 | content = await response.text() 437 | 438 | soup = BeautifulSoup(content, 'html.parser') 439 | 440 | script_items = soup.find_all('div', class_='tv-feed__item') 441 | if not script_items: 442 | return # Return if there are no more script items on the page 443 | 444 | for item in script_items: 445 | if item.get('data-widget-type') == 'idea': 446 | script_data = item.get('data-widget-data') 447 | script_name = item.find('a', class_='tv-widget-idea__title').text.strip() 448 | script_author = item.find('span', class_='tv-card-user-info__name').text.strip() 449 | script_description = item.find('p', class_='tv-widget-idea__description-row').text.strip() 450 | print('Script Name:', script_name) 451 | print('Author:', script_author) 452 | print('Description:', script_description) 453 | print('---') 454 | 455 | # Replace invalid characters in script name 456 | invalid_characters = '/\\?%*:|"<>' 457 | for char in invalid_characters: 458 | script_name = script_name.replace(char, '-') 459 | 460 | # Create directory if it doesn't exist 461 | directory = 'PineScripts' 462 | os.makedirs(directory, exist_ok=True) 463 | 464 | # Check if the file already exists 465 | filename = os.path.join(directory, f"{script_name}.pine") 466 | if os.path.exists(filename): 467 | print(f"Skipping page {page}. File already exists: {filename}\n") 468 | continue 469 | 470 | # Extract PineScript code 471 | script_url = item.find('a', class_='tv-widget-idea__title').get('href') 472 | script_full_url = 'https://www.tradingview.com' + script_url 473 | 474 | # Open the script page with Firefox 475 | driver.get(script_full_url) 476 | time.sleep(2) # Add a delay to allow the page to fully load 477 | 478 | try: 479 | # Find and expand the collapse buttons 480 | collapse_buttons = driver.find_elements(By.CLASS_NAME, "collapseBtn") 481 | for button in collapse_buttons: 482 | driver.execute_script("arguments[0].click();", button) 483 | time.sleep(1) # Add a delay after expanding each collapse button 484 | 485 | # Get the PineScript code element 486 | script_code_element = driver.find_element(By.CLASS_NAME, "tv-chart-view__script-wrap") 487 | 488 | # Get the PineScript code text using innerText property 489 | script_code = script_code_element.get_property("innerText").strip() 490 | 491 | # Save PineScript to a file with .pine extension 492 | with open(filename, 'w') as file: 493 | file.write(f"Script Name: {script_name}\n") 494 | file.write(f"Author: {script_author}\n") 495 | file.write(f"Description: {script_description}\n") 496 | file.write("PineScript code:\n") 497 | file.write(script_code) 498 | 499 | print(f"Saved PineScript code to {filename}\n") 500 | 501 | except NoSuchElementException: 502 | print(f"Element not found. Skipping script: {script_name}") 503 | 504 | async def process_pages(): 505 | async with aiohttp.ClientSession() as session: 506 | page = 1 507 | while True: 508 | await download_page(session, page) 509 | page += 1 510 | 511 | # Number of pages to download concurrently 512 | num_concurrent_pages = 3 513 | 514 | # Run the event loop 515 | loop = asyncio.get_event_loop() 516 | tasks = [process_pages() for _ in range(num_concurrent_pages)] 517 | loop.run_until_complete(asyncio.gather(*tasks)) 518 | 519 | # Close the WebDriver 520 | driver.quit() 521 | 522 | ======================================== 523 | File: /home/hasnocool/Github/active/tradingview-script-downloader/backup-data.sh 524 | ======================================== 525 | #!/bin/bash 526 | 527 | # Set the source directory and filename 528 | source_dir="PineScripts" 529 | filename="Backups/PineScripts_$(date +'%Y%m%d_%H%M%S').tar.gz" 530 | cleaned_source_dir="PineScripts_Cleaned" 531 | cleaned_filename="Backups/PineScripts_Cleaned_$(date +'%Y%m%d_%H%M%S').tar.gz" 532 | # Create the tar.gz archive 533 | tar -czf "$filename" "$source_dir" 534 | tar -czf "$cleaned_filename" "$cleaned_source_dir" 535 | echo "Archive created: $filename" 536 | echo "Archive created: $cleaned_filename" 537 | 538 | ======================================== 539 | File: /home/hasnocool/Github/active/tradingview-script-downloader/docker-run.sh 540 | ======================================== 541 | docker run -v "$(pwd):/app" tradingview_pinescript_downloader 542 | 543 | ======================================== 544 | File: /home/hasnocool/Github/active/tradingview-script-downloader/Dockerfile 545 | ======================================== 546 | # Use an official Python runtime as the base image 547 | FROM python:3.9 548 | 549 | # Set the working directory in the container 550 | WORKDIR /app 551 | 552 | # Install Selenium, Requests, and BeautifulSoup 553 | RUN pip install selenium requests beautifulsoup4 554 | 555 | # Copy the Python script to the container 556 | COPY fetch.py . 557 | 558 | # Set the volume to the current directory 559 | VOLUME /app 560 | 561 | # Run the Python script when the container launches 562 | CMD ["python", "fetch.py"] 563 | 564 | ======================================== 565 | File: /home/hasnocool/Github/active/tradingview-script-downloader/docker-build.sh 566 | ======================================== 567 | docker build -t tradingview_pinescript_downloader . 568 | 569 | ======================================== 570 | File: /home/hasnocool/Github/active/tradingview-script-downloader/fetch_indicators.py 571 | ======================================== 572 | import os 573 | import asyncio 574 | import aiohttp 575 | from bs4 import BeautifulSoup 576 | from selenium import webdriver 577 | from selenium.webdriver.common.by import By 578 | from selenium.common.exceptions import NoSuchElementException 579 | from selenium.webdriver.firefox.options import Options 580 | import time 581 | 582 | # Set the path to the geckodriver executable 583 | geckodriver_path = os.path.join(os.getcwd(), 'geckodriver') 584 | 585 | # Configure Firefox options for headless browsing 586 | firefox_options = Options() 587 | firefox_options.add_argument("--headless") 588 | 589 | # Launch Firefox WebDriver with headless options 590 | driver = webdriver.Firefox(executable_path=geckodriver_path, options=firefox_options) 591 | 592 | base_url = 'https://www.tradingview.com/scripts/indicators/' 593 | 594 | async def download_page(session, page): 595 | url = base_url + f'page-{page}/?script_type=strategies&script_access=open&sort=month_popular&route_range=1' 596 | 597 | async with session.get(url) as response: 598 | content = await response.text() 599 | 600 | soup = BeautifulSoup(content, 'html.parser') 601 | 602 | script_items = soup.find_all('div', class_='tv-feed__item') 603 | if not script_items: 604 | return # Return if there are no more script items on the page 605 | 606 | for item in script_items: 607 | if item.get('data-widget-type') == 'idea': 608 | script_data = item.get('data-widget-data') 609 | script_name = item.find('a', class_='tv-widget-idea__title').text.strip() 610 | script_author = item.find('span', class_='tv-card-user-info__name').text.strip() 611 | script_description = item.find('p', class_='tv-widget-idea__description-row').text.strip() 612 | print('Script Name:', script_name) 613 | print('Author:', script_author) 614 | print('Description:', script_description) 615 | print('---') 616 | 617 | # Replace invalid characters in script name 618 | invalid_characters = '/\\?%*:|"<>' 619 | for char in invalid_characters: 620 | script_name = script_name.replace(char, '-') 621 | 622 | # Create directory if it doesn't exist 623 | directory = 'PineScripts_Indicators' 624 | os.makedirs(directory, exist_ok=True) 625 | 626 | # Check if the file already exists 627 | filename = os.path.join(directory, f"{script_name}.pine") 628 | if os.path.exists(filename): 629 | print(f"Skipping page {page}. File already exists: {filename}\n") 630 | continue 631 | 632 | # Extract PineScript code 633 | script_url = item.find('a', class_='tv-widget-idea__title').get('href') 634 | script_full_url = 'https://www.tradingview.com' + script_url 635 | 636 | # Open the script page with Firefox 637 | driver.get(script_full_url) 638 | time.sleep(2) # Add a delay to allow the page to fully load 639 | 640 | try: 641 | # Find and expand the collapse buttons 642 | collapse_buttons = driver.find_elements(By.CLASS_NAME, "collapseBtn") 643 | for button in collapse_buttons: 644 | driver.execute_script("arguments[0].click();", button) 645 | time.sleep(1) # Add a delay after expanding each collapse button 646 | 647 | # Get the PineScript code element 648 | script_code_element = driver.find_element(By.CLASS_NAME, "tv-chart-view__script-wrap") 649 | 650 | # Get the PineScript code text using innerText property 651 | script_code = script_code_element.get_property("innerText").strip() 652 | 653 | # Save PineScript to a file with .pine extension 654 | with open(filename, 'w') as file: 655 | file.write(f"Script Name: {script_name}\n") 656 | file.write(f"Author: {script_author}\n") 657 | file.write(f"Description: {script_description}\n") 658 | file.write("PineScript code:\n") 659 | file.write(script_code) 660 | 661 | print(f"Saved PineScript code to {filename}\n") 662 | 663 | except NoSuchElementException: 664 | print(f"Element not found. Skipping script: {script_name}") 665 | 666 | async def process_pages(): 667 | async with aiohttp.ClientSession() as session: 668 | page = 1 669 | while True: 670 | await download_page(session, page) 671 | page += 1 672 | 673 | # Number of pages to download concurrently 674 | num_concurrent_pages = 3 675 | 676 | # Run the event loop 677 | loop = asyncio.get_event_loop() 678 | tasks = [process_pages() for _ in range(num_concurrent_pages)] 679 | loop.run_until_complete(asyncio.gather(*tasks)) 680 | 681 | # Close the WebDriver 682 | driver.quit() 683 | 684 | ======================================== 685 | File: /home/hasnocool/Github/active/tradingview-script-downloader/docker-compose.yaml 686 | ======================================== 687 | 688 | ======================================== 689 | File: /home/hasnocool/Github/active/tradingview-script-downloader/fetch_strategies.py 690 | ======================================== 691 | import os 692 | import asyncio 693 | import aiohttp 694 | from bs4 import BeautifulSoup 695 | from selenium import webdriver 696 | from selenium.webdriver.common.by import By 697 | from selenium.common.exceptions import NoSuchElementException 698 | from selenium.webdriver.firefox.options import Options 699 | import time 700 | 701 | # Set the path to the geckodriver executable 702 | geckodriver_path = os.path.join(os.getcwd(), 'geckodriver') 703 | 704 | # Configure Firefox options for headless browsing 705 | firefox_options = Options() 706 | firefox_options.add_argument("--headless") 707 | 708 | # Launch Firefox WebDriver with headless options 709 | driver = webdriver.Firefox(executable_path=geckodriver_path, options=firefox_options) 710 | 711 | base_url = 'https://www.tradingview.com/scripts/strategies/' 712 | 713 | async def download_page(session, page): 714 | url = base_url + f'page-{page}/?script_type=strategies&script_access=open&sort=month_popular&route_range=1' 715 | 716 | async with session.get(url) as response: 717 | content = await response.text() 718 | 719 | soup = BeautifulSoup(content, 'html.parser') 720 | 721 | script_items = soup.find_all('div', class_='tv-feed__item') 722 | if not script_items: 723 | return # Return if there are no more script items on the page 724 | 725 | for item in script_items: 726 | if item.get('data-widget-type') == 'idea': 727 | script_data = item.get('data-widget-data') 728 | script_name = item.find('a', class_='tv-widget-idea__title').text.strip() 729 | script_author = item.find('span', class_='tv-card-user-info__name').text.strip() 730 | script_description = item.find('p', class_='tv-widget-idea__description-row').text.strip() 731 | print('Script Name:', script_name) 732 | print('Author:', script_author) 733 | print('Description:', script_description) 734 | print('---') 735 | 736 | # Replace invalid characters in script name 737 | invalid_characters = '/\\?%*:|"<>' 738 | for char in invalid_characters: 739 | script_name = script_name.replace(char, '-') 740 | 741 | # Create directory if it doesn't exist 742 | directory = 'PineScripts_Indicators' 743 | os.makedirs(directory, exist_ok=True) 744 | 745 | # Check if the file already exists 746 | filename = os.path.join(directory, f"{script_name}.pine") 747 | if os.path.exists(filename): 748 | print(f"Skipping page {page}. File already exists: {filename}\n") 749 | continue 750 | 751 | # Extract PineScript code 752 | script_url = item.find('a', class_='tv-widget-idea__title').get('href') 753 | script_full_url = 'https://www.tradingview.com' + script_url 754 | 755 | # Open the script page with Firefox 756 | driver.get(script_full_url) 757 | time.sleep(2) # Add a delay to allow the page to fully load 758 | 759 | try: 760 | # Find and expand the collapse buttons 761 | collapse_buttons = driver.find_elements(By.CLASS_NAME, "collapseBtn") 762 | for button in collapse_buttons: 763 | driver.execute_script("arguments[0].click();", button) 764 | time.sleep(1) # Add a delay after expanding each collapse button 765 | 766 | # Get the PineScript code element 767 | script_code_element = driver.find_element(By.CLASS_NAME, "tv-chart-view__script-wrap") 768 | 769 | # Get the PineScript code text using innerText property 770 | script_code = script_code_element.get_property("innerText").strip() 771 | 772 | # Save PineScript to a file with .pine extension 773 | with open(filename, 'w') as file: 774 | file.write(f"Script Name: {script_name}\n") 775 | file.write(f"Author: {script_author}\n") 776 | file.write(f"Description: {script_description}\n") 777 | file.write("PineScript code:\n") 778 | file.write(script_code) 779 | 780 | print(f"Saved PineScript code to {filename}\n") 781 | 782 | except NoSuchElementException: 783 | print(f"Element not found. Skipping script: {script_name}") 784 | 785 | async def process_pages(): 786 | async with aiohttp.ClientSession() as session: 787 | page = 1 788 | while True: 789 | await download_page(session, page) 790 | page += 1 791 | 792 | # Number of pages to download concurrently 793 | num_concurrent_pages = 3 794 | 795 | # Run the event loop 796 | loop = asyncio.get_event_loop() 797 | tasks = [process_pages() for _ in range(num_concurrent_pages)] 798 | loop.run_until_complete(asyncio.gather(*tasks)) 799 | 800 | # Close the WebDriver 801 | driver.quit() 802 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Directories 2 | node_modules/ 3 | dist/ 4 | Backups/ 5 | PineScripts/ 6 | PineScripts_Cleaned/ 7 | PineScripts_Editors-Picks/ 8 | PineScripts_Indicators/ 9 | 10 | # Files 11 | .DS_Store 12 | .env 13 | geckodriver 14 | geckodriver.log 15 | README.md 16 | todo.md 17 | git.py 18 | git.sh 19 | # Python bytecode files 20 | 21 | *.pyc 22 | __pycache__/ 23 | -------------------------------------------------------------------------------- /.repo_summary/consolidated_summary.txt: -------------------------------------------------------------------------------- 1 | Here's the structured summary with placeholders replaced: 2 | 3 | 1. **Project Name:** Tradingview Script Downloader 4 | 2. **Short Description:** A Python script that downloads PineScript code from TradingView. 5 | 3. **Overall Description:** The Tradingview Script Downloader project aims to automate the process of collecting and saving PineScript codes from TradingView, a popular platform for traders and investors. It utilizes various technologies such as Selenium WebDriver and aiohttp to navigate through web pages, extract relevant information, and save scripts to local files. This project is useful for those who want to explore and learn from various trading strategies without having to manually copy and paste code. 6 | 4. **Main Purpose:** To create a tool that simplifies the process of downloading and saving PineScript codes from TradingView. 7 | 8 | 5. **Key Technologies:** 9 | - Selenium WebDriver 10 | - aiohttp 11 | - asyncio 12 | 13 | 6. **Unique Features:** 14 | - Utilizes aiohttp for asynchronous page downloads 15 | - Employs Selenium WebDriver for navigating through web pages and extracting information 16 | 17 | 7. **Future Improvements:** 18 | - Enhance error handling mechanisms to improve stability 19 | - Explore other platforms for downloading scripts (e.g., GitHub, Bitbucket) 20 | 21 | 8. **Personal Notes:** This project has the potential to be extended into a more comprehensive tool that provides additional features such as script filtering, categorization, and visualization. 22 | 23 | 9. **GitHub Tags:** tradingview,pinescript,selenium,aiohttp,asyncio,web-scraping,trading-strategies 24 | 25 | 10. **Installable Modules:** aiohttp, selenium, time, requests, re, concurrent, asyncio, bs4, os 26 | 27 | 11. **Suggested Folder Names:** scripts, downloads, data, resources, tradingview-downloader 28 | 29 | 12. **Root Project Folder Name Suggestions:** Tradingview Script Downloader, PineScript Collector, Trading Strategy Toolbox 30 | 31 | # Extracted Variables: 32 | PROJECT_NAME = "Tradingview Script Downloader" 33 | SHORT_DESCRIPTION = "A Python script that downloads PineScript code from TradingView." 34 | OVERALL_DESCRIPTION = "The Tradingview Script Downloader project aims to automate the process of collecting and saving PineScript codes from TradingView, a popular platform for traders and investors. It utilizes various technologies such as Selenium WebDriver and aiohttp to navigate through web pages, extract relevant information, and save scripts to local files. This project is useful for those who want to explore and learn from various trading strategies without having to manually copy and paste code." 35 | MAIN_PURPOSE = "To create a tool that simplifies the process of downloading and saving PineScript codes from TradingView." 36 | KEY_TECHNOLOGIES = "Selenium WebDriver, aiohttp, asyncio" 37 | UNIQUE_FEATURES = "Utilizes aiohttp for asynchronous page downloads, Employs Selenium WebDriver for navigating through web pages and extracting information" 38 | FUTURE_IMPROVEMENTS = "Enhance error handling mechanisms to improve stability, Explore other platforms for downloading scripts (e.g., GitHub, Bitbucket)" 39 | PERSONAL_NOTES = "This project has the potential to be extended into a more comprehensive tool that provides additional features such as script filtering, categorization, and visualization." 40 | GITHUB_TAGS = "tradingview, pinescript, selenium, aiohttp, asyncio, web, scraping, trading, strategies" 41 | INSTALLABLE_MODULES = "aiohttp, requests, selenium, bs4" 42 | SUGGESTED_FOLDER_NAMES = "scripts, downloads, data, resources, tradingview, downloader" 43 | ROOT_PROJECT_FOLDER_NAME_SUGGESTIONS = "Tradingview Script Downloader, PineScript Collector, Trading Strategy Toolbox" 44 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Use an official Python runtime as the base image 2 | FROM python:3.9 3 | 4 | # Set the working directory in the container 5 | WORKDIR /app 6 | 7 | # Install Selenium, Requests, and BeautifulSoup 8 | RUN pip install selenium requests beautifulsoup4 9 | 10 | # Copy the Python script to the container 11 | COPY fetch.py . 12 | 13 | # Set the volume to the current directory 14 | VOLUME /app 15 | 16 | # Run the Python script when the container launches 17 | CMD ["python", "fetch.py"] 18 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | **Project Title** 2 | ================ 3 | TradingView PineScript Downloader 4 | 5 | **Description** 6 | --------------- 7 | A concurrent web scraper written in Python using Selenium and aiohttp to download PineScripts from TradingView, a popular online community of traders. 8 | 9 | **Features** 10 | ------------ 11 | 12 | * **Concurrent Page Downloading**: I built this to take advantage of multi-core processors by downloading multiple pages concurrently. 13 | * **PineScript Code Extraction**: One cool feature is the ability to extract and save PineScript code from TradingView's script pages, which can then be used for backtesting or other purposes. 14 | * **Headless Browsing**: I configured Firefox to run in headless mode using geckodriver, making it possible to scrape content without displaying any browser windows. 15 | * **Directory Management**: The project creates a designated directory for storing downloaded PineScripts and checks if files already exist before saving them again. 16 | 17 | **Installation** 18 | ---------------- 19 | Before running the script, make sure you have: 20 | 21 | 1. Python installed on your machine (`python --version`) 22 | 2. `geckodriver` executable in your system's PATH (download from [here](https://github.com/mozilla/geckodriver/releases)) 23 | 3. Required libraries: 24 | * `selenium` 25 | * `aiohttp` 26 | * `BeautifulSoup4` 27 | 28 | You can install them using pip: 29 | 30 | ```bash 31 | pip install -r requirements.txt 32 | ``` 33 | 34 | **Usage** 35 | --------- 36 | Run the script with Python, and it will start downloading PineScripts from TradingView in a concurrent manner. You can adjust the number of concurrent pages by modifying the `num_concurrent_pages` variable. 37 | 38 | ```python 39 | python main.py 40 | ``` 41 | 42 | **Contributing** 43 | ---------------- 44 | 45 | If you'd like to contribute or report any issues, feel free to open a pull request on GitHub: [TradingView PineScript Downloader](https://github.com/your-username/tradingview-pinescript-downloader) 46 | 47 | **License** 48 | ---------- 49 | 50 | This project is licensed under the MIT License. See LICENSE file for details. 51 | 52 | **Tags/Keywords** 53 | ----------------- 54 | 55 | * TradingView 56 | * PineScript 57 | * Web Scraping 58 | * Selenium 59 | * Aiohttp 60 | * Concurrent Downloading -------------------------------------------------------------------------------- /backup-data.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Set the source directory and filename 4 | source_dir="PineScripts" 5 | filename="Backups/PineScripts_$(date +'%Y%m%d_%H%M%S').tar.gz" 6 | cleaned_source_dir="PineScripts_Cleaned" 7 | cleaned_filename="Backups/PineScripts_Cleaned_$(date +'%Y%m%d_%H%M%S').tar.gz" 8 | # Create the tar.gz archive 9 | tar -czf "$filename" "$source_dir" 10 | tar -czf "$cleaned_filename" "$cleaned_source_dir" 11 | echo "Archive created: $filename" 12 | echo "Archive created: $cleaned_filename" 13 | -------------------------------------------------------------------------------- /clean-data.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import os 4 | import re 5 | 6 | # Function to clean the PineScript 7 | def clean_pinescript(script): 8 | lines = script.split('\n') 9 | 10 | # Remove numbered lines dynamically 11 | lines = [line for line in lines if not re.match(r"^\d+\s*$", line.strip())] 12 | 13 | # Find the index of "Copy code" line 14 | copy_code_index = -1 15 | for i, line in enumerate(lines): 16 | if line.strip() == 'Copy code': 17 | copy_code_index = i 18 | break 19 | 20 | # Remove the last line if it contains the word "Expand" followed by parentheses and a number 21 | if re.match(r"Expand \(\d+ lines\)", lines[-1].strip()): 22 | lines = lines[:-1] 23 | 24 | # Add comments to the lines preceding "Copy code" 25 | comment_lines = [] 26 | for i in range(copy_code_index): 27 | line = lines[i].strip() 28 | if line: 29 | comment_lines.append(f"// {line}") 30 | lines = comment_lines + lines[copy_code_index + 1:] 31 | 32 | # Remove unnecessary lines and format PineScript code properly 33 | cleaned_script = '\n'.join(lines).strip() 34 | cleaned_script = cleaned_script.replace('\n\n', '\n') 35 | cleaned_script = cleaned_script.replace('=', ' = ') 36 | cleaned_script = cleaned_script.replace(':', ': ') 37 | cleaned_script = cleaned_script.replace(' ', ' ') 38 | return cleaned_script 39 | 40 | # Specify the folder paths 41 | input_folder = 'PineScripts' 42 | output_folder = 'PineScripts_Cleaned' 43 | 44 | # Create the output folder if it doesn't exist 45 | os.makedirs(output_folder, exist_ok=True) 46 | 47 | # Iterate through the files in the input folder 48 | for file_name in os.listdir(input_folder): 49 | if file_name.endswith('.pine'): 50 | input_file_path = os.path.join(input_folder, file_name) 51 | output_file_path = os.path.join(output_folder, file_name) 52 | 53 | with open(input_file_path, 'r') as file: 54 | script = file.read() 55 | 56 | cleaned_script = clean_pinescript(script) 57 | 58 | with open(output_file_path, 'w') as cleaned_file: 59 | cleaned_file.write(cleaned_script) 60 | 61 | print(f"Cleaned script saved to: {output_file_path}") 62 | -------------------------------------------------------------------------------- /count.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import os 4 | 5 | def get_directory_size(directory): 6 | total_size = 0 7 | for dirpath, _, filenames in os.walk(directory): 8 | for filename in filenames: 9 | file_path = os.path.join(dirpath, filename) 10 | total_size += os.path.getsize(file_path) 11 | return total_size 12 | 13 | def format_size(size): 14 | # Adapted from https://stackoverflow.com/a/1094933 15 | units = ["B", "KB", "MB", "GB", "TB"] 16 | index = 0 17 | while size >= 1024 and index < len(units) - 1: 18 | size /= 1024 19 | index += 1 20 | return f"{size:.2f} {units[index]}" 21 | 22 | # Specify the directory path you want to count files in 23 | directory_path = "PineScripts" 24 | 25 | file_count = sum([len(files) for _, _, files in os.walk(directory_path)]) 26 | directory_size = get_directory_size(directory_path) 27 | 28 | print(f"The number of files in '{directory_path}' is: {file_count}") 29 | print(f"The total size of the directory is: {format_size(directory_size)}") 30 | -------------------------------------------------------------------------------- /docker-build.sh: -------------------------------------------------------------------------------- 1 | docker build -t tradingview_pinescript_downloader . 2 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hasnocool/tradingview-script-downloader/697dd58cc23fee39a7d2442f76895e979124cbe3/docker-compose.yaml -------------------------------------------------------------------------------- /docker-run.sh: -------------------------------------------------------------------------------- 1 | docker run -v "$(pwd):/app" tradingview_pinescript_downloader 2 | -------------------------------------------------------------------------------- /fetch.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import os 4 | import requests 5 | from bs4 import BeautifulSoup 6 | from selenium import webdriver 7 | from selenium.webdriver.common.by import By 8 | from selenium.common.exceptions import NoSuchElementException 9 | import time 10 | 11 | # Set the path to the geckodriver executable 12 | geckodriver_path = os.path.join(os.getcwd(), 'geckodriver') 13 | 14 | # Launch Firefox WebDriver 15 | driver = webdriver.Firefox(executable_path=geckodriver_path) 16 | 17 | base_url = 'https://www.tradingview.com/scripts/' 18 | page = 1 19 | 20 | while True: 21 | url = base_url + f'page-{page}/?script_type=strategies&script_access=open&sort=month_popular&route_range=1' 22 | response = requests.get(url) 23 | soup = BeautifulSoup(response.content, 'html.parser') 24 | 25 | script_items = soup.find_all('div', class_='tv-feed__item') 26 | if not script_items: 27 | break # Break the loop if there are no more script items on the page 28 | 29 | for item in script_items: 30 | if item.get('data-widget-type') == 'idea': 31 | script_data = item.get('data-widget-data') 32 | script_name = item.find('a', class_='tv-widget-idea__title').text.strip() 33 | script_author = item.find('span', class_='tv-card-user-info__name').text.strip() 34 | script_description = item.find('p', class_='tv-widget-idea__description-row').text.strip() 35 | print('Script Name:', script_name) 36 | print('Author:', script_author) 37 | print('Description:', script_description) 38 | print('---') 39 | 40 | # Extract PineScript code 41 | script_url = item.find('a', class_='tv-widget-idea__title').get('href') 42 | script_full_url = 'https://www.tradingview.com' + script_url 43 | 44 | # Open the script page with Firefox 45 | driver.get(script_full_url) 46 | time.sleep(2) # Add a delay to allow the page to fully load 47 | 48 | try: 49 | # Find and expand the collapse buttons 50 | collapse_buttons = driver.find_elements(By.CLASS_NAME, "collapseBtn") 51 | for button in collapse_buttons: 52 | driver.execute_script("arguments[0].click();", button) 53 | time.sleep(1) # Add a delay after expanding each collapse button 54 | 55 | # Get the PineScript code element 56 | script_code_element = driver.find_element(By.CLASS_NAME, "tv-chart-view__script-wrap") 57 | 58 | # Get the PineScript code text using innerText property 59 | script_code = script_code_element.get_property("innerText").strip() 60 | 61 | # Create directory if it doesn't exist 62 | directory = 'PineScripts' 63 | os.makedirs(directory, exist_ok=True) 64 | 65 | # Replace invalid characters in script name 66 | invalid_characters = '/\\?%*:|"<>' 67 | for char in invalid_characters: 68 | script_name = script_name.replace(char, '-') 69 | 70 | # Save PineScript to a file with .pine extension 71 | filename = os.path.join(directory, f"{script_name}.pine") 72 | with open(filename, 'w') as file: 73 | file.write(f"Script Name: {script_name}\n") 74 | file.write(f"Author: {script_author}\n") 75 | file.write(f"Description: {script_description}\n") 76 | file.write("PineScript code:\n\n") 77 | file.write(script_code) 78 | 79 | print(f"Saved PineScript code to {filename}\n") 80 | 81 | except NoSuchElementException: 82 | print(f"Element not found. Skipping script: {script_name}") 83 | 84 | page += 1 85 | 86 | # Close the WebDriver 87 | driver.quit() 88 | -------------------------------------------------------------------------------- /fetch_editors-picks.py: -------------------------------------------------------------------------------- 1 | import os 2 | import asyncio 3 | import aiohttp 4 | from bs4 import BeautifulSoup 5 | from selenium import webdriver 6 | from selenium.webdriver.common.by import By 7 | from selenium.common.exceptions import NoSuchElementException 8 | from selenium.webdriver.firefox.options import Options 9 | import time 10 | 11 | # Set the path to the geckodriver executable 12 | geckodriver_path = os.path.join(os.getcwd(), 'geckodriver') 13 | 14 | # Configure Firefox options for headless browsing 15 | firefox_options = Options() 16 | firefox_options.add_argument("--headless") 17 | 18 | # Launch Firefox WebDriver with headless options 19 | driver = webdriver.Firefox(executable_path=geckodriver_path, options=firefox_options) 20 | 21 | base_url = 'https://www.tradingview.com/scripts/editors-picks/' 22 | 23 | async def download_page(session, page): 24 | url = base_url + f'page-{page}/?script_type=strategies&script_access=open&sort=month_popular&route_range=1' 25 | 26 | async with session.get(url) as response: 27 | content = await response.text() 28 | 29 | soup = BeautifulSoup(content, 'html.parser') 30 | 31 | script_items = soup.find_all('div', class_='tv-feed__item') 32 | if not script_items: 33 | return # Return if there are no more script items on the page 34 | 35 | for item in script_items: 36 | if item.get('data-widget-type') == 'idea': 37 | script_data = item.get('data-widget-data') 38 | script_name = item.find('a', class_='tv-widget-idea__title').text.strip() 39 | script_author = item.find('span', class_='tv-card-user-info__name').text.strip() 40 | script_description = item.find('p', class_='tv-widget-idea__description-row').text.strip() 41 | print('Script Name:', script_name) 42 | print('Author:', script_author) 43 | print('Description:', script_description) 44 | print('---') 45 | 46 | # Replace invalid characters in script name 47 | invalid_characters = '/\\?%*:|"<>' 48 | for char in invalid_characters: 49 | script_name = script_name.replace(char, '-') 50 | 51 | # Create directory if it doesn't exist 52 | directory = 'PineScripts_Editors-Picks' 53 | os.makedirs(directory, exist_ok=True) 54 | 55 | # Check if the file already exists 56 | filename = os.path.join(directory, f"{script_name}.pine") 57 | if os.path.exists(filename): 58 | print(f"Skipping page {page}. File already exists: {filename}\n") 59 | continue 60 | 61 | # Extract PineScript code 62 | script_url = item.find('a', class_='tv-widget-idea__title').get('href') 63 | script_full_url = 'https://www.tradingview.com' + script_url 64 | 65 | # Open the script page with Firefox 66 | driver.get(script_full_url) 67 | time.sleep(2) # Add a delay to allow the page to fully load 68 | 69 | try: 70 | # Find and expand the collapse buttons 71 | collapse_buttons = driver.find_elements(By.CLASS_NAME, "collapseBtn") 72 | for button in collapse_buttons: 73 | driver.execute_script("arguments[0].click();", button) 74 | time.sleep(1) # Add a delay after expanding each collapse button 75 | 76 | # Get the PineScript code element 77 | script_code_element = driver.find_element(By.CLASS_NAME, "tv-chart-view__script-wrap") 78 | 79 | # Get the PineScript code text using innerText property 80 | script_code = script_code_element.get_property("innerText").strip() 81 | 82 | # Save PineScript to a file with .pine extension 83 | with open(filename, 'w') as file: 84 | file.write(f"Script Name: {script_name}\n") 85 | file.write(f"Author: {script_author}\n") 86 | file.write(f"Description: {script_description}\n") 87 | file.write("PineScript code:\n") 88 | file.write(script_code) 89 | 90 | print(f"Saved PineScript code to {filename}\n") 91 | 92 | except NoSuchElementException: 93 | print(f"Element not found. Skipping script: {script_name}") 94 | 95 | async def process_pages(): 96 | async with aiohttp.ClientSession() as session: 97 | page = 1 98 | while True: 99 | await download_page(session, page) 100 | page += 1 101 | 102 | # Number of pages to download concurrently 103 | num_concurrent_pages = 3 104 | 105 | # Run the event loop 106 | loop = asyncio.get_event_loop() 107 | tasks = [process_pages() for _ in range(num_concurrent_pages)] 108 | loop.run_until_complete(asyncio.gather(*tasks)) 109 | 110 | # Close the WebDriver 111 | driver.quit() 112 | -------------------------------------------------------------------------------- /fetch_indicators.py: -------------------------------------------------------------------------------- 1 | import os 2 | import asyncio 3 | import aiohttp 4 | from bs4 import BeautifulSoup 5 | from selenium import webdriver 6 | from selenium.webdriver.common.by import By 7 | from selenium.common.exceptions import NoSuchElementException 8 | from selenium.webdriver.firefox.options import Options 9 | import time 10 | 11 | # Set the path to the geckodriver executable 12 | geckodriver_path = os.path.join(os.getcwd(), 'geckodriver') 13 | 14 | # Configure Firefox options for headless browsing 15 | firefox_options = Options() 16 | firefox_options.add_argument("--headless") 17 | 18 | # Launch Firefox WebDriver with headless options 19 | driver = webdriver.Firefox(executable_path=geckodriver_path, options=firefox_options) 20 | 21 | base_url = 'https://www.tradingview.com/scripts/indicators/' 22 | 23 | async def download_page(session, page): 24 | url = base_url + f'page-{page}/?script_type=strategies&script_access=open&sort=month_popular&route_range=1' 25 | 26 | async with session.get(url) as response: 27 | content = await response.text() 28 | 29 | soup = BeautifulSoup(content, 'html.parser') 30 | 31 | script_items = soup.find_all('div', class_='tv-feed__item') 32 | if not script_items: 33 | return # Return if there are no more script items on the page 34 | 35 | for item in script_items: 36 | if item.get('data-widget-type') == 'idea': 37 | script_data = item.get('data-widget-data') 38 | script_name = item.find('a', class_='tv-widget-idea__title').text.strip() 39 | script_author = item.find('span', class_='tv-card-user-info__name').text.strip() 40 | script_description = item.find('p', class_='tv-widget-idea__description-row').text.strip() 41 | print('Script Name:', script_name) 42 | print('Author:', script_author) 43 | print('Description:', script_description) 44 | print('---') 45 | 46 | # Replace invalid characters in script name 47 | invalid_characters = '/\\?%*:|"<>' 48 | for char in invalid_characters: 49 | script_name = script_name.replace(char, '-') 50 | 51 | # Create directory if it doesn't exist 52 | directory = 'PineScripts_Indicators' 53 | os.makedirs(directory, exist_ok=True) 54 | 55 | # Check if the file already exists 56 | filename = os.path.join(directory, f"{script_name}.pine") 57 | if os.path.exists(filename): 58 | print(f"Skipping page {page}. File already exists: {filename}\n") 59 | continue 60 | 61 | # Extract PineScript code 62 | script_url = item.find('a', class_='tv-widget-idea__title').get('href') 63 | script_full_url = 'https://www.tradingview.com' + script_url 64 | 65 | # Open the script page with Firefox 66 | driver.get(script_full_url) 67 | time.sleep(2) # Add a delay to allow the page to fully load 68 | 69 | try: 70 | # Find and expand the collapse buttons 71 | collapse_buttons = driver.find_elements(By.CLASS_NAME, "collapseBtn") 72 | for button in collapse_buttons: 73 | driver.execute_script("arguments[0].click();", button) 74 | time.sleep(1) # Add a delay after expanding each collapse button 75 | 76 | # Get the PineScript code element 77 | script_code_element = driver.find_element(By.CLASS_NAME, "tv-chart-view__script-wrap") 78 | 79 | # Get the PineScript code text using innerText property 80 | script_code = script_code_element.get_property("innerText").strip() 81 | 82 | # Save PineScript to a file with .pine extension 83 | with open(filename, 'w') as file: 84 | file.write(f"Script Name: {script_name}\n") 85 | file.write(f"Author: {script_author}\n") 86 | file.write(f"Description: {script_description}\n") 87 | file.write("PineScript code:\n") 88 | file.write(script_code) 89 | 90 | print(f"Saved PineScript code to {filename}\n") 91 | 92 | except NoSuchElementException: 93 | print(f"Element not found. Skipping script: {script_name}") 94 | 95 | async def process_pages(): 96 | async with aiohttp.ClientSession() as session: 97 | page = 1 98 | while True: 99 | await download_page(session, page) 100 | page += 1 101 | 102 | # Number of pages to download concurrently 103 | num_concurrent_pages = 3 104 | 105 | # Run the event loop 106 | loop = asyncio.get_event_loop() 107 | tasks = [process_pages() for _ in range(num_concurrent_pages)] 108 | loop.run_until_complete(asyncio.gather(*tasks)) 109 | 110 | # Close the WebDriver 111 | driver.quit() 112 | -------------------------------------------------------------------------------- /fetch_more.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import os 4 | import requests 5 | from bs4 import BeautifulSoup 6 | from selenium import webdriver 7 | from selenium.webdriver.common.by import By 8 | from selenium.common.exceptions import NoSuchElementException 9 | from selenium.webdriver.firefox.options import Options 10 | import time 11 | import concurrent.futures 12 | 13 | # Set the path to the geckodriver executable 14 | geckodriver_path = os.path.join(os.getcwd(), 'geckodriver') 15 | 16 | # Configure Firefox options for headless browsing 17 | firefox_options = Options() 18 | firefox_options.add_argument("--headless") 19 | 20 | # Launch Firefox WebDriver with headless options 21 | driver = webdriver.Firefox(executable_path=geckodriver_path, options=firefox_options) 22 | 23 | base_url = 'https://www.tradingview.com/scripts/' 24 | 25 | def download_page(page): 26 | url = base_url + f'page-{page}/?script_type=strategies&script_access=open&sort=month_popular&route_range=1' 27 | response = requests.get(url) 28 | soup = BeautifulSoup(response.content, 'html.parser') 29 | 30 | script_items = soup.find_all('div', class_='tv-feed__item') 31 | if not script_items: 32 | return # Return if there are no more script items on the page 33 | 34 | for item in script_items: 35 | if item.get('data-widget-type') == 'idea': 36 | script_data = item.get('data-widget-data') 37 | script_name = item.find('a', class_='tv-widget-idea__title').text.strip() 38 | script_author = item.find('span', class_='tv-card-user-info__name').text.strip() 39 | script_description = item.find('p', class_='tv-widget-idea__description-row').text.strip() 40 | print('Script Name:', script_name) 41 | print('Author:', script_author) 42 | print('Description:', script_description) 43 | print('---') 44 | 45 | # Extract PineScript code 46 | script_url = item.find('a', class_='tv-widget-idea__title').get('href') 47 | script_full_url = 'https://www.tradingview.com' + script_url 48 | 49 | # Open the script page with Firefox 50 | driver.get(script_full_url) 51 | time.sleep(1) # Add a delay to allow the page to fully load 52 | 53 | try: 54 | # Find and expand the collapse buttons 55 | collapse_buttons = driver.find_elements(By.CLASS_NAME, "collapseBtn") 56 | for button in collapse_buttons: 57 | driver.execute_script("arguments[0].click();", button) 58 | time.sleep(0.5) # Add a delay after expanding each collapse button 59 | 60 | # Get the PineScript code element 61 | script_code_element = driver.find_element(By.CLASS_NAME, "tv-chart-view__script-wrap") 62 | 63 | # Get the PineScript code text using innerText property 64 | script_code = script_code_element.get_property("innerText").strip() 65 | 66 | # Create directory if it doesn't exist 67 | directory = 'PineScripts' 68 | os.makedirs(directory, exist_ok=True) 69 | 70 | # Replace invalid characters in script name 71 | invalid_characters = '/\\?%*:|"<>' 72 | for char in invalid_characters: 73 | script_name = script_name.replace(char, '-') 74 | 75 | # Save PineScript to a file with .pine extension 76 | filename = os.path.join(directory, f"{script_name}.pine") 77 | with open(filename, 'w') as file: 78 | file.write(f"Script Name: {script_name}\n") 79 | file.write(f"Author: {script_author}\n") 80 | file.write(f"Description: {script_description}\n") 81 | file.write("PineScript code:\n") 82 | file.write(script_code) 83 | 84 | print(f"Saved PineScript code to {filename}\n") 85 | 86 | except NoSuchElementException: 87 | print(f"Element not found. Skipping script: {script_name}") 88 | 89 | # Number of pages to download concurrently 90 | num_pages = 3 91 | 92 | with concurrent.futures.ThreadPoolExecutor() as executor: 93 | page_range = range(1, num_pages + 1) 94 | executor.map(download_page, page_range) 95 | 96 | # Close the WebDriver 97 | driver.quit() 98 | -------------------------------------------------------------------------------- /fetch_scripts.py: -------------------------------------------------------------------------------- 1 | import os 2 | import asyncio 3 | import aiohttp 4 | from bs4 import BeautifulSoup 5 | from selenium import webdriver 6 | from selenium.webdriver.common.by import By 7 | from selenium.common.exceptions import NoSuchElementException 8 | from selenium.webdriver.firefox.options import Options 9 | import time 10 | 11 | # Set the path to the geckodriver executable 12 | geckodriver_path = os.path.join(os.getcwd(), 'geckodriver') 13 | 14 | # Configure Firefox options for headless browsing 15 | firefox_options = Options() 16 | firefox_options.add_argument("--headless") 17 | 18 | # Launch Firefox WebDriver with headless options 19 | driver = webdriver.Firefox(executable_path=geckodriver_path, options=firefox_options) 20 | 21 | base_url = 'https://www.tradingview.com/scripts/' 22 | 23 | async def download_page(session, page): 24 | url = base_url + f'page-{page}/?script_type=strategies&script_access=open&sort=month_popular&route_range=1' 25 | 26 | async with session.get(url) as response: 27 | content = await response.text() 28 | 29 | soup = BeautifulSoup(content, 'html.parser') 30 | 31 | script_items = soup.find_all('div', class_='tv-feed__item') 32 | if not script_items: 33 | return # Return if there are no more script items on the page 34 | 35 | for item in script_items: 36 | if item.get('data-widget-type') == 'idea': 37 | script_data = item.get('data-widget-data') 38 | script_name = item.find('a', class_='tv-widget-idea__title').text.strip() 39 | script_author = item.find('span', class_='tv-card-user-info__name').text.strip() 40 | script_description = item.find('p', class_='tv-widget-idea__description-row').text.strip() 41 | print('Script Name:', script_name) 42 | print('Author:', script_author) 43 | print('Description:', script_description) 44 | print('---') 45 | 46 | # Replace invalid characters in script name 47 | invalid_characters = '/\\?%*:|"<>' 48 | for char in invalid_characters: 49 | script_name = script_name.replace(char, '-') 50 | 51 | # Create directory if it doesn't exist 52 | directory = 'PineScripts' 53 | os.makedirs(directory, exist_ok=True) 54 | 55 | # Check if the file already exists 56 | filename = os.path.join(directory, f"{script_name}.pine") 57 | if os.path.exists(filename): 58 | print(f"Skipping page {page}. File already exists: {filename}\n") 59 | continue 60 | 61 | # Extract PineScript code 62 | script_url = item.find('a', class_='tv-widget-idea__title').get('href') 63 | script_full_url = 'https://www.tradingview.com' + script_url 64 | 65 | # Open the script page with Firefox 66 | driver.get(script_full_url) 67 | time.sleep(2) # Add a delay to allow the page to fully load 68 | 69 | try: 70 | # Find and expand the collapse buttons 71 | collapse_buttons = driver.find_elements(By.CLASS_NAME, "collapseBtn") 72 | for button in collapse_buttons: 73 | driver.execute_script("arguments[0].click();", button) 74 | time.sleep(1) # Add a delay after expanding each collapse button 75 | 76 | # Get the PineScript code element 77 | script_code_element = driver.find_element(By.CLASS_NAME, "tv-chart-view__script-wrap") 78 | 79 | # Get the PineScript code text using innerText property 80 | script_code = script_code_element.get_property("innerText").strip() 81 | 82 | # Save PineScript to a file with .pine extension 83 | with open(filename, 'w') as file: 84 | file.write(f"Script Name: {script_name}\n") 85 | file.write(f"Author: {script_author}\n") 86 | file.write(f"Description: {script_description}\n") 87 | file.write("PineScript code:\n") 88 | file.write(script_code) 89 | 90 | print(f"Saved PineScript code to {filename}\n") 91 | 92 | except NoSuchElementException: 93 | print(f"Element not found. Skipping script: {script_name}") 94 | 95 | async def process_pages(): 96 | async with aiohttp.ClientSession() as session: 97 | page = 1 98 | while True: 99 | await download_page(session, page) 100 | page += 1 101 | 102 | # Number of pages to download concurrently 103 | num_concurrent_pages = 3 104 | 105 | # Run the event loop 106 | loop = asyncio.get_event_loop() 107 | tasks = [process_pages() for _ in range(num_concurrent_pages)] 108 | loop.run_until_complete(asyncio.gather(*tasks)) 109 | 110 | # Close the WebDriver 111 | driver.quit() 112 | -------------------------------------------------------------------------------- /fetch_strategies.py: -------------------------------------------------------------------------------- 1 | import os 2 | import asyncio 3 | import aiohttp 4 | from bs4 import BeautifulSoup 5 | from selenium import webdriver 6 | from selenium.webdriver.common.by import By 7 | from selenium.common.exceptions import NoSuchElementException 8 | from selenium.webdriver.firefox.options import Options 9 | import time 10 | 11 | # Set the path to the geckodriver executable 12 | geckodriver_path = os.path.join(os.getcwd(), 'geckodriver') 13 | 14 | # Configure Firefox options for headless browsing 15 | firefox_options = Options() 16 | firefox_options.add_argument("--headless") 17 | 18 | # Launch Firefox WebDriver with headless options 19 | driver = webdriver.Firefox(executable_path=geckodriver_path, options=firefox_options) 20 | 21 | base_url = 'https://www.tradingview.com/scripts/strategies/' 22 | 23 | async def download_page(session, page): 24 | url = base_url + f'page-{page}/?script_type=strategies&script_access=open&sort=month_popular&route_range=1' 25 | 26 | async with session.get(url) as response: 27 | content = await response.text() 28 | 29 | soup = BeautifulSoup(content, 'html.parser') 30 | 31 | script_items = soup.find_all('div', class_='tv-feed__item') 32 | if not script_items: 33 | return # Return if there are no more script items on the page 34 | 35 | for item in script_items: 36 | if item.get('data-widget-type') == 'idea': 37 | script_data = item.get('data-widget-data') 38 | script_name = item.find('a', class_='tv-widget-idea__title').text.strip() 39 | script_author = item.find('span', class_='tv-card-user-info__name').text.strip() 40 | script_description = item.find('p', class_='tv-widget-idea__description-row').text.strip() 41 | print('Script Name:', script_name) 42 | print('Author:', script_author) 43 | print('Description:', script_description) 44 | print('---') 45 | 46 | # Replace invalid characters in script name 47 | invalid_characters = '/\\?%*:|"<>' 48 | for char in invalid_characters: 49 | script_name = script_name.replace(char, '-') 50 | 51 | # Create directory if it doesn't exist 52 | directory = 'PineScripts_Indicators' 53 | os.makedirs(directory, exist_ok=True) 54 | 55 | # Check if the file already exists 56 | filename = os.path.join(directory, f"{script_name}.pine") 57 | if os.path.exists(filename): 58 | print(f"Skipping page {page}. File already exists: {filename}\n") 59 | continue 60 | 61 | # Extract PineScript code 62 | script_url = item.find('a', class_='tv-widget-idea__title').get('href') 63 | script_full_url = 'https://www.tradingview.com' + script_url 64 | 65 | # Open the script page with Firefox 66 | driver.get(script_full_url) 67 | time.sleep(2) # Add a delay to allow the page to fully load 68 | 69 | try: 70 | # Find and expand the collapse buttons 71 | collapse_buttons = driver.find_elements(By.CLASS_NAME, "collapseBtn") 72 | for button in collapse_buttons: 73 | driver.execute_script("arguments[0].click();", button) 74 | time.sleep(1) # Add a delay after expanding each collapse button 75 | 76 | # Get the PineScript code element 77 | script_code_element = driver.find_element(By.CLASS_NAME, "tv-chart-view__script-wrap") 78 | 79 | # Get the PineScript code text using innerText property 80 | script_code = script_code_element.get_property("innerText").strip() 81 | 82 | # Save PineScript to a file with .pine extension 83 | with open(filename, 'w') as file: 84 | file.write(f"Script Name: {script_name}\n") 85 | file.write(f"Author: {script_author}\n") 86 | file.write(f"Description: {script_description}\n") 87 | file.write("PineScript code:\n") 88 | file.write(script_code) 89 | 90 | print(f"Saved PineScript code to {filename}\n") 91 | 92 | except NoSuchElementException: 93 | print(f"Element not found. Skipping script: {script_name}") 94 | 95 | async def process_pages(): 96 | async with aiohttp.ClientSession() as session: 97 | page = 1 98 | while True: 99 | await download_page(session, page) 100 | page += 1 101 | 102 | # Number of pages to download concurrently 103 | num_concurrent_pages = 3 104 | 105 | # Run the event loop 106 | loop = asyncio.get_event_loop() 107 | tasks = [process_pages() for _ in range(num_concurrent_pages)] 108 | loop.run_until_complete(asyncio.gather(*tasks)) 109 | 110 | # Close the WebDriver 111 | driver.quit() 112 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | aiohttp 2 | bs4 3 | requests 4 | selenium --------------------------------------------------------------------------------