├── requirements.txt ├── docs ├── cinder │ ├── img │ │ ├── grid1.png │ │ ├── grid10.png │ │ ├── grid11.png │ │ ├── grid12.png │ │ ├── grid13.png │ │ ├── grid14.png │ │ ├── grid15.png │ │ ├── grid16.png │ │ ├── grid17.png │ │ ├── grid18.png │ │ ├── grid19.png │ │ ├── grid2.png │ │ ├── grid20.png │ │ ├── grid3.png │ │ ├── grid4.png │ │ ├── grid5.png │ │ ├── grid6.png │ │ ├── grid7.png │ │ ├── grid8.png │ │ ├── grid9.png │ │ └── favicon.ico │ ├── fonts │ │ ├── fontawesome-webfont.eot │ │ ├── fontawesome-webfont.ttf │ │ └── fontawesome-webfont.woff │ ├── content.html │ ├── mkdocs_theme.yml │ ├── main.html │ ├── 404.html │ ├── nav-sub.html │ ├── toc.html │ ├── css │ │ ├── highlight.min.css │ │ ├── cinder.min.css │ │ ├── highlight.css │ │ ├── cinder.css │ │ ├── base.min.css │ │ └── base.css │ ├── search-modal.html │ ├── keyboard-modal.html │ ├── nav.html │ ├── base.html │ └── js │ │ ├── base.js │ │ └── bootstrap-3.0.3.min.js ├── docs │ ├── overview │ │ ├── dependencies.md │ │ ├── external-apis.md │ │ └── parameters.md │ ├── plugins │ │ ├── ip-location.md │ │ ├── subdomains.md │ │ ├── page-details.md │ │ ├── extract-colors.md │ │ ├── robots.md │ │ ├── virustotal.md │ │ ├── whois.md │ │ ├── detect-fonts.md │ │ ├── screenshot.md │ │ └── apis │ │ │ ├── imgur.md │ │ │ ├── google-fonts.md │ │ │ └── virustotal.md │ ├── basics │ │ ├── core.md │ │ ├── cookies.md │ │ ├── headers.md │ │ ├── get-emails.md │ │ ├── get-links.md │ │ └── static │ │ │ ├── js.md │ │ │ ├── css.md │ │ │ └── images.md │ ├── settings │ │ └── env.md │ └── index.md └── mkdocs.yml ├── linkscraper ├── helper │ ├── configs.py │ ├── apis.py │ └── regex.py ├── exceptions │ └── settings_exception.py ├── core │ ├── static │ │ ├── autoload.py │ │ ├── images.py │ │ ├── js.py │ │ └── css.py │ ├── write_env.py │ ├── cookies.py │ ├── headers.py │ ├── core.py │ ├── plugins.py │ ├── scraper.py │ └── download_resources.py ├── utils │ ├── date_time.py │ ├── file_size.py │ ├── file_ext.py │ ├── locale.py │ ├── url.py │ ├── file.py │ └── http.py ├── layout │ ├── table.py │ └── layout.py ├── classes │ ├── env.py │ ├── startup.py │ └── settings.py ├── plugins │ ├── whois.py │ ├── ip_location.py │ ├── robots.py │ ├── extract_colors.py │ ├── subdomains.py │ ├── virustotal.py │ ├── screenshot.py │ ├── detect_fonts.py │ └── page_details.py ├── __main__.py └── apis │ ├── virustotal.py │ ├── imgur.py │ └── google_fonts.py ├── .env-example ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md └── FUNDING.yml ├── LICENSE ├── linkscraper.yml ├── .gitignore └── README.md /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kremilly/Linkscraper/HEAD/requirements.txt -------------------------------------------------------------------------------- /docs/cinder/img/grid1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kremilly/Linkscraper/HEAD/docs/cinder/img/grid1.png -------------------------------------------------------------------------------- /docs/cinder/img/grid10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kremilly/Linkscraper/HEAD/docs/cinder/img/grid10.png -------------------------------------------------------------------------------- /docs/cinder/img/grid11.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kremilly/Linkscraper/HEAD/docs/cinder/img/grid11.png -------------------------------------------------------------------------------- /docs/cinder/img/grid12.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kremilly/Linkscraper/HEAD/docs/cinder/img/grid12.png -------------------------------------------------------------------------------- /docs/cinder/img/grid13.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kremilly/Linkscraper/HEAD/docs/cinder/img/grid13.png -------------------------------------------------------------------------------- /docs/cinder/img/grid14.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kremilly/Linkscraper/HEAD/docs/cinder/img/grid14.png -------------------------------------------------------------------------------- /docs/cinder/img/grid15.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kremilly/Linkscraper/HEAD/docs/cinder/img/grid15.png -------------------------------------------------------------------------------- /docs/cinder/img/grid16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kremilly/Linkscraper/HEAD/docs/cinder/img/grid16.png -------------------------------------------------------------------------------- /docs/cinder/img/grid17.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kremilly/Linkscraper/HEAD/docs/cinder/img/grid17.png -------------------------------------------------------------------------------- /docs/cinder/img/grid18.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kremilly/Linkscraper/HEAD/docs/cinder/img/grid18.png -------------------------------------------------------------------------------- /docs/cinder/img/grid19.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kremilly/Linkscraper/HEAD/docs/cinder/img/grid19.png -------------------------------------------------------------------------------- /docs/cinder/img/grid2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kremilly/Linkscraper/HEAD/docs/cinder/img/grid2.png -------------------------------------------------------------------------------- /docs/cinder/img/grid20.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kremilly/Linkscraper/HEAD/docs/cinder/img/grid20.png -------------------------------------------------------------------------------- /docs/cinder/img/grid3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kremilly/Linkscraper/HEAD/docs/cinder/img/grid3.png -------------------------------------------------------------------------------- /docs/cinder/img/grid4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kremilly/Linkscraper/HEAD/docs/cinder/img/grid4.png -------------------------------------------------------------------------------- /docs/cinder/img/grid5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kremilly/Linkscraper/HEAD/docs/cinder/img/grid5.png -------------------------------------------------------------------------------- /docs/cinder/img/grid6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kremilly/Linkscraper/HEAD/docs/cinder/img/grid6.png -------------------------------------------------------------------------------- /docs/cinder/img/grid7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kremilly/Linkscraper/HEAD/docs/cinder/img/grid7.png -------------------------------------------------------------------------------- /docs/cinder/img/grid8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kremilly/Linkscraper/HEAD/docs/cinder/img/grid8.png -------------------------------------------------------------------------------- /docs/cinder/img/grid9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kremilly/Linkscraper/HEAD/docs/cinder/img/grid9.png -------------------------------------------------------------------------------- /docs/cinder/img/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kremilly/Linkscraper/HEAD/docs/cinder/img/favicon.ico -------------------------------------------------------------------------------- /docs/cinder/fonts/fontawesome-webfont.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kremilly/Linkscraper/HEAD/docs/cinder/fonts/fontawesome-webfont.eot -------------------------------------------------------------------------------- /docs/cinder/fonts/fontawesome-webfont.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kremilly/Linkscraper/HEAD/docs/cinder/fonts/fontawesome-webfont.ttf -------------------------------------------------------------------------------- /docs/cinder/fonts/fontawesome-webfont.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kremilly/Linkscraper/HEAD/docs/cinder/fonts/fontawesome-webfont.woff -------------------------------------------------------------------------------- /docs/cinder/content.html: -------------------------------------------------------------------------------- 1 | {% if page.meta.source %} 2 |
3 | {% for filename in page.meta.source %} 4 | {{ filename }} 5 | {% endfor %} 6 |
7 | {% endif %} 8 | 9 | {{ page.content }} 10 | -------------------------------------------------------------------------------- /docs/docs/overview/dependencies.md: -------------------------------------------------------------------------------- 1 | # Dependencies 2 | 3 | The Linkscraper requires the following libraries to function: 4 | 5 | * beautifulsoup4 6 | * cloudscraper 7 | * pyfiglet 8 | * pyperclip 9 | * requests 10 | * selenium 11 | * whois 12 | * rich 13 | * python-dotenv 14 | -------------------------------------------------------------------------------- /linkscraper/helper/configs.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | class Configs: 4 | 5 | VERSION = '2.6.8' 6 | 7 | APP_NAME = 'Linkscraper' 8 | 9 | CONFIGS_FILE = './linkscraper.yml' 10 | 11 | HOMEPAGE = 'https://github.com/Kremilly/Linkscraper' 12 | -------------------------------------------------------------------------------- /docs/cinder/mkdocs_theme.yml: -------------------------------------------------------------------------------- 1 | cinder_theme: true 2 | 3 | static_templates: 4 | - 404.html 5 | 6 | include_search_page: false 7 | search_index_only: false 8 | 9 | shortcuts: 10 | help: 191 # ? 11 | next: 78 # n 12 | previous: 80 # p 13 | search: 83 # s 14 | -------------------------------------------------------------------------------- /linkscraper/exceptions/settings_exception.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | import traceback 4 | 5 | class SettingsException(Exception): 6 | 7 | def __init__(cls, message): 8 | cls.message = message 9 | super().__init__(message) 10 | traceback.print_exc() 11 | -------------------------------------------------------------------------------- /.env-example: -------------------------------------------------------------------------------- 1 | # Env file created with Linkscraper (v.2.5.1) 2 | # File generated in: 2024-02-21 09:21:58 3 | 4 | # Get key: https://www.virustotal.com/gui/my-apikey 5 | VIRUSTOTAL_KEY=YOUR_VIRUSTOTAL_KEY 6 | 7 | # Get key: https://api.imgur.com/oauth2/addclient 8 | IMGUR_CLIENT_API=YOUR_CLIENT_API 9 | 10 | # Get key: https://console.cloud.google.com/apis/credentials 11 | GOOGLE_FONTS_KEY=YOUR_GOOGLE_FONTS_KEY 12 | -------------------------------------------------------------------------------- /docs/docs/overview/external-apis.md: -------------------------------------------------------------------------------- 1 | # External API's use 2 | 3 | The Linkscraper utilizes the following APIs to provide additional features through the tool's plugins: 4 | 5 | * [Imgur](https://imgur.com) 6 | * [VirusTotal](https://virustotal.com) 7 | * [IP-API](https://ip-api.com/) 8 | * [who.is](https://who.is/) 9 | * [threatcrowd](https://ci-www.threatcrowd.org/) 10 | * [Google Fonts](https://fonts.google.com/) 11 | -------------------------------------------------------------------------------- /docs/docs/plugins/ip-location.md: -------------------------------------------------------------------------------- 1 | ## IP-location 2 | 3 | With the ip-location plugin, you can gather geographic information about the IP of the specified URL. To use this command, simply type it into your terminal: 4 | 5 | ```shell 6 | python linkscraper -u https://example.com -a get-plugins -p ip-location 7 | ``` 8 | 9 | The plugin utilizes the api from [ip-api.com](). Its free version doesn't require any API key. 10 | -------------------------------------------------------------------------------- /docs/docs/plugins/subdomains.md: -------------------------------------------------------------------------------- 1 | ## Subdomains 2 | 3 | With the subdomains plugin, you can list all subdomains of the given URL. To use this command, simply type it into your terminal: 4 | 5 | ```shell 6 | python linkscraper -u https://example.com -a get-plugins -p subdomains 7 | ``` 8 | 9 | The plugin utilizes the api from [threatcrowd](https://ci-www.threatcrowd.org/). Its free version doesn't require any API key. 10 | -------------------------------------------------------------------------------- /docs/cinder/main.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | 3 | {# 4 | The entry point for the MkDocs Theme. 5 | 6 | Any theme customisations should override this file to redefine blocks defined in 7 | the various templates. The custom theme should only need to define a main.html 8 | which `{% extends "base.html" %}` and defines various blocks which will replace 9 | the blocks defined in base.html and its included child templates. 10 | #} -------------------------------------------------------------------------------- /docs/cinder/404.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | 3 | {% block content %} 4 | 5 |
6 |
7 |

404

8 |

Page not found

9 |

Home

10 |
11 |
12 | 13 | {% endblock %} 14 | -------------------------------------------------------------------------------- /docs/docs/basics/core.md: -------------------------------------------------------------------------------- 1 | ## Core 2 | 3 | The core function is the first function that can be executed by the application, it provides the main functions concerning the connection to a certain URL that was provided by you. 4 | 5 | You can execute the function by running the following command: 6 | 7 | ```shell 8 | python linkscraper -u https://example.com -a get-core 9 | ``` 10 | 11 | When you run the above command, it will fetch and request headers of the URL `https://example.com`. 12 | -------------------------------------------------------------------------------- /docs/cinder/nav-sub.html: -------------------------------------------------------------------------------- 1 | {% if not nav_item.children %} 2 |
  • 3 | {{ nav_item.title }} 4 |
    • 5 | {% else %} 6 |
  • 7 | {{ nav_item.title }} 8 | 13 |
    • 14 | {% endif %} 15 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **Screenshots** 14 | If applicable, add screenshots to help explain your problem. 15 | 16 | **Desktop (please complete the following information):** 17 | - OS: [e.g. iOS] 18 | 19 | **Additional context** 20 | Add any other context about the problem here. 21 | -------------------------------------------------------------------------------- /docs/docs/plugins/page-details.md: -------------------------------------------------------------------------------- 1 | ## Page-details 2 | 3 | The `page-details` plugin extracts all metadata from the given URL, excluding CSS, JS files, and fonts, since they necessitate additional `linkscraper` resources. 4 | 5 | ```shell 6 | python linkscraper -u https://example.com -a get-plugins -p page-details 7 | ``` 8 | 9 | With the plugin, you can gather the following metadata: 10 | 11 | * Title 12 | * Description 13 | * Robots directives 14 | * Viewport 15 | * Charset 16 | * WordPress 17 | * WordPress version 18 | * OG metadata ([read documentation](https://ogp.me/)) 19 | -------------------------------------------------------------------------------- /linkscraper/core/static/autoload.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | import requests, time 4 | 5 | from classes.settings import Settings 6 | 7 | from utils.file import File 8 | from utils.file_ext import FileExt 9 | from utils.file_size import FileSize 10 | from core.download_resources import DownloadResources 11 | 12 | from urllib.parse import urljoin 13 | from bs4 import BeautifulSoup as bs 14 | 15 | from layout.table import Table 16 | from layout.layout import Layout 17 | 18 | session = requests.Session() 19 | session.headers["User-Agent"] = Settings.get('general.default_user_agent', 'STRING') 20 | -------------------------------------------------------------------------------- /docs/docs/plugins/extract-colors.md: -------------------------------------------------------------------------------- 1 | ## Extract-colors 2 | 3 | To extract the colors used in the creation of the page from the URL provided by the user, you can utilize the `extract-colors` plugin. Its usage is straightforward; simply enter the following command into your terminal: 4 | 5 | ```shell 6 | python linkscraper -u https://example.com -a get-plugins -p extract-colors 7 | ``` 8 | 9 | This plugin has a limitation: it can only recognize colors that fit into four specific patterns. See some examples below: 10 | 11 | * `#fff` 12 | * `#ffffff` 13 | * `rgb(255, 255, 255)` 14 | * `rgba(255, 255, 255, 1)` 15 | -------------------------------------------------------------------------------- /linkscraper/utils/date_time.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | import time 4 | from datetime import datetime 5 | 6 | from classes.settings import Settings 7 | 8 | class DateTime: 9 | 10 | @classmethod 11 | def get_datetime(cls): 12 | return datetime.now().strftime(Settings.get('format_dates.datetime', 'STRING')) 13 | 14 | @classmethod 15 | def today_datetime(cls): 16 | return datetime.today().strftime(Settings.get('format_dates.today_datetime', 'STRING')) 17 | 18 | @classmethod 19 | def calculate_interval(cls, start_time): 20 | return '{:.2f}'.format(time.time() - start_time) 21 | -------------------------------------------------------------------------------- /docs/docs/plugins/robots.md: -------------------------------------------------------------------------------- 1 | ## Robots 2 | 3 | The `robots.txt` file is a standard used by websites to communicate with web crawlers and other web robots. The file indicates which areas of the site should not be processed or scanned. These rules are set by the site administrator to ensure certain pages or directories aren't crawled and to specify a delay for crawling, among other functions. 4 | 5 | To view the `robots.txt` file of a domain, it's straightforward. Simply use the `robots` plugin by executing the following command in your terminal: 6 | 7 | ```shell 8 | python linkscraper -u https://example.com -a get-plugins -p robots 9 | ``` 10 | -------------------------------------------------------------------------------- /docs/docs/basics/cookies.md: -------------------------------------------------------------------------------- 1 | ## Cookies 2 | 3 | Retrieves all cookies from the site. 4 | 5 | ```shell 6 | python linkscraper -u https://example.com -a get-cookies 7 | ``` 8 | 9 | This will return a list of all the cookies used by `https://example.com`. 10 | 11 | ### Filter 12 | 13 | The `-filter` option allows users to refine their results by including only those entries that match the filter keyword. 14 | 15 | ```shell 16 | python linkscraper -u https://example.com -a get-cookies -filter cookie 17 | ``` 18 | 19 | To collect cookies from the website "`https://example.com`" and then filter the results to only show those related to the term "`cookie`" 20 | -------------------------------------------------------------------------------- /linkscraper/helper/apis.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | class Apis: 4 | 5 | IP_API_REQUEST = 'http://ip-api.com/json/' 6 | IMGUR_API_REQUEST = 'https://api.imgur.com/3/image' 7 | VIRUSTOTAL_API_REQUEST = 'https://www.virustotal.com/api/v3/urls' 8 | GOOGLE_FONTS_API_REQUEST = 'https://www.googleapis.com/webfonts/v1/webfonts' 9 | THREATCROWD_API_REQUEST = 'http://ci-www.threatcrowd.org/searchApi/v2/domain/report' 10 | 11 | IMGUR_API_KEY_URL = 'https://api.imgur.com/oauth2/addclient' 12 | VIRUSTOTAL_API_KEY_URL = 'https://www.virustotal.com/gui/my-apikey' 13 | GOOGLE_FONTS_API_KEY_URL = 'https://console.cloud.google.com/apis/credentials' 14 | -------------------------------------------------------------------------------- /linkscraper/helper/regex.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | class Regex: 4 | 5 | COLORS_CODE = [ 6 | r'#[0-9a-fA-F]{6}', # e.g., #FFFFFF 7 | r'#[0-9a-fA-F]{3}', # e.g., #FFF 8 | r'rgb\(\s*\d+\s*,\s*\d+\s*,\s*\d+\s*\)', # e.g., rgb(255, 255, 255) 9 | r'rgba\(\s*\d+\s*,\s*\d+\s*,\s*\d+\s*,\s*[\d.]+\s*\)', # e.g., rgba(255, 255, 255, 0.5) 10 | ] 11 | 12 | URL_PATTERN_PROTOCOL = "^[-a-zA-Z0-9@:%._\\+~#=]{1,256}\\.[a-zA-Z0-9()]{1,6}\\b(?:[-a-zA-Z0-9()@:%_\\+.~#?&\\/=]*)$" 13 | URL_PATTERN_CHECK_PROTOCOL = "^https?:\\/\\/(?:www\\.)?[-a-zA-Z0-9@:%._\\+~#=]{1,256}\\.[a-zA-Z0-9()]{1,6}\\b(?:[-a-zA-Z0-9()@:%_\\+.~#?&\\/=]*)$" 14 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /docs/docs/basics/headers.md: -------------------------------------------------------------------------------- 1 | ## Headers 2 | 3 | Fetches and displays the headers of the specified URL. 4 | 5 | ```shell 6 | python linkscraper -u https://example.com -a get-headers 7 | ``` 8 | 9 | When you run the above command, it will fetch and display the headers of the URL `https://example.com`. 10 | 11 | ### Filter headers 12 | 13 | Upon running the command, the tool will visit the webpage at https://example.com, scrape the links found on the page, and retrieve the headers associated with those links. The results will then be filtered to only display links that match the filter criteria specified by header. 14 | 15 | ```shell 16 | python linkscraper -u https://example.com -a get-headers -filter header 17 | ``` 18 | -------------------------------------------------------------------------------- /docs/docs/plugins/virustotal.md: -------------------------------------------------------------------------------- 1 | ## VirusTotal 2 | 3 | VirusTotal is a free online service that analyzes files and URLs to detect viruses, worms, trojans, and other kinds of malicious content. It uses multiple antivirus engines and website scanners to provide a comprehensive report on the potential threats associated with the uploaded content. 4 | 5 | To scan a url using the VirusTotal tool, simply enter the following command: 6 | 7 | ```shell 8 | python linkscraper -u https://example.com -a get-plugins -p virustotal 9 | ``` 10 | 11 | However, remember that to utilize VirusTotal services within Linkscraper, you'll need an VirusTotal API key, which can be obtained for free. [Click here](apis/virustotal.md) to learn how. 12 | -------------------------------------------------------------------------------- /docs/cinder/toc.html: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: Kremilly 4 | patreon: Kremilly 5 | open_collective: # Replace with a single Open Collective username 6 | ko_fi: # Replace with a single Ko-fi username 7 | tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel 8 | community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry 9 | liberapay: # Replace with a single Liberapay username 10 | issuehunt: # Replace with a single IssueHunt username 11 | otechie: # Replace with a single Otechie username 12 | lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry 13 | custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] 14 | -------------------------------------------------------------------------------- /docs/docs/plugins/whois.md: -------------------------------------------------------------------------------- 1 | ## Whois 2 | 3 | WHOIS is a query and response protocol that is used for querying databases that store an Internet resource's registered users or assignees. These resources include domain names, IP address blocks and autonomous systems, but it is also used for a wider range of other information. 4 | 5 | The `whois` plugin allows you to extract details about a particular domain, including ownership information, unless it's set to private. To utilize this plugin, execute the command below: 6 | 7 | ```shell 8 | python linkscraper -u https://example.com -a get-plugins -p whois 9 | ``` 10 | 11 | This plugin provides the subsequent domain-related details: 12 | 13 | * Domain name 14 | * Domain registrar 15 | * WHOIS server 16 | * Domain creation date 17 | * Expiration date 18 | -------------------------------------------------------------------------------- /linkscraper/utils/file_size.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | import os, requests 4 | 5 | class FileSize: 6 | 7 | @classmethod 8 | def format(cls, bytes, units=[ ' bytes','KB','MB','GB','TB', 'PB', 'EB' ]): 9 | return str(bytes) + units[0] if bytes < 1024 else cls.format( 10 | bytes >> 10, units[1:] 11 | ) 12 | 13 | @classmethod 14 | def local_file(cls, file): 15 | file_size = os.stat(file) 16 | return cls.format(file_size.st_size) 17 | 18 | @classmethod 19 | def remote_file(cls, url): 20 | try: 21 | req_headers = requests.get(url) 22 | 23 | return cls.format( 24 | int(req_headers.headers["Content-Length"]) 25 | ) 26 | except: 27 | return None 28 | -------------------------------------------------------------------------------- /docs/docs/settings/env.md: -------------------------------------------------------------------------------- 1 | ## Environments 2 | 3 | The `.env` file is a plain-text configuration file where environment variables are stored. Typically used in the context of software development, these files allow developers to set environment-specific settings without changing application code. 4 | 5 | In essence, the `.env` file serves as a convenient and secure method for managing application configurations. 6 | 7 | To save your API keys, it's straightforward. Just enter the following command into your terminal: 8 | 9 | ```shell 10 | python linkscraper -we 11 | ``` 12 | 13 | If you prefer not to use the automatic .env file-writing tool, you can manually create a .env file in the root directory of Linkscraper with the following content: 14 | 15 | 16 | -------------------------------------------------------------------------------- /docs/cinder/css/highlight.min.css: -------------------------------------------------------------------------------- 1 | .hljs{display:block;overflow-x:auto;padding:.5em;color:#333;background:#fcfdff}.hljs-comment,.hljs-quote{color:#998;font-style:italic}.hljs-keyword,.hljs-selector-tag,.hljs-subst{color:#333;font-weight:bold}.hljs-number,.hljs-literal,.hljs-variable,.hljs-template-variable,.hljs-tag .hljs-attr{color:teal}.hljs-string,.hljs-doctag{color:#d14}.hljs-title,.hljs-section,.hljs-selector-id{color:#900;font-weight:bold}.hljs-subst{font-weight:normal}.hljs-type,.hljs-class .hljs-title{color:#458;font-weight:bold}.hljs-tag,.hljs-name,.hljs-attribute{color:navy;font-weight:normal}.hljs-regexp,.hljs-link{color:#009926}.hljs-symbol,.hljs-bullet{color:#990073}.hljs-built_in,.hljs-builtin-name{color:#0086b3}.hljs-meta{color:#999;font-weight:bold}.hljs-deletion{background:#fdd}.hljs-addition{background:#dfd}.hljs-emphasis{font-style:italic}.hljs-strong{font-weight:bold} 2 | -------------------------------------------------------------------------------- /linkscraper/utils/file_ext.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | import os, requests 4 | 5 | from classes.settings import Settings 6 | 7 | class FileExt: 8 | 9 | @classmethod 10 | def remove(cls, file): 11 | return file.rsplit('.', 1)[0] 12 | 13 | @classmethod 14 | def is_valid(cls, file): 15 | _, extension = os.path.splitext(file) 16 | dataset_allowed_ext = Settings.get('dataset.list_allowed_extensions', 'STRING') 17 | 18 | ext = requests.get(dataset_allowed_ext).json() 19 | all_ext = ext['scripts'] + ext['styles'] + ext['images'] 20 | 21 | return extension.lower() in all_ext 22 | 23 | @classmethod 24 | def get(cls, file): 25 | ext = os.path.splitext(file) 26 | 27 | if ext != '' or ext != '.': 28 | return ext 29 | 30 | return None 31 | -------------------------------------------------------------------------------- /docs/docs/index.md: -------------------------------------------------------------------------------- 1 | ## Introduction 2 | 3 | Dive deep into the web's intricate layers with Linkscraper! Whether you're a researcher, developer, or a curious explorer, our tool efficiently scans web pages to fetch links, images, emails, and much more. Powered by an array of versatile plugins and a user-friendly interface, Linkscraper streamlines the process of extracting and managing web data. From headers to JavaScript files, and from cookies to CSS – uncover the digital signatures of the web with ease. Join us on this journey and uncover the treasures hidden in plain sight on the web. 4 | 5 | ### Requirements 6 | 7 | * Python >= 3.10 ([Download](https://www.python.org/downloads/)) 8 | * PIP 9 | 10 | ### Getting Started 11 | 12 | Clone this repository. 13 | 14 | ```shell 15 | git clone https://github.com/kremilly/linkscraper.git 16 | ``` 17 | 18 | To install dependencies. 19 | 20 | ```python 21 | pip install -r requirements.txt 22 | ``` 23 | -------------------------------------------------------------------------------- /linkscraper/layout/table.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | from rich.console import Console 4 | from rich.table import Table as RichTable 5 | 6 | class Table: 7 | 8 | table = RichTable(box=None) 9 | console = Console(record=True) 10 | 11 | @classmethod 12 | def header(cls, headers): 13 | for header in headers: 14 | text = header[0] 15 | style = header[1] 16 | no_wrap = header[2] 17 | 18 | if style: 19 | cls.table.add_column(text, style=f"{style}", no_wrap=no_wrap) 20 | else: 21 | cls.table.add_column(text, no_wrap=no_wrap) 22 | 23 | @classmethod 24 | def row(cls, *args): 25 | cls.table.add_row(*args) 26 | 27 | @classmethod 28 | def caption(cls, text): 29 | cls.table.caption = text 30 | 31 | @classmethod 32 | def display(cls): 33 | cls.console.print(cls.table) 34 | -------------------------------------------------------------------------------- /linkscraper/classes/env.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | import os 4 | from dotenv import load_dotenv 5 | 6 | from helper.apis import Apis 7 | from helper.configs import Configs 8 | 9 | from utils.date_time import DateTime 10 | 11 | class Env: 12 | 13 | file_name = '.env' 14 | 15 | @classmethod 16 | def write(cls, content): 17 | with open(cls.file_name, 'w') as f: 18 | f.write(f"# Env file created with {Configs.APP_NAME}\n" 19 | f"# File generated in: {DateTime.get_datetime()}\n\n" 20 | f"# Get key: {Apis.VIRUSTOTAL_API_KEY_URL}\nVIRUSTOTAL_KEY={content['virustotal']}\n\n" 21 | f"# Get key: {Apis.IMGUR_API_KEY_URL}\nIMGUR_CLIENT_API={content['imgur']}\n\n" 22 | f"# Get key: {Apis.GOOGLE_FONTS_API_KEY_URL}\nGOOGLE_FONTS_KEY={content['google_fonts']}\n") 23 | 24 | @classmethod 25 | def get(cls, var_env): 26 | load_dotenv() 27 | return os.getenv(var_env) 28 | -------------------------------------------------------------------------------- /linkscraper/utils/locale.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | import requests 4 | 5 | from classes.settings import Settings 6 | 7 | class Locale: 8 | 9 | @classmethod 10 | def get_param(cls, string, slice): 11 | if string.find("_") != -1: 12 | return string.split("_")[slice].lower() 13 | else: 14 | return string.lower() 15 | 16 | @classmethod 17 | def country(cls, country): 18 | country = cls.get_param(country, 1) 19 | 20 | r = requests.get(Settings.get('dataset.list_countries', 'STRING')) 21 | r = r.json() 22 | 23 | for code in r: 24 | if code['alpha2'] == country: 25 | return code['name'] 26 | 27 | @classmethod 28 | def language(cls, lang): 29 | lang = cls.get_param(lang, 0) 30 | 31 | r = requests.get(Settings.get('dataset.list_languages', 'STRING')) 32 | r = r.json() 33 | 34 | for code in r: 35 | if code['code'] == lang: 36 | return code['name'] 37 | -------------------------------------------------------------------------------- /linkscraper/plugins/whois.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | import whois, time 4 | 5 | from utils.http import HTTP 6 | from utils.date_time import DateTime 7 | 8 | from layout.table import Table 9 | 10 | class Whois: 11 | 12 | @classmethod 13 | def run(cls, url): 14 | start_time = time.time() 15 | domain_name = HTTP.get_hostname(url) 16 | whois_info = whois.whois(domain_name) 17 | 18 | Table.header([ 19 | ('Name', 'cyan', True), 20 | ('Value', 'white', False) 21 | ]) 22 | 23 | Table.row('Domain name', f'{whois_info.domain_name}') 24 | Table.row('Domain registrar', f'{whois_info.registrar}') 25 | Table.row('WHOIS server', f'{whois_info.whois_server}') 26 | Table.row('Domain creation date', f'{str(whois_info.creation_date)}') 27 | Table.row('Expiration date', f'{str(whois_info.expiration_date)}') 28 | 29 | Table.caption(f'Time taken: {DateTime.calculate_interval(start_time)} seconds') 30 | Table.display() 31 | -------------------------------------------------------------------------------- /docs/docs/plugins/detect-fonts.md: -------------------------------------------------------------------------------- 1 | ## Detect-fonts 2 | 3 | To detect fonts in the provided URL, you need to use the `detect-fonts` plugin. To utilize this plugin, simply enter the following command into your terminal: 4 | 5 | ```shell 6 | python linkscraper -u https://example.com -a get-plugins -p detect-fonts 7 | ``` 8 | 9 | ### Google Fonts 10 | 11 | To collect variations of a specific font, simply use the `-gf` flag, type the font name, and press Enter. Doing this, Linkscraper will list all font files indexed by Google Fonts. 12 | 13 | ```shell 14 | python linkscraper -u https://example.com -a get-plugins -p detect-fonts -gf 15 | ``` 16 | 17 | However, remember that to utilize Google Fonts services within Linkscraper, you'll need an Google Fonts API key, which can be obtained for free. [Click here](apis/google-fonts.md) to learn how. 18 | 19 | ### Download 20 | 21 | To download all variants of the font, you just need to add the `-d` flag. 22 | 23 | ```shell 24 | python linkscraper -u https://example.com -a get-plugins -p detect-fonts -gf -d 25 | ``` 26 | -------------------------------------------------------------------------------- /linkscraper/plugins/ip_location.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | import requests, time 4 | 5 | from layout.table import Table 6 | 7 | from helper.apis import Apis 8 | 9 | from utils.http import HTTP 10 | from utils.date_time import DateTime 11 | 12 | class IPLocation: 13 | 14 | @classmethod 15 | def ip_data(cls, data): 16 | response = requests.get(f'{Apis.IP_API_REQUEST}') 17 | return response.json()[data] 18 | 19 | @classmethod 20 | def run(cls, url): 21 | start_time = time.time() 22 | ip = HTTP.get_ip(url) 23 | 24 | response = requests.get(f'{Apis.IP_API_REQUEST}{ip}') 25 | resp_json = response.json() 26 | 27 | Table.header([ 28 | ('Name', 'cyan', True), 29 | ('Value', 'white', False) 30 | ]) 31 | 32 | for data in resp_json: 33 | if resp_json[data] != '': 34 | Table.row(data, str(resp_json[data])) 35 | 36 | Table.caption(f'Time taken: {DateTime.calculate_interval(start_time)} seconds') 37 | Table.display() 38 | -------------------------------------------------------------------------------- /docs/docs/basics/get-emails.md: -------------------------------------------------------------------------------- 1 | ## Get Emails 2 | 3 | By using various parameters, users can define specific actions, like collecting email addresses from the web pages. In the context of the provided command: 4 | 5 | ```shell 6 | python linkscraper -u https://example.com -a get-emails 7 | ``` 8 | 9 | The flag is directed to scrape the website "`https://example.com`" with the specific action (`-a`) of retrieving email addresses (`get-emails`). This allows users to gather emails present on the given site, which can be useful for various purposes, including research, auditing, or data collection. As always, such a tool should be used ethically and with proper permissions to avoid any legal or ethical violations. 10 | 11 | ### Filter 12 | 13 | The `-filter` option allows users to refine their results by including only those entries that match the filter keyword. 14 | 15 | ```shell 16 | python linkscraper -u https://example.com -a get-emails -filter example@domain.com 17 | ``` 18 | 19 | To collect emails from the website "`https://example.com`" and then filter the results to only show those related to the email "`example@doamin.com`" 20 | -------------------------------------------------------------------------------- /linkscraper/core/write_env.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | from helper.apis import Apis 4 | 5 | from classes.env import Env 6 | 7 | from rich.prompt import Prompt 8 | 9 | from layout.layout import Layout 10 | 11 | class WriteEnv: 12 | 13 | @classmethod 14 | def run(cls): 15 | imgur_key = Prompt.ask(f'Enter your Imgur Client ID (get ' 16 | f'[blue bold][link={Apis.IMGUR_API_KEY_URL}]here[/link][/blue bold])') 17 | 18 | virustotal_key = Prompt.ask(f'Enter your VirusTotal API key (get ' 19 | f'[blue bold][link={Apis.VIRUSTOTAL_API_KEY_URL}]here[/link][/blue bold])') 20 | 21 | google_fonts_key = Prompt.ask(f'Enter your Google Fonts API Key (get ' 22 | f'[blue bold][link={Apis.GOOGLE_FONTS_API_KEY_URL}]here[/link][/blue bold])') 23 | 24 | Env.write({ 25 | 'imgur': imgur_key, 26 | 'virustotal': virustotal_key, 27 | 'google_fonts': google_fonts_key, 28 | }) 29 | 30 | Layout.success('Env file written successfully.', True) 31 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Emily Silva 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /docs/docs/basics/get-links.md: -------------------------------------------------------------------------------- 1 | ## Get Links 2 | 3 | The `-get-links` command is designed to extract all links from a user-specified URL. Along with its primary function, it supports three additional sub-commands that we will detail below. 4 | 5 | The command allows for tailored actions through different parameters. A common use case is to harvest links from web pages, as demonstrated: 6 | 7 | ```shell 8 | python linkscraper -u https://example.com -a get-links 9 | ``` 10 | 11 | ### Filter 12 | 13 | The `-filter` option allows users to refine their results by including only those entries that match the filter keyword. 14 | 15 | ```shell 16 | python linkscraper -u https://example.com -a get-links -filter domain.com 17 | ``` 18 | 19 | ### Only external links 20 | 21 | The `-oel` option allows users to refine their results by including only those entries that match the links outside from `-u`. 22 | 23 | ```shell 24 | python linkscraper -u https://example.com -a get-emails -oel 25 | ``` 26 | 27 | ### Show status code 28 | 29 | The `-ssc` option show the status code of all links listed 30 | 31 | ```shell 32 | python linkscraper -u https://example.com -a get-emails -ssc 33 | ``` 34 | -------------------------------------------------------------------------------- /docs/docs/basics/static/js.md: -------------------------------------------------------------------------------- 1 | ## JavaScript 2 | 3 | The `-get-js-files` command is designed to extract and list all JavaScript (JS) files from a user-specified URL. This is particularly useful for web developers, designers, and security professionals who wish to review or analyze the style assets of a website. 4 | 5 | To fetch all JS files from the website `https://example.com`, you can execute the following command: 6 | 7 | ```shell 8 | python linkscraper -u https://example.com -a get-js-files 9 | ``` 10 | 11 | ### Filter 12 | 13 | The `-filter` option allows users to refine their results by including only those entries that match the filter keyword. 14 | 15 | ```shell 16 | python linkscraper -u https://example.com -a get-js-files -filter example.css 17 | ``` 18 | 19 | ### Show minify files 20 | 21 | The `-smf` option filters the listed files to display only minified ones identified as `.min.css`. 22 | 23 | ```shell 24 | python linkscraper -u https://example.com -a get-js-files -smf 25 | ``` 26 | 27 | ### Download 28 | 29 | You can also download all the listed files easily; simply use the `-d` flag. 30 | 31 | ```shell 32 | python linkscraper -u https://example.com -a get-js-files -d 33 | ``` 34 | -------------------------------------------------------------------------------- /docs/docs/basics/static/css.md: -------------------------------------------------------------------------------- 1 | ## CSS 2 | 3 | The `-get-css-files` command is designed to extract and list all Cascading Style Sheets (CSS) files from a user-specified URL. This is particularly useful for web developers, designers, and security professionals who wish to review or analyze the style assets of a website. 4 | 5 | To fetch all CSS files from the website `https://example.com`, you can execute the following command: 6 | 7 | ```shell 8 | python linkscraper -u https://example.com -a get-css-files 9 | ``` 10 | 11 | ### Filter 12 | 13 | The `-filter` option allows users to refine their results by including only those entries that match the filter keyword. 14 | 15 | ```shell 16 | python linkscraper -u https://example.com -a get-css-files -filter example.css 17 | ``` 18 | 19 | ### Show minify files 20 | 21 | The `-smf` option filters the listed files to display only minified ones identified as `.min.css`. 22 | 23 | ```shell 24 | python linkscraper -u https://example.com -a get-css-files -smf 25 | ``` 26 | 27 | ### Download 28 | 29 | You can also download all the listed files easily; simply use the `-d` flag. 30 | 31 | ```shell 32 | python linkscraper -u https://example.com -a get-css-files -d 33 | ``` 34 | -------------------------------------------------------------------------------- /linkscraper/core/cookies.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | import requests, time 4 | 5 | from layout.table import Table 6 | from layout.layout import Layout 7 | 8 | from utils.date_time import DateTime 9 | 10 | class Cookies: 11 | 12 | @classmethod 13 | def get_cookies(cls, url, filter_data = None): 14 | start_time = time.time() 15 | response = requests.get(url) 16 | 17 | Table.header([ 18 | ('Name', 'cyan', True), 19 | ('Value', 'white', False) 20 | ]) 21 | 22 | cookie_dict = response.cookies.get_dict() 23 | 24 | if filter_data: 25 | cookie_dict = { 26 | k: v for k, v in cookie_dict.items() if k.find(filter_data) 27 | } 28 | 29 | for name, value in cookie_dict.items(): 30 | Table.row(name, value) 31 | 32 | Table.caption(f'Total of cookies on page: {len(cookie_dict)} - ' 33 | f'Time taken: {DateTime.calculate_interval(start_time)} seconds') 34 | 35 | Table.display() 36 | 37 | @classmethod 38 | def section(cls, url, filter_data): 39 | Layout.header_section('Cookies') 40 | cls.get_cookies(url, filter_data) 41 | -------------------------------------------------------------------------------- /linkscraper/plugins/robots.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | import time, requests 4 | 5 | from layout.layout import Layout 6 | 7 | class Robots: 8 | 9 | @classmethod 10 | def run(cls, url): 11 | start_time = time.time() 12 | robots_url = url + '/robots.txt' 13 | 14 | try: 15 | response = requests.get(robots_url) 16 | response.raise_for_status() 17 | robots = response.text 18 | 19 | for line in robots.split('\n'): 20 | match line: 21 | case _ if 'Allow' in line: 22 | Layout.print(None, line, 'green') 23 | case _ if 'Disallow' in line: 24 | Layout.print(None, line, 'red') 25 | case _ if 'Sitemap' in line: 26 | Layout.print(None, line, 'italic cyan') 27 | case _ if 'User-agent' in line: 28 | Layout.print(None, line, 'italic yellow') 29 | case _: 30 | Layout.print(None, line, 'white') 31 | 32 | Layout.time_taken(start_time, True) 33 | 34 | except requests.RequestException as e: 35 | Layout.error(e, False, True) 36 | -------------------------------------------------------------------------------- /docs/cinder/search-modal.html: -------------------------------------------------------------------------------- 1 | 28 | -------------------------------------------------------------------------------- /linkscraper.yml: -------------------------------------------------------------------------------- 1 | general: 2 | enable_logs: true # Boolean (true/false) 3 | enable_plugins: true # Boolean (true/false) 4 | default_user_agent: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36' # String 5 | 6 | storage: 7 | logs: './logs' # String 8 | downloads: './data/downloads' # String 9 | screenshots: './data/screenshots' # String 10 | 11 | format_dates: 12 | datetime: '%Y-%m-%d %H:%M:%S' # String 13 | today_datetime: '%a, %b %d %Y - %I:%M:%S %p' # String 14 | 15 | dataset: 16 | list_http_status: 'https://gist.githubusercontent.com/kremilly/31aed3c28577f78bcaace2a377f8aa17/raw/4e765d040b739f0d7bc60ef8c2c8f37c17eae81c/http-status-code.json' # String 17 | list_countries: 'https://gist.githubusercontent.com/kremilly/c468fb230d6fcf97de827e37f91f2f6c/raw/3e037cefa50d0381956e862de478c5e5cce758ab/countries.json' # String 18 | list_languages: 'https://gist.githubusercontent.com/kremilly/fd5e5dd45d3480a8da57d56218cecd1e/raw/221c38e4a7e83e2bb9bab92cd8101c9c9adebaaf/languages.json' # String 19 | list_allowed_extensions: 'https://gist.githubusercontent.com/kremilly/5ce0a3c85bc7f6c5b2c49a52aa8ecd36/raw/551cc92e5f3a62f2c7e631add0c5ccb19312a6df/allowed-extensions.json' # String 20 | 21 | advanced: 22 | max_workers: 5 # Integer (unsigned) 23 | -------------------------------------------------------------------------------- /docs/docs/plugins/screenshot.md: -------------------------------------------------------------------------------- 1 | ## Screenshot 2 | 3 | With the `screenshot` plugin, you can capture screenshots of a URL using the Selenium library. To utilize this plugin, enter the following command in your terminal: 4 | 5 | ```shell 6 | python linkscraper -u https://example.com -a get-plugins -p screenshot -b firefox 7 | ``` 8 | 9 | *The `-b` flag specifies which browser you'll use for the screenshot capture (ensure the software is installed on your machine). You have two browser options: Google Chrome and Mozilla Firefox. However, th* 10 | 11 | *# Ignore the path site/ere's a crucial caveat when using Google Chrome with this feature, as noted below.* 12 | 13 | > *Versions starting from 114 of the Google Chrome browser are incompatible with this feature; we suggest using the Mozilla Firefox browser.* 14 | 15 | ### Upload to Imgur 16 | 17 | To upload the screenshot to Imgur, simply use the `-up` flag. However, remember that to utilize Imgur services within Linkscraper, you'll need an Imgur API key, which can be obtained for free. [Click here](apis/imgur.md) to learn how. 18 | 19 | ```shell 20 | python linkscraper -u https://example.com -a get-plugins -p screenshot -b firefox -up -t "Title of post here" 21 | ``` 22 | 23 | *The `-t` flag sets the title for the post. By default, its value is '`Screenshot made by Linkscraper`'.* 24 | -------------------------------------------------------------------------------- /docs/docs/basics/static/images.md: -------------------------------------------------------------------------------- 1 | ## Images 2 | 3 | The `-get-images-files` command is designed to extract and list all JavaScript (JS) files from a user-specified URL. This is particularly useful for web developers, designers, and security professionals who wish to review or analyze the style assets of a website. 4 | 5 | To fetch all JS files from the website `https://example.com`, you can execute the following command: 6 | 7 | ```shell 8 | python linkscraper -u https://example.com -a get-images-files 9 | ``` 10 | 11 | ### Filter 12 | 13 | The `-filter` option allows users to refine their results by including only those entries that match the filter keyword. 14 | 15 | ```shell 16 | python linkscraper -u https://example.com -a get-images-files -filter example.png 17 | ``` 18 | 19 | ### Download 20 | 21 | You can also download all the listed files easily; simply use the `-d` flag. 22 | 23 | ```shell 24 | python linkscraper -u https://example.com -a get-images-files -d 25 | ``` 26 | 27 | ### Formats of images 28 | 29 | Linkscraper is compatible with the main image formats used on the modern internet and also supports some formats that aren't widely used today, aiming to enhance the command's compatibility and to ensure no image format is left out. 30 | 31 | * PNG 32 | * SVG 33 | * TIFF 34 | * WebP 35 | * AVIF 36 | * JPEG 37 | * JPEG XR 38 | * JPEG 2000 39 | -------------------------------------------------------------------------------- /linkscraper/utils/url.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | import re 4 | 5 | from helper.regex import Regex 6 | from urllib.parse import urlparse, parse_qs 7 | 8 | from utils.http import HTTP 9 | 10 | from layout.layout import Layout 11 | 12 | class URL: 13 | 14 | @classmethod 15 | def remove_query(cls, url): 16 | split = url.split('?')[1] 17 | return url.replace('?' + split, '') 18 | 19 | @classmethod 20 | def extract_query_params(cls, url): 21 | parsed_url = urlparse(url) 22 | return parse_qs(parsed_url.query) 23 | 24 | @classmethod 25 | def is_url(cls, url, check_protocol=True): 26 | if url == None: 27 | return False 28 | 29 | if check_protocol == True: 30 | if re.match(Regex.URL_PATTERN_CHECK_PROTOCOL, url) != None: 31 | return True 32 | 33 | return False 34 | 35 | if re.match(Regex.URL_PATTERN_PROTOCOL, url) != None: 36 | return True 37 | 38 | return False 39 | 40 | @classmethod 41 | def check_url_and_connection(cls, url): 42 | if not cls.is_url(url): 43 | Layout.error('URL is missing', False, True) 44 | 45 | if not HTTP.check_connection(url): 46 | Layout.error('connection is not established', False, True) 47 | -------------------------------------------------------------------------------- /docs/docs/plugins/apis/imgur.md: -------------------------------------------------------------------------------- 1 | ## Imgur API Integration 2 | 3 | To integrate with Imgur's API, follow this guide: 4 | 5 | 1. **Sign Up for an Account**: If you're not an Imgur user, [register here](https://imgur.com/). 6 | 2. **Application Registration**: Go to [Imgur's developer portal](https://api.imgur.com/oauth2/addclient) and register your app. This grants API access and provides your Client ID and Secret. 7 | 3. **Form Details**: 8 | 1. **Application Name**: Assign a distinctive name to your app. 9 | 2. **Authorization Type**: Select your preferred authorization flow. Typically, "OAuth 2 without a callback URL" suffices. 10 | 3. **Contact & Overview**: Provide your email and a concise app description. 11 | 4. **Finalize Registration**: Submit the form. 12 | 5. **Secure Your Credentials**: Post-registration, you'll receive a `Client ID` and `Client Secret`. Store these securely, as they're essential for API interactions. 13 | 6. **API Calls**: With your `Client ID`, you're set to interact with the Imgur API. Always respect Imgur's API practices and terms. 14 | 15 | ***Reminder**: The free-tier has rate limits. For extensive API usage, consider premium plans or optimize your app to stay within free limits. Always refer to Imgur's official documentation for current details.* 16 | 17 | > *Visit the [Environments](../../settings/env.md) page to learn how to save the key in your .env file so that Linkscraper starts using your key.* 18 | -------------------------------------------------------------------------------- /linkscraper/plugins/extract_colors.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | import requests, time, re 4 | from layout.table import Table 5 | 6 | from helper.regex import Regex 7 | 8 | from utils.date_time import DateTime 9 | 10 | class ExtractColors: 11 | 12 | @classmethod 13 | def run(cls, url): 14 | start_time = time.time() 15 | 16 | response = requests.get(url) 17 | content = response.text 18 | 19 | colors = [] 20 | color_id = 0 21 | 22 | patterns = [ 23 | re.compile(Regex.COLORS_CODE[0]), # e.g., #FFFFFF 24 | re.compile(Regex.COLORS_CODE[1]), # e.g., #FFF 25 | re.compile(Regex.COLORS_CODE[2]), # e.g., rgb(255, 255, 255) 26 | re.compile(Regex.COLORS_CODE[3]) # e.g., rgba(255, 255, 255, 0.5) 27 | ] 28 | 29 | Table.header([ 30 | ('#', 'cyan', True), 31 | ('Value', 'white', False) 32 | ]) 33 | 34 | for pattern in patterns: 35 | colors.extend( 36 | pattern.findall(content) 37 | ) 38 | 39 | colors = list(set(colors)) 40 | 41 | if len(colors) > 0: 42 | for color in colors: 43 | Table.row(f'Color #{color_id}', color) 44 | color_id += 1 45 | 46 | Table.caption(f'Total of colors: {len(colors)} - ' 47 | f'Time taken: {DateTime.calculate_interval(start_time)} seconds') 48 | 49 | Table.display() 50 | -------------------------------------------------------------------------------- /linkscraper/core/headers.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | import requests, time, json 4 | 5 | from rich.json import JSON 6 | 7 | from layout.table import Table 8 | from layout.layout import Layout 9 | 10 | from utils.date_time import DateTime 11 | 12 | class Headers: 13 | 14 | @classmethod 15 | def is_json(cls, string): 16 | try: 17 | if json.loads(string): 18 | return True 19 | 20 | return False 21 | 22 | except ValueError as e: 23 | return False 24 | 25 | @classmethod 26 | def get_headers(cls, url, filter_data=None): 27 | start_time = time.time() 28 | 29 | response = requests.get(url) 30 | headers_dict = response.headers 31 | 32 | if filter_data: 33 | headers_dict = {k: v for k, v in headers_dict.items() if k.find(filter_data)} 34 | 35 | Table.header([ 36 | ('Name', 'cyan', True), 37 | ('Value', 'white', False) 38 | ]) 39 | 40 | for header_name, header_value in headers_dict.items(): 41 | formatted_value = JSON(header_value) if cls.is_json(header_value) else header_value 42 | Table.row(header_name, formatted_value) 43 | 44 | Table.caption(f'Total of headers: {len(headers_dict)} - ' 45 | f'Time taken: {DateTime.calculate_interval(start_time)} seconds') 46 | 47 | Table.display() 48 | 49 | @classmethod 50 | def section(cls, url, filter_data): 51 | Layout.header_section('Headers') 52 | cls.get_headers(url, filter_data) 53 | -------------------------------------------------------------------------------- /docs/cinder/css/cinder.min.css: -------------------------------------------------------------------------------- 1 | body{font-family:"Open Sans","Helvetica Neue",Helvetica,Arial,sans-serif;font-size:16px;line-height:1.7;background-color:#FFF;color:#343838}h1,h2,h3,h4,h5,h6{font-family:'Inter','Helvetica Neue',Helvetica,Arial,sans-serif;color:#222}h1 small,h2 small,h3 small,h4 small,h5 small,h6 small,.h1 small,.h2 small,.h3 small,.h4 small,.h5 small,.h6 small,h1 .small,h2 .small,h3 .small,h4 .small,h5 .small,h6 .small,.h1 .small,.h2 .small,.h3 .small,.h4 .small,.h5 .small,.h6 .small{color:#b1b7b9}h2{margin-top:35px}h1,h2{font-weight:700}h4{font-family:'Inter','Helvetica Neue',Helvetica,Arial,sans-serif;font-weight:300;margin-top:20px;font-style:italic}h5{font-family:'Inter','Helvetica Neue',Helvetica,Arial,sans-serif;font-weight:300;font-variant:small-caps}pre,code{background-color:#fcfdff}pre>code{font-size:13px}pre{margin-top:25px;margin-bottom:25px}.lead{font-family:"Inter","Helvetica Neue",Helvetica,Arial,sans-serif;font-weight:400;line-height:1.4;letter-spacing:.0312em;color:#b1b7b9}.navbar-default{background-color:#343838;border-bottom:8px #ebf2f2 solid}.bs-sidenav{background-image:url("../img/grid11.png");background-repeat:repeat;font-family:Inter,"Helvetica Neue",Helvetica,Arial,sans-serif;font-size:13px}.well{background-color:#fcfdff}.btn-default{background-color:#fcfdff}.table-striped>tbody>tr:nth-child(2n+1)>td,.table-striped>tbody>tr:nth-child(2n+1)>th{background-color:#fcfdff}#mkdocs-search-query:focus{outline:0;-webkit-box-shadow:none;box-shadow:none}#mkdocs-search-query{font-family:"Inter","Helvetica Neue",Helvetica,Arial,sans-serif;font-size:20px;font-weight:700;color:#343838;height:45px}footer>hr{width:35%} 2 | -------------------------------------------------------------------------------- /linkscraper/utils/file.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | import os, base64 4 | from urllib.parse import urlparse 5 | 6 | from utils.url import URL 7 | 8 | class File: 9 | 10 | @classmethod 11 | def open_read(cls, file): 12 | f = open(file, 'r') 13 | return f.read() 14 | 15 | @classmethod 16 | def to_base64(cls, file): 17 | with open(file, 'rb') as f: 18 | output = base64.b64encode(f.read()) 19 | 20 | return output 21 | 22 | @classmethod 23 | def get_file_name(cls, string): 24 | name = os.path.split(string)[1] 25 | 26 | if string.find('?'): 27 | return URL.remove_query(name) 28 | 29 | return name 30 | 31 | @classmethod 32 | def get_remote_file_name(cls, url): 33 | a = urlparse(url) 34 | basename = os.path.basename(a.path) 35 | 36 | if basename.find('.'): 37 | return basename 38 | 39 | for file in a.path.split('/'): 40 | if basename.find('.') and len(file) > 1: 41 | return file 42 | 43 | @classmethod 44 | def create_path(cls, folder): 45 | if os.path.isdir(folder) != True: 46 | os.makedirs(folder) 47 | 48 | @classmethod 49 | def open(cls, file): 50 | if os.path.exists(file): 51 | os.startfile( 52 | os.getcwd() + '\\' + file 53 | ) 54 | 55 | @classmethod 56 | def write(cls, file_name, content, mode, enconding): 57 | with open(f'{file_name}', mode, encoding=enconding) as f: 58 | f.write(content) 59 | -------------------------------------------------------------------------------- /linkscraper/core/core.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | import time 4 | 5 | from utils.http import HTTP 6 | from utils.date_time import DateTime 7 | 8 | from layout.layout import Layout 9 | 10 | from plugins.ip_location import IPLocation 11 | 12 | class Core: 13 | 14 | @classmethod 15 | def basic(cls, url): 16 | start_time = time.time() 17 | Layout.header_section('Core') 18 | 19 | Layout.print('IP Address:', HTTP.get_ip(url)) 20 | Layout.print('HTTP Code:', HTTP.code(url)) 21 | Layout.print('HTTP Code Message:', HTTP.code_list(HTTP.code(url))) 22 | 23 | if HTTP.check_protocol_url(url, True): 24 | Layout.print('HTTPS Status:', 'Secure', 'bold green') 25 | elif HTTP.check_protocol_url(url): 26 | Layout.print('HTTPS Status', 'Not secure', 'bold red') 27 | 28 | Layout.time_taken(start_time, True) 29 | 30 | @classmethod 31 | def home(cls, url): 32 | location = f"{IPLocation.ip_data('city')}, {IPLocation.ip_data('regionName')} - {IPLocation.ip_data('country')}" 33 | 34 | Layout.print('Public IP:', IPLocation.ip_data('query'), 'italic white') 35 | Layout.print('ISP:', IPLocation.ip_data('isp'), 'italic cyan') 36 | Layout.print('Location:', f'{ location }', 'italic green') 37 | Layout.separator() 38 | 39 | Layout.print('Target:', url, 'bold green') 40 | Layout.print('Hostname:', HTTP.get_hostname(url), 'bold blue') 41 | Layout.print('Scan:', DateTime.today_datetime(), 'italic cyan') 42 | -------------------------------------------------------------------------------- /linkscraper/core/static/images.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | from core.static.autoload import * 4 | 5 | from utils.date_time import DateTime 6 | 7 | class Images: 8 | 9 | @classmethod 10 | def images_files(cls, url, *args): 11 | if args.download: 12 | return DownloadResources.download(url, 'images', None, args.filter_data) 13 | 14 | start_time = time.time() 15 | html = session.get(url).content 16 | soup = bs(html, 'html.parser') 17 | 18 | Table.header([ 19 | ('Filename', 'cyan', True), 20 | ('URL', 'bold blue', False), 21 | ('Size', 'green', False) 22 | ]) 23 | 24 | links = [] 25 | 26 | for img in soup.find_all('img'): 27 | img_url = urljoin(url, img.attrs.get('src')) 28 | 29 | if args.filter_data: 30 | if img_url.find(args.filter_data): 31 | links.append(img_url) 32 | else: 33 | links.append(img_url) 34 | 35 | list_images = list(set(links)) 36 | 37 | for img_url in list_images: 38 | Table.row(File.get_remote_file_name(img_url), img_url, FileSize.remote_file(img_url)) 39 | 40 | Table.caption(f'Total images files on page: {len(list_images)} - ' 41 | f'Time taken: {DateTime.calculate_interval(start_time)} seconds') 42 | 43 | Table.display() 44 | 45 | @classmethod 46 | def section(cls, url, filter_data, download): 47 | Layout.header_section('Images') 48 | cls.images_files(url, filter_data, download) 49 | -------------------------------------------------------------------------------- /docs/cinder/css/highlight.css: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | github.com style (c) Vasily Polovnyov 4 | 5 | */ 6 | 7 | .hljs { 8 | display: block; 9 | overflow-x: auto; 10 | padding: 0.5em; 11 | color: #333; 12 | background: #FCFDFF; 13 | } 14 | 15 | .hljs-comment, 16 | .hljs-quote { 17 | color: #998; 18 | font-style: italic; 19 | } 20 | 21 | .hljs-keyword, 22 | .hljs-selector-tag, 23 | .hljs-subst { 24 | color: #333; 25 | font-weight: bold; 26 | } 27 | 28 | .hljs-number, 29 | .hljs-literal, 30 | .hljs-variable, 31 | .hljs-template-variable, 32 | .hljs-tag .hljs-attr { 33 | color: #008080; 34 | } 35 | 36 | .hljs-string, 37 | .hljs-doctag { 38 | color: #d14; 39 | } 40 | 41 | .hljs-title, 42 | .hljs-section, 43 | .hljs-selector-id { 44 | color: #900; 45 | font-weight: bold; 46 | } 47 | 48 | .hljs-subst { 49 | font-weight: normal; 50 | } 51 | 52 | .hljs-type, 53 | .hljs-class .hljs-title { 54 | color: #458; 55 | font-weight: bold; 56 | } 57 | 58 | .hljs-tag, 59 | .hljs-name, 60 | .hljs-attribute { 61 | color: #000080; 62 | font-weight: normal; 63 | } 64 | 65 | .hljs-regexp, 66 | .hljs-link { 67 | color: #009926; 68 | } 69 | 70 | .hljs-symbol, 71 | .hljs-bullet { 72 | color: #990073; 73 | } 74 | 75 | .hljs-built_in, 76 | .hljs-builtin-name { 77 | color: #0086b3; 78 | } 79 | 80 | .hljs-meta { 81 | color: #999; 82 | font-weight: bold; 83 | } 84 | 85 | .hljs-deletion { 86 | background: #fdd; 87 | } 88 | 89 | .hljs-addition { 90 | background: #dfd; 91 | } 92 | 93 | .hljs-emphasis { 94 | font-style: italic; 95 | } 96 | 97 | .hljs-strong { 98 | font-weight: bold; 99 | } 100 | -------------------------------------------------------------------------------- /docs/cinder/keyboard-modal.html: -------------------------------------------------------------------------------- 1 | 41 | -------------------------------------------------------------------------------- /docs/docs/plugins/apis/google-fonts.md: -------------------------------------------------------------------------------- 1 | ## Google Fonts Integration 2 | 3 | **Obtaining an API Key for Google Fonts Developer API:** 4 | 5 | 1. **Google Cloud Console**: 6 | 7 | - Go to the [Google Cloud Console](https://console.cloud.google.com/). 8 | - If you haven’t already, sign in with your Google account. 9 | - Create a new project or select an existing one. 10 | 2. **Enable the API**: 11 | 12 | - In the navigation menu, select "APIs & Services" > "Library". 13 | - In the search bar, type "Google Fonts Developer API". 14 | - Click on the Google Fonts Developer API from the search results. 15 | - Click on the "Enable" button. 16 | 3. **Create Credentials**: 17 | 18 | - After enabling the API, you’ll be directed to the API page. If not, go back to the dashboard of your project. 19 | - Click on "APIs & Services" > "Credentials". 20 | - Click the "Create Credentials" dropdown button and select "API key". 21 | - Your new API key will be displayed. Copy this key for your use. 22 | 4. **(Optional) Restrict the API Key**: 23 | 24 | - For added security, you might want to restrict the API key so that it can only be used by certain IPs, apps, or other specific conditions. 25 | - On the "Credentials" page, click on the API key you've just created. 26 | - Under "Key restriction", choose the option that suits your needs, and follow the on-screen instructions to set the restrictions. 27 | 5. **Usage**: 28 | 29 | - With the API key in hand, you can use it to make calls to the Google Fonts Developer API following the official documentation. 30 | 31 | > *Visit the [Environments](../../settings/env.md) page to learn how to save the key in your .env file so that Linkscraper starts using your key.* 32 | -------------------------------------------------------------------------------- /linkscraper/__main__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | from helper.configs import Configs 4 | 5 | from classes.startup import Startup 6 | 7 | if __name__ == '__main__': 8 | Startup('E.g.: python linkscraper -u http://example.com', [ 9 | {'short': 'u', 'long': 'url', 'help': 'URL to scan', 'required': False}, 10 | {'short': 'f', 'long': 'filter', 'help': 'Filter data', 'required': False}, 11 | {'short': 'a', 'long': 'action', 'help': 'Run an action', 'required': False}, 12 | {'short': 'p', 'long': 'plugin', 'help': 'Load a plugin', 'required': False}, 13 | {'short': 'b', 'long': 'browser', 'help': 'Set browser to take screenshot', 'required': False}, 14 | {'short': 't', 'long': 'title', 'help': 'Set title the screenshot on Imgur', 'required': False}, 15 | {'short': 'd', 'long': 'download', 'help': 'Download static files', 'action': 'store_true', 'required': False}, 16 | {'short': 'up', 'long': 'upload', 'help': 'Upload the screenshot to Imgur', 'action': 'store_true', 'required': False}, 17 | {'short': 'v', 'long': 'version', 'help': 'Show current version', 'action': 'version', 'version': f'{Configs.VERSION}'}, 18 | {'short': 'we', 'long': 'write-env', 'help': 'Write environments file (.env)', 'action': 'store_true', 'required': False}, 19 | {'short': 'ssc', 'long': 'show-status-code', 'help': 'Show status code', 'required': False, 'action': 'store_true', 'default': 'false'}, 20 | {'short': 'smf', 'long': 'show-minify-files', 'help': 'Show only minify files', 'required': False, 'action': 'store_true', 'default': 'false'}, 21 | {'short': 'oel', 'long': 'only-external-links', 'help': 'Show only external links', 'required': False, 'action': 'store_true', 'default': 'false'}, 22 | ]).run() 23 | -------------------------------------------------------------------------------- /docs/docs/plugins/apis/virustotal.md: -------------------------------------------------------------------------------- 1 | ## VirusTotal Integration 2 | 3 | **Obtaining the VirusTotal API Key:** 4 | 5 | 1. **Sign Up/Log In** : 6 | 7 | * Go to the [VirusTotal website](https://www.virustotal.com/). 8 | * If you don't already have an account, you'll need to sign up. If you do, simply log in. 9 | 10 | 1. **Navigate to the API Section** : 11 | 12 | * Once logged in, head to your profile or account section. 13 | * Look for an option related to API or API Key. The exact navigation might vary as websites update their UI, but generally, it's within the user's profile or settings area. 14 | 15 | 1. **Generate or View API Key** : 16 | 17 | * Depending on the platform's design, you might either directly see your API key or have an option to generate one. If provided an option to generate, click on it. 18 | 19 | 1. **Note and Store Safely** : 20 | 21 | * Once you have your API key, make sure to note it down and store it safely. API keys should be treated like passwords, as they provide access to the services and their capabilities. 22 | 23 | 1. **Usage Limits** : 24 | 25 | * VirusTotal provides a free tier with limited requests per day. If you need more requests or additional features, you might need to look into their paid or premium offerings. 26 | 27 | 1. **Implementation** : 28 | 29 | * When implementing the API in your application or tool, ensure that the key is stored securely and is not exposed, especially if you're distributing the application to others. 30 | 31 | ***Remember** : The above steps provide a general guide, and the exact steps might slightly vary based on changes made to the VirusTotal website or its services. Always refer to official documentation or user guides provided by the platform for the most accurate and up-to-date information.* 32 | 33 | > *Visit the [Environments](../../settings/env.md) page to learn how to save the key in your .env file so that Linkscraper starts using your key.* 34 | -------------------------------------------------------------------------------- /linkscraper/core/plugins.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | from layout.layout import Layout 4 | 5 | from classes.settings import Settings 6 | 7 | from plugins.whois import Whois 8 | from plugins.robots import Robots 9 | from plugins.virustotal import VT 10 | from plugins.screenshot import Screenshot 11 | from plugins.subdomains import Subdomains 12 | from plugins.ip_location import IPLocation 13 | from plugins.page_details import PageDetails 14 | from plugins.detect_fonts import DetectFonts 15 | from plugins.extract_colors import ExtractColors 16 | 17 | class Plugins: 18 | 19 | @classmethod 20 | def run(cls, plugin, url, args): 21 | cfg_plugin = Settings.get('general.enable_plugins', 'boolean') 22 | 23 | if cfg_plugin is True: 24 | Layout.header_plugin(plugin) 25 | 26 | match plugin: 27 | case 'whois': 28 | Whois.run(url) 29 | case 'robots': 30 | Robots.run(url) 31 | case 'page-details': 32 | PageDetails.run(url) 33 | case 'virustotal': 34 | VT.run(url) 35 | case 'ip-location': 36 | IPLocation.run(url) 37 | case 'subdomains': 38 | Subdomains.run(url) 39 | case 'screenshot': 40 | Screenshot.run(url, args) 41 | case 'detect-fonts': 42 | DetectFonts.run(url, args) 43 | case 'extract-colors': 44 | ExtractColors.run(url) 45 | case _: 46 | Layout.error('Plugin invalid', False, True) 47 | else: 48 | Layout.print('[bold green]Message[/bold green]:', f'The plugins are disabled. ' 49 | f'Enable that on [blue]linkscraper.yml[/blue] file', None, True) 50 | -------------------------------------------------------------------------------- /linkscraper/apis/virustotal.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | import requests 4 | 5 | from http import HTTPStatus 6 | 7 | from helper.apis import Apis 8 | 9 | from utils.http import HTTP 10 | 11 | from layout.layout import Layout 12 | 13 | class VirusTotal: 14 | 15 | @classmethod 16 | def stats(cls, params): 17 | Layout.print('\t\t', 'Stats:') 18 | Layout.separator() 19 | 20 | Layout.print('[bold green]Harmless[/bold green]:', str(params['harmless'])) 21 | Layout.print('[bold red]Malicious[/bold red]:', str(params['malicious'])) 22 | Layout.print('[bold yellow]Suspicious[/bold yellow]:', str(params['suspicious'])) 23 | Layout.print('[bold cyan]Undetected[/bold cyan]:', str(params['undetected'])) 24 | 25 | @classmethod 26 | def error(cls, message, additional_print = True): 27 | if additional_print: 28 | return Layout.error(message, False, True, { 29 | 'style': 'bold blue', 30 | 'text': 'Get your VirusTotal key here:', 31 | 'value': Apis.VIRUSTOTAL_API_KEY_URL, 32 | }) 33 | 34 | return Layout.error(message, False, True) 35 | 36 | @classmethod 37 | def request(cls, url, key): 38 | response = requests.post(Apis.VIRUSTOTAL_API_REQUEST, data='url=' + HTTP.strip_scheme(url), headers={ 39 | 'x-apikey': key, 40 | 'accept': 'application/json', 41 | 'content-type': 'application/x-www-form-urlencoded' 42 | }) 43 | 44 | if not response.status_code is HTTPStatus.OK: 45 | if response.json()['error']['code'] == 'WrongCredentialsError': 46 | cls.error('Key is invalid') 47 | 48 | cls.error(response.json()['error']['message'], False) 49 | 50 | response = requests.get(response.json()['data']['links']['self'], headers={ 51 | 'x-apikey': key, 52 | 'accept': 'application/json', 53 | }) 54 | 55 | return response.json() 56 | -------------------------------------------------------------------------------- /linkscraper/core/static/js.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | from core.static.autoload import * 4 | 5 | from utils.date_time import DateTime 6 | 7 | class JS: 8 | 9 | @classmethod 10 | def js_files(cls, url, *args): 11 | if args.download: 12 | return DownloadResources.download(url, 'js', args.minify_files, args.filter_data) 13 | 14 | start_time = time.time() 15 | html = session.get(url).content 16 | soup = bs(html, 'html.parser') 17 | 18 | Table.header([ 19 | ('Filename', 'cyan', True), 20 | ('URL', 'bold blue', False), 21 | ('Size', 'green', False), 22 | ]) 23 | 24 | links = [] 25 | 26 | for script in soup.find_all('script'): 27 | if script.attrs.get('src'): 28 | script_url = urljoin(url, script.attrs.get('src')) 29 | 30 | if script_url.find('.js') != -1: 31 | if args.filter_data: 32 | if script_url.find(args.filter_data): 33 | links.append(script_url) 34 | 35 | if args.minify_files: 36 | if script_url.find('.min.js'): 37 | links.append(script_url) 38 | 39 | links.append(script_url) 40 | 41 | list_scripts = list(set(links)) 42 | 43 | for script_url in list_scripts: 44 | Table.row(File.get_remote_file_name(script_url), script_url, FileSize.remote_file(script_url)) 45 | 46 | Table.caption(f'Total script files on page: {len(list_scripts)} - ' 47 | f'Time taken: {DateTime.calculate_interval(start_time)} seconds') 48 | 49 | Table.display() 50 | 51 | @classmethod 52 | def section(cls, url, minify_files, filter_data, download): 53 | Layout.header_section('Scripts JavaScript') 54 | cls.js_files(url, minify_files, filter_data, download) 55 | -------------------------------------------------------------------------------- /linkscraper/plugins/subdomains.py: -------------------------------------------------------------------------------- 1 | import requests, time 2 | from concurrent.futures import ThreadPoolExecutor 3 | 4 | from rich.progress import Progress 5 | 6 | from utils.http import HTTP 7 | from utils.date_time import DateTime 8 | 9 | from helper.apis import Apis 10 | 11 | from layout.table import Table 12 | 13 | from classes.settings import Settings 14 | 15 | class Subdomains: 16 | 17 | @classmethod 18 | def get_status(cls, item, protocol): 19 | url = protocol + item 20 | http_code = HTTP.code(url) 21 | http_reason = HTTP.code_list(http_code) 22 | 23 | if http_code == False: 24 | http_reason = 'Offline' 25 | 26 | Table.row(item, str(http_code), http_reason, url) 27 | 28 | @classmethod 29 | def run(cls, url): 30 | start_time = time.time() 31 | max_workers = Settings.get('advanced.max_workers', 'int') 32 | 33 | domain = HTTP.strip_scheme(url) 34 | protocol = HTTP.get_scheme(url) + '://' 35 | 36 | resp_json = requests.get(Apis.THREATCROWD_API_REQUEST, params={ 37 | 'domain': domain 38 | }).json() 39 | 40 | total_subdomains = len(resp_json['subdomains']) 41 | 42 | Table.header([ 43 | ('Subdomain', 'cyan', True), 44 | ('Status code', 'white', False), 45 | ('HTTP message', 'yellow', False), 46 | ('Link', 'bold blue', False) 47 | ]) 48 | 49 | with Progress() as progress: 50 | task = progress.add_task("[cyan]Fetching subdomains...", total=total_subdomains) 51 | 52 | with ThreadPoolExecutor(max_workers=max_workers) as executor: 53 | for _ in executor.map(lambda item: cls.get_status(item, protocol), resp_json['subdomains']): 54 | progress.advance(task) 55 | 56 | Table.caption(f'Total of subdomains: {total_subdomains} - ' 57 | f'Time taken: {DateTime.calculate_interval(start_time)} seconds') 58 | 59 | Table.display() 60 | -------------------------------------------------------------------------------- /linkscraper/core/static/css.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | from core.static.autoload import * 4 | 5 | from utils.date_time import DateTime 6 | 7 | class CSS: 8 | 9 | @classmethod 10 | def css_files(cls, url, *args): 11 | if args.download: 12 | return DownloadResources.download(url, 'css', args.minify_files, args.filter_data) 13 | 14 | start_time = time.time() 15 | 16 | html = session.get(url).content 17 | soup = bs(html, 'html.parser') 18 | 19 | Table.header([ 20 | ('Filename', 'cyan', True), 21 | ('URL', 'bold blue', False), 22 | ('Size', 'green', False) 23 | ]) 24 | 25 | links = [] 26 | 27 | for css in soup.find_all('link'): 28 | if css.attrs.get('href'): 29 | css_url = urljoin(url, css.attrs.get('href')) 30 | 31 | if css_url.find('.css') != -1: 32 | if args.filter_data: 33 | if css_url.find('.css') and css_url.find(args.filter_data): 34 | links.append(css_url) 35 | 36 | if args.minify_files: 37 | if css_url.find('.css') and css_url.find('.min.css'): 38 | links.append(css_url) 39 | 40 | if css_url.find('.css'): 41 | links.append(css_url) 42 | 43 | list_css = list(set(links)) 44 | 45 | for css_url in list_css: 46 | Table.row(File.get_remote_file_name(css_url), css_url, FileSize.remote_file(css_url)) 47 | 48 | Table.caption(f'Total CSS files on page: {len(list_css)} - ' 49 | f'Time taken: {DateTime.calculate_interval(start_time)} seconds') 50 | 51 | Table.display() 52 | 53 | @classmethod 54 | def section(cls, url, minify_files, filter_data, download): 55 | Layout.header_section('CSS Files') 56 | cls.css_files(url, minify_files, filter_data, download) 57 | -------------------------------------------------------------------------------- /docs/docs/overview/parameters.md: -------------------------------------------------------------------------------- 1 | ## Parameters 2 | 3 | ### Core 4 | 5 | | Parameter | Description | Required | Default | 6 | | --------------------------- | ------------------------------ | --------------- | ------------ | 7 | | -u, --url | URL to scan | ✅ in live mode | | 8 | | -a, --action | Run an action | No | ``get-core`` | 9 | | -p, --plugin | Load a plugin | No | | 10 | | -oel, --only-external-links | Show only external links | No | `Null` | 11 | | -ssc, --show-status-code | Show status code | No | `Null` | 12 | | -smf, --show-minify-files | Show only minify files | No | `Null` | 13 | | -f, --filter | Filter data | No | | 14 | | -d, --download | Download static files | No | `Null` | 15 | | -we, --write-env | Write environments file (.env) | No | `Null` | 16 | | -v, --version | Show current version | No | `Null` | 17 | 18 | > *The parameters `-oel`, `-ssc`, `-smf`, `-we`, and `-d` cannot take values.* 19 | 20 | ### Plugins 21 | 22 | These parameters are only useful when used with some plugin. 23 | 24 | | Parameter | Description | Required | Default | 25 | | ------------------- | ---------------------------------- | -------- | ---------------------------------- | 26 | | -b, --browser | Set the browser to take screenshot | No | ``firefox`` | 27 | | -t, --title | Set title the screenshot on Imgur | No | ``Screenshot made by Linkscraper`` | 28 | | -gf, --google-fonts | Download fonts from Google Fonts | No | `Null` | 29 | | -up, --upload | Upload the screenshot to Imgur | No | `Null` | 30 | 31 | > *The parameters `-up` and `-gf` cannot take values.* 32 | -------------------------------------------------------------------------------- /linkscraper/plugins/virustotal.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | import time 4 | 5 | from classes.env import Env 6 | 7 | from apis.virustotal import VirusTotal 8 | 9 | from layout.table import Table 10 | from layout.layout import Layout 11 | 12 | from utils.date_time import DateTime 13 | 14 | class VT: 15 | 16 | @classmethod 17 | def run(cls, url): 18 | key = Env.get('VIRUSTOTAL_KEY') 19 | 20 | if not key: 21 | return VirusTotal.error('Key is requiired') 22 | 23 | start_time = time.time() 24 | resp_json = VirusTotal.request(url, key) 25 | permalink = resp_json['data']['links']['item'].replace('api/v3/urls', 'gui/url') 26 | 27 | VirusTotal.stats({ 28 | 'harmless': resp_json['data']['attributes']['stats']['harmless'], 29 | 'malicious': resp_json['data']['attributes']['stats']['malicious'], 30 | 'suspicious': resp_json['data']['attributes']['stats']['suspicious'], 31 | 'undetected': resp_json['data']['attributes']['stats']['undetected'], 32 | }) 33 | 34 | Layout.separator() 35 | Layout.print('Permalink', permalink, 'bold green') 36 | 37 | Layout.header_section('Result') 38 | 39 | Table.header([ 40 | ('Engine', 'cyan', True), 41 | ('Result', 'white', False), 42 | ('Category', 'white', False) 43 | ]) 44 | 45 | for engine in resp_json['data']['attributes']['results']: 46 | result = resp_json['data']['attributes']['results'][engine]['result'] 47 | category = resp_json['data']['attributes']['results'][engine]['category'] 48 | 49 | match result: 50 | case 'clean': 51 | Table.row(engine, f'[bold green]{result}[/bold green]') 52 | case 'unrated': 53 | Table.row(engine, f'[bold cyan]{result}[/bold cyan]') 54 | case _: 55 | Table.row(engine, f'[bold red]{result}[/bold red]', f'[bold red]{category}[/bold red]') 56 | 57 | Table.caption(f'Time taken: {DateTime.calculate_interval(start_time)} seconds') 58 | Table.display() 59 | -------------------------------------------------------------------------------- /docs/cinder/css/cinder.css: -------------------------------------------------------------------------------- 1 | /* 2 | Cinder Theme for MkDocs | Copyright 2015 Christopher Simpkins | MIT License 3 | */ 4 | 5 | body { 6 | font-family:"Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif; 7 | font-size: 16px; 8 | line-height: 1.7; 9 | background-color: #FFF; 10 | color: #343838; 11 | } 12 | h1, h2, h3, h4, h5, h6 { 13 | font-family:'Inter', 'Helvetica Neue', Helvetica, Arial, sans-serif; 14 | color: #222; 15 | } 16 | h1 small, h2 small, h3 small, h4 small, h5 small, h6 small, .h1 small, .h2 small, .h3 small, .h4 small, .h5 small, .h6 small, h1 .small, h2 .small, h3 .small, h4 .small, h5 .small, h6 .small, .h1 .small, .h2 .small, .h3 .small, .h4 .small, .h5 .small, .h6 .small { 17 | color: #B1B7B9; 18 | } 19 | 20 | h2 { 21 | margin-top: 35px; 22 | } 23 | 24 | h1, h2 { 25 | font-weight: 700; 26 | } 27 | h4 { 28 | font-family: 'Inter', 'Helvetica Neue', Helvetica, Arial, sans-serif; 29 | font-weight: 300; 30 | margin-top: 20px; 31 | font-style: italic; 32 | } 33 | h5 { 34 | font-family: 'Inter', 'Helvetica Neue', Helvetica, Arial, sans-serif; 35 | font-weight: 300; 36 | font-variant: small-caps; 37 | } 38 | pre, code { 39 | background-color: #FCFDFF; 40 | } 41 | pre>code { 42 | font-size: 13px; 43 | } 44 | pre { 45 | margin-top: 25px; 46 | margin-bottom: 25px; 47 | } 48 | .lead { 49 | font-family:"Inter", "Helvetica Neue", Helvetica, Arial, sans-serif; 50 | font-weight: 400; 51 | line-height: 1.4; 52 | letter-spacing: 0.0312em; 53 | color: #B1B7B9; 54 | } 55 | .navbar-default { 56 | background-color: #343838; 57 | border-bottom: 8px #EBF2F2 solid; 58 | } 59 | .bs-sidenav { 60 | background-image: url("../img/grid11.png"); 61 | background-repeat: repeat; 62 | font-family: Inter,"Helvetica Neue",Helvetica,Arial,sans-serif; 63 | font-size: 13px; 64 | } 65 | .well { 66 | background-color: #FCFDFF; 67 | } 68 | .btn-default { 69 | background-color:#FCFDFF; 70 | } 71 | .table-striped > tbody > tr:nth-child(2n+1) > td, .table-striped > tbody > tr:nth-child(2n+1) > th { 72 | background-color: #FCFDFF; 73 | } 74 | #mkdocs-search-query:focus { 75 | outline: none; 76 | -webkit-box-shadow: none; 77 | box-shadow: none; 78 | } 79 | #mkdocs-search-query { 80 | font-family:"Inter", "Helvetica Neue", Helvetica, Arial, sans-serif; 81 | font-size: 20px; 82 | font-weight: 700; 83 | color: #343838; 84 | height: 45px; 85 | } 86 | footer > hr { 87 | width: 35%; 88 | } 89 | -------------------------------------------------------------------------------- /linkscraper/classes/startup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | import argparse, sys 4 | 5 | from utils.url import URL 6 | 7 | from layout.layout import Layout 8 | 9 | from core.headers import Headers 10 | from core.cookies import Cookies 11 | from core.scraper import Scraper 12 | 13 | from core.static.js import JS 14 | from core.static.css import CSS 15 | from core.static.images import Images 16 | 17 | from core.core import Core 18 | from core.plugins import Plugins 19 | 20 | from core.write_env import WriteEnv 21 | 22 | class Startup: 23 | 24 | @classmethod 25 | def __init__(cls, desc, flags): 26 | cls.args = cls.parser(desc, flags) 27 | 28 | @classmethod 29 | def parser(cls, desc, args): 30 | parser = argparse.ArgumentParser(description=desc) 31 | 32 | for arg in args: 33 | parser.add_argument( 34 | f"-{arg.pop('short', None)}", f"--{arg.pop('long', None)}", **arg 35 | ) 36 | 37 | return parser.parse_args() 38 | 39 | @classmethod 40 | def run(cls): 41 | Layout.header() 42 | 43 | if cls.args.write_env: 44 | WriteEnv.run() 45 | sys.exit() 46 | 47 | base_url = cls.args.url 48 | URL.check_url_and_connection(base_url) 49 | 50 | Core.home(base_url) 51 | 52 | match cls.args.action: 53 | case 'get-core' | 'core': 54 | Core.basic(base_url) 55 | case 'get-headers' | 'headers': 56 | Headers.section(base_url, cls.args.filter) 57 | case 'get-cookies' | 'cookies': 58 | Cookies.section(base_url, cls.args.filter) 59 | case 'get-js-files' | 'js-files': 60 | JS.section(base_url, *cls.args) 61 | case 'get-css-files' | 'css-files': 62 | CSS.section(base_url, cls.args.show_minify_files, cls.args.filter, cls.args.download) 63 | case 'get-images-files' | 'images-files': 64 | Images.section(base_url, cls.args.filter, cls.args.download) 65 | case 'get-links' | 'links': 66 | Scraper.section_links(base_url, cls.args.only_external_links, cls.args.show_status_code, cls.args.filter) 67 | case 'get-emails' | 'emails': 68 | Scraper.section_emails(base_url, cls.args.filter) 69 | case 'get-plugins' | 'plugins': 70 | Plugins.run(cls.args.plugin, base_url, cls.args) 71 | case _: 72 | Layout.error('Action invalid', True, True) 73 | -------------------------------------------------------------------------------- /linkscraper/utils/http.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | import socket, requests 3 | 4 | from urllib.parse import urlparse 5 | from http.client import HTTPConnection, HTTPSConnection 6 | 7 | from helper.configs import Configs 8 | 9 | from classes.settings import Settings 10 | 11 | class HTTP: 12 | 13 | @classmethod 14 | def get_scheme(cls, url): 15 | parsed = urlparse(url) 16 | return parsed.scheme 17 | 18 | @classmethod 19 | def strip_scheme(cls, url): 20 | parsed = urlparse(url) 21 | 22 | return parsed.geturl().replace( 23 | "%s://" % parsed.scheme, '', 1 24 | ) 25 | 26 | @classmethod 27 | def code_list(cls, c): 28 | r = requests.get(Settings.get('dataset.list_http_status', 'STRING')) 29 | r = r.json() 30 | 31 | for code in r: 32 | if code == str(c): 33 | return "[italic yellow][link=" + r[code]["spec_href"] + "]" + r[code]["message"] + "[/link][/italic yellow]" 34 | 35 | @classmethod 36 | def code(cls, url): 37 | try: 38 | return str( 39 | requests.head(url).status_code 40 | ) 41 | 42 | except requests.ConnectionError: 43 | return False 44 | 45 | @classmethod 46 | def get_ip(cls, url): 47 | url = cls.strip_scheme(url) 48 | return socket.gethostbyname(url) 49 | 50 | @classmethod 51 | def get_hostname(cls, url): 52 | return urlparse(url).netloc 53 | 54 | @classmethod 55 | def check_protocol_url(cls, url, https = False): 56 | link = f'http://{url}' 57 | 58 | protocol_url = urlparse(link) 59 | connection = HTTPConnection(protocol_url.netloc) 60 | 61 | if https: 62 | link = link.replace('http', 'https') 63 | connection = HTTPSConnection(protocol_url.netloc) 64 | 65 | try: 66 | connection.request('HEAD', protocol_url.path) 67 | 68 | if connection.getresponse(): 69 | return True 70 | 71 | return False 72 | 73 | except: 74 | return False 75 | 76 | @classmethod 77 | def check_connection(cls, host): 78 | try: 79 | response = requests.get(host) 80 | if response.status_code == requests.codes.ok: 81 | return True 82 | else: 83 | return False 84 | 85 | except: 86 | return False 87 | -------------------------------------------------------------------------------- /linkscraper/plugins/screenshot.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | import os, time, random 4 | 5 | from utils.http import HTTP 6 | from utils.file import File 7 | 8 | from apis.imgur import Imgur 9 | from layout.layout import Layout 10 | 11 | from classes.settings import Settings 12 | 13 | from selenium import webdriver 14 | from selenium.webdriver.firefox.options import Options as FirefoxOptions 15 | 16 | class Screenshot: 17 | 18 | @classmethod 19 | def generate_id(cls, size): 20 | random_string = '' 21 | random_str_seq = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' 22 | 23 | for _ in range(0, size): 24 | random_string += str( 25 | random_str_seq[ 26 | random.randint( 27 | 0, len(random_str_seq) - 1 28 | ) 29 | ] 30 | ) 31 | 32 | return random_string 33 | 34 | @classmethod 35 | def browser_chrome(cls, url, file): 36 | options = webdriver.ChromeOptions() 37 | 38 | options.add_argument('--headless') 39 | driver = webdriver.Chrome(options=options) 40 | 41 | driver.get(url) 42 | driver.save_screenshot(file) 43 | driver.quit() 44 | 45 | @classmethod 46 | def browser_firefox(cls, url, file): 47 | options = FirefoxOptions() 48 | options.add_argument('--headless') 49 | 50 | driver = webdriver.Firefox(options=options) 51 | driver.get(url) 52 | driver.save_screenshot(file) 53 | driver.quit() 54 | 55 | @classmethod 56 | def run(cls, url, args): 57 | start_time = time.time() 58 | path = f"{Settings.get('storage.screenshots', 'STRING')}/{HTTP.get_hostname(url)}/" 59 | 60 | File.create_path(path) 61 | 62 | file = path + f'{cls.generate_id(12)}.png' 63 | 64 | match args.browser: 65 | case 'chrome': 66 | cls.browser_chrome(url, file) 67 | 68 | case 'firefox': 69 | cls.browser_firefox(url, file) 70 | 71 | case _: 72 | Layout.error('Browser is invalid', False, True) 73 | 74 | if os.path.exists(file): 75 | Layout.success('screenshot saved with successfully.') 76 | 77 | if not args.upload: 78 | File.open(file) 79 | return Layout.time_taken(start_time) 80 | 81 | return Imgur.upload(file, args.title) 82 | -------------------------------------------------------------------------------- /linkscraper/classes/settings.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | import yaml 4 | 5 | from helper.configs import Configs 6 | 7 | from exceptions.settings_exception import SettingsException 8 | 9 | class Settings: 10 | 11 | @classmethod 12 | def search_property_in_file(cls, prop): 13 | with open(Configs.CONFIGS_FILE, 'r') as file: 14 | lines = file.readlines() 15 | 16 | for line_number, line in enumerate(lines, start=1): 17 | if prop.split('.')[-1] in line: 18 | return f"{line_number} -> '{line.strip()}'" 19 | 20 | @classmethod 21 | def get_wrong_property_position(cls, prop, open_file=False): 22 | configs_file = Configs.CONFIGS_FILE 23 | line_position = cls.search_property_in_file(prop) 24 | 25 | if open_file: 26 | line_position = line_position.split(' -> ')[0] 27 | return f'{configs_file}:{line_position}' 28 | 29 | return f"Please fix it in: {configs_file.replace('./', '')}:{line_position}" 30 | 31 | @classmethod 32 | def is_valid(cls, prop, value, data_type): 33 | if type(value) == str: 34 | value_type = 'STRING' 35 | elif type(value) == int: 36 | value_type = 'INT' 37 | elif type(value) == float: 38 | value_type = 'FLOAT' 39 | elif type(value) == bool: 40 | value_type = 'BOOLEAN' 41 | elif type(value) == list: 42 | value_type = 'LIST' 43 | 44 | data_type = data_type.upper() 45 | property_position = cls.get_wrong_property_position(prop) 46 | 47 | if data_type != value_type: 48 | raise SettingsException(f"The '{prop}' configuration is invalid. Expected type {data_type}, but " 49 | f"instead a {value_type} was passed. {property_position}.") 50 | 51 | return value 52 | 53 | @classmethod 54 | def get(cls, prop, data_type): 55 | try: 56 | with open(Configs.CONFIGS_FILE, 'r') as content: 57 | data = yaml.safe_load(content) 58 | 59 | value = data 60 | property_parts = prop.split('.') 61 | 62 | for part in property_parts: 63 | value = value[part] 64 | 65 | return cls.is_valid(prop, value, data_type) 66 | 67 | except FileNotFoundError: 68 | raise SettingsException(f"File '{Configs.CONFIGS_FILE.replace('./', '')}' not found.") 69 | 70 | except yaml.YAMLError as e: 71 | raise SettingsException(f'Error while parsing the YAML file.: {e}') 72 | -------------------------------------------------------------------------------- /linkscraper/layout/layout.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | import sys, pyfiglet 4 | 5 | from rich.prompt import Prompt 6 | from rich.console import Console 7 | 8 | from helper.configs import Configs 9 | 10 | from utils.date_time import DateTime 11 | 12 | class Layout: 13 | 14 | prompt = Prompt 15 | console = Console(record=True) 16 | 17 | @classmethod 18 | def separator(cls): 19 | return cls.console.print("-" * 60) 20 | 21 | @classmethod 22 | def header(cls): 23 | cls.console.print("[bold blue]" + pyfiglet.figlet_format(Configs.APP_NAME) + "[/bold blue]") 24 | 25 | cls.separator() 26 | cls.console.print(f"Homepage: [bold green]{Configs.HOMEPAGE}[/bold green]") 27 | cls.separator() 28 | 29 | cls.console.print(f"\t\tv.[bold green]{Configs.VERSION}[/bold green]") 30 | cls.separator() 31 | 32 | @classmethod 33 | def success(cls, text, separator = False): 34 | if separator: 35 | cls.separator() 36 | 37 | return cls.console.print(f"[bold green]Success[/bold green]: {text}") 38 | 39 | @classmethod 40 | def error(cls, error, separator = False, exit = False, additional_print = None): 41 | if separator: 42 | cls.separator() 43 | 44 | cls.console.print(f"[bold red]Error[/bold red]: {error}") 45 | 46 | if additional_print: 47 | return cls.print( 48 | additional_print["text"], 49 | additional_print["value"], 50 | additional_print["style"], 51 | ) 52 | 53 | if exit: 54 | return sys.exit() 55 | 56 | @classmethod 57 | def header_section(cls, text): 58 | cls.separator() 59 | cls.console.print(f"{text}") 60 | cls.separator() 61 | 62 | @classmethod 63 | def print(cls, text, value, style = None, separator = False): 64 | if separator: 65 | cls.separator() 66 | 67 | if not text == None: 68 | if style: 69 | return cls.console.print(text, f"[{style}]{value}[/{style}]") 70 | 71 | return cls.console.print(text, value) 72 | 73 | return cls.console.print(f"[{style}]{value}[/{style}]") 74 | 75 | @classmethod 76 | def time_taken(cls, start_time, separator = False): 77 | if separator: 78 | cls.separator() 79 | 80 | return cls.print("Time taken:", f"{DateTime.calculate_interval(start_time)} seconds") 81 | 82 | @classmethod 83 | def header_plugin(cls, plugin): 84 | return cls.header_section(f"Plugin: [bold cyan]{plugin}[/bold cyan]") 85 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mypy 121 | .mypy_cache/ 122 | .dmypy.json 123 | dmypy.json 124 | 125 | # Pyre type checker 126 | .pyre/ 127 | 128 | # Remove all test files 129 | *test* 130 | 131 | # Ignore the IDE's path 132 | .* 133 | 134 | # Ignore DB files 135 | *.db 136 | *.sql 137 | 138 | # Ignore .env file 139 | .env 140 | 141 | # Ignore the download path 142 | download/** 143 | 144 | # Ignore the screenshots path 145 | screenshots/** 146 | 147 | # Mkdocs documentation 148 | docs/site/** 149 | 150 | # Ignore the venv path 151 | lsc/ 152 | 153 | # Ignore data path 154 | data/ 155 | -------------------------------------------------------------------------------- /linkscraper/apis/imgur.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | import requests, time 4 | import pyperclip as Pyperclip 5 | 6 | from http import HTTPStatus 7 | 8 | from utils.file import File 9 | from utils.file_ext import FileExt 10 | 11 | from classes.env import Env 12 | from layout.layout import Layout 13 | 14 | from helper.apis import Apis 15 | from helper.configs import Configs 16 | 17 | class Imgur: 18 | 19 | @classmethod 20 | def get_title(cls, title): 21 | if not title: 22 | return f'Screenshot made by {Configs.APP_NAME}' 23 | 24 | return title 25 | 26 | @classmethod 27 | def embed_code(cls, params): 28 | Layout.header_section("Embed codes") 29 | Layout.print("[italic yellow]Imgur Post[/italic yellow]:", f'
    {params["title"]}
    ') 30 | 31 | Layout.print("[italic yellow]HTML[/italic yellow]:", f"{params[") 32 | Layout.print("[italic yellow]Markdown[/italic yellow]:", f"![{params['title']}]({params['direct_link']})") 33 | 34 | @classmethod 35 | def upload(cls, file, title): 36 | Layout.separator() 37 | 38 | title = cls.get_title(title) 39 | key = Env.get('IMGUR_CLIENT_API') 40 | 41 | if not key: 42 | return Layout.error('Key is required', False, True, { 43 | "style": 'bold blue', 44 | "text": 'Get your client id here:', 45 | "value": Apis.IMGUR_API_KEY_URL, 46 | }) 47 | 48 | start_time = time.time() 49 | 50 | response = requests.request('POST', Apis.IMGUR_API_REQUEST, headers={ 51 | 'Authorization': f'Client-ID {key}' 52 | }, data={ 53 | 'title': title, 54 | 'type': 'base64', 55 | 'image': File.to_base64(file), 56 | }) 57 | 58 | if response.status_code != HTTPStatus.OK: 59 | return Layout.error(f'{response.status_code} - Failed to upload image to Imgur.', False, True) 60 | 61 | callback = response.json() 62 | if callback['success'] == True: 63 | direct_link = callback['data']['link'] 64 | imgur_page = direct_link.replace('i.', '') 65 | imgur_code_img = FileExt.remove(imgur_page).replace('https://imgur.com/', '') 66 | 67 | Layout.print('Imgur page:', FileExt.remove(imgur_page), 'bold blue') 68 | Layout.print('Link Direct:', direct_link, 'bold blue') 69 | 70 | cls.embed_code({ 71 | 'title': title, 72 | 'imgur_page': imgur_page, 73 | 'direct_link': direct_link, 74 | 'imgur_code_url': imgur_code_img, 75 | }) 76 | 77 | Layout.separator() 78 | Pyperclip.copy(direct_link) 79 | 80 | Layout.print(None, f'Link copied to clipboard', 'cyan') 81 | Layout.time_taken(start_time, True) 82 | 83 | return True 84 | 85 | Layout.error(f"{callback['status']} - {callback['data']['error']}", False, True) 86 | return False 87 | -------------------------------------------------------------------------------- /docs/cinder/css/base.min.css: -------------------------------------------------------------------------------- 1 | html{scroll-padding-top:70px}body{padding-top:70px}p>img{max-width:100%;height:auto}ul.nav li.first-level{font-weight:bold}ul.nav li.third-level{padding-left:12px}div.col-md-3{padding-left:0}div.col-md-9{padding-bottom:100px}div.source-links{float:right}.bs-sidebar.affix{position:static}.bs-sidebar.well{padding:0}.bs-sidenav{margin-top:30px;margin-bottom:30px;padding-top:10px;padding-bottom:10px;border-radius:5px}.bs-sidebar .nav>li>a{display:block;padding:5px 20px;z-index:1}.bs-sidebar .nav>li>a:hover,.bs-sidebar .nav>li>a:focus{text-decoration:none;border-right:1px solid}.bs-sidebar .nav>.active>a,.bs-sidebar .nav>.active:hover>a,.bs-sidebar .nav>.active:focus>a{font-weight:bold;background-color:transparent;border-right:1px solid}.bs-sidebar .nav .nav{display:none;margin-bottom:8px}.bs-sidebar .nav .nav>li>a{padding-top:3px;padding-bottom:3px;padding-left:30px;font-size:90%}@media(min-width:992px){.bs-sidebar .nav>.active>ul{display:block}.bs-sidebar.affix,.bs-sidebar.affix-bottom{width:213px}.bs-sidebar.affix{position:fixed;top:80px;max-height:calc(100% - 180px);overflow-y:auto}.bs-sidebar.affix-bottom{position:absolute}.bs-sidebar.affix-bottom .bs-sidenav,.bs-sidebar.affix .bs-sidenav{margin-top:0;margin-bottom:0}}@media(min-width:1200px){.bs-sidebar.affix-bottom,.bs-sidebar.affix{width:263px}}.dropdown-submenu{position:relative}.dropdown-submenu>.dropdown-menu{top:0;left:100%;margin-top:0;margin-left:0}.dropdown-submenu:hover>.dropdown-menu{display:block}.dropdown-submenu>a:after{display:block;content:" ";float:right;width:0;height:0;border-color:transparent;border-style:solid;border-width:5px 0 5px 5px;border-left-color:#ccc;margin-top:5px;margin-right:-10px}.dropdown-submenu:hover>a:after{border-left-color:#fff}.dropdown-submenu.pull-left{float:none}.dropdown-submenu.pull-left>.dropdown-menu{left:-100%;margin-left:00px}.bs-callout{padding:20px;margin:20px 0;border:1px solid #eee;border-left-width:5px;border-radius:3px;background-color:#fcfdff}.bs-callout h4{font-style:normal;font-weight:400;margin-top:0;margin-bottom:5px}.bs-callout p:last-child{margin-bottom:0}.bs-callout code{border-radius:3px}.bs-callout+.bs-callout{margin-top:-5px}.bs-callout-default{border-left-color:#fa023c}.bs-callout-default h4{color:#fa023c}.bs-callout-primary{border-left-color:#428bca}.bs-callout-primary h4{color:#428bca}.bs-callout-success{border-left-color:#5cb85c}.bs-callout-success h4{color:#5cb85c}.bs-callout-danger{border-left-color:#d9534f}.bs-callout-danger h4{color:#d9534f}.bs-callout-warning{border-left-color:#f0ad4e}.bs-callout-warning h4{color:#f0ad4e}.bs-callout-info{border-left-color:#5bc0de}.bs-callout-info h4{color:#5bc0de}.headerlink{display:none;padding-left:.5em}h1:hover .headerlink,h2:hover .headerlink,h3:hover .headerlink,h4:hover .headerlink,h5:hover .headerlink,h6:hover .headerlink{display:inline-block}.admonition{padding:20px;margin:20px 0;border:1px solid #eee;border-left-width:5px;border-radius:3px;background-color:#fcfdff}.admonition p:last-child{margin-bottom:0}.admonition code{border-radius:3px}.admonition+.admonition{margin-top:-5px}.admonition.note{border-left-color:#428bca}.admonition.warning{border-left-color:#f0ad4e}.admonition.danger{border-left-color:#d9534f}.admonition-title{font-size:19px;font-style:normal;font-weight:400;margin-top:0;margin-bottom:5px}.admonition.note>.admonition-title{color:#428bca}.admonition.warning>.admonition-title{color:#f0ad4e}.admonition.danger>.admonition-title{color:#d9534f} 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |
    2 | 3 |

    4 | A powerful tool to scrape and manage links from web pages. 5 |
    6 | 7 |
    8 | 9 |
    10 | 📚 Documentation • 11 | 📝 License • 12 | 💡 Sponsor 13 |
    14 | 15 | ## **📑 Table of Contents:** 16 | 17 | * 🔍 Overview 18 | * [Introduction](https://linkscraper.kremilly.com/#introduction) 19 | * [Requirements](https://linkscraper.kremilly.com/#requirements) 20 | * [Getting started](https://linkscraper.kremilly.com/#getting-started) 21 | * [Parameters](https://linkscraper.kremilly.com/overview/parameters/) 22 | * [Dependencies](https://linkscraper.kremilly.com/overview/dependencies/) 23 | * [External apis use](https://linkscraper.kremilly.com/overview/external-apis/) 24 | * 🛠 Basics 25 | * [Core data](https://linkscraper.kremilly.com/basics/core) 26 | * [Get headers](https://linkscraper.kremilly.com/basics/headers) 27 | * [Get cookies](https://linkscraper.kremilly.com/basics/cookies) 28 | * [Get links](https://linkscraper.kremilly.com/basics/get-links) 29 | * [Get emails](https://linkscraper.kremilly.com/basics/get-emails/) 30 | * Static files 31 | * [CSS](https://linkscraper.kremilly.com/basics/static/css/) 32 | * [JS](https://linkscraper.kremilly.com/basics/static/js/) 33 | * [Images](https://linkscraper.kremilly.com/basics/static/images/) 34 | * 🧩 Plugins 35 | * [whois](https://linkscraper.kremilly.com/plugins/whois/) 36 | * [page-details](https://linkscraper.kremilly.com/plugins/page-details/) 37 | * [robots](https://linkscraper.kremilly.com/plugins/robots/) 38 | * [screenshot](https://linkscraper.kremilly.com/plugins/screenshot/) 39 | * [virustotal](https://linkscraper.kremilly.com/plugins/virustotal/) 40 | * [subdomains](https://linkscraper.kremilly.com/plugins/subdomains/) 41 | * [ip-location](https://linkscraper.kremilly.com/plugins/ip-location/) 42 | * [detect-fonts](https://linkscraper.kremilly.com/plugins/detect-fonts/) 43 | * [extract-colors](https://linkscraper.kremilly.com/plugins/extract-colors/) 44 | * How to get keys 45 | * [Imgur](https://linkscraper.kremilly.com/plugins/apis/imgur/) 46 | * [Google Fonts](https://linkscraper.kremilly.com/plugins/apis/google-fonts/) 47 | * [VirusTotal](https://linkscraper.kremilly.com/plugins/apis/virustotal/) 48 | * ⚙️ Settings 49 | * [Environments](https://linkscraper.kremilly.com/settings/env/) 50 | * 🤝 Contribute 51 | * [Sponsor](https://github.com/sponsors/kremilly) 52 | * [Report a bug](https://github.com/kremilly/linkscraper/issues) 53 | * 📝 [License](#-license) 54 | * 💡 [Sponsor](#-sponsor) 55 | 56 | ## 📝 License 57 | 58 | Code licensed under [MIT License](blob/main/LICENSE) 59 | 60 | ## 💡 Sponsor 61 | 62 | Hello Linkscraper user! 63 | 64 | Creating and maintaining Linkscraper requires time and resources. If you find it valuable, please consider supporting its development. Your contribution, no matter the size, will help keep Linkscraper updated and improve with new features. 65 | 66 | By contributing, you’re helping make web data extraction accessible and efficient for everyone. [Click here](https://github.com/sponsors/kremilly) to support Linkscraper. Every bit counts, and we appreciate your generosity. 🚀❤️ 67 | 68 | Thank you for being part of our journey! 69 | -------------------------------------------------------------------------------- /docs/mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: Linkscraper 2 | repo_url: https://github.com/kremilly/Linkscraper 3 | site_author: Kremilly 4 | copyright: "Linkscraper is licensed under the MIT license" 5 | theme: 6 | name: material 7 | features: 8 | - navigation.tabs 9 | - navigation.instant 10 | - navigation.sections 11 | - navigation.footer 12 | - content.tabs.link 13 | - content.footnote.tooltips 14 | 15 | palette: 16 | - media: "(prefers-color-scheme: light)" 17 | scheme: default 18 | primary: black 19 | toggle: 20 | icon: material/brightness-7 21 | name: Switch to dark mode 22 | 23 | - media: "(prefers-color-scheme: dark)" 24 | scheme: slate 25 | primary: black 26 | toggle: 27 | icon: material/brightness-4 28 | name: Switch to light mode 29 | 30 | hljs_languages: 31 | - shell 32 | - python 33 | - environments 34 | 35 | nav: 36 | - Overview: 37 | - Getting Started: index.md # Done 38 | - Parameters: overview/parameters.md # Done 39 | - Dependencies: overview/dependencies.md # Done 40 | - External API's use: overview/external-apis.md # Done 41 | 42 | - Basics: 43 | Core data: basics/core.md # Done 44 | Get headers: basics/headers.md # Done 45 | Get cookies: basics/cookies.md # Done 46 | Get links: basics/get-links.md # Done 47 | Get emails: basics/get-emails.md # Done 48 | 49 | Static files: 50 | CSS: basics/static/css.md # Done 51 | Images: basics/static/images.md # Done 52 | JavaScript: basics/static/js.md # Done 53 | 54 | - Plugins: 55 | Plugins: 56 | whois: plugins/whois.md # Done 57 | page-details: plugins/page-details.md # Done 58 | robots: plugins/robots.md # Done 59 | screenshot: plugins/screenshot.md # Done 60 | virustotal: plugins/virustotal.md # Done 61 | subdomains: plugins/subdomains.md # Done 62 | ip-location: plugins/ip-location.md # Done 63 | detect-fonts: plugins/detect-fonts.md # Done 64 | extract-colors: plugins/extract-colors.md # Done 65 | 66 | How to get keys: 67 | Imgur: plugins/apis/imgur.md # Done 68 | Google Fonts: plugins/apis/google-fonts.md # Done 69 | VirusTotal: plugins/apis/virustotal.md # Done 70 | 71 | - Settings: 72 | Environments: settings/env.md # Done 73 | 74 | - Contribute: 75 | GitHub: https://github.com/kremilly/linkscraper 76 | Sponsor: https://github.com/sponsors/kremilly 77 | Report a bug: https://github.com/kremilly/linkscraper/issues 78 | markdown_extensions: 79 | - tables 80 | - attr_list 81 | - md_in_html 82 | - pymdownx.superfences 83 | - footnotes 84 | - pymdownx.superfences 85 | - def_list 86 | - pymdownx.tasklist: 87 | custom_checkbox: true 88 | - pymdownx.tabbed: 89 | alternate_style: true 90 | - pymdownx.arithmatex: 91 | generic: true 92 | - pymdownx.superfences: 93 | custom_fences: 94 | - name: mermaid 95 | class: mermaid 96 | 97 | extra: 98 | social: 99 | - icon: fontawesome/solid/blog 100 | link: https://kremilly.com 101 | name: Blog 102 | 103 | - icon: fontawesome/brands/x-twitter 104 | link: https://x.com/import_kremilly 105 | name: Twitter 106 | 107 | generator: false 108 | -------------------------------------------------------------------------------- /linkscraper/apis/google_fonts.py: -------------------------------------------------------------------------------- 1 | import time, requests 2 | 3 | from http import HTTPStatus 4 | 5 | from helper.apis import Apis 6 | 7 | from classes.env import Env 8 | 9 | from classes.settings import Settings 10 | 11 | from utils.http import HTTP 12 | from utils.file import File 13 | from utils.date_time import DateTime 14 | from utils.file_size import FileSize 15 | 16 | from layout.table import Table 17 | from layout.layout import Layout 18 | 19 | class GoogleFonts: 20 | 21 | @classmethod 22 | def create_path(cls, path): 23 | path = f"{Settings.get('storage.downloads', 'string')}/{path}" 24 | File.create_path(path) 25 | return path 26 | 27 | @classmethod 28 | def download_font(cls, link, path): 29 | response = requests.get(link) 30 | font_path = f'{path + File.get_remote_file_name(link)}' 31 | File.write(font_path, response.content, 'wb', 'utf-8') 32 | 33 | @classmethod 34 | def get_font_files(cls, font_name): 35 | key = Env.get('GOOGLE_FONTS_KEY') 36 | 37 | if not key: 38 | return Layout.error('Key is required', False, True, { 39 | "style": 'bold blue', 40 | "text": 'Get your key here:', 41 | "value": Apis.GOOGLE_FONTS_API_KEY_URL, 42 | }) 43 | 44 | response = requests.get(Apis.GOOGLE_FONTS_API_REQUEST, params={ 45 | "key": key, 46 | "sort": 'alpha' 47 | }) 48 | 49 | if response.status_code != HTTPStatus.OK: 50 | Layout.error(f'{response.status_code}: {response.reason}', False, True, { 51 | 'style': 'bold blue', 52 | 'text': 'Get your key here:', 53 | 'value': Apis.GOOGLE_FONTS_API_KEY_URL, 54 | }) 55 | 56 | fonts = response.json().get('items', []) 57 | for font in fonts: 58 | if font['family'].lower() == font_name.lower(): 59 | return font['files'] 60 | 61 | return None 62 | 63 | @classmethod 64 | def get_list(cls, font_name): 65 | links = cls.get_font_files(font_name) 66 | 67 | if links: 68 | return links 69 | 70 | return None 71 | 72 | @classmethod 73 | def list(cls, url, font_name, *args): 74 | start_time = time.time() 75 | list = cls.get_list(font_name) 76 | 77 | Layout.header_section(f'Google Fonts: [bold blue]{font_name}[/bold blue]') 78 | 79 | if list is not None: 80 | if args.download is not False: 81 | path = cls.create_path(f'{HTTP.get_hostname(url)}/fonts/{font_name}/') 82 | 83 | Table.header([ 84 | ('Style', 'white', True), 85 | ('URL', 'bold blue', True), 86 | ('Size', 'green', False) 87 | ]) 88 | 89 | for style, file_url in list.items(): 90 | Table.row(style, file_url, FileSize.remote_file(file_url)) 91 | 92 | if args.download is not False: 93 | cls.download_font(file_url, path) 94 | 95 | Table.caption(f"Total of variants: {len(list)} - " 96 | f"Time taken: {DateTime.calculate_interval(start_time)} seconds") 97 | 98 | return Table.display() 99 | 100 | return Layout.error('This font is not found on Google Fonts', False, True) 101 | -------------------------------------------------------------------------------- /linkscraper/core/scraper.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | import re, requests, time 4 | from bs4 import BeautifulSoup as bs 5 | 6 | from utils.url import URL 7 | from utils.http import HTTP 8 | from utils.date_time import DateTime 9 | 10 | from layout.table import Table 11 | from layout.layout import Layout 12 | 13 | class Scraper: 14 | 15 | @classmethod 16 | def get_links(cls, url, external_links, status_code, filter_data): 17 | start_time = time.time() 18 | 19 | reqs = requests.get(url).text 20 | soup = bs(reqs, 'html.parser') 21 | 22 | headers = [ 23 | ('Domain', 'cyan', True), 24 | ('URL', 'bold blue', False) 25 | ] 26 | 27 | if status_code: 28 | headers = [ 29 | ('Domain', 'cyan', True), 30 | ('URL', 'bold blue', False), 31 | ('Status', 'bold', False) 32 | ] 33 | 34 | Table.header(headers) 35 | 36 | links = [] 37 | 38 | for link in soup.find_all('a'): 39 | if link.get('href') != None: 40 | if filter_data: 41 | if URL.is_url(link.get('href')) and link.get('href').find(filter_data) != -1: 42 | links.append(link.get('href')) 43 | 44 | if not external_links: 45 | if URL.is_url(link.get('href')): 46 | links.append(link.get('href')) 47 | 48 | if URL.is_url(link.get('href')) and HTTP.get_hostname(link.get('href')).find(HTTP.get_hostname(url)) == -1: 49 | links.append(link.get('href')) 50 | 51 | links_list = list(set(links)) 52 | 53 | for link in list(set(links)): 54 | if status_code: 55 | Table.row(HTTP.get_hostname(link), link, HTTP.code(link)) 56 | 57 | Table.row(HTTP.get_hostname(link), link) 58 | 59 | Table.caption(f'Total of links in page: {len(links_list)} - ' 60 | f'Time taken: {DateTime.calculate_interval(start_time)} seconds') 61 | 62 | Table.display() 63 | 64 | @classmethod 65 | def get_emails(cls, url, filter_data): 66 | start_time = time.time() 67 | text = requests.get(url).text 68 | 69 | soup = str( 70 | bs(text,'html.parser').body 71 | ) 72 | 73 | Table.header([ 74 | ('Domain', 'cyan', True), 75 | ('Email', 'bold blue', False) 76 | ]) 77 | 78 | emails = re.findall(r'[\w.+-]+@[\w-]+\.[\w.-]+', soup) 79 | list_emails = list(set(emails)) 80 | 81 | for email in list_emails: 82 | if filter_data: 83 | if email.find(filter_data): 84 | Table.row(email.split('@')[1], email) 85 | 86 | Table.row(email.split('@')[1], email) 87 | 88 | Table.caption(f'Total of emails on page: {len(list_emails)} - ' 89 | f'Time taken: {DateTime.calculate_interval(start_time)} seconds') 90 | 91 | Table.display() 92 | 93 | @classmethod 94 | def section_links(cls, url, external_links, status_code, filter_data): 95 | Layout.header_section('Links') 96 | cls.get_links(url, external_links, status_code, filter_data) 97 | 98 | @classmethod 99 | def section_emails(cls, url, filter_data): 100 | Layout.header_section('Emails') 101 | cls.get_emails(url, filter_data) 102 | -------------------------------------------------------------------------------- /linkscraper/plugins/detect_fonts.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | import requests, time, re 4 | from bs4 import BeautifulSoup 5 | 6 | from rich.prompt import Prompt 7 | 8 | from layout.table import Table 9 | 10 | from apis.google_fonts import GoogleFonts 11 | 12 | from utils.date_time import DateTime 13 | 14 | class DetectFonts: 15 | 16 | @classmethod 17 | def process_font_string(cls, font_str): 18 | replacements = [''', ''', ')', 'var(', '!important'] 19 | 20 | for replacement in replacements: 21 | font_str = font_str.replace(replacement, '') 22 | 23 | return font_str.strip() 24 | 25 | @classmethod 26 | def get_css_links(cls, url): 27 | response = requests.get(url) 28 | soup = BeautifulSoup(response.content, 'html.parser') 29 | 30 | css_links = [] 31 | for link in soup.find_all('link', rel='stylesheet'): 32 | href = link.get('href') 33 | 34 | if href: 35 | if 'http' not in href: 36 | href = requests.compat.urljoin(url, href) 37 | 38 | css_links.append(href) 39 | 40 | return css_links 41 | 42 | @classmethod 43 | def get_fonts_from_css_files(cls, url): 44 | css_links = cls.get_css_links(url) 45 | font_families = [] 46 | 47 | for link in css_links: 48 | response = requests.get(link) 49 | css_content = response.text 50 | 51 | matches = re.findall( 52 | r'font-family\s*:\s*([^;}]+)', css_content 53 | ) 54 | 55 | for match in matches: 56 | fonts = [cls.process_font_string(f) for f in match.split(',')] 57 | 58 | fonts = [ 59 | f for f in fonts if f not in [ 60 | 'important', 'inherit', 'var' 61 | ] 62 | ] 63 | 64 | font_families.extend(fonts) 65 | 66 | return list( 67 | set(font_families) 68 | ) 69 | 70 | @classmethod 71 | def get_fonts_from_html(cls, url): 72 | response = requests.get(url) 73 | soup = BeautifulSoup(response.content, 'html.parser') 74 | 75 | fonts = '' 76 | 77 | body_style = soup.find('body').attrs.get('style', '') 78 | if 'font-family' in body_style: 79 | fonts = body_style.split('font-family:')[1].split(';')[0].strip().split('}')[0].replace('"', '') 80 | 81 | for style_tag in soup.find_all('style'): 82 | if 'font-family' in style_tag.string: 83 | fonts = style_tag.string.split('font-family:')[1].split(';')[0].strip().split('}')[0].replace('"', '') 84 | 85 | list_fonts = fonts.split(',') 86 | 87 | return list( 88 | set(list_fonts) 89 | ) 90 | 91 | @classmethod 92 | def run(cls, url, *args) -> GoogleFonts|Table: 93 | start_time = time.time() 94 | font_families = cls.get_fonts_from_css_files(url) 95 | 96 | if len(font_families) == 0: 97 | font_families = cls.get_fonts_from_html(url) 98 | 99 | if any('google_fonts' in arg for arg in args): 100 | font_name = Prompt.ask(f'Enter the font name', choices=font_families) 101 | return GoogleFonts.list(url, font_name, args.download) 102 | 103 | Table.header([ 104 | ('Name', 'cyan', True), 105 | ('Value', 'white', False) 106 | ]) 107 | 108 | for font_name in font_families: 109 | Table.row('font-family', font_name.strip()) 110 | 111 | Table.caption(f'Total of fonts: {len(font_families)} - ' 112 | f'Time taken: {DateTime.calculate_interval(start_time)} seconds') 113 | 114 | return Table.display() 115 | -------------------------------------------------------------------------------- /linkscraper/plugins/page_details.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | import requests, cloudscraper, time 4 | 5 | from bs4 import BeautifulSoup 6 | from urllib.parse import urljoin 7 | 8 | from utils.locale import Locale 9 | 10 | from classes.settings import Settings 11 | 12 | from layout.table import Table 13 | from layout.layout import Layout 14 | 15 | class PageDetails: 16 | 17 | @classmethod 18 | def language_country(cls, code, param): 19 | if param == 'locale': 20 | return f'{Locale.language(code)} ({Locale.country(code)})' 21 | 22 | return False 23 | 24 | @classmethod 25 | def wp_detect(cls, url): 26 | session = requests.Session() 27 | session.headers['User-Agent'] = Settings.get('general.default_user_agent', 'STRING') 28 | 29 | soup = BeautifulSoup(session.get(url).content, 'html.parser') 30 | metas = soup.find_all('meta') 31 | 32 | wp_detected = False 33 | 34 | wp_meta_generator = [ 35 | meta.attrs['content'] for meta in metas if 'name' in meta.attrs and meta.attrs['name'] == 'generator' 36 | ] 37 | 38 | if len(wp_meta_generator) >= 1 and wp_meta_generator[0].find('WordPress'): 39 | wp_detected = True 40 | wp_version = wp_meta_generator[0] 41 | 42 | for css in soup.find_all('link'): 43 | if css.attrs.get('href'): 44 | css_url = urljoin(url, css.attrs.get('href')) 45 | 46 | if css_url.find('wp-content') or css_url.find('wp-includes'): 47 | wp_detected = True 48 | 49 | wp_detected = False 50 | 51 | break 52 | 53 | if wp_detected: 54 | Layout.print(f'[blue]WordPress[/blue]:', 'detected', 'green') 55 | Layout.print(f'[blue]WordPress version[/blue]:', wp_version, 'green') 56 | 57 | Layout.print(f'[blue]WordPress[/blue]:', 'not detected', 'red') 58 | 59 | @classmethod 60 | def run(cls, url): 61 | start_time = time.time() 62 | scraper = cloudscraper.create_scraper() 63 | 64 | html = scraper.get(url) 65 | soup = BeautifulSoup(html.text, 'html.parser') 66 | 67 | metatitle = (soup.find('title')).get_text() 68 | metadescription = soup.find('meta',attrs={'name':'description'}) 69 | robots_directives = soup.find('meta',attrs={'name':'robots'}) 70 | viewport = soup.find('meta',attrs={'name':'viewport'}) 71 | charset = soup.find('meta',attrs={'charset':True}) 72 | open_graph = [[a['property'].replace('og:',''),a['content']] for a in soup.select('meta[property^=og]')] 73 | 74 | Layout.print('[blue]Title:[/blue]', metatitle) 75 | if metadescription: Layout.print('[blue]Description:[/blue]', metadescription['content']) 76 | if robots_directives: Layout.print('[blue]Robots directives:[/blue]', robots_directives['content'].split(',')) 77 | if viewport: Layout.print('[blue]Viewport:[/blue]', viewport['content']) 78 | if charset: Layout.print('[blue]Charset:[/blue]', charset['charset']) 79 | 80 | cls.wp_detect(url) 81 | 82 | if open_graph: 83 | Layout.separator() 84 | Layout.print(f'[blue]What is Open Graph?[/blue]', 'The Open Graph protocol enables any web page to become a rich object in a social graph. For instance, this is used on Facebook to allow any web page to have the same functionality as any other object on Facebook.') 85 | Layout.print(f'[blue]Documentation[/blue]:', 'https://ogp.me', 'bold blue') 86 | Layout.separator() 87 | 88 | Table.header([ 89 | ('Name', 'cyan', True), 90 | ('Value', 'white', False) 91 | ]) 92 | 93 | for info in open_graph: 94 | if not cls.language_country(info[1], info[0]): 95 | Table.row('og:' + info[0], info[1]) 96 | else: 97 | Table.row('og:' + info[0], cls.language_country(info[1], info[0])) 98 | 99 | Table.display() 100 | 101 | Layout.time_taken(start_time, True) 102 | -------------------------------------------------------------------------------- /linkscraper/core/download_resources.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | 3 | import os, requests, time 4 | 5 | from urllib.parse import urljoin 6 | from bs4 import BeautifulSoup as bs 7 | 8 | from classes.settings import Settings 9 | 10 | from helper.configs import Configs 11 | 12 | from layout.table import Table 13 | from layout.layout import Layout 14 | 15 | from utils.http import HTTP 16 | from utils.file import File 17 | from utils.file_ext import FileExt 18 | from utils.file_size import FileSize 19 | from utils.date_time import DateTime 20 | 21 | class DownloadResources: 22 | 23 | session = requests.Session() 24 | session.headers['User-Agent'] = Settings.get('general.default_user_agent', 'STRING') 25 | 26 | @classmethod 27 | def download(cls, url, resource_type, minify_files=None, filter_data=None): 28 | start_time = time.time() 29 | domain = HTTP.get_hostname(url) 30 | path = f"{Settings.get('storage.downloads')}/{domain}/{resource_type}/" 31 | 32 | File.create_path(path) 33 | 34 | html = cls.session.get(url).content 35 | soup = bs(html, 'html.parser') 36 | 37 | Table.header([ 38 | ('Filename', 'cyan', True), 39 | ('URL', 'bold blue', False), 40 | ('Size', 'green', False), 41 | ('Status', 'white', False), 42 | ]) 43 | 44 | links = cls.extract_links( 45 | url, 46 | soup, 47 | resource_type, 48 | minify_files, 49 | filter_data 50 | ) 51 | 52 | for link in list(set(links)): 53 | if FileExt.is_valid(File.get_remote_file_name(link)): 54 | file_name = path + File.get_remote_file_name(link) 55 | content = cls.session.get(link).content if resource_type == 'images' else cls.session.get(link).text 56 | 57 | try: 58 | mode = 'wb' if resource_type == 'images' else 'w' 59 | File.write(file_name, content, mode, 'utf-8') 60 | 61 | except PermissionError: 62 | Layout.error(f'Permission denied when trying to write to: {file_name}', False, True) 63 | 64 | status = '[bold green]Download completed[/bold green]' if os.path.exists(file_name) else '[bold red]Download failed[/bold red]' 65 | 66 | Table.row( 67 | File.get_remote_file_name(link), link, FileSize.local_file(file_name), status 68 | ) 69 | 70 | File.open(path) 71 | 72 | Table.caption(f'Total of downloaded files: {len(links)} - ' 73 | f'Time taken: {DateTime.calculate_interval(start_time)} seconds') 74 | 75 | Table.display() 76 | 77 | @classmethod 78 | def extract_links(cls, url, soup, resource_type, minify_files, filter_data): 79 | links = [] 80 | 81 | match resource_type: 82 | case 'js': 83 | tags = soup.find_all('script') 84 | attr_name = 'src' 85 | filter_str = '.min' if minify_files == 'true' else None 86 | 87 | case 'css': 88 | tags = soup.find_all('link') 89 | attr_name = 'href' 90 | filter_str = '.min.css' if minify_files == 'true' else '.css' 91 | 92 | case 'images': 93 | tags = soup.find_all('img') 94 | attr_name = 'src' 95 | filter_str = None 96 | 97 | case _: 98 | raise ValueError(f'Unsupported resource type: {resource_type}') 99 | 100 | for tag in tags: 101 | if tag.attrs.get(attr_name): 102 | link = urljoin( 103 | url, tag.attrs.get(attr_name) 104 | ) 105 | 106 | if not filter_data == None: 107 | if filter_data and not link.find(filter_data): 108 | continue 109 | 110 | if filter_str and not link.find(filter_data): 111 | continue 112 | 113 | links.append(link) 114 | 115 | return links 116 | -------------------------------------------------------------------------------- /docs/cinder/nav.html: -------------------------------------------------------------------------------- 1 | 110 | -------------------------------------------------------------------------------- /docs/cinder/base.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | {% if config.site_description %}{% endif %} 9 | {% if config.site_author %}{% endif %} 10 | {% if page.canonical_url %}{% endif %} 11 | 12 | 13 | {% block htmltitle %} 14 | {% if page.title %}{{ page.title }} - {% endif %}{{ config.site_name }} 15 | {% endblock %} 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | {% if config.theme.highlightjs is defined and config.theme.highlightjs is sameas false %} 27 | 28 | {% else %} 29 | {% if config.theme.colorscheme %} 30 | 31 | {% else %} 32 | 33 | {% endif %} 34 | {% endif %} 35 | 36 | {%- for path in config['extra_css'] %} 37 | 38 | {%- endfor %} 39 | 40 | 41 | 45 | 46 | {% if config.google_analytics %} 47 | 63 | {% endif %} 64 | 65 | {% block extrahead %} {% endblock %} 66 | 67 | 68 | 69 | 70 | {% include "nav.html" %} 71 | 72 |
    73 | {% block content %} 74 | {% if page.meta.disable_toc %} 75 |
    {% include "content.html" %}
    76 | {% else %} 77 |
    {% include "toc.html" %}
    78 |
    {% include "content.html" %}
    79 | {% endif %} 80 | {% endblock %} 81 |
    82 | 83 | {% if not config.theme.disable_footer %} 84 |
    85 | {% block footer %} 86 | {% if not config.theme.disable_footer_except_revision %} 87 |
    88 |

    {% if config.copyright %} 89 | {{ config.copyright }}
    90 | {% endif %} 91 | Documentation built with MkDocs. 92 |

    93 | {% endif %} 94 | 95 | {% if page and page.meta.revision_date %} 96 | {% if config.theme.disable_footer_except_revision %}
    {% else %}
    {% endif %} 97 | Revised on: {{ page.meta.revision_date }} 98 | {% endif %} 99 | {% endblock %} 100 |
    101 | {% endif %} 102 | 103 | {%- block scripts %} 104 | 105 | 106 | 107 | {% if config.theme.highlightjs is defined and config.theme.highlightjs is sameas false %} 108 | 109 | {% else %} 110 | 111 | {% if config.theme.hljs_languages %} 112 | {%- for lang in config.theme.hljs_languages %} 113 | 114 | {%- endfor %} 115 | {% endif %} 116 | 117 | {% endif %} 118 | 119 | 120 | {% if config.shortcuts %} 121 | 122 | {% endif %} 123 | 124 | {%- for path in config['extra_javascript'] %} 125 | 126 | {%- endfor %} 127 | {%- endblock %} 128 | 129 | {% if 'search' in config['plugins'] %}{%- include "search-modal.html" %}{% endif %} 130 | {%- include "keyboard-modal.html" %} 131 | 132 | 133 | 134 | {% if page and page.is_homepage %} 135 | 139 | {% endif %} 140 | -------------------------------------------------------------------------------- /docs/cinder/js/base.js: -------------------------------------------------------------------------------- 1 | function getSearchTerm() { 2 | var sPageURL = window.location.search.substring(1); 3 | var sURLVariables = sPageURL.split('&'); 4 | for (var i = 0; i < sURLVariables.length; i++) { 5 | var sParameterName = sURLVariables[i].split('='); 6 | if (sParameterName[0] == 'q') { 7 | return sParameterName[1]; 8 | } 9 | } 10 | } 11 | 12 | $(document).ready(function() { 13 | /** 14 | * ------------------------------------------------------------------------ 15 | * Taken from themes/mkdocs/js/base.js 16 | * ------------------------------------------------------------------------ 17 | */ 18 | var search_term = getSearchTerm(), 19 | $search_modal = $('#mkdocs_search_modal'), 20 | $keyboard_modal = $('#mkdocs_keyboard_modal'); 21 | 22 | if (search_term) { 23 | $search_modal.modal(); 24 | } 25 | 26 | // make sure search input gets autofocus everytime modal opens. 27 | $search_modal.on('shown.bs.modal', function() { 28 | $search_modal.find('#mkdocs-search-query').focus(); 29 | }); 30 | 31 | // Close search modal when result is selected 32 | // The links get added later so listen to parent 33 | $('#mkdocs-search-results').click(function(e) { 34 | if ($(e.target).is('a')) { 35 | $search_modal.modal('hide'); 36 | } 37 | }); 38 | 39 | if (typeof shortcuts !== 'undefined') { 40 | // Populate keyboard modal with proper Keys 41 | $keyboard_modal.find('.help.shortcut kbd')[0].innerHTML = keyCodes[shortcuts.help]; 42 | $keyboard_modal.find('.prev.shortcut kbd')[0].innerHTML = keyCodes[shortcuts.previous]; 43 | $keyboard_modal.find('.next.shortcut kbd')[0].innerHTML = keyCodes[shortcuts.next]; 44 | $keyboard_modal.find('.search.shortcut kbd')[0].innerHTML = keyCodes[shortcuts.search]; 45 | 46 | // Keyboard navigation 47 | document.addEventListener("keydown", function(e) { 48 | if ($(e.target).is(':input')) return true; 49 | var key = e.which || e.key || window.event && window.event.key; 50 | var page; 51 | switch (key) { 52 | case shortcuts.next: 53 | page = $('.navbar a[rel="next"]:first').prop('href'); 54 | break; 55 | case shortcuts.previous: 56 | page = $('.navbar a[rel="prev"]:first').prop('href'); 57 | break; 58 | case shortcuts.search: 59 | e.preventDefault(); 60 | $keyboard_modal.modal('hide'); 61 | $search_modal.modal('show'); 62 | $search_modal.find('#mkdocs-search-query').focus(); 63 | break; 64 | case shortcuts.help: 65 | $search_modal.modal('hide'); 66 | $keyboard_modal.modal('show'); 67 | break; 68 | default: 69 | break; 70 | } 71 | if (page) { 72 | $keyboard_modal.modal('hide'); 73 | window.location.href = page; 74 | } 75 | }); 76 | } 77 | 78 | $('table').addClass('table table-striped table-hover'); 79 | 80 | // Improve the scrollspy behaviour when users click on a TOC item. 81 | $(".bs-sidenav a").on("click", function() { 82 | var clicked = this; 83 | setTimeout(function() { 84 | var active = $('.nav li.active a'); 85 | active = active[active.length - 1]; 86 | if (clicked !== active) { 87 | $(active).parent().removeClass("active"); 88 | $(clicked).parent().addClass("active"); 89 | } 90 | }, 50); 91 | }); 92 | }); 93 | 94 | 95 | /** 96 | * ------------------------------------------------------------------------ 97 | * Taken from themes/mkdocs/js/base.js 98 | * ------------------------------------------------------------------------ 99 | */ 100 | 101 | $('body').scrollspy({ 102 | target: '.bs-sidebar', 103 | offset: 100 104 | }); 105 | 106 | /* Prevent disabled links from causing a page reload */ 107 | $("li.disabled a").click(function() { 108 | event.preventDefault(); 109 | }); 110 | 111 | // See https://www.cambiaresearch.com/articles/15/javascript-char-codes-key-codes 112 | // We only list common keys below. Obscure keys are omited and their use is discouraged. 113 | var keyCodes = { 114 | 8: 'backspace', 115 | 9: 'tab', 116 | 13: 'enter', 117 | 16: 'shift', 118 | 17: 'ctrl', 119 | 18: 'alt', 120 | 19: 'pause/break', 121 | 20: 'caps lock', 122 | 27: 'escape', 123 | 32: 'spacebar', 124 | 33: 'page up', 125 | 34: 'page down', 126 | 35: 'end', 127 | 36: 'home', 128 | 37: '←', 129 | 38: '↑', 130 | 39: '→', 131 | 40: '↓', 132 | 45: 'insert', 133 | 46: 'delete', 134 | 48: '0', 135 | 49: '1', 136 | 50: '2', 137 | 51: '3', 138 | 52: '4', 139 | 53: '5', 140 | 54: '6', 141 | 55: '7', 142 | 56: '8', 143 | 57: '9', 144 | 65: 'a', 145 | 66: 'b', 146 | 67: 'c', 147 | 68: 'd', 148 | 69: 'e', 149 | 70: 'f', 150 | 71: 'g', 151 | 72: 'h', 152 | 73: 'i', 153 | 74: 'j', 154 | 75: 'k', 155 | 76: 'l', 156 | 77: 'm', 157 | 78: 'n', 158 | 79: 'o', 159 | 80: 'p', 160 | 81: 'q', 161 | 82: 'r', 162 | 83: 's', 163 | 84: 't', 164 | 85: 'u', 165 | 86: 'v', 166 | 87: 'w', 167 | 88: 'x', 168 | 89: 'y', 169 | 90: 'z', 170 | 91: 'Left Windows Key / Left ⌘', 171 | 92: 'Right Windows Key', 172 | 93: 'Windows Menu / Right ⌘', 173 | 96: 'numpad 0', 174 | 97: 'numpad 1', 175 | 98: 'numpad 2', 176 | 99: 'numpad 3', 177 | 100: 'numpad 4', 178 | 101: 'numpad 5', 179 | 102: 'numpad 6', 180 | 103: 'numpad 7', 181 | 104: 'numpad 8', 182 | 105: 'numpad 9', 183 | 106: 'multiply', 184 | 107: 'add', 185 | 109: 'subtract', 186 | 110: 'decimal point', 187 | 111: 'divide', 188 | 112: 'f1', 189 | 113: 'f2', 190 | 114: 'f3', 191 | 115: 'f4', 192 | 116: 'f5', 193 | 117: 'f6', 194 | 118: 'f7', 195 | 119: 'f8', 196 | 120: 'f9', 197 | 121: 'f10', 198 | 122: 'f11', 199 | 123: 'f12', 200 | 124: 'f13', 201 | 125: 'f14', 202 | 126: 'f15', 203 | 127: 'f16', 204 | 128: 'f17', 205 | 129: 'f18', 206 | 130: 'f19', 207 | 131: 'f20', 208 | 132: 'f21', 209 | 133: 'f22', 210 | 134: 'f23', 211 | 135: 'f24', 212 | 144: 'num lock', 213 | 145: 'scroll lock', 214 | 186: ';', 215 | 187: '=', 216 | 188: ',', 217 | 189: '‐', 218 | 190: '.', 219 | 191: '?', 220 | 192: '`', 221 | 219: '[', 222 | 220: '\', 223 | 221: ']', 224 | 222: ''', 225 | }; 226 | -------------------------------------------------------------------------------- /docs/cinder/css/base.css: -------------------------------------------------------------------------------- 1 | html { 2 | scroll-padding-top: 70px; 3 | } 4 | 5 | body { 6 | padding-top: 70px; 7 | } 8 | 9 | p > img { 10 | max-width: 100%; 11 | height: auto; 12 | } 13 | 14 | ul.nav li.first-level { 15 | font-weight: bold; 16 | } 17 | 18 | ul.nav li.third-level { 19 | padding-left: 12px; 20 | } 21 | 22 | div.col-md-3 { 23 | padding-left: 0; 24 | } 25 | 26 | div.col-md-9 { 27 | padding-bottom: 100px; 28 | } 29 | 30 | div.source-links { 31 | float: right; 32 | } 33 | 34 | /* 35 | * Side navigation 36 | * 37 | * Scrollspy and affixed enhanced navigation to highlight sections and secondary 38 | * sections of docs content. 39 | */ 40 | 41 | /* By default it's not affixed in mobile views, so undo that */ 42 | .bs-sidebar.affix { 43 | position: static; 44 | } 45 | 46 | .bs-sidebar.well { 47 | padding: 0; 48 | } 49 | 50 | /* First level of nav */ 51 | .bs-sidenav { 52 | margin-top: 30px; 53 | margin-bottom: 30px; 54 | padding-top: 10px; 55 | padding-bottom: 10px; 56 | border-radius: 5px; 57 | } 58 | 59 | /* All levels of nav */ 60 | .bs-sidebar .nav > li > a { 61 | display: block; 62 | padding: 5px 20px; 63 | z-index: 1; 64 | } 65 | .bs-sidebar .nav > li > a:hover, 66 | .bs-sidebar .nav > li > a:focus { 67 | text-decoration: none; 68 | border-right: 1px solid; 69 | } 70 | .bs-sidebar .nav > .active > a, 71 | .bs-sidebar .nav > .active:hover > a, 72 | .bs-sidebar .nav > .active:focus > a { 73 | font-weight: bold; 74 | background-color: transparent; 75 | border-right: 1px solid; 76 | } 77 | 78 | /* Nav: second level (shown on .active) */ 79 | .bs-sidebar .nav .nav { 80 | display: none; /* Hide by default, but at >768px, show it */ 81 | margin-bottom: 8px; 82 | } 83 | .bs-sidebar .nav .nav > li > a { 84 | padding-top: 3px; 85 | padding-bottom: 3px; 86 | padding-left: 30px; 87 | font-size: 90%; 88 | } 89 | 90 | /* Show and affix the side nav when space allows it */ 91 | @media (min-width: 992px) { 92 | .bs-sidebar .nav > .active > ul { 93 | display: block; 94 | } 95 | /* Widen the fixed sidebar */ 96 | .bs-sidebar.affix, 97 | .bs-sidebar.affix-bottom { 98 | width: 213px; 99 | } 100 | .bs-sidebar.affix { 101 | position: fixed; /* Undo the static from mobile first approach */ 102 | top: 80px; 103 | max-height: calc(100% - 180px); 104 | overflow-y: auto; 105 | } 106 | .bs-sidebar.affix-bottom { 107 | position: absolute; /* Undo the static from mobile first approach */ 108 | } 109 | .bs-sidebar.affix-bottom .bs-sidenav, 110 | .bs-sidebar.affix .bs-sidenav { 111 | margin-top: 0; 112 | margin-bottom: 0; 113 | } 114 | } 115 | @media (min-width: 1200px) { 116 | /* Widen the fixed sidebar again */ 117 | .bs-sidebar.affix-bottom, 118 | .bs-sidebar.affix { 119 | width: 263px; 120 | } 121 | } 122 | 123 | 124 | /* Added to support >2 level nav in drop down */ 125 | 126 | .dropdown-submenu { 127 | position: relative; 128 | } 129 | 130 | .dropdown-submenu>.dropdown-menu { 131 | top: 0; 132 | left: 100%; 133 | margin-top: 0px; 134 | margin-left: 0px; 135 | } 136 | 137 | .dropdown-submenu:hover>.dropdown-menu { 138 | display: block; 139 | } 140 | 141 | .dropdown-submenu>a:after { 142 | display: block; 143 | content: " "; 144 | float: right; 145 | width: 0; 146 | height: 0; 147 | border-color: transparent; 148 | border-style: solid; 149 | border-width: 5px 0 5px 5px; 150 | border-left-color: #ccc; 151 | margin-top: 5px; 152 | margin-right: -10px; 153 | } 154 | 155 | .dropdown-submenu:hover>a:after { 156 | border-left-color: #fff; 157 | } 158 | 159 | .dropdown-submenu.pull-left { 160 | float: none; 161 | } 162 | 163 | .dropdown-submenu.pull-left>.dropdown-menu { 164 | left: -100%; 165 | margin-left: 00px; 166 | } 167 | /* Start Bootstrap Callouts CSS Source by Chris Pratt (https://codepen.io/chrisdpratt/pen/IAymB) MIT License*/ 168 | .bs-callout { 169 | padding: 20px; 170 | margin: 20px 0; 171 | border: 1px solid #eee; 172 | border-left-width: 5px; 173 | border-radius: 3px; 174 | background-color: #FCFDFF; 175 | } 176 | .bs-callout h4 { 177 | font-style: normal; 178 | font-weight: 400; 179 | margin-top: 0; 180 | margin-bottom: 5px; 181 | } 182 | .bs-callout p:last-child { 183 | margin-bottom: 0; 184 | } 185 | .bs-callout code { 186 | border-radius: 3px; 187 | } 188 | .bs-callout+.bs-callout { 189 | margin-top: -5px; 190 | } 191 | .bs-callout-default { 192 | border-left-color: #FA023C; /*modified from upstream default by Christopher Simpkins*/ 193 | } 194 | .bs-callout-default h4 { 195 | color: #FA023C; /*modified from upstream default by Christopher Simpkins*/ 196 | } 197 | .bs-callout-primary { 198 | border-left-color: #428bca; 199 | } 200 | .bs-callout-primary h4 { 201 | color: #428bca; 202 | } 203 | .bs-callout-success { 204 | border-left-color: #5cb85c; 205 | } 206 | .bs-callout-success h4 { 207 | color: #5cb85c; 208 | } 209 | .bs-callout-danger { 210 | border-left-color: #d9534f; 211 | } 212 | .bs-callout-danger h4 { 213 | color: #d9534f; 214 | } 215 | .bs-callout-warning { 216 | border-left-color: #f0ad4e; 217 | } 218 | .bs-callout-warning h4 { 219 | color: #f0ad4e; 220 | } 221 | .bs-callout-info { 222 | border-left-color: #5bc0de; 223 | } 224 | .bs-callout-info h4 { 225 | color: #5bc0de; 226 | } 227 | /* End Bootstrap Callouts CSS Source by Chris Pratt */ 228 | 229 | /* Headerlinks */ 230 | .headerlink { 231 | display: none; 232 | padding-left: .5em; 233 | } 234 | 235 | h1:hover .headerlink, h2:hover .headerlink, h3:hover .headerlink, h4:hover .headerlink, h5:hover .headerlink, h6:hover .headerlink { 236 | display: inline-block; 237 | } 238 | 239 | /* Admonitions */ 240 | .admonition { 241 | padding: 20px; 242 | margin: 20px 0; 243 | border: 1px solid #eee; 244 | border-left-width: 5px; 245 | border-radius: 3px; 246 | background-color: #FCFDFF; 247 | } 248 | 249 | .admonition p:last-child { 250 | margin-bottom: 0; 251 | } 252 | .admonition code { 253 | border-radius: 3px; 254 | } 255 | .admonition+.admonition { 256 | margin-top: -5px; 257 | } 258 | 259 | .admonition.note { /* csslint allow: adjoining-classes */ 260 | border-left-color: #428bca; 261 | } 262 | 263 | .admonition.warning { /* csslint allow: adjoining-classes */ 264 | border-left-color: #f0ad4e; 265 | } 266 | 267 | .admonition.danger { /* csslint allow: adjoining-classes */ 268 | border-left-color: #d9534f; 269 | } 270 | 271 | .admonition-title { 272 | font-size: 19px; 273 | font-style: normal; 274 | font-weight: 400; 275 | margin-top: 0; 276 | margin-bottom: 5px; 277 | } 278 | 279 | .admonition.note > .admonition-title { 280 | color: #428bca; 281 | } 282 | 283 | .admonition.warning > .admonition-title { 284 | color: #f0ad4e; 285 | } 286 | 287 | .admonition.danger > .admonition-title { 288 | color: #d9534f; 289 | } 290 | -------------------------------------------------------------------------------- /docs/cinder/js/bootstrap-3.0.3.min.js: -------------------------------------------------------------------------------- 1 | /*! 2 | * Bootstrap v3.0.3 (http://getbootstrap.com) 3 | * Copyright 2013 Twitter, Inc. 4 | * Licensed under http://www.apache.org/licenses/LICENSE-2.0 5 | */ 6 | 7 | if("undefined"==typeof jQuery)throw new Error("Bootstrap requires jQuery");+function(a){"use strict";function b(){var a=document.createElement("bootstrap"),b={WebkitTransition:"webkitTransitionEnd",MozTransition:"transitionend",OTransition:"oTransitionEnd otransitionend",transition:"transitionend"};for(var c in b)if(void 0!==a.style[c])return{end:b[c]}}a.fn.emulateTransitionEnd=function(b){var c=!1,d=this;a(this).one(a.support.transition.end,function(){c=!0});var e=function(){c||a(d).trigger(a.support.transition.end)};return setTimeout(e,b),this},a(function(){a.support.transition=b()})}(jQuery),+function(a){"use strict";var b='[data-dismiss="alert"]',c=function(c){a(c).on("click",b,this.close)};c.prototype.close=function(b){function c(){f.trigger("closed.bs.alert").remove()}var d=a(this),e=d.attr("data-target");e||(e=d.attr("href"),e=e&&e.replace(/.*(?=#[^\s]*$)/,""));var f=a(e);b&&b.preventDefault(),f.length||(f=d.hasClass("alert")?d:d.parent()),f.trigger(b=a.Event("close.bs.alert")),b.isDefaultPrevented()||(f.removeClass("in"),a.support.transition&&f.hasClass("fade")?f.one(a.support.transition.end,c).emulateTransitionEnd(150):c())};var d=a.fn.alert;a.fn.alert=function(b){return this.each(function(){var d=a(this),e=d.data("bs.alert");e||d.data("bs.alert",e=new c(this)),"string"==typeof b&&e[b].call(d)})},a.fn.alert.Constructor=c,a.fn.alert.noConflict=function(){return a.fn.alert=d,this},a(document).on("click.bs.alert.data-api",b,c.prototype.close)}(jQuery),+function(a){"use strict";var b=function(c,d){this.$element=a(c),this.options=a.extend({},b.DEFAULTS,d)};b.DEFAULTS={loadingText:"loading..."},b.prototype.setState=function(a){var b="disabled",c=this.$element,d=c.is("input")?"val":"html",e=c.data();a+="Text",e.resetText||c.data("resetText",c[d]()),c[d](e[a]||this.options[a]),setTimeout(function(){"loadingText"==a?c.addClass(b).attr(b,b):c.removeClass(b).removeAttr(b)},0)},b.prototype.toggle=function(){var a=this.$element.closest('[data-toggle="buttons"]'),b=!0;if(a.length){var c=this.$element.find("input");"radio"===c.prop("type")&&(c.prop("checked")&&this.$element.hasClass("active")?b=!1:a.find(".active").removeClass("active")),b&&c.prop("checked",!this.$element.hasClass("active")).trigger("change")}b&&this.$element.toggleClass("active")};var c=a.fn.button;a.fn.button=function(c){return this.each(function(){var d=a(this),e=d.data("bs.button"),f="object"==typeof c&&c;e||d.data("bs.button",e=new b(this,f)),"toggle"==c?e.toggle():c&&e.setState(c)})},a.fn.button.Constructor=b,a.fn.button.noConflict=function(){return a.fn.button=c,this},a(document).on("click.bs.button.data-api","[data-toggle^=button]",function(b){var c=a(b.target);c.hasClass("btn")||(c=c.closest(".btn")),c.button("toggle"),b.preventDefault()})}(jQuery),+function(a){"use strict";var b=function(b,c){this.$element=a(b),this.$indicators=this.$element.find(".carousel-indicators"),this.options=c,this.paused=this.sliding=this.interval=this.$active=this.$items=null,"hover"==this.options.pause&&this.$element.on("mouseenter",a.proxy(this.pause,this)).on("mouseleave",a.proxy(this.cycle,this))};b.DEFAULTS={interval:5e3,pause:"hover",wrap:!0},b.prototype.cycle=function(b){return b||(this.paused=!1),this.interval&&clearInterval(this.interval),this.options.interval&&!this.paused&&(this.interval=setInterval(a.proxy(this.next,this),this.options.interval)),this},b.prototype.getActiveIndex=function(){return this.$active=this.$element.find(".item.active"),this.$items=this.$active.parent().children(),this.$items.index(this.$active)},b.prototype.to=function(b){var c=this,d=this.getActiveIndex();return b>this.$items.length-1||0>b?void 0:this.sliding?this.$element.one("slid.bs.carousel",function(){c.to(b)}):d==b?this.pause().cycle():this.slide(b>d?"next":"prev",a(this.$items[b]))},b.prototype.pause=function(b){return b||(this.paused=!0),this.$element.find(".next, .prev").length&&a.support.transition.end&&(this.$element.trigger(a.support.transition.end),this.cycle(!0)),this.interval=clearInterval(this.interval),this},b.prototype.next=function(){return this.sliding?void 0:this.slide("next")},b.prototype.prev=function(){return this.sliding?void 0:this.slide("prev")},b.prototype.slide=function(b,c){var d=this.$element.find(".item.active"),e=c||d[b](),f=this.interval,g="next"==b?"left":"right",h="next"==b?"first":"last",i=this;if(!e.length){if(!this.options.wrap)return;e=this.$element.find(".item")[h]()}this.sliding=!0,f&&this.pause();var j=a.Event("slide.bs.carousel",{relatedTarget:e[0],direction:g});if(!e.hasClass("active")){if(this.$indicators.length&&(this.$indicators.find(".active").removeClass("active"),this.$element.one("slid.bs.carousel",function(){var b=a(i.$indicators.children()[i.getActiveIndex()]);b&&b.addClass("active")})),a.support.transition&&this.$element.hasClass("slide")){if(this.$element.trigger(j),j.isDefaultPrevented())return;e.addClass(b),e[0].offsetWidth,d.addClass(g),e.addClass(g),d.one(a.support.transition.end,function(){e.removeClass([b,g].join(" ")).addClass("active"),d.removeClass(["active",g].join(" ")),i.sliding=!1,setTimeout(function(){i.$element.trigger("slid.bs.carousel")},0)}).emulateTransitionEnd(600)}else{if(this.$element.trigger(j),j.isDefaultPrevented())return;d.removeClass("active"),e.addClass("active"),this.sliding=!1,this.$element.trigger("slid.bs.carousel")}return f&&this.cycle(),this}};var c=a.fn.carousel;a.fn.carousel=function(c){return this.each(function(){var d=a(this),e=d.data("bs.carousel"),f=a.extend({},b.DEFAULTS,d.data(),"object"==typeof c&&c),g="string"==typeof c?c:f.slide;e||d.data("bs.carousel",e=new b(this,f)),"number"==typeof c?e.to(c):g?e[g]():f.interval&&e.pause().cycle()})},a.fn.carousel.Constructor=b,a.fn.carousel.noConflict=function(){return a.fn.carousel=c,this},a(document).on("click.bs.carousel.data-api","[data-slide], [data-slide-to]",function(b){var c,d=a(this),e=a(d.attr("data-target")||(c=d.attr("href"))&&c.replace(/.*(?=#[^\s]+$)/,"")),f=a.extend({},e.data(),d.data()),g=d.attr("data-slide-to");g&&(f.interval=!1),e.carousel(f),(g=d.attr("data-slide-to"))&&e.data("bs.carousel").to(g),b.preventDefault()}),a(window).on("load",function(){a('[data-ride="carousel"]').each(function(){var b=a(this);b.carousel(b.data())})})}(jQuery),+function(a){"use strict";var b=function(c,d){this.$element=a(c),this.options=a.extend({},b.DEFAULTS,d),this.transitioning=null,this.options.parent&&(this.$parent=a(this.options.parent)),this.options.toggle&&this.toggle()};b.DEFAULTS={toggle:!0},b.prototype.dimension=function(){var a=this.$element.hasClass("width");return a?"width":"height"},b.prototype.show=function(){if(!this.transitioning&&!this.$element.hasClass("in")){var b=a.Event("show.bs.collapse");if(this.$element.trigger(b),!b.isDefaultPrevented()){var c=this.$parent&&this.$parent.find("> .panel > .in");if(c&&c.length){var d=c.data("bs.collapse");if(d&&d.transitioning)return;c.collapse("hide"),d||c.data("bs.collapse",null)}var e=this.dimension();this.$element.removeClass("collapse").addClass("collapsing")[e](0),this.transitioning=1;var f=function(){this.$element.removeClass("collapsing").addClass("in")[e]("auto"),this.transitioning=0,this.$element.trigger("shown.bs.collapse")};if(!a.support.transition)return f.call(this);var g=a.camelCase(["scroll",e].join("-"));this.$element.one(a.support.transition.end,a.proxy(f,this)).emulateTransitionEnd(350)[e](this.$element[0][g])}}},b.prototype.hide=function(){if(!this.transitioning&&this.$element.hasClass("in")){var b=a.Event("hide.bs.collapse");if(this.$element.trigger(b),!b.isDefaultPrevented()){var c=this.dimension();this.$element[c](this.$element[c]())[0].offsetHeight,this.$element.addClass("collapsing").removeClass("collapse").removeClass("in"),this.transitioning=1;var d=function(){this.transitioning=0,this.$element.trigger("hidden.bs.collapse").removeClass("collapsing").addClass("collapse")};return a.support.transition?(this.$element[c](0).one(a.support.transition.end,a.proxy(d,this)).emulateTransitionEnd(350),void 0):d.call(this)}}},b.prototype.toggle=function(){this[this.$element.hasClass("in")?"hide":"show"]()};var c=a.fn.collapse;a.fn.collapse=function(c){return this.each(function(){var d=a(this),e=d.data("bs.collapse"),f=a.extend({},b.DEFAULTS,d.data(),"object"==typeof c&&c);e||d.data("bs.collapse",e=new b(this,f)),"string"==typeof c&&e[c]()})},a.fn.collapse.Constructor=b,a.fn.collapse.noConflict=function(){return a.fn.collapse=c,this},a(document).on("click.bs.collapse.data-api","[data-toggle=collapse]",function(b){var c,d=a(this),e=d.attr("data-target")||b.preventDefault()||(c=d.attr("href"))&&c.replace(/.*(?=#[^\s]+$)/,""),f=a(e),g=f.data("bs.collapse"),h=g?"toggle":d.data(),i=d.attr("data-parent"),j=i&&a(i);g&&g.transitioning||(j&&j.find('[data-toggle=collapse][data-parent="'+i+'"]').not(d).addClass("collapsed"),d[f.hasClass("in")?"addClass":"removeClass"]("collapsed")),f.collapse(h)})}(jQuery),+function(a){"use strict";function b(){a(d).remove(),a(e).each(function(b){var d=c(a(this));d.hasClass("open")&&(d.trigger(b=a.Event("hide.bs.dropdown")),b.isDefaultPrevented()||d.removeClass("open").trigger("hidden.bs.dropdown"))})}function c(b){var c=b.attr("data-target");c||(c=b.attr("href"),c=c&&/#/.test(c)&&c.replace(/.*(?=#[^\s]*$)/,""));var d=c&&a(c);return d&&d.length?d:b.parent()}var d=".dropdown-backdrop",e="[data-toggle=dropdown]",f=function(b){a(b).on("click.bs.dropdown",this.toggle)};f.prototype.toggle=function(d){var e=a(this);if(!e.is(".disabled, :disabled")){var f=c(e),g=f.hasClass("open");if(b(),!g){if("ontouchstart"in document.documentElement&&!f.closest(".navbar-nav").length&&a('
    ').insertAfter(a(this)).on("click",b),f.trigger(d=a.Event("show.bs.dropdown")),d.isDefaultPrevented())return;f.toggleClass("open").trigger("shown.bs.dropdown"),e.focus()}return!1}},f.prototype.keydown=function(b){if(/(38|40|27)/.test(b.keyCode)){var d=a(this);if(b.preventDefault(),b.stopPropagation(),!d.is(".disabled, :disabled")){var f=c(d),g=f.hasClass("open");if(!g||g&&27==b.keyCode)return 27==b.which&&f.find(e).focus(),d.click();var h=a("[role=menu] li:not(.divider):visible a",f);if(h.length){var i=h.index(h.filter(":focus"));38==b.keyCode&&i>0&&i--,40==b.keyCode&&i').appendTo(document.body),this.$element.on("click.dismiss.modal",a.proxy(function(a){a.target===a.currentTarget&&("static"==this.options.backdrop?this.$element[0].focus.call(this.$element[0]):this.hide.call(this))},this)),d&&this.$backdrop[0].offsetWidth,this.$backdrop.addClass("in"),!b)return;d?this.$backdrop.one(a.support.transition.end,b).emulateTransitionEnd(150):b()}else!this.isShown&&this.$backdrop?(this.$backdrop.removeClass("in"),a.support.transition&&this.$element.hasClass("fade")?this.$backdrop.one(a.support.transition.end,b).emulateTransitionEnd(150):b()):b&&b()};var c=a.fn.modal;a.fn.modal=function(c,d){return this.each(function(){var e=a(this),f=e.data("bs.modal"),g=a.extend({},b.DEFAULTS,e.data(),"object"==typeof c&&c);f||e.data("bs.modal",f=new b(this,g)),"string"==typeof c?f[c](d):g.show&&f.show(d)})},a.fn.modal.Constructor=b,a.fn.modal.noConflict=function(){return a.fn.modal=c,this},a(document).on("click.bs.modal.data-api",'[data-toggle="modal"]',function(b){var c=a(this),d=c.attr("href"),e=a(c.attr("data-target")||d&&d.replace(/.*(?=#[^\s]+$)/,"")),f=e.data("modal")?"toggle":a.extend({remote:!/#/.test(d)&&d},e.data(),c.data());b.preventDefault(),e.modal(f,this).one("hide",function(){c.is(":visible")&&c.focus()})}),a(document).on("show.bs.modal",".modal",function(){a(document.body).addClass("modal-open")}).on("hidden.bs.modal",".modal",function(){a(document.body).removeClass("modal-open")})}(jQuery),+function(a){"use strict";var b=function(a,b){this.type=this.options=this.enabled=this.timeout=this.hoverState=this.$element=null,this.init("tooltip",a,b)};b.DEFAULTS={animation:!0,placement:"top",selector:!1,template:'
    ',trigger:"hover focus",title:"",delay:0,html:!1,container:!1},b.prototype.init=function(b,c,d){this.enabled=!0,this.type=b,this.$element=a(c),this.options=this.getOptions(d);for(var e=this.options.trigger.split(" "),f=e.length;f--;){var g=e[f];if("click"==g)this.$element.on("click."+this.type,this.options.selector,a.proxy(this.toggle,this));else if("manual"!=g){var h="hover"==g?"mouseenter":"focus",i="hover"==g?"mouseleave":"blur";this.$element.on(h+"."+this.type,this.options.selector,a.proxy(this.enter,this)),this.$element.on(i+"."+this.type,this.options.selector,a.proxy(this.leave,this))}}this.options.selector?this._options=a.extend({},this.options,{trigger:"manual",selector:""}):this.fixTitle()},b.prototype.getDefaults=function(){return b.DEFAULTS},b.prototype.getOptions=function(b){return b=a.extend({},this.getDefaults(),this.$element.data(),b),b.delay&&"number"==typeof b.delay&&(b.delay={show:b.delay,hide:b.delay}),b},b.prototype.getDelegateOptions=function(){var b={},c=this.getDefaults();return this._options&&a.each(this._options,function(a,d){c[a]!=d&&(b[a]=d)}),b},b.prototype.enter=function(b){var c=b instanceof this.constructor?b:a(b.currentTarget)[this.type](this.getDelegateOptions()).data("bs."+this.type);return clearTimeout(c.timeout),c.hoverState="in",c.options.delay&&c.options.delay.show?(c.timeout=setTimeout(function(){"in"==c.hoverState&&c.show()},c.options.delay.show),void 0):c.show()},b.prototype.leave=function(b){var c=b instanceof this.constructor?b:a(b.currentTarget)[this.type](this.getDelegateOptions()).data("bs."+this.type);return clearTimeout(c.timeout),c.hoverState="out",c.options.delay&&c.options.delay.hide?(c.timeout=setTimeout(function(){"out"==c.hoverState&&c.hide()},c.options.delay.hide),void 0):c.hide()},b.prototype.show=function(){var b=a.Event("show.bs."+this.type);if(this.hasContent()&&this.enabled){if(this.$element.trigger(b),b.isDefaultPrevented())return;var c=this.tip();this.setContent(),this.options.animation&&c.addClass("fade");var d="function"==typeof this.options.placement?this.options.placement.call(this,c[0],this.$element[0]):this.options.placement,e=/\s?auto?\s?/i,f=e.test(d);f&&(d=d.replace(e,"")||"top"),c.detach().css({top:0,left:0,display:"block"}).addClass(d),this.options.container?c.appendTo(this.options.container):c.insertAfter(this.$element);var g=this.getPosition(),h=c[0].offsetWidth,i=c[0].offsetHeight;if(f){var j=this.$element.parent(),k=d,l=document.documentElement.scrollTop||document.body.scrollTop,m="body"==this.options.container?window.innerWidth:j.outerWidth(),n="body"==this.options.container?window.innerHeight:j.outerHeight(),o="body"==this.options.container?0:j.offset().left;d="bottom"==d&&g.top+g.height+i-l>n?"top":"top"==d&&g.top-l-i<0?"bottom":"right"==d&&g.right+h>m?"left":"left"==d&&g.left-h

    '}),b.prototype=a.extend({},a.fn.tooltip.Constructor.prototype),b.prototype.constructor=b,b.prototype.getDefaults=function(){return b.DEFAULTS},b.prototype.setContent=function(){var a=this.tip(),b=this.getTitle(),c=this.getContent();a.find(".popover-title")[this.options.html?"html":"text"](b),a.find(".popover-content")[this.options.html?"html":"text"](c),a.removeClass("fade top bottom left right in"),a.find(".popover-title").html()||a.find(".popover-title").hide()},b.prototype.hasContent=function(){return this.getTitle()||this.getContent()},b.prototype.getContent=function(){var a=this.$element,b=this.options;return a.attr("data-content")||("function"==typeof b.content?b.content.call(a[0]):b.content)},b.prototype.arrow=function(){return this.$arrow=this.$arrow||this.tip().find(".arrow")},b.prototype.tip=function(){return this.$tip||(this.$tip=a(this.options.template)),this.$tip};var c=a.fn.popover;a.fn.popover=function(c){return this.each(function(){var d=a(this),e=d.data("bs.popover"),f="object"==typeof c&&c;e||d.data("bs.popover",e=new b(this,f)),"string"==typeof c&&e[c]()})},a.fn.popover.Constructor=b,a.fn.popover.noConflict=function(){return a.fn.popover=c,this}}(jQuery),+function(a){"use strict";function b(c,d){var e,f=a.proxy(this.process,this);this.$element=a(c).is("body")?a(window):a(c),this.$body=a("body"),this.$scrollElement=this.$element.on("scroll.bs.scroll-spy.data-api",f),this.options=a.extend({},b.DEFAULTS,d),this.selector=(this.options.target||(e=a(c).attr("href"))&&e.replace(/.*(?=#[^\s]+$)/,"")||"")+" .nav li > a",this.offsets=a([]),this.targets=a([]),this.activeTarget=null,this.refresh(),this.process()}b.DEFAULTS={offset:10},b.prototype.refresh=function(){var b=this.$element[0]==window?"offset":"position";this.offsets=a([]),this.targets=a([]);var c=this;this.$body.find(this.selector).map(function(){var d=a(this),e=d.data("target")||d.attr("href"),f=/^#\w/.test(e)&&a(e);return f&&f.length&&[[f[b]().top+(!a.isWindow(c.$scrollElement.get(0))&&c.$scrollElement.scrollTop()),e]]||null}).sort(function(a,b){return a[0]-b[0]}).each(function(){c.offsets.push(this[0]),c.targets.push(this[1])})},b.prototype.process=function(){var a,b=this.$scrollElement.scrollTop()+this.options.offset,c=this.$scrollElement[0].scrollHeight||this.$body[0].scrollHeight,d=c-this.$scrollElement.height(),e=this.offsets,f=this.targets,g=this.activeTarget;if(b>=d)return g!=(a=f.last()[0])&&this.activate(a);for(a=e.length;a--;)g!=f[a]&&b>=e[a]&&(!e[a+1]||b<=e[a+1])&&this.activate(f[a])},b.prototype.activate=function(b){this.activeTarget=b,a(this.selector).parents(".active").removeClass("active");var c=this.selector+'[data-target="'+b+'"],'+this.selector+'[href="'+b+'"]',d=a(c).parents("li").addClass("active");d.parent(".dropdown-menu").length&&(d=d.closest("li.dropdown").addClass("active")),d.trigger("activate.bs.scrollspy")};var c=a.fn.scrollspy;a.fn.scrollspy=function(c){return this.each(function(){var d=a(this),e=d.data("bs.scrollspy"),f="object"==typeof c&&c;e||d.data("bs.scrollspy",e=new b(this,f)),"string"==typeof c&&e[c]()})},a.fn.scrollspy.Constructor=b,a.fn.scrollspy.noConflict=function(){return a.fn.scrollspy=c,this},a(window).on("load",function(){a('[data-spy="scroll"]').each(function(){var b=a(this);b.scrollspy(b.data())})})}(jQuery),+function(a){"use strict";var b=function(b){this.element=a(b)};b.prototype.show=function(){var b=this.element,c=b.closest("ul:not(.dropdown-menu)"),d=b.data("target");if(d||(d=b.attr("href"),d=d&&d.replace(/.*(?=#[^\s]*$)/,"")),!b.parent("li").hasClass("active")){var e=c.find(".active:last a")[0],f=a.Event("show.bs.tab",{relatedTarget:e});if(b.trigger(f),!f.isDefaultPrevented()){var g=a(d);this.activate(b.parent("li"),c),this.activate(g,g.parent(),function(){b.trigger({type:"shown.bs.tab",relatedTarget:e})})}}},b.prototype.activate=function(b,c,d){function e(){f.removeClass("active").find("> .dropdown-menu > .active").removeClass("active"),b.addClass("active"),g?(b[0].offsetWidth,b.addClass("in")):b.removeClass("fade"),b.parent(".dropdown-menu")&&b.closest("li.dropdown").addClass("active"),d&&d()}var f=c.find("> .active"),g=d&&a.support.transition&&f.hasClass("fade");g?f.one(a.support.transition.end,e).emulateTransitionEnd(150):e(),f.removeClass("in")};var c=a.fn.tab;a.fn.tab=function(c){return this.each(function(){var d=a(this),e=d.data("bs.tab");e||d.data("bs.tab",e=new b(this)),"string"==typeof c&&e[c]()})},a.fn.tab.Constructor=b,a.fn.tab.noConflict=function(){return a.fn.tab=c,this},a(document).on("click.bs.tab.data-api",'[data-toggle="tab"], [data-toggle="pill"]',function(b){b.preventDefault(),a(this).tab("show")})}(jQuery),+function(a){"use strict";var b=function(c,d){this.options=a.extend({},b.DEFAULTS,d),this.$window=a(window).on("scroll.bs.affix.data-api",a.proxy(this.checkPosition,this)).on("click.bs.affix.data-api",a.proxy(this.checkPositionWithEventLoop,this)),this.$element=a(c),this.affixed=this.unpin=null,this.checkPosition()};b.RESET="affix affix-top affix-bottom",b.DEFAULTS={offset:0},b.prototype.checkPositionWithEventLoop=function(){setTimeout(a.proxy(this.checkPosition,this),1)},b.prototype.checkPosition=function(){if(this.$element.is(":visible")){var c=a(document).height(),d=this.$window.scrollTop(),e=this.$element.offset(),f=this.options.offset,g=f.top,h=f.bottom;"object"!=typeof f&&(h=g=f),"function"==typeof g&&(g=f.top()),"function"==typeof h&&(h=f.bottom());var i=null!=this.unpin&&d+this.unpin<=e.top?!1:null!=h&&e.top+this.$element.height()>=c-h?"bottom":null!=g&&g>=d?"top":!1;this.affixed!==i&&(this.unpin&&this.$element.css("top",""),this.affixed=i,this.unpin="bottom"==i?e.top-d:null,this.$element.removeClass(b.RESET).addClass("affix"+(i?"-"+i:"")),"bottom"==i&&this.$element.offset({top:document.body.offsetHeight-h-this.$element.height()}))}};var c=a.fn.affix;a.fn.affix=function(c){return this.each(function(){var d=a(this),e=d.data("bs.affix"),f="object"==typeof c&&c;e||d.data("bs.affix",e=new b(this,f)),"string"==typeof c&&e[c]()})},a.fn.affix.Constructor=b,a.fn.affix.noConflict=function(){return a.fn.affix=c,this},a(window).on("load",function(){a('[data-spy="affix"]').each(function(){var b=a(this),c=b.data();c.offset=c.offset||{},c.offsetBottom&&(c.offset.bottom=c.offsetBottom),c.offsetTop&&(c.offset.top=c.offsetTop),b.affix(c)})})}(jQuery); --------------------------------------------------------------------------------