├── .github └── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md ├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── README.md ├── index.d.ts ├── index.js ├── package-lock.json ├── package.json ├── src ├── Constants.js ├── Scraper.js └── Util.js └── test ├── Test.js └── index.js /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **Environment** 14 | Node Version: ... 15 | NPM Version: ... 16 | Additional info related to your environment... 17 | 18 | **To Reproduce** 19 | Steps to reproduce the behavior: 20 | 1. Go to '...' 21 | 2. Click on '....' 22 | 3. Scroll down to '....' 23 | 4. See error 24 | 25 | **Expected behavior** 26 | A clear and concise description of what you expected to happen. 27 | 28 | **Screenshots** 29 | If applicable, add screenshots to help explain your problem. 30 | 31 | **Additional context** 32 | Add any other context about the problem here. 33 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Yimura/Scraper Changelog 2 | 3 | ## Versioning Policy 4 | 5 | Following: 6 | **major.minor.patch** 7 | 8 | * **major** is almost never used unless major breaking changes were pushed or a complete rewrite happened of the code 9 | * **minor** used whenever a single file was rewritten or significant change happened 10 | * **patch** may only be bumped after a bug was resolved as whole 11 | 12 | ## 2021-05-25, Version 1.2.2, @GeopJr 13 | 14 | * fix(idToThumbnail): always use maxres, thanks @GeopJr 15 | 16 | ## 2021-03-25, Version 1.2.1, @omarkhatibco 17 | 18 | ### Changes 19 | 20 | * fixed TypeScript annotations, thanks @omarkhatibco 21 | 22 | ## 2021-03-25, Version 1.2.0, @Lioness100 23 | 24 | ### Changes 25 | 26 | * Added typings, thanks @Lioness100 27 | 28 | ## 2021-02-19, Version 1.1.0, @Yimura 29 | 30 | ### Changes 31 | 32 | * Fixed an issue with verified channel states where it would only check for verified artist but not verified channel. 33 | * Added support for channel search. 34 | * Fixed typo's in README 35 | 36 | ## 2021-02-15, Version 1.0.0, @Yimura 37 | 38 | ### Changes 39 | 40 | * Updated Constants with pre-decoded SearchTypes, this to prevent problems where URLSearchParams would encode the value again. 41 | * The Constants are accessible and export in the index if you needed these for some reason. 42 | * YT Scraper has been updated and support has been added for the following search types: 43 | * any 44 | * live 45 | * movie 46 | * playlist 47 | * video 48 | * Tests have been updated for the new functionalities 49 | * README includes an entire response object of possible results 50 | 51 | ## 2020-12-16, Version 0.2.3, @Yimura 52 | 53 | ### Changes 54 | 55 | * Fixed an edge case were data from YouTube would be incorrectly parsed. 56 | 57 | ## 2020-11-25, Version 0.2.2, @Yimura 58 | 59 | ### Changes 60 | 61 | * Fixed a change in the YouTube webpage preventing from parsing a valid result 62 | 63 | ## 2020-10-28, Version 0.2.1, @Yimura 64 | 65 | ### Changes 66 | 67 | * Updated README 68 | 69 | ## 2020-10-28, Version 0.2.0, @Yimura 70 | 71 | ### Changes 72 | 73 | * Added a language option 74 | * Added a limit option 75 | * Fixed #2 76 | * Made testing more in-depth of features 77 | * Updated README to showcase the options 78 | 79 | ## 2020-10-27, Version 0.1.0, @Yimura 80 | 81 | ### Changes 82 | 83 | * Initial Beta Release 84 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Yimura 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # YouTube Scraper 2 | 3 | ## Table of Contents 4 | 5 | * [Why use this package](#why-use-this-package) 6 | - [Timings](#timings) 7 | * [Options](#options) 8 | - [Example Options](#example-options) 9 | * [Example Code](#example-code) 10 | * [Output](#output) 11 | * [Return Object Structure](#return-object-structure) 12 | 13 | ## Why use this package? 14 | 15 | This is a YouTube scraper with zero dependencies. 16 | Everything has been coded to have a minimal footprint creating a small package that's aimed at being as fast as possible. 17 | 18 | ### Timings 19 | 20 | These are the timings I would get on average over 20 tests, ofcourse the Fetch time depends on how good your connection is to YouTube and how loaded YouTube is at that point. 21 | 22 | | Type | Fetch Time | Processing Time | 23 | |---|---|---| 24 | | `video` | 585.632055 ms | 3.117175 ms | 25 | | `channel` | 494.026065 ms | `not tested` | 26 | | `playlist` | 569.424545 ms | `not tested` | 27 | 28 | [Check here](https://prnt.sc/1018ttl) 29 | 30 | ## Options 31 | 32 | | Property | Default | Description | 33 | |---|---|---| 34 | | language | `en` | Set the language that you would like for results to be returned in. A list of supported language types can be found [here](http://www.iana.org/assignments/language-subtag-registry/language-subtag-registry). | 35 | | searchType | `video` | Which type to search for on YouTube, supported types are `any`, `channel`, `live`, `movie`, `playlist` and `video` | 36 | ``` 37 | "Sort by" has not been implemented as of now. 38 | All data is sorted in the default order that YouTube returns these in. 39 | ``` 40 | 41 | ### Example Options 42 | You can set the global language which YouTube should return results in or set the return language per search/request: 43 | ```js 44 | import youtube from '@yimura/scraper' 45 | 46 | // This will set the language to French from France globally 47 | const yt = new youtube.default('fr-FR'); 48 | 49 | // Sets the language communicated to YouTube to Dutch from Belgium for this search 50 | const results = yt.search('Never gonna give you up', { 51 | language: 'nl-BE', 52 | searchType: 'video' // video is the default search type 53 | }); 54 | ``` 55 | 56 | ## Example Code 57 | 58 | **CommonJS:** 59 | ```js 60 | const Scraper = require('@yimura/scraper').default; 61 | 62 | const youtube = new Scraper(); 63 | 64 | youtube.search('Never gonna give you up').then(results => { 65 | console.log(results.videos[0]); 66 | }); 67 | ``` 68 | 69 | **ESModule:** 70 | ```js 71 | import youtube from '@yimura/scraper' 72 | 73 | const yt = new youtube.default(); 74 | yt.search('Never gonna give you up').then(results => { 75 | console.log(results.videos[0]); 76 | }); 77 | ``` 78 | 79 | ### Output 80 | 81 | ```js 82 | { 83 | channel: { 84 | name: 'Official Rick Astley', 85 | link: 'https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw', 86 | verified: true 87 | }, 88 | description: "Rick Astley's official music video for “Never Gonna Give You Up” Listen to Rick Astley: https://RickAstley.lnk.to/_listenYD Subscribe ...", 89 | duration: 213, 90 | duration_raw: "3:33", 91 | id: 'dQw4w9WgXcQ', 92 | link: 'https://www.youtube.com/watch?v=dQw4w9WgXcQ', 93 | thumbnail: 'https://i.ytimg.com/vi/dQw4w9WgXcQhqdefault.jpg', 94 | shareLink: 'https://youtu.be/dQw4w9WgXcQ', 95 | title: 'Rick Astley - Never Gonna Give You Up (Video)', 96 | uploaded: '11 years ago', 97 | views: 788551856 98 | } 99 | ``` 100 | 101 | ## Return Object Structure 102 | ```js 103 | { 104 | channels: [ 105 | { 106 | channelId: String, 107 | description: String, 108 | link: String, 109 | thumbnails: [ 110 | { 111 | url: String, 112 | width: Number, 113 | height: Number 114 | } 115 | ], 116 | subscribed: Boolean, 117 | uploadedVideos: Number, 118 | verified: Boolean 119 | } 120 | ], 121 | playlists: [ 122 | { 123 | preview: [ 124 | { 125 | duration: Number, 126 | duration_raw: String, 127 | views: Number, 128 | id: String, 129 | link: String, 130 | thumbnail: String, 131 | title: String, 132 | shareLink: String 133 | } 134 | ], 135 | id: String, 136 | link: String, 137 | thumbnail: String, 138 | title: String, 139 | videoCount: Number 140 | } 141 | ], 142 | streams: [ 143 | { 144 | watching: Number, 145 | channel: { 146 | name: String, 147 | link: String, 148 | verified: Boolean 149 | }, 150 | id: String, 151 | link: String, 152 | thumbnail: String, 153 | title: String, 154 | shareLink: String 155 | } 156 | ], 157 | videos: [ 158 | { 159 | description: String, 160 | duration: Number, 161 | duration_raw: String, 162 | uploaded: String, 163 | views: Number, 164 | channel: { 165 | name: String, 166 | link: String, 167 | verified: Boolean 168 | }, 169 | id: String, 170 | link: String, 171 | thumbnail: String, 172 | title: String, 173 | shareLink: String 174 | } 175 | ] 176 | } 177 | ``` 178 | 179 | -------------------------------------------------------------------------------- /index.d.ts: -------------------------------------------------------------------------------- 1 | export interface SearchTypes { 2 | ANY: "CAA%3D"; 3 | CHANNEL: "EgIQAg%3D%3D"; 4 | LIVE: "EgJAAQ%3D%3D"; 5 | MOVIE: "EgIQBA%3D%3D"; 6 | PLAYLIST: "EgIQAw%3D%3D"; 7 | VIDEO: "EgIQAQ%3D%3D"; 8 | } 9 | 10 | export interface Thumbnail { 11 | url: string; 12 | width: number; 13 | height: number; 14 | } 15 | 16 | export interface GeneralData { 17 | id: string; 18 | link: string; 19 | thumbnail: string; 20 | title: string; 21 | } 22 | 23 | export interface ShareableGeneralData extends GeneralData { 24 | shareLink: string; 25 | } 26 | 27 | export interface VideoPreview extends ShareableGeneralData { 28 | duration: number; 29 | duration_raw: string; 30 | views: number; 31 | } 32 | 33 | export interface ChannelPreview { 34 | name: string; 35 | link: string; 36 | verified: boolean; 37 | } 38 | 39 | export interface Video extends ShareableGeneralData { 40 | description: string; 41 | duration: number; 42 | duration_raw: string; 43 | uploaded: string; 44 | views: number; 45 | channel: ChannelPreview; 46 | } 47 | 48 | export interface Playlist extends GeneralData { 49 | preview: VideoPreview[]; 50 | duration: number; 51 | duration_raw: string; 52 | } 53 | 54 | export interface Stream extends ShareableGeneralData { 55 | watching: number; 56 | channel: ChannelPreview; 57 | } 58 | 59 | export interface Channel { 60 | channelId: string; 61 | description: string; 62 | link: string; 63 | thumbnails: Thumbnail[]; 64 | subscribed: boolean; 65 | uploadedVideos: number; 66 | verified: boolean; 67 | } 68 | 69 | export interface Results { 70 | channels: Channel[]; 71 | playlists: Playlist[]; 72 | streams: Stream[]; 73 | videos: Video[]; 74 | } 75 | 76 | export interface SearchOptions { 77 | searchType?: keyof SearchTypes; 78 | language?: string; 79 | } 80 | 81 | declare class Scraper { 82 | private _lang: string; 83 | 84 | public constructor(language?: string); 85 | 86 | private _extractData( 87 | json: Record 88 | ): Record[]; 89 | private _fetch( 90 | search_query: string, 91 | searchType?: keyof SearchTypes, 92 | requestedLang?: string 93 | ): Promise; 94 | private _getSearchData(webPage: string): Record; 95 | private _parseData(data: Record[]): Results; 96 | 97 | public search(query: string, options?: SearchOptions): Promise; 98 | public setLang(language?: string): void; 99 | } 100 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | "use strict" 2 | const Constants = require('./src/Constants.js'); 3 | const Scraper = require('./src/Scraper.js'); 4 | 5 | module.exports = { 6 | Constants, 7 | default: Scraper, 8 | Scraper 9 | }; 10 | -------------------------------------------------------------------------------- /package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@yimura/scraper", 3 | "version": "1.2.4", 4 | "lockfileVersion": 2, 5 | "requires": true, 6 | "packages": { 7 | "": { 8 | "name": "@yimura/scraper", 9 | "version": "1.2.3", 10 | "license": "MIT" 11 | } 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@yimura/scraper", 3 | "version": "1.2.4", 4 | "description": "A YouTube scraper using zero dependencies", 5 | "main": "index.js", 6 | "types": "index.d.ts", 7 | "scripts": { 8 | "test": "node test/index.js" 9 | }, 10 | "repository": { 11 | "type": "git", 12 | "url": "https://github.com/Yimura/Scraper" 13 | }, 14 | "bugs": { 15 | "url": "https://github.com/Yimura/Scraper/issues" 16 | }, 17 | "homepage": "https://github.com/Yimura/Scraper#readme", 18 | "keywords": [ 19 | "youtube search", 20 | "search youtube", 21 | "yt", 22 | "music", 23 | "youtube", 24 | "scrape", 25 | "youtube-scraper", 26 | "scrape-youtube", 27 | "search", 28 | "discord", 29 | "bot" 30 | ], 31 | "author": "yimura", 32 | "license": "MIT", 33 | "dependencies": {} 34 | } 35 | -------------------------------------------------------------------------------- /src/Constants.js: -------------------------------------------------------------------------------- 1 | "use strict" 2 | exports.SearchTypes = { 3 | ANY: "CAA%3D", 4 | CHANNEL: "EgIQAg%3D%3D", 5 | LIVE: "EgJAAQ%3D%3D", 6 | MOVIE: "EgIQBA%3D%3D", 7 | PLAYLIST: "EgIQAw%3D%3D", 8 | VIDEO: "EgIQAQ%3D%3D" 9 | }; 10 | exports.YouTubeURL = new URL('https://www.youtube.com/results'); 11 | -------------------------------------------------------------------------------- /src/Scraper.js: -------------------------------------------------------------------------------- 1 | "use strict" 2 | const https = require('https'); 3 | const { SearchTypes, YouTubeURL } = require('./Constants.js'); 4 | const Util = require('./Util.js'); 5 | 6 | class Scraper { 7 | /** 8 | * @param {string} [language = 'en'] An IANA Language Subtag, see => http://www.iana.org/assignments/language-subtag-registry/language-subtag-registry 9 | */ 10 | constructor(language = 'en') { 11 | this._lang = language; 12 | } 13 | 14 | /** 15 | * @param {Object} json 16 | */ 17 | _extractData(json) { 18 | json = json 19 | .contents 20 | .twoColumnSearchResultsRenderer 21 | .primaryContents; 22 | 23 | let contents = []; 24 | 25 | if (json.sectionListRenderer) { 26 | contents = json.sectionListRenderer.contents.filter((item) => 27 | item?.itemSectionRenderer?.contents.filter(x => x.videoRenderer || x.playlistRenderer || x.channelRenderer) 28 | ).shift().itemSectionRenderer.contents; 29 | } 30 | 31 | if (json.richGridRenderer) { 32 | contents = json.richGridRenderer.contents.filter((item) => 33 | item.richItemRenderer && item.richItemRenderer.content 34 | ).map(item => item.richItemRenderer.content); 35 | } 36 | 37 | return contents; 38 | } 39 | 40 | /** 41 | * @private 42 | * @param {string} search_query 43 | * @param {string} [requestedLang=null] 44 | * @returns {Promise} The entire YouTube webpage as a string 45 | */ 46 | _fetch(search_query, searchType = 'VIDEO', requestedLang = this._lang) { 47 | if (requestedLang && typeof requestedLang !== 'string') { 48 | throw new TypeError('The request language property was not a string while a valid IANA language subtag is expected.'); 49 | } 50 | 51 | const sp = SearchTypes[searchType.toUpperCase()] || SearchTypes['VIDEO']; 52 | 53 | YouTubeURL.search = new URLSearchParams({ 54 | search_query, 55 | sp 56 | }); 57 | 58 | return new Promise((resolve, reject) => { 59 | https.get(YouTubeURL, { 60 | headers: { 61 | 'Accept-Language': requestedLang 62 | } 63 | }, res => { 64 | res.setEncoding('utf8'); 65 | 66 | let data = ''; 67 | 68 | res.on('data', chunk => { 69 | data += chunk; 70 | }); 71 | res.on('end', () => resolve(data)); 72 | }).on('error', reject); 73 | }); 74 | } 75 | 76 | /** 77 | * @private 78 | * @param {string} webPage The YouTube webpage with search results 79 | * @returns The search data 80 | */ 81 | _getSearchData(webPage) { 82 | const startString = 'var ytInitialData = '; 83 | const start = webPage.indexOf(startString); 84 | const end = webPage.indexOf(';', start); 85 | 86 | const data = webPage.substring(start + startString.length, end); 87 | 88 | try { 89 | return JSON.parse(data); 90 | } catch (e) { 91 | throw new Error('Failed to parse YouTube search data. YouTube might have updated their site or no results returned.'); 92 | } 93 | } 94 | 95 | _parseData(data) { 96 | const results = { 97 | channels: [], 98 | playlists: [], 99 | streams: [], 100 | videos: [] 101 | }; 102 | 103 | for (const item of data) { 104 | // Ordered in which they would occur the most frequently to decrease cost of these if else statements 105 | if (Util.isVideo(item)) 106 | results.videos.push(Util.getVideoData(item)); 107 | else if (Util.isPlaylist(item)) 108 | results.playlists.push(Util.getPlaylistData(item)); 109 | else if (Util.isStream(item)) 110 | results.streams.push(Util.getStreamData(item)); 111 | else if (Util.isChannel(item)) 112 | results.channels.push(Util.getChannelData(item)); 113 | } 114 | 115 | return results; 116 | } 117 | 118 | /** 119 | * @param {string} query The string to search for on youtube 120 | */ 121 | async search(query, options = {}) { 122 | const webPage = await this._fetch(query, options.searchType, options.language); 123 | 124 | const parsedJson = this._getSearchData(webPage); 125 | 126 | const extracted = this._extractData(parsedJson); 127 | const parsed = this._parseData(extracted); 128 | 129 | return parsed; 130 | } 131 | 132 | /** 133 | * @param {string} [language='en'] 134 | */ 135 | setLang(language = 'en') { 136 | this._lang = language; 137 | } 138 | } 139 | module.exports = Scraper; 140 | -------------------------------------------------------------------------------- /src/Util.js: -------------------------------------------------------------------------------- 1 | "use strict" 2 | function assign(...args) { 3 | return Object.assign(...args); 4 | } 5 | 6 | const compress = (key) => { 7 | return (key && key['runs'] ? key['runs'].map((v) => v.text) : []).join(''); 8 | } 9 | 10 | const isChannelVerified = (vRender) => { 11 | const badges = (vRender['ownerBadges'] ? 12 | vRender['ownerBadges'].map((badge) => badge['metadataBadgeRenderer']['style']) : [] 13 | ); 14 | return badges.includes('BADGE_STYLE_TYPE_VERIFIED') || badges.includes('BADGE_STYLE_TYPE_VERIFIED_ARTIST'); 15 | } 16 | 17 | const getChannelData = (vRender) => { 18 | const channel = vRender.ownerText?.runs[0]; 19 | if (!channel) return { 20 | name: 'Unknown Channel', 21 | link: null, 22 | verified: false 23 | }; 24 | 25 | return { 26 | name: channel.text, 27 | link: 'https://www.youtube.com'+ channel['navigationEndpoint']['commandMetadata']['webCommandMetadata']['url'], 28 | verified: isChannelVerified(vRender) 29 | }; 30 | } 31 | 32 | const getChannelLink = (cRender) => { 33 | return 'https://www.youtube.com' + cRender.navigationEndpoint.browseEndpoint.canonicalBaseUrl; 34 | } 35 | 36 | const getChannelVideoCount = (cRender) => { 37 | if (!cRender.videoCountText?.runs) return 0; 38 | return +cRender.videoCountText.runs[0].text; 39 | } 40 | 41 | const getGeneralData = (renderer) => { 42 | return { 43 | channel: getChannelData(renderer), 44 | id: renderer.videoId, 45 | link: shareLink(renderer.videoId, false), 46 | thumbnail: idToThumbnail(renderer.videoId), 47 | title: compress(renderer.title), 48 | shareLink: shareLink(renderer.videoId) 49 | }; 50 | } 51 | 52 | const getPlaylistGeneralData = (pRender) => { 53 | const id = pRender.playlistId; 54 | 55 | return { 56 | channel: getChannelData(pRender), 57 | id, 58 | link: 'https://www.youtube.com/playlist?list=' + id, 59 | thumbnail: getPlaylistThumbnail(pRender), 60 | title: pRender.title.simpleText 61 | }; 62 | } 63 | 64 | const getPlaylistThumbnail = (pRender) => { 65 | return idToThumbnail(pRender.navigationEndpoint.watchEndpoint.videoId); 66 | } 67 | 68 | const getPlaylistVideoData = (renderer) => { 69 | return { 70 | duration: parseDuration(renderer), 71 | duration_raw: renderer.lengthText ? renderer.lengthText.simpleText : '00:00:00', 72 | id: renderer.videoId, 73 | link: shareLink(renderer.videoId, false), 74 | thumbnail: idToThumbnail(renderer.videoId), 75 | title: compress(renderer), 76 | shareLink: shareLink(renderer.videoId) 77 | }; 78 | } 79 | 80 | const getUploadDate = (vRender) => { 81 | return vRender['publishedTimeText'] ? vRender['publishedTimeText']['simpleText'] : ''; 82 | } 83 | 84 | const getViews = (vRender) => { 85 | if (!vRender.viewCountText?.simpleText) return 0; 86 | return +vRender.viewCountText.simpleText.replace(/[^0-9]/g, ''); 87 | } 88 | 89 | const getWatching = (vRender) => { 90 | if (!vRender.viewCountText?.runs) return 0; 91 | return +vRender.viewCountText.runs[0].text.replace(/[^0-9]/g, ''); 92 | } 93 | 94 | const idToThumbnail = function(id) { 95 | return 'https://i.ytimg.com/vi/'+ id +'/maxresdefault.jpg'; 96 | } 97 | 98 | const parseDuration = (vRender) => { 99 | if (!vRender.lengthText?.simpleText) return 0; 100 | 101 | const nums = vRender.lengthText.simpleText.split(':'); 102 | let time = nums.reduce((a, t) => (60 * a) + +t) * 1e3; 103 | 104 | return time; 105 | } 106 | 107 | const shareLink = (id, short = true) => { 108 | return short ? 'https://youtu.be/'+ id : 'https://www.youtube.com/watch?v='+ id; 109 | } 110 | 111 | exports.getChannelData = (item) => { 112 | const cRender = item.channelRenderer; 113 | const id = cRender.channelId; 114 | 115 | return { 116 | channelId: id, 117 | description: compress(cRender.descriptionSnippet), 118 | link: getChannelLink(cRender), 119 | thumbnails: cRender.thumbnail.thumbnails, 120 | subscribed: cRender.subscriptionButton.subscribed, 121 | uploadedVideos: getChannelVideoCount(cRender), 122 | verified: isChannelVerified(cRender) 123 | }; 124 | }; 125 | 126 | exports.getPlaylistData = (item) => { 127 | const pRender = item.playlistRenderer; 128 | const preview = []; 129 | 130 | pRender.videos.forEach(video => preview.push( 131 | getPlaylistVideoData(video.childVideoRenderer) 132 | )); 133 | 134 | return assign({ 135 | preview, 136 | videoCount: +pRender['videoCount'] 137 | }, getPlaylistGeneralData(pRender)); 138 | } 139 | 140 | exports.getStreamData = (item) => { 141 | const vRender = item.videoRenderer; 142 | 143 | return assign({ 144 | watching: getWatching(vRender) 145 | }, getGeneralData(vRender)) 146 | } 147 | 148 | exports.getVideoData = (item) => { 149 | const vRender = item.videoRenderer; 150 | 151 | return assign({ 152 | description: compress(vRender.descriptionSnippet), 153 | duration: parseDuration(vRender), 154 | duration_raw: vRender.lengthText ? vRender.lengthText.simpleText : '00:00:00', 155 | uploaded: getUploadDate(vRender), 156 | views: getViews(vRender) 157 | }, getGeneralData(vRender)); 158 | }; 159 | 160 | exports.isChannel = (item) => typeof item.channelRenderer !== 'undefined'; 161 | exports.isPlaylist = (item) => typeof item.playlistRenderer !== 'undefined'; 162 | exports.isStream = (item) => item.videoRenderer && !item.videoRenderer.lengthText; 163 | exports.isVideo = (item) => item.videoRenderer && item.videoRenderer.lengthText; 164 | -------------------------------------------------------------------------------- /test/Test.js: -------------------------------------------------------------------------------- 1 | "use strict" 2 | const Scraper = require('../index.js').default; 3 | 4 | const youtube = new Scraper(); 5 | 6 | exports.language = () => { 7 | return youtube.search('Never gonna give you up', { language: 'fr-FR' }); 8 | } 9 | 10 | exports.search = (q = 'Never gonna give you up') => { 11 | return youtube.search(q); 12 | }; 13 | 14 | exports.searchChannel = (q = 'NCS') => { 15 | return youtube.search(q, { 16 | searchType: 'channel' 17 | }); 18 | }; 19 | 20 | exports.searchLive = (q = 'NCS') => { 21 | return youtube.search(q, { 22 | searchType: 'live' 23 | }); 24 | } 25 | 26 | exports.searchPlaylist = (q = 'NCS') => { 27 | return youtube.search(q, { 28 | searchType: 'playlist' 29 | }); 30 | } -------------------------------------------------------------------------------- /test/index.js: -------------------------------------------------------------------------------- 1 | "use strict" 2 | const Test = require('./Test.js'); 3 | 4 | const time = () => process.hrtime.bigint(); 5 | const average = (arr) => { 6 | let acc = BigInt(0); 7 | arr.forEach(val => acc += val); 8 | 9 | return acc / BigInt(arr.length); 10 | }; 11 | const size = 20; 12 | 13 | const channelTimings = async () => { 14 | const timings = []; 15 | const queries = ['NCS', 'Rick Astley', 'Freddie Mercury', 'ilmango', 'DGR']; 16 | 17 | for (let i = 0; i < size; i++) { 18 | const t = time(); 19 | 20 | await Test.searchChannel(queries[Math.floor(Math.random() * queries.length)]); 21 | 22 | timings.push(time() - t); 23 | } 24 | 25 | return ['channel', average(timings)]; 26 | }; 27 | 28 | const playlistTimings = async () => { 29 | const timings = []; 30 | const queries = ['NCS', 'Gaming Music', 'GTA V Chaos Mod', 'Nature']; 31 | 32 | for (let i = 0; i < size; i++) { 33 | const t = time(); 34 | 35 | await Test.searchPlaylist(queries[Math.floor(Math.random() * queries.length)]); 36 | 37 | timings.push(time() - t); 38 | } 39 | 40 | return ['playlist', average(timings)]; 41 | }; 42 | 43 | const videoTimings = async () => { 44 | const timings = []; 45 | const queries = ['Flo Rida - Right Round', 'Never Gonna Wake You Up', 'Donut-shaped C code that generates a 3D spinning donut', 'LazyPurple Clips: a horribly unfortunate spy']; 46 | 47 | for (let i = 0; i < size; i++) { 48 | const t = time(); 49 | 50 | await Test.search(queries[Math.floor(Math.random() * queries.length)]); 51 | 52 | timings.push(time() - t); 53 | } 54 | 55 | return ['video', average(timings)]; 56 | }; 57 | 58 | const main = async () => { 59 | console.log('Running parallel Tests...\n'); 60 | 61 | const promises = []; 62 | 63 | promises.push(channelTimings()); 64 | promises.push(playlistTimings()); 65 | promises.push(videoTimings()); 66 | 67 | const results = await Promise.all(promises); 68 | 69 | console.log('Type'.padStart(10), ' | ', 'Avg Time (ms)'); 70 | results.forEach(([type, nanos]) => console.log(type.padStart(10), ' | ', Number(nanos) / 1e6)); 71 | 72 | console.log('\nFinished all tests, batch size: ', size); 73 | }; 74 | 75 | main(); 76 | --------------------------------------------------------------------------------