├── .deepsource.toml
├── .eslintrc.js
├── .github
├── FUNDING.yml
├── ISSUE_TEMPLATE
│ ├── bug-report.yml
│ ├── config.yml
│ ├── feature-request.yml
│ └── question.yml
├── dependabot.yml
└── workflows
│ ├── CI.yml
│ ├── codeql.yml
│ ├── coverage.yml
│ └── test-report.yml
├── .gitignore
├── .prettierrc.json
├── .yarnclean
├── CONTRIBUTING.md
├── LICENSE
├── README.md
├── codecov.yml
├── contrib-readme
└── CONTRIBUTING.ms.md
├── examples
├── homepage.ts
├── searchpage.ts
└── vidInfo.ts
├── jest.config.js
├── package.json
├── readme
└── readme.ms.md
├── reports
└── jest-junit.xml
├── src
├── extractors
│ ├── youtube-new
│ │ ├── README.md
│ │ ├── parser
│ │ │ ├── __tests__
│ │ │ │ ├── parserHelperTest.test.ts
│ │ │ │ └── parserTest.test.ts
│ │ │ ├── index.ts
│ │ │ ├── parser.const.ts
│ │ │ ├── parserHelpers.ts
│ │ │ ├── parsers.ts
│ │ │ ├── types.ts
│ │ │ └── types
│ │ │ │ ├── README.md
│ │ │ │ ├── aliases.ts
│ │ │ │ ├── appliedRule.ts
│ │ │ │ ├── common.ts
│ │ │ │ ├── condition.ts
│ │ │ │ ├── default.ts
│ │ │ │ ├── examples.ts
│ │ │ │ ├── flaten.ts
│ │ │ │ ├── index.ts
│ │ │ │ ├── jsonpath.ts
│ │ │ │ ├── objectProps.ts
│ │ │ │ ├── remap.ts
│ │ │ │ ├── required.ts
│ │ │ │ ├── strict.ts
│ │ │ │ └── utils.ts
│ │ ├── proto
│ │ │ ├── __tests__
│ │ │ │ ├── protoTest.test.ts
│ │ │ │ └── testCases.json
│ │ │ ├── index.ts
│ │ │ ├── types.ts
│ │ │ └── youtube.proto
│ │ ├── requester
│ │ │ ├── __tests__
│ │ │ │ ├── controllerTest.test.ts
│ │ │ │ └── initializerTest.test.ts
│ │ │ ├── controllers
│ │ │ │ ├── base.controller.model.ts
│ │ │ │ ├── browse.ts
│ │ │ │ └── homepage.controller.ts
│ │ │ ├── index.ts
│ │ │ └── initializer
│ │ │ │ ├── config.ts
│ │ │ │ ├── consts
│ │ │ │ └── context.const.ts
│ │ │ │ ├── context.ts
│ │ │ │ ├── index.ts
│ │ │ │ ├── initializer.ts
│ │ │ │ └── options.ts
│ │ ├── rules
│ │ │ ├── __tests__
│ │ │ │ └── ruleFactory.test.ts
│ │ │ ├── continuations.const.ts
│ │ │ └── index.ts
│ │ └── utils
│ │ │ ├── constants.ts
│ │ │ └── types.ts
│ └── youtube
│ │ ├── README.md
│ │ ├── controllers
│ │ ├── basicController.ts
│ │ ├── commentController.ts
│ │ ├── homePageController.ts
│ │ ├── searchController.ts
│ │ └── videoPageController.ts
│ │ ├── core
│ │ ├── initializer.ts
│ │ └── requester.ts
│ │ ├── index.ts
│ │ ├── parsers
│ │ ├── __tests__
│ │ │ └── parserTest.test.ts
│ │ ├── abstractParser.ts
│ │ ├── index.ts
│ │ ├── mixins
│ │ │ └── section.ts
│ │ ├── modelParsers
│ │ │ ├── CellDividerParser.ts
│ │ │ ├── ChannelRendererParser.ts
│ │ │ ├── Continuations.ts
│ │ │ ├── Thumbnail.ts
│ │ │ ├── VideoContextParser.ts
│ │ │ └── index.ts
│ │ ├── pageParser.ts
│ │ ├── stratIdentifiers.ts
│ │ └── strats
│ │ │ ├── homePage.ts
│ │ │ ├── index.ts
│ │ │ ├── searchPage.ts
│ │ │ ├── searchSuggestions.ts
│ │ │ └── ytVideo.ts
│ │ ├── proto
│ │ ├── __tests__
│ │ │ ├── protoTest.test.ts
│ │ │ └── testCases.json
│ │ ├── conversion.ts
│ │ ├── index.ts
│ │ ├── types.ts
│ │ └── youtube.proto
│ │ ├── types.ts
│ │ └── utils
│ │ ├── constants.ts
│ │ ├── errors.ts
│ │ ├── index.ts
│ │ └── youtubeHTTPOptions.ts
├── index.ts
├── types.ts
└── utils
│ ├── __tests__
│ └── helpers.test.ts
│ ├── errors.ts
│ ├── helpers.ts
│ └── index.ts
├── tests
├── cases.ts
└── main.test.ts
├── tsconfig.json
└── yarn.lock
/.deepsource.toml:
--------------------------------------------------------------------------------
1 | version = 1
2 |
3 | [[analyzers]]
4 | name = "javascript"
5 | enabled = true
--------------------------------------------------------------------------------
/.eslintrc.js:
--------------------------------------------------------------------------------
1 | module.exports = {
2 | root: true,
3 | parser: "@typescript-eslint/parser",
4 | plugins: ["@typescript-eslint"],
5 | extends: ["eslint:recommended", "plugin:@typescript-eslint/recommended"],
6 | rules: {
7 | "@typescript-eslint/no-explicit-any": "off",
8 | "no-unsafe-optional-chaining": "off",
9 | },
10 | };
11 |
--------------------------------------------------------------------------------
/.github/FUNDING.yml:
--------------------------------------------------------------------------------
1 | ko_fi: vuetube
--------------------------------------------------------------------------------
/.github/ISSUE_TEMPLATE/bug-report.yml:
--------------------------------------------------------------------------------
1 | name: 🐞 Issue Report
2 | description: Report a issue in VueTube Extractor
3 | labels: [bug]
4 | body:
5 |
6 | - type: textarea
7 | id: reproduce-steps
8 | attributes:
9 | label: Steps to reproduce
10 | description: Provide an example of the issue.
11 | placeholder: |
12 | Example:
13 | 1. First step
14 | 2. Second step
15 | 3. Issue here
16 | validations:
17 | required: true
18 |
19 | - type: textarea
20 | id: expected-behavior
21 | attributes:
22 | label: Expected behavior
23 | placeholder: |
24 | Example:
25 | "This should happen..."
26 | validations:
27 | required: true
28 |
29 | - type: textarea
30 | id: actual-behavior
31 | attributes:
32 | label: Actual behavior
33 | placeholder: |
34 | Example:
35 | "This happened instead..."
36 | validations:
37 | required: true
38 |
39 | - type: input
40 | id: extractor-version
41 | attributes:
42 | label: Extractor version
43 | description: |
44 | You can find the version within any error report or within package.json.
45 | placeholder: |
46 | Example: "0.1.0"
47 | validations:
48 | required: true
49 |
50 | - type: input
51 | id: node-version
52 | attributes:
53 | label: Node version
54 | description: |
55 | run node --version from your terminal to check your version.
56 | placeholder: |
57 | Example: "v16.14.0"
58 | validations:
59 | required: true
60 |
61 | - type: input
62 | id: locality
63 | attributes:
64 | label: Nation or locality
65 | description: |
66 | Where are you from? Some API responses may vary depending on your location. Optional.
67 | placeholder: |
68 | Example: "Australia"
69 | validations:
70 | required: false
71 |
72 | - type: textarea
73 | id: other-details
74 | attributes:
75 | label: Other details
76 | placeholder: |
77 | Additional details and attachments.
78 |
79 | - type: checkboxes
80 | id: acknowledgements
81 | attributes:
82 | label: Acknowledgements
83 | description: Your issue will be closed if you haven't done these steps.
84 | options:
85 | - label: I have searched the existing issues and this is a new ticket, **NOT** a duplicate or related to another open issue.
86 | required: true
87 | - label: I have written a short but informative title.
88 | required: true
89 | - label: I have updated to the latest version of the extractor and confirmed this issue still exists.
90 | required: true
91 | - label: I will fill out all of the requested information in this form.
92 | required: true
93 |
--------------------------------------------------------------------------------
/.github/ISSUE_TEMPLATE/config.yml:
--------------------------------------------------------------------------------
1 | blank_issues_enabled: false
2 | contact_links:
3 | - name: 💬 Discord
4 | url: https://vuetube.app/discord
5 | about: Join the Discord server to chat and ask questions
6 | - name: 💬 Telegram
7 | url: https://t.me/vuetube
8 | about: Join the Telegram group to chat and ask questions
9 | - name: 💬 VueTube website
10 | url: https://vuetube.app/
11 | about: For example, to check FAQ before asking frecuently asked questions (wait... then they won't be frequently asked questions?)
12 |
--------------------------------------------------------------------------------
/.github/ISSUE_TEMPLATE/feature-request.yml:
--------------------------------------------------------------------------------
1 | name: ⭐ Feature request
2 | description: Suggest a feature to improve the extractor
3 | labels: [feature request]
4 | body:
5 |
6 | - type: textarea
7 | id: feature-description
8 | attributes:
9 | label: Describe your suggested feature
10 | description: How can an existing source be improved?
11 | placeholder: |
12 | Example:
13 | "It should work like this..."
14 | validations:
15 | required: true
16 |
17 | - type: textarea
18 | id: other-details
19 | attributes:
20 | label: Other details
21 | placeholder: |
22 | Additional details and attachments.
23 |
24 | - type: checkboxes
25 | id: acknowledgements
26 | attributes:
27 | label: Acknowledgements
28 | description: Your issue will be closed if you haven't done these steps.
29 | options:
30 | - label: I have searched the existing issues and this is a new ticket, **NOT** a duplicate or related to another open issue.
31 | required: true
32 | - label: I have written a short but informative title.
33 | required: true
34 | - label: I will fill out all of the requested information in this form.
35 | required: true
36 |
--------------------------------------------------------------------------------
/.github/ISSUE_TEMPLATE/question.yml:
--------------------------------------------------------------------------------
1 | name: ❓ Question
2 | description: Ask a question related to the extractor
3 | labels: [question]
4 | body:
5 |
6 | - type: textarea
7 | id: question
8 | attributes:
9 | label: Ask your question
10 | description: What do you want to know?
11 | placeholder: |
12 | Example:
13 | "How do I search through videos in a channel?"
14 | validations:
15 | required: true
16 |
17 | - type: textarea
18 | id: aditional-info
19 | attributes:
20 | label: Aditional information
21 | placeholder: |
22 | Additional useful information, for example, a screenshot.
23 |
24 | - type: checkboxes
25 | id: acknowledgements
26 | attributes:
27 | label: Acknowledgements
28 | description: Your question will be closed if you haven't done these steps.
29 | options:
30 | - label: I have searched the existing issues and this is a new question, **NOT** a duplicate or related to another open issue.
31 | required: true
32 | - label: I have written a short but informative title.
33 | required: true
34 | - label: I will fill out all of the requested information in this form.
35 | required: true
36 |
--------------------------------------------------------------------------------
/.github/dependabot.yml:
--------------------------------------------------------------------------------
1 | # To get started with Dependabot version updates, you'll need to specify which
2 | # package ecosystems to update and where the package manifests are located.
3 | # Please see the documentation for all configuration options:
4 | # https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
5 |
6 | version: 2
7 | updates:
8 | - package-ecosystem: "npm"
9 | directory: "/"
10 | schedule:
11 | interval: "weekly"
12 | - package-ecosystem: "github-actions"
13 | directory: "/"
14 | schedule:
15 | interval: "weekly"
16 |
--------------------------------------------------------------------------------
/.github/workflows/CI.yml:
--------------------------------------------------------------------------------
1 | # This is a basic workflow to help you get started with Actions
2 |
3 | name: CI
4 | # Controls when the workflow will run
5 | on:
6 | # Triggers the workflow on push or pull request events but only for the master branch
7 | push:
8 | pull_request:
9 |
10 | # Allows you to run this workflow manually from the Actions tab
11 | workflow_dispatch:
12 |
13 | jobs:
14 | build:
15 | runs-on: ubuntu-latest
16 |
17 | strategy:
18 | matrix:
19 | node-version: [16.x]
20 |
21 | steps:
22 | - name: Checkout repository
23 | uses: actions/checkout@v3 # checkout the repo
24 |
25 | - name: Set up Node.js ${{ matrix.node-version }}
26 | uses: actions/setup-node@v3
27 | with:
28 | node-version: ${{ matrix.node-version }}
29 |
30 | - name: Install dependencies
31 | run: yarn # install packages
32 |
33 | - name: Run the tests
34 | run: yarn ci # run tests (configured to use jest-junit reporter)
35 |
36 | - name: Upload results
37 | uses: actions/upload-artifact@v3 # upload test results
38 | if: success() || failure() # run this step even if previous step failed
39 | with:
40 | name: test-results
41 | path: reports/jest-junit.xml
42 |
--------------------------------------------------------------------------------
/.github/workflows/codeql.yml:
--------------------------------------------------------------------------------
1 | name: "CodeQL"
2 |
3 | on:
4 | push:
5 | branches: [ "master" ]
6 | pull_request:
7 | branches: [ "master" ]
8 | schedule:
9 | - cron: "14 23 * * 4"
10 |
11 | jobs:
12 | analyze:
13 | name: Analyze
14 | runs-on: ubuntu-latest
15 | permissions:
16 | actions: read
17 | contents: read
18 | security-events: write
19 |
20 | strategy:
21 | fail-fast: false
22 | matrix:
23 | language: [ javascript ]
24 |
25 | steps:
26 | - name: Checkout
27 | uses: actions/checkout@v3
28 |
29 | - name: Initialize CodeQL
30 | uses: github/codeql-action/init@v2
31 | with:
32 | languages: ${{ matrix.language }}
33 | queries: +security-and-quality
34 |
35 | - name: Autobuild
36 | uses: github/codeql-action/autobuild@v2
37 |
38 | - name: Perform CodeQL Analysis
39 | uses: github/codeql-action/analyze@v2
40 | with:
41 | category: "/language:${{ matrix.language }}"
42 |
--------------------------------------------------------------------------------
/.github/workflows/coverage.yml:
--------------------------------------------------------------------------------
1 | # This is a basic workflow to help you get started with Actions
2 |
3 | name: "Coverage"
4 | # Controls when the workflow will run
5 | on:
6 | # Triggers the workflow on pull request or push events but only for the master branch
7 | pull_request:
8 | branches: [master]
9 | push:
10 | branches: [master]
11 |
12 | # Allows you to run this workflow manually from the Actions tab
13 | workflow_dispatch:
14 |
15 | jobs:
16 | build:
17 | runs-on: ubuntu-latest
18 |
19 | strategy:
20 | matrix:
21 | node-version: [16.x]
22 |
23 | steps:
24 | - name: Checkout repository
25 | uses: actions/checkout@v3
26 |
27 | - name: Set up Node.js ${{ matrix.node-version }}
28 | uses: actions/setup-node@v3
29 | with:
30 | node-version: ${{ matrix.node-version }}
31 |
32 | - name: Install dependencies
33 | run: yarn
34 |
35 | - name: Run the tests
36 | run: yarn test --coverage
37 |
38 | - name: Upload coverage to Codecov
39 | uses: codecov/codecov-action@v3
40 |
--------------------------------------------------------------------------------
/.github/workflows/test-report.yml:
--------------------------------------------------------------------------------
1 | name: "Test Report"
2 | on:
3 | workflow_run:
4 | workflows: ["CI"] # runs after CI workflow
5 | types:
6 | - completed
7 | jobs:
8 | report:
9 | permissions:
10 | id-token: write
11 | contents: read
12 | checks: write
13 | runs-on: ubuntu-latest
14 | steps:
15 | - uses: dorny/test-reporter@v1
16 | with:
17 | artifact: test-results # artifact name
18 | name: JEST Tests # Name of the check run which will be created
19 | path: "*.xml" # Path to test results (inside artifact .zip)
20 | reporter: jest-junit # Format of test results
21 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | /node_modules/
2 | /build/
3 | /lib/
4 | /dist/
5 | /docs/
6 | .idea/*
7 | /.vscode/
8 |
9 | .DS_Store
10 | coverage
11 | *.log
--------------------------------------------------------------------------------
/.prettierrc.json:
--------------------------------------------------------------------------------
1 | {
2 | "singleQuote": true,
3 | "arrowParens": "avoid",
4 | "endOfLine": "auto",
5 | "overrides": [
6 | {
7 | "files": "*.ts",
8 | "options": {
9 | "printWidth": 140
10 | }
11 | }
12 | ]
13 | }
14 |
--------------------------------------------------------------------------------
/.yarnclean:
--------------------------------------------------------------------------------
1 | # test directories
2 | __tests__
3 | test
4 | tests
5 | powered-test
6 |
7 | # asset directories
8 | docs
9 | doc
10 | website
11 | images
12 | assets
13 | !istanbul-reports/lib/html/assets
14 |
15 | # examples
16 | example
17 | examples
18 |
19 | # code coverage directories
20 | coverage
21 | .nyc_output
22 |
23 | # build scripts
24 | Makefile
25 | Gulpfile.js
26 | Gruntfile.js
27 |
28 | # configs
29 | appveyor.yml
30 | circle.yml
31 | codeship-services.yml
32 | codeship-steps.yml
33 | wercker.yml
34 | .tern-project
35 | .gitattributes
36 | .editorconfig
37 | .*ignore
38 | .eslintrc
39 | .jshintrc
40 | .flowconfig
41 | .documentup.json
42 | .yarn-metadata.json
43 | .travis.yml
44 |
45 | # misc
46 | *.md
47 |
--------------------------------------------------------------------------------
/CONTRIBUTING.md:
--------------------------------------------------------------------------------
1 |
2 |
3 |
5 |
6 |
7 |
8 | # Contributing to VueTube Extractor
9 | Please refer to our website for contribution best practises as well as how to get started on your first contribution to VueTube
10 | * [Contributing Guidelines](https://vuetube.app/contributing/)
11 | * [Contributing to the Extractor](https://vuetube.app/contributing/extractor.html)
12 |
13 | Before you make a pull request, please ensure that all tests pass, and that your code is linted.
14 |
15 | Please state clearly what you are trying to do in the pull request title, and describe more in the body. It is also
16 | recommended that you add a test for your pull request if you are adding new features.
17 |
18 | Thank you for your contribution!
19 |
20 | ### Connect with us!
21 |
2 |
3 |
4 |
14 |
15 |
17 | Show Readme credits
18 |
19 | VueTube Logo by @afnzmn
20 | English Readme contributors: @404-Program-not-found
21 | , @AdamIskandarAI
22 |
23 |
24 |
25 | VueTube extractor is a library designed to extract data from streaming services. Designed for use in VueTube.
26 | here.
28 |
29 |
30 |
31 |
32 |
34 |
35 |
37 |
38 |
40 |
41 |
43 |
44 |
46 |
47 |
49 |
51 |
52 | **Read this page in other languages**: [English,](readme.md) [Bahasa Melayu](/readme/readme.ms.md)
53 |
54 | VueTube Extractor is currently under active development, and is not a part of the main VueTube app. It is planned to
55 | completely replace the current API wrapper employed by VueTube upon completion.
56 |
57 | ## Getting Started & Contributing
58 |
59 | See [the wiki](https://github.com/VueTubeApp/VueTube-Extractor/wiki) as well as [CONTRIBUTING.md](CONTRIBUTING.md) for more information.
60 |
61 | ## Contributors
62 |
63 |
64 |
66 |
67 | Made with [contrib.rocks](https://contrib.rocks).
68 |
69 | ## Disclaimer
70 |
71 | The VueTube project and its contents are not affiliated with, funded, authorized, endorsed by, or in any way associated
72 | with YouTube, Google LLC or any of its affiliates and subsidiaries. The official YouTube website can be found
73 | at [www.youtube.com](https://www.youtube.com).
74 |
75 | Any trademark, service mark, trade name, or other intellectual property rights used in the VueTube project are owned by
76 | the respective owners.
77 |
--------------------------------------------------------------------------------
/codecov.yml:
--------------------------------------------------------------------------------
1 | coverage:
2 | status:
3 | project:
4 | default:
5 | target: 85%
6 | threshold: 5%
7 |
--------------------------------------------------------------------------------
/contrib-readme/CONTRIBUTING.ms.md:
--------------------------------------------------------------------------------
1 |
2 |
3 |
5 |
6 |
7 |
8 | # Menyumbang kepada VueTube Extractor
9 | Sila rujuk laman sesawang kami untuk amalan terbaik semasa menyumbang ke projek ini serta bagaimana untuk memulakan sumbangan pertama ke VueTube
10 | * [Garis Panduan Sumbangan](https://vuetube.app/contributing/)
11 | * [Menyumbang ke Extractor](https://vuetube.app/contributing/extractor.html)
12 |
13 | Sebelum anda membuat *pull request*, sila pastikan bahawa semua ujian lulus, dan kod anda sudah di *lint* kan.
14 |
15 | Sila nyatakan dengan jelas apa yang anda cuba lakukan dalam tajuk *pull request*, dan terangkan lebih lagi dalam *body*. Ia juga digalakkan untuk anda menambah ujian untuk *pull request* anda jika anda menambah ciri baru.
16 |
17 | Terima kasih untuk sumbangan anda!
18 |
19 | ### Berhubung dengan kami!
20 | "],
3 | testMatch: [
4 | "**/__tests__/**/*.+(ts|tsx|js)",
5 | "**/?(*.)+(spec|test).+(ts|tsx|js)",
6 | "!**/youtube/**",
7 | ],
8 | transform: {
9 | "^.+\\.(ts|tsx)$": "ts-jest",
10 | },
11 | moduleFileExtensions: ["ts", "tsx", "js"],
12 | moduleNameMapper: {
13 | "@utils": "/src/utils",
14 | "@types": "/src/types",
15 | "@package": "/package.json",
16 | },
17 | reporters: ["default", "jest-junit"],
18 | collectCoverageFrom: ["./src/**"],
19 | coveragePathIgnorePatterns: [
20 | "node_modules",
21 | "test-config",
22 | "interfaces",
23 | "jestGlobalMocks.ts",
24 | ".module.ts",
25 | ".mock.ts",
26 | ".json",
27 | "__tests__",
28 | "./src/extractors/youtube/"
29 | ],
30 | };
31 |
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "vuetube-extractor",
3 | "version": "0.1.0",
4 | "description": "Core API wrapper for VueTube",
5 | "main": "dict/index.js",
6 | "types": "dist/index.d.ts",
7 | "author": "404-Program-not-found (Alex)",
8 | "license": "GPL-3.0",
9 | "private": true,
10 | "devDependencies": {
11 | "@types/jest": "^27.5.0",
12 | "@types/node": "^18.15.11",
13 | "@types/user-agents": "^1.0.2",
14 | "@typescript-eslint/eslint-plugin": "^5.59.5",
15 | "@typescript-eslint/parser": "^5.59.6",
16 | "copyfiles": "^2.4.1",
17 | "eslint": "^8.39.0",
18 | "isomorphic-fetch": "^3.0.0",
19 | "jest": "^28.1.0",
20 | "jest-junit": "^16.0.0",
21 | "ts-jest": "^28.0.1",
22 | "ts-node": "^10.9.1",
23 | "tsconfig-paths": "^4.2.0",
24 | "typescript": "^5.0.4"
25 | },
26 | "jest-junit": {
27 | "outputDirectory": "reports",
28 | "outputName": "jest-junit.xml",
29 | "ancestorSeparator": " › ",
30 | "uniqueOutputName": "false",
31 | "suiteNameTemplate": "{filepath}",
32 | "classNameTemplate": "{classname}",
33 | "titleTemplate": "{title}"
34 | },
35 | "scripts": {
36 | "test": "jest",
37 | "ci": "jest --ci --reporters=default --reporters=jest-junit",
38 | "test-build": "npm run build && npm run test",
39 | "lint": "eslint src/ --ext .js,.jsx,.ts,.tsx",
40 | "build": "npm run clean && tsc -p tsconfig.json && npm run copy-files",
41 | "clean": "rm -rf dist build",
42 | "copy-files": "copyfiles --error --up 1 src/youtube/proto/youtube.proto dist"
43 | },
44 | "dependencies": {
45 | "@vuetubeapp/http": "^1.4.2",
46 | "prettier": "^2.8.8",
47 | "protobufjs": "^7.2.3",
48 | "user-agents": "^1.0.1381"
49 | }
50 | }
51 |
--------------------------------------------------------------------------------
/readme/readme.ms.md:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
14 |
15 |
17 | Tunjukkan kredit Readme
18 |
19 | Logo VueTube oleh @afnzmn
20 | Penyumbang-penyumbang Readme Bahasa Melayu: @AdamIskandarAI
21 |
22 |
23 |
24 | VueTube extractor adalah sebuah pustaka (library) yang direka untuk mengekstrak daripada perkhidmatan penstriman (misalnya YouTube). Direka untuk digunakan dalam VueTube.
25 | di sini.
27 |
28 |
29 |
30 |
31 |
33 |
34 |
36 |
37 |
39 |
40 |
42 |
43 |
45 |
46 |
48 |
50 |
51 | **Baca halaman ini dalam bahasa lain**: [English,](readme.md) [Bahasa Melayu](/readme/readme.ms.md)
52 |
53 | VueTube Extractor sedang aktif dibangunkan, dan ia bukan sebahagian daripada apl VueTube. Ia dirancang untuk
54 | menggantikan pembalut API (API wrapper) sedia ada yang digunakan oleh VueTube setelah disiapkan.
55 |
56 | ## Bermula & Menyumbang
57 |
58 | Lihat [halaman wiki](https://github.com/VueTubeApp/VueTube-Extractor/wiki) serta [CONTRIBUTING.md](CONTRIBUTING.md) untuk maklumat lanjut.
59 |
60 | ## Penyumbang-penyumbang
61 |
62 |
63 |
65 |
66 | Dibuat dengan [contrib.rocks](https://contrib.rocks).
67 |
68 | ## Penafian
69 |
70 | Projek VueTube dan kandungannya tidak bergabung dengan, dibiayai, diberi izin, disokong oleh, atau dalam apa jua cara dikaitkan dengan YouTube, Google LLC atau mana-mana sekutu dan anak syarikatnya. Laman web rasmi YouTube boleh didapati di [www.youtube.com](https://www.youtube.com).
71 |
72 | Sebarang tanda dagangan, tanda perkhidmatan, nama dagangan atau hak harta intelek lain yang digunakan dalam projek VueTube dimiliki oleh pemilik masing-masing.
73 |
--------------------------------------------------------------------------------
/reports/jest-junit.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
--------------------------------------------------------------------------------
/src/extractors/youtube-new/parser/__tests__/parserHelperTest.test.ts:
--------------------------------------------------------------------------------
1 | import {arrayRule, objectRule} from "../types";
2 | import {ArrayRuleHelper, ObjectRuleHelper} from "../parserHelpers";
3 |
4 | describe("Helper Tests", () => {
5 | describe("ObjectRuleHelper Tests", () => {
6 | test("if fillRule correctly defaults a given rule", () => {
7 | // @ts-expect-error no name
8 | const rule: objectRule = {
9 | type: "object",
10 | properties: {
11 | test: {
12 | type: "string",
13 | },
14 | },
15 | };
16 | const Helper = new ObjectRuleHelper(rule);
17 | expect(Helper.fillRule()).toEqual({
18 | type: "object",
19 | flatten: false,
20 | properties: {
21 | test: {
22 | type: "string",
23 | required: true,
24 | },
25 | },
26 | strict: true,
27 | keymap: {},
28 | condition: expect.any(Function),
29 | });
30 | })
31 | test("if flattenObject correctly flattens a given object", () => {
32 | // @ts-expect-error no name
33 | const Helper = new ObjectRuleHelper({
34 | type: "object",
35 | properties: {
36 | test: {
37 | type: "string",
38 | },
39 | },
40 | });
41 | expect(Helper.flattenConvertObject({
42 | test: {
43 | test: "test",
44 | },
45 | })).toEqual({
46 | "test-test": "test",
47 | });
48 | });
49 | test("if flattenObject correctly flattens a given object with a keymap", () => {
50 | // @ts-expect-error no name
51 | const Helper = new ObjectRuleHelper({
52 | type: "object",
53 | properties: {
54 | test: {
55 | type: "string",
56 | },
57 | },
58 | keymap: {
59 | "test-test": "test2",
60 | },
61 | });
62 | expect(Helper.flattenConvertObject({
63 | test: {
64 | test: "test",
65 | },
66 | })).toEqual({
67 | "test2": "test",
68 | });
69 | });
70 | test("if jsonPathToObject functions as expected", () => {
71 | // @ts-expect-error no name
72 | const Helper = new ObjectRuleHelper({
73 | type: "object",
74 | properties: {
75 | test: {
76 | type: "string",
77 | }
78 | }
79 | })
80 | const jsonPath = "test.test[0]"
81 | const invalidJsonPath = "test.test[0].test"
82 | const testObject = {
83 | test: {
84 | test: ["test"]
85 | }
86 | }
87 | expect(Helper.jsonPathToObject(jsonPath, testObject)).toEqual("test")
88 | expect(Helper.jsonPathToObject(invalidJsonPath, testObject)).toEqual(undefined)
89 | });
90 | });
91 | describe("ArrayRuleHelper Tests", () => {
92 | test("if fillRule correctly defaults a given rule", () => {
93 | const rule: arrayRule = {
94 | type: "array",
95 | // @ts-expect-error no name
96 | items: {
97 | type: "object",
98 | properties: {
99 | test: {
100 | type: "string",
101 | }
102 | }
103 | },
104 | };
105 | const Helper = new ArrayRuleHelper(rule);
106 | expect(Helper.fillRule()).toEqual({
107 | type: "array",
108 | limit: 0,
109 | items: {
110 | type: "object",
111 | properties: {
112 | test: {
113 | type: "string",
114 | }
115 | }
116 | },
117 | strict: true,
118 | condition: expect.any(Function),
119 | });
120 | });
121 | });
122 | describe("General Helper Tests", () => {
123 | let Helper: ObjectRuleHelper;
124 | beforeAll(() => {
125 | // @ts-expect-error no name
126 | const rule: objectRule = {
127 | type: "object",
128 | properties: {
129 | test: {
130 | type: "string",
131 | },
132 | },
133 | keymap: {
134 | test: "test2",
135 | }
136 | };
137 | Helper = new ObjectRuleHelper(rule);
138 | });
139 | test("if checkTypeGuard correctly checks a type guard", () => {
140 | expect(Helper.checkTypeGuard("test", "string")).toBe(true);
141 | expect(Helper.checkTypeGuard(1, "string")).toBe(false);
142 | expect(Helper.checkTypeGuard(1, "any")).toBe(true);
143 | expect(Helper.checkTypeGuard([], "object")).toBe(false);
144 | expect(Helper.checkTypeGuard([], "array")).toBe(true);
145 | });
146 | test("if followKeymap correctly follows a keymap", () => {
147 | expect(Helper.followKeymap("test")).toBe("test2");
148 | expect(Helper.followKeymap("test2")).toBe("test2");
149 | })
150 | });
151 | });
--------------------------------------------------------------------------------
/src/extractors/youtube-new/parser/index.ts:
--------------------------------------------------------------------------------
1 | export {parseRule} from "./parsers";
2 |
3 |
4 |
--------------------------------------------------------------------------------
/src/extractors/youtube-new/parser/parser.const.ts:
--------------------------------------------------------------------------------
1 | export const PRIMITIVES = {
2 | STRING: 'string',
3 | NUMBER: 'number',
4 | BOOLEAN: 'boolean',
5 | } as const;
6 |
7 | export const SUPPORTED_TYPES = {
8 | OBJECT: 'object',
9 | ARRAY: 'array',
10 | RULE: 'rule',
11 | } as const;
12 |
13 | export const PROPERTY_TYPES = {
14 | ...SUPPORTED_TYPES,
15 | ...PRIMITIVES,
16 | ANY: 'any',
17 | } as const;
18 |
--------------------------------------------------------------------------------
/src/extractors/youtube-new/parser/parserHelpers.ts:
--------------------------------------------------------------------------------
1 | import {arrayRule, conditionalFunction, conditionalRule, objectRule, Rule} from "./types";
2 | import {ErrorMessages, utilityErrors} from "@utils";
3 |
4 | /**
5 | * A helper class for parsers. Abstract class, do not instantiate.
6 | * @abstract
7 | * @param {rule} rule - The rule to parse
8 | */
9 | abstract class ParserHelper {
10 |
11 | protected rule: Rule;
12 |
13 | protected constructor(rule: Rule) {
14 | this.rule = rule;
15 | }
16 |
17 | /**
18 | * Checks if a given value matches a type guard
19 | * @param {any} toCheck - The object to check
20 | * @param {string} typeGuard - The type guard to check against
21 | * @returns {boolean} Whether the object matches the type guard
22 | */
23 | public checkTypeGuard(toCheck: any, typeGuard: string): boolean {
24 | // Check if type guard is a supported typeof type
25 | if (typeGuard == "any") {
26 | return true;
27 | }
28 | if (['string', 'number', 'boolean', 'object', 'array'].includes(typeGuard)) {
29 | if (Array.isArray(toCheck)) {
30 | return typeGuard == 'array';
31 | } else {
32 | return typeof toCheck == typeGuard;
33 | }
34 | } else {
35 | const errorMessage = ErrorMessages.invalidTypeGuard(typeGuard);
36 | throw new utilityErrors.VueTubeExtractorError(errorMessage);
37 | }
38 | }
39 |
40 | /**
41 | * Fill a rule with default values
42 | * @abstract
43 | * @returns {Rule} The filled rule
44 | */
45 | public abstract fillRule(): Rule;
46 |
47 | /**
48 | * Evaluates a given condition
49 | * @param {any} toCheck - The object to check
50 | * @param {conditionalFunction | { [key: string]: conditionalRule }} condition - The condition to check against
51 | * @returns {boolean} Whether the condition is true
52 | */
53 | public evaluateCondition(toCheck: any, condition: conditionalFunction | { [key: string]: conditionalRule }): boolean {
54 | if (typeof condition == 'function') {
55 | return condition(toCheck);
56 | } else {
57 | return false // TODO: Implement conditional rules
58 | }
59 | }
60 |
61 | /**
62 | * Function to check if a rule is strictly required. If true throws a given error, if false returns the default value
63 | * @param {boolean} isStrictlyRequired - Whether the rule is strictly required
64 | * @param {Error} error - The error to throw if the rule is strictly required
65 | * @param {any} defaultValue - The default value to return if the rule is not strictly required
66 | * @returns {any} The default value if the rule is not strictly required, otherwise throws an error
67 | */
68 | public checkStrictlyRequired(isStrictlyRequired: boolean, error: Error, defaultValue: any): any {
69 | if (isStrictlyRequired) {
70 | throw error;
71 | } else {
72 | return defaultValue;
73 | }
74 | }
75 |
76 | /**
77 | * Returns a function wrapped in a try-catch block. Used for sub-rules
78 | * @returns The result of the function
79 | * @param {function} runCode - The function to run
80 | * @param {string} key - The key to use for the error message.
81 | * @param thisArg - Pass the context to the function
82 | * @param args - The arguments to pass to the function
83 | */
84 | public wrapFunction(runCode: (...args: T) => R, key: string, thisArg: any, ...args: T): R | undefined {
85 | try {
86 | return runCode.bind(thisArg)(...args);
87 | } catch (error) {
88 | let message = `error from sub-rule`;
89 | message = ErrorMessages.appendAdditionalInfoIfPresent(message, key);
90 | if (error instanceof Error) message += `: ${error.message}`;
91 | throw new utilityErrors.VueTubeExtractorError(message);
92 | }
93 | }
94 | }
95 |
96 |
97 | /**
98 | * A helper class for object rules. Array rules should refer to `ArrayRuleHelper` instead
99 | * @augments ParserHelper
100 | * @param {objectRule} rule - The rule to parse
101 | */
102 | export class ObjectRuleHelper extends ParserHelper {
103 |
104 | protected rule: objectRule;
105 |
106 | constructor(rule: objectRule) {
107 | super(rule);
108 | }
109 |
110 | /**
111 | * Follows a keymap and returns the value of the key
112 | * @param {string} key - The key to convert
113 | * @returns {string} The converted key
114 | */
115 | public followKeymap(key: string): string {
116 | if (this.rule.keymap && key in this.rule.keymap) {
117 | return this.rule.keymap[key];
118 | }
119 | return key;
120 | }
121 |
122 |
123 | /**
124 | * Fills a rule with default values if they are not present
125 | * @returns {objectRule} The filled rule
126 | */
127 | public fillRule(): objectRule {
128 | this.rule.keymap ??= {};
129 | this.rule.strict ??= true;
130 | this.rule.condition ??= () => true;
131 | this.rule.flatten ??= false;
132 | for (const KEY of Object.keys(this.rule.properties)) {
133 | this.rule.properties[KEY].required ??= true;
134 | }
135 | return this.rule;
136 | }
137 |
138 | /**
139 | * Flattens & converts a given object via keymap
140 | * @param {[key: string]: any} obj - The object to convert
141 | * @returns {[key: string]: any} The converted object
142 | */
143 | public flattenConvertObject(obj: { [key: string]: any }): { [key: string]: any } {
144 | const flattened = this.flattenObject(obj);
145 | const converted: { [key: string]: any } = {};
146 | for (const key of Object.keys(flattened)) {
147 | converted[this.followKeymap(key)] = flattened[key];
148 | }
149 | return converted;
150 | }
151 |
152 | /**
153 | * Flattens a given object
154 | * @param obj - The object to flatten
155 | * @param path - Optional. The path to the object. For naming conflicts
156 | */
157 | private flattenObject(obj: { [key: string]: any }, path = ''): { [key: string]: any } {
158 | const result: { [key: string]: any } = {};
159 | for (const key of Object.keys(obj)) {
160 | const value = obj[key];
161 | const newPath = path ? `${path}-${key}` : key;
162 | if (typeof value === 'object' && value !== null) {
163 | Object.assign(result, this.flattenObject(value, newPath));
164 | } else {
165 | result[newPath] = value;
166 | }
167 | }
168 | return result;
169 | }
170 |
171 | /**
172 | * JSONPath to object. Returns undefined if the path does not exist
173 | * @param {string} path - The path to the object
174 | * @param {any} obj - The object to search
175 | * @returns {any|undefined} The object at the given path
176 | */
177 | public jsonPathToObject(path: string, obj: any): any | undefined {
178 | const pathArray = path.split('.');
179 | let current = obj;
180 | for (const key of pathArray) {
181 | if (current === undefined) {
182 | return undefined;
183 | } else if (key.includes('[')) {
184 | const index = parseInt(key.split('[')[1].split(']')[0]);
185 | current = current[key.split('[')[0]][index];
186 | } else {
187 | current = current[key];
188 | }
189 | }
190 | return current;
191 | }
192 | }
193 |
194 | /**
195 | * A helper class for array rules. Object rules should refer to `ObjectRuleHelper` instead
196 | * @constructor
197 | * @augments ParserHelper
198 | * @param {arrayRule} rule - The rule to parse
199 | */
200 | export class ArrayRuleHelper extends ParserHelper {
201 |
202 | protected rule: arrayRule;
203 |
204 | constructor(rule: arrayRule) {
205 | super(rule);
206 | }
207 |
208 | /**
209 | * Fills a rule with default values if they are not present
210 | * @returns {arrayRule} The filled rule
211 | */
212 | public fillRule(): arrayRule {
213 | const DEFAULTED_RULE: Rule = this.rule;
214 | DEFAULTED_RULE.strict ??= true;
215 | DEFAULTED_RULE.condition ??= () => true;
216 | DEFAULTED_RULE.limit ??= 0;
217 | return DEFAULTED_RULE;
218 | }
219 | }
--------------------------------------------------------------------------------
/src/extractors/youtube-new/parser/parsers.ts:
--------------------------------------------------------------------------------
1 | import { arrayRule, conditionalFunction, conditionalRule, groupedRule, objectRule, propertyRule, Rule } from './types';
2 | import { ErrorMessages, utilityErrors } from '@utils';
3 | import { ArrayRuleHelper, ObjectRuleHelper } from './parserHelpers';
4 | import RuleFactory from '../rules';
5 | export { ObjectRuleHelper, ArrayRuleHelper } from './parserHelpers';
6 |
7 | interface GenericRuleParser {
8 | parse(): any;
9 | }
10 |
11 | /**
12 | * Class to parse object rules
13 | * @param {object} toParse - The object to parse according to the given rule
14 | * @param {objectRule} rule - The rule to parse the object with
15 | */
16 | export class ObjectRuleParser implements GenericRuleParser {
17 | private readonly TO_PARSE: { [key: string]: any };
18 | private KEYMAP: { [key: string]: string };
19 | private readonly isStrict: boolean;
20 | private readonly condition: conditionalFunction | { [key: string]: conditionalRule };
21 | private readonly RULE_NAME: string;
22 | private readonly RULE_TYPE: string;
23 | private readonly flatten: boolean;
24 | private readonly PROPERTIES: { [key: string]: propertyRule };
25 | private readonly PROCESSED_OBJECT: { [key: string]: any };
26 | private Helper: ObjectRuleHelper;
27 |
28 | constructor(toParse: { [key: string]: any }, rule: objectRule) {
29 | this.TO_PARSE = toParse;
30 | this.RULE_TYPE = rule.type;
31 | this.RULE_NAME = rule.name || 'objectRule';
32 | this.PROPERTIES = rule.properties;
33 | this.guardClauses();
34 | this.Helper = new ObjectRuleHelper(rule);
35 | const filledRule = new RuleFactory().getSubRules(this.Helper.fillRule(), this.TO_PARSE);
36 | this.KEYMAP = filledRule.keymap as { [key: string]: string };
37 | this.isStrict = filledRule.strict as boolean;
38 | this.condition = filledRule.condition as conditionalFunction | { [key: string]: conditionalRule };
39 | this.PROPERTIES = filledRule.properties;
40 | this.flatten = filledRule.flatten as boolean;
41 | this.PROCESSED_OBJECT = {};
42 | }
43 |
44 | /**
45 | * Method to parse a basic property. Simply returns the value of the given key in TO_PARSE
46 | * @param {string} key - The key to parse
47 | * @param {propertyRule} rule - The rule to parse the key with
48 | * @param {boolean} isStrictlyRequired - Whether the key is required and the rule is strict
49 | * @returns {any | undefined} - The value of the key in TO_PARSE. If the key is not present and the rule is strict, returns undefined
50 | */
51 | private parseBasicProperty(key: string, rule: propertyRule, isStrictlyRequired: boolean): any | undefined {
52 | if (!(key in this.TO_PARSE)) {
53 | const errorMessage = ErrorMessages.missingRequired('key', key, 'applyObjectRule');
54 | return this.Helper.checkStrictlyRequired(isStrictlyRequired, new utilityErrors.VueTubeExtractorError(errorMessage), rule.default);
55 | }
56 | const result = this.Helper.jsonPathToObject(key, this.TO_PARSE);
57 | if (!this.Helper.checkTypeGuard(result, rule.type)) {
58 | const errorMessage = ErrorMessages.typeGuardMismatch(rule.type, typeof result, key);
59 | return this.Helper.checkStrictlyRequired(isStrictlyRequired, new TypeError(errorMessage), rule.default);
60 | }
61 | return result;
62 | }
63 |
64 | /**
65 | * Method to parse a property that is a rule. Essentially, it recursively calls applyObjectRule on the value of the key in TO_PARSE
66 | * @param {string} key - The key to parse
67 | * @param {propertyRule} rule - The rule to parse the key with
68 | * @param {boolean} isStrictlyRequired - Whether the key is required and the rule is strict
69 | * @returns {any | undefined} - The parsed result. If the key is not present and the rule is strict, returns undefined
70 | */
71 | private parseRecursiveProperty(key: string, rule: propertyRule, isStrictlyRequired: boolean): any | undefined {
72 | if (!rule.rule || rule.type !== 'rule') {
73 | throw new utilityErrors.VueTubeExtractorError(ErrorMessages.missingValuesInRule(key, 'rule'));
74 | }
75 | let SUB_RULE_RESULT = this.Helper.wrapFunction(parseRule, key, undefined, this.Helper.jsonPathToObject(key, this.TO_PARSE), rule.rule);
76 | SUB_RULE_RESULT ??= this.Helper.checkStrictlyRequired(
77 | isStrictlyRequired,
78 | new utilityErrors.VueTubeExtractorError(ErrorMessages.missingRequired('key', key, 'applyObjectRule')),
79 | rule.default
80 | );
81 | return SUB_RULE_RESULT;
82 | }
83 |
84 | /**
85 | * Identifies the type of rule and calls the appropriate method to parse the property
86 | * @param {string} key - The key to parse
87 | * @param {propertyRule} rule - The rule to parse the key with
88 | * @returns {any | undefined} - The parsed result. If the key is not present and the rule is strict, returns undefined
89 | */
90 | private parseProperty(key: string, rule: propertyRule): any | undefined {
91 | const isStrictlyRequired = this.isRuleStrictlyRequired(rule);
92 | if (rule.type === 'rule') {
93 | return this.parseRecursiveProperty(key, rule, isStrictlyRequired);
94 | }
95 | return this.parseBasicProperty(key, rule, isStrictlyRequired);
96 | }
97 |
98 | /**
99 | * A series of checks to determine if the given rule is valid and can be parsed. Throws an error if the rule is invalid
100 | * @returns {void}
101 | */
102 | private guardClauses(): void {
103 | const errorMessage: string[] = [];
104 | if (this.RULE_TYPE !== 'object') {
105 | errorMessage.push(ErrorMessages.invalidRuleType('object', this.RULE_TYPE));
106 | }
107 | if (!this.PROPERTIES) {
108 | errorMessage.push(ErrorMessages.missingValuesInRule(this.RULE_NAME || 'object', 'properties'));
109 | }
110 | if (typeof this.TO_PARSE !== 'object' || Array.isArray(this.TO_PARSE)) {
111 | errorMessage.push(ErrorMessages.ruleTypeMismatch('object', Array.isArray(this.TO_PARSE) ? 'array' : typeof this.TO_PARSE));
112 | }
113 | if (errorMessage.length > 0) {
114 | throw new utilityErrors.VueTubeExtractorError(errorMessage.join('\n'));
115 | }
116 | }
117 |
118 | private isRuleStrictlyRequired(rule: propertyRule | groupedRule): boolean {
119 | return (rule.required ?? true) && (this.isStrict ?? true) && !rule.default;
120 | }
121 |
122 | /**
123 | * Public method to parse the object. Returns the parsed object
124 | * @returns {[key: string]: any} - The parsed object. If for whatever reason the result is an empty object, returns empty object
125 | */
126 | public parse(): { [key: string]: any } {
127 | if (this.condition && !this.Helper.evaluateCondition(this.TO_PARSE, this.condition)) {
128 | return {};
129 | }
130 | for (const [key, value] of Object.entries(this.PROPERTIES)) {
131 | const parsedProperty = this.parseProperty(key, value);
132 | if (parsedProperty) this.PROCESSED_OBJECT[this.Helper.followKeymap(key)] = parsedProperty;
133 | }
134 | if (this.flatten) return this.Helper.flattenConvertObject(this.PROCESSED_OBJECT);
135 | return this.PROCESSED_OBJECT;
136 | }
137 | }
138 |
139 | /**
140 | * Class to parse array rules
141 | * @param {Array} toParse - The object to parse
142 | * @param {arrayRule} rule - The rule to parse the object with
143 | */
144 | export class ArrayRuleParser implements GenericRuleParser {
145 | private readonly TO_PARSE: Array;
146 | private readonly ITEMS: Rule;
147 | private PROCESSED_ARRAY: Array = [];
148 | private readonly RULE_TYPE: string;
149 | private readonly RULE_NAME: string;
150 | private readonly isStrict: boolean;
151 | private readonly condition: conditionalFunction | { [p: string]: conditionalRule };
152 | private Helper: ArrayRuleHelper;
153 |
154 | constructor(toParse: Array, rule: arrayRule) {
155 | this.TO_PARSE = toParse;
156 | this.RULE_TYPE = rule.type;
157 | this.ITEMS = rule.items;
158 | this.Helper = new ArrayRuleHelper(rule);
159 | this.guardClauses();
160 | const filledRule = this.Helper.fillRule();
161 | this.RULE_NAME = filledRule.name || 'arrayRule';
162 | this.isStrict = filledRule.strict as boolean;
163 | this.condition = filledRule.condition as conditionalFunction | { [key: string]: conditionalRule };
164 | }
165 |
166 | /**
167 | * A series of checks to determine if the given rule is valid and can be parsed. Throws an error if the rule is invalid
168 | * @returns {void}
169 | */
170 | private guardClauses(): void {
171 | const errorMessage: string[] = [];
172 | if (this.RULE_TYPE !== 'array') {
173 | errorMessage.push(ErrorMessages.invalidRuleType('array', this.RULE_TYPE));
174 | }
175 | if (!this.ITEMS) {
176 | errorMessage.push(ErrorMessages.missingValuesInRule(this.RULE_NAME || 'array', 'items'));
177 | }
178 | if (!Array.isArray(this.TO_PARSE)) {
179 | errorMessage.push(ErrorMessages.ruleTypeMismatch('array', typeof this.TO_PARSE));
180 | }
181 | if (errorMessage.length > 0) {
182 | throw new utilityErrors.VueTubeExtractorError(errorMessage.join('\n'));
183 | }
184 | }
185 |
186 | /**
187 | * Public method to parse the array. Returns the parsed array
188 | * @returns {Array | undefined} - The parsed array. If for whatever reason the result is an empty array, returns undefined
189 | */
190 | public parse(): Array | undefined {
191 | for (const item of this.TO_PARSE) {
192 | if (!this.Helper.checkTypeGuard(item, this.ITEMS.type)) {
193 | const errorMessage = ErrorMessages.subRuleError(this.RULE_NAME, ErrorMessages.typeGuardMismatch(this.ITEMS.type, typeof item));
194 | if (this.Helper.checkStrictlyRequired(this.isStrict, new TypeError(errorMessage), true)) continue;
195 | }
196 | if (this.condition && !this.Helper.evaluateCondition(item, this.condition)) {
197 | continue;
198 | }
199 | const parsedItem = this.Helper.wrapFunction(parseRule, this.RULE_NAME, undefined, item, this.ITEMS);
200 | if (parsedItem) this.PROCESSED_ARRAY.push(parsedItem);
201 | }
202 | return this.PROCESSED_ARRAY.length > 0 ? this.PROCESSED_ARRAY : undefined;
203 | }
204 | }
205 |
206 | /**
207 | * Function that selects the appropriate parser for the given rule type
208 | * @param {any} toParse - The object to parse
209 | * @param {Rule} rule - The rule to parse the object with
210 | * @returns {any | undefined} - The parsed object. If for whatever reason the result is an empty object, returns undefined
211 | */
212 | export function parseRule(toParse: any, rule: Rule): any | undefined {
213 | let Parser: GenericRuleParser;
214 | if (rule.type === 'object') {
215 | Parser = new ObjectRuleParser(toParse, rule);
216 | } else if (rule.type === 'array') {
217 | Parser = new ArrayRuleParser(toParse, rule);
218 | } else {
219 | throw new utilityErrors.VueTubeExtractorError(ErrorMessages.invalidRuleType('object or array', (rule as Rule).type || 'invalid rule'));
220 | }
221 | return Parser.parse();
222 | }
223 |
--------------------------------------------------------------------------------
/src/extractors/youtube-new/parser/types.ts:
--------------------------------------------------------------------------------
1 | import { PROPERTY_TYPES, SUPPORTED_TYPES } from './parser.const';
2 |
3 | type supportedTypes = (typeof SUPPORTED_TYPES)[keyof typeof SUPPORTED_TYPES];
4 | type propertyType = (typeof PROPERTY_TYPES)[keyof typeof PROPERTY_TYPES];
5 |
6 | export type conditionalFunction = (item: any) => boolean;
7 |
8 | type baseRule = {
9 | type: supportedTypes;
10 | name: string;
11 | aliases?: string[];
12 | strict?: boolean;
13 | isDiscoverable?: boolean;
14 | condition?: conditionalFunction | { [key: string]: conditionalRule };
15 | };
16 |
17 | export interface objectRule extends baseRule {
18 | type: 'object';
19 | flatten?: boolean;
20 |
21 | properties: {
22 | [key: string]: propertyRule;
23 | };
24 |
25 | keymap?: {
26 | [key: string]: string;
27 | };
28 | }
29 |
30 | export interface arrayRule extends baseRule {
31 | type: 'array';
32 | limit?: number;
33 | items: Rule;
34 | }
35 |
36 | export type Rule = objectRule | arrayRule;
37 |
38 | export interface propertyRule {
39 | type: propertyType;
40 | required?: boolean;
41 | rule?: Rule;
42 | default?: any;
43 | }
44 |
45 | export interface conditionalRule extends Omit {
46 | expected: any;
47 | }
48 |
49 | export interface groupedRule extends Omit {
50 | type: 'group';
51 | properties: {
52 | [key: string]: propertyRule;
53 | };
54 | }
55 |
--------------------------------------------------------------------------------
/src/extractors/youtube-new/parser/types/aliases.ts:
--------------------------------------------------------------------------------
1 | import type { ObjectRule, PropertyRule, ObjectRuleProps } from "./common";
2 | import type { UnionToIntersection, PickNever } from "./utils";
3 |
4 | type PropAliases = Prop['aliases'];
5 |
6 | type AliasesUnion =
7 | PropAliases extends readonly string[] ?
8 | {
9 | [Key in keyof PropAliases]: PropAliases[Key]
10 | }[number] :
11 | never;
12 |
13 | type ApplyAliasToProp = {
14 | [Key in AliasesUnion]: Prop
15 | };
16 |
17 | type AppliedRuleAliasesWithNever = Omit & {
18 | properties: ObjectRuleProps & UnionToIntersection<{
19 | [Key in keyof ObjectRuleProps]: ApplyAliasToProp[Key]>;
20 | }[keyof ObjectRuleProps]>
21 | };
22 |
23 | export type AppliedRuleAliases =
24 | Rule extends ObjectRule ?
25 | (
26 | keyof PickNever>> extends never ?
27 | AppliedRuleAliasesWithNever :
28 | never
29 | ) :
30 | never;
31 |
--------------------------------------------------------------------------------
/src/extractors/youtube-new/parser/types/appliedRule.ts:
--------------------------------------------------------------------------------
1 | import type { PropertyRule, ObjectRule, ArrayRule, TypeMap, Rule } from "./common";
2 | import type { RuleKeyRemap } from "./remap";
3 | import type { ObjectProps } from "./objectProps";
4 | import type { AppliedCondition } from "./condition";
5 | import type { AppliedRuleAliases } from "./aliases";
6 | import type { AppliedFlattenObjectRule } from "./flaten";
7 | import type { AppliedJsonPath } from "./jsonpath";
8 |
9 | export type IndexType =
10 | Prop['type'] extends keyof TypeMap ?
11 | TypeMap[Prop['type']] :
12 | Prop extends ObjectRule ?
13 | AppliedObjectRule :
14 | Prop extends ArrayRule ?
15 | AppliedArrayRule :
16 | never;
17 |
18 | type AppliedObjectRuleWithoutCondition =
19 | ObjectProps<
20 | AppliedFlattenObjectRule<
21 | AppliedRuleAliases<
22 | RuleKeyRemap<
23 | AppliedJsonPath<
24 | Rule
25 | >>>>
26 | >;
27 |
28 | export type AppliedObjectRule = AppliedCondition<
29 | Rule,
30 | AppliedObjectRuleWithoutCondition
31 | >;
32 |
33 | // If object condition is inside array rule, then have to remove object
34 | // from the array, and it cant be nullable
35 |
36 | export type AppliedArrayRule = Array<
37 | Rule['items'] extends ObjectRule ?
38 | AppliedObjectRuleWithoutCondition :
39 | AppliedRule
40 | >;
41 |
42 | export type AppliedRule =
43 | RuleType extends ObjectRule ?
44 | AppliedObjectRule :
45 | RuleType extends ArrayRule ?
46 | AppliedArrayRule :
47 | never;
48 |
--------------------------------------------------------------------------------
/src/extractors/youtube-new/parser/types/common.ts:
--------------------------------------------------------------------------------
1 | export type TypeMap = {
2 | string: string;
3 | number: number;
4 | boolean: boolean;
5 | any: any;
6 | };
7 |
8 | export type IndexTypeMap =
9 | Key extends keyof TypeMap ?
10 | TypeMap[Key] :
11 | never;
12 |
13 | export type ConditionalFn = (item: any) => boolean;
14 |
15 | export interface ObjectRule {
16 | type: 'object';
17 | strict?: boolean;
18 | flatten?: boolean;
19 | flattenAll?: boolean;
20 | properties: Record;
21 | keymap?: Record;
22 | condition?: ConditionalFn;
23 | }
24 |
25 | export interface ArrayRule {
26 | type: 'array';
27 | limit?: number;
28 | items: Rule;
29 | condition?: ConditionalFn;
30 | }
31 |
32 | export type Rule = ObjectRule | ArrayRule;
33 |
34 | type PropertyBase = {
35 | required?: boolean;
36 | aliases?: readonly string[];
37 | }
38 |
39 | type MappedPrimitive = PropertyBase & ({
40 | type: Key;
41 | default: TypeMap[Key];
42 | expected?: never;
43 | } | {
44 | type: Key;
45 | default?: never;
46 | expected: TypeMap[Key];
47 | } | {
48 | type: Key;
49 | default?: never;
50 | expected?: never;
51 | });
52 |
53 | type PrimitivePropertyRule = {
54 | [Key in keyof TypeMap]: MappedPrimitive
55 | }[keyof TypeMap];
56 |
57 | export type ObjectPropertyRule = ObjectRule & PropertyBase;
58 |
59 | type ArrayPropertyRule = ArrayRule & PropertyBase;
60 |
61 | export type PropertyRule = ArrayPropertyRule | ObjectPropertyRule | PrimitivePropertyRule;
62 |
63 | export type ObjectRuleProps = Rule['properties'];
64 |
--------------------------------------------------------------------------------
/src/extractors/youtube-new/parser/types/condition.ts:
--------------------------------------------------------------------------------
1 | import type { ObjectRule, ConditionalFn, ObjectRuleProps, IndexTypeMap } from './common';
2 |
3 | type ConditionalKeysSet = keyof {
4 | [Key in keyof ObjectRuleProps as ObjectRuleProps[Key] extends { expected: IndexTypeMap[Key]['type']> }
5 | ? Key
6 | : never]: unknown;
7 | };
8 |
9 | type HasConditionalKeysSet = ConditionalKeysSet extends never ? Second : First;
10 |
11 | export type HasCondition = Rule extends { condition: ConditionalFn }
12 | ? First
13 | : HasConditionalKeysSet;
14 |
15 | export type AppliedCondition = Rule extends ObjectRule ? HasCondition : never;
16 |
--------------------------------------------------------------------------------
/src/extractors/youtube-new/parser/types/default.ts:
--------------------------------------------------------------------------------
1 | import type { PropertyRule, IndexTypeMap } from './common';
2 |
3 | export type PropDefaultSet = Prop extends { default: IndexTypeMap } ? First : Second;
--------------------------------------------------------------------------------
/src/extractors/youtube-new/parser/types/examples.ts:
--------------------------------------------------------------------------------
1 | import type { Rule } from "./common";
2 | import type { AppliedRule } from "./appliedRule";
3 |
4 | // Rewritten continuation rule from "../../rules/continuations.const.ts"
5 |
6 | const continuation = {
7 | type: 'object',
8 | properties: {
9 | continuation: {
10 | type: 'string',
11 | },
12 | },
13 | } as const satisfies Rule;
14 |
15 | export const CONTINUATIONS = {
16 | type: 'array',
17 | items: {
18 | type: 'object',
19 | properties: {
20 | nextContinuationData: {
21 | ...continuation,
22 | aliases: ['reloadContinuationData']
23 | }
24 | },
25 | },
26 | } as const satisfies Rule;
27 |
28 | // Examples
29 |
30 | const subRule = {
31 | type: 'object',
32 | properties: {
33 | item: {
34 | type: 'number',
35 | default: 3,
36 | },
37 | item2: {
38 | required: true,
39 | type: 'object',
40 | properties: {
41 | prop1: {
42 | type: 'string',
43 | default: 'someDefault',
44 | },
45 | prop2: {
46 | type: 'object',
47 | properties: {
48 | prop: {
49 | type: 'number',
50 | default: 4
51 | }
52 | }
53 | }
54 | }
55 | }
56 | }
57 | } as const satisfies Rule;
58 |
59 | const rule = {
60 | type: 'object',
61 | flatten: true,
62 | keymap: {
63 | continuation: 'remapedContuniation',
64 | nonPrimitive: 'remapedNonPrimitive',
65 | },
66 | properties: {
67 | "obj.another.arr[0].obj": {
68 | type: 'string',
69 | required: true,
70 | },
71 | obj: {
72 | type: 'object',
73 | properties: {
74 | prop: {
75 | type: 'number'
76 | }
77 | }
78 | },
79 | continuation: {
80 | aliases: ['contAlias'],
81 | type: 'string',
82 | required: true,
83 | default: '',
84 | },
85 | reloadContinuationData: {
86 | aliases: ['continuation'],
87 | type: 'number',
88 | required: false,
89 | },
90 | nonPrimitive: {
91 | ...subRule,
92 | required: false,
93 | flatten: true,
94 | },
95 | },
96 | } as const satisfies Rule;
97 |
98 | function typedFunc(obj: any, rule: RuleType): AppliedRule {
99 | // ...
100 | return obj;
101 | }
102 |
103 | // Here you can analyse what properties item will have after rule is applied
104 |
105 | const item = typedFunc({}, rule);
106 |
--------------------------------------------------------------------------------
/src/extractors/youtube-new/parser/types/flaten.ts:
--------------------------------------------------------------------------------
1 | import type { ObjectRule, ObjectPropertyRule, ObjectRuleProps } from './common';
2 | import type { UnionToIntersection } from './utils';
3 | import type { PropRequired } from './required';
4 | import type { RuleStrictMode } from './strict';
5 | import type { HasCondition } from './condition';
6 |
7 | // Helpers
8 | // ========================
9 |
10 | type IsFlatten = Rule extends { flatten: true } ? First : Second;
11 |
12 | type IsFullFlatten = Rule extends { flattenAll: true } ? First : Second;
13 |
14 | type MakeRequiredFalse> = Omit<
15 | ObjectRuleProps[Key],
16 | 'required' | 'default'
17 | > & { required: false };
18 |
19 | type PropagateOptionalProperty> = PropRequired<
20 | Rule,
21 | RuleStrictMode, ObjectRuleProps[Key]>, MakeRequiredFalse>,
22 | MakeRequiredFalse
23 | >;
24 |
25 | // Filter keys
26 | // ========================
27 |
28 | type ObjectRuleKeys = Exclude<
29 | keyof {
30 | [Key in keyof ObjectRuleProps as ObjectRuleProps[Key] extends ObjectPropertyRule ? Key : never]: unknown;
31 | },
32 | symbol
33 | >;
34 |
35 | // Object props that have { flatten: true }
36 | type FlattenKeys = Exclude<
37 | keyof {
38 | [Key in keyof ObjectRuleProps as ObjectRuleProps[Key] extends ObjectRule
39 | ? IsFlatten[Key], Key, never>
40 | : never]: unknown;
41 | },
42 | symbol
43 | >;
44 |
45 | // Flatten types
46 | // ========================
47 |
48 | type FlattenProp = {
49 | [Key in Exclude, symbol> as `${PropName}-${Key}`]: PropagateOptionalProperty;
50 | };
51 |
52 | // flattenAll: true
53 |
54 | type OneLevelDeepFlattenRule = Omit & {
55 | properties: Omit, ObjectRuleKeys> &
56 | UnionToIntersection<
57 | {
58 | [Key in ObjectRuleKeys]: ObjectRuleProps[Key] extends ObjectPropertyRule
59 | ? FlattenProp[Key]>
60 | : never;
61 | }[ObjectRuleKeys]
62 | >;
63 | };
64 |
65 | type DeepFlattenRule = Rule extends OneLevelDeepFlattenRule
66 | ? Rule
67 | : DeepFlattenRule>;
68 |
69 | // flatten: true
70 |
71 | type OneLevelFlattenRule = Omit & {
72 | properties: Omit, FlattenKeys> &
73 | UnionToIntersection<
74 | {
75 | [Key in FlattenKeys]: ObjectRuleProps[Key] extends ObjectPropertyRule
76 | ? FlattenProp[Key]>
77 | : never;
78 | }[FlattenKeys]
79 | >;
80 | };
81 |
82 | type FlattenRule = Rule extends OneLevelFlattenRule ? Rule : FlattenRule>;
83 |
84 | // Applied type
85 | // ========================
86 |
87 | export type AppliedFlattenObjectRule = Rule extends ObjectRule
88 | ? IsFullFlatten<
89 | Rule,
90 | DeepFlattenRule,
91 | IsFlatten<
92 | Rule,
93 | OneLevelDeepFlattenRule>, // Performing one level of flattening
94 | Rule
95 | >
96 | >
97 | : never;
98 |
--------------------------------------------------------------------------------
/src/extractors/youtube-new/parser/types/index.ts:
--------------------------------------------------------------------------------
1 | export { Rule, ObjectRule, ArrayRule, PropertyRule } from "./common";
2 | export { AppliedRule } from "./appliedRule";
--------------------------------------------------------------------------------
/src/extractors/youtube-new/parser/types/jsonpath.ts:
--------------------------------------------------------------------------------
1 | import type { ObjectRule, PropertyRule, ObjectRuleProps } from "./common";
2 | import type { PickNever } from "./utils";
3 |
4 | type ArraySeparator = '[0]';
5 |
6 | type ObjectSeparator = '.';
7 |
8 | type ExtractKey =
9 | Key extends `${infer Prefix}${ObjectSeparator}${string | number}` ?
10 | Prefix extends `${infer InnerPrefix}${ArraySeparator}` ?
11 | InnerPrefix :
12 | Prefix :
13 | Key;
14 |
15 | type Extract =
16 | Key extends `${infer Prefix}${ObjectSeparator}${infer Postfix}` ?
17 | Prefix extends `${string | number}${ArraySeparator}` ?
18 | {
19 | type: 'array',
20 | items: {
21 | type: 'object';
22 | properties: {
23 | [Key in ExtractKey]: Extract
24 | }
25 | }
26 | } :
27 | {
28 | type: 'object',
29 | properties: {
30 | [Key in ExtractKey]: Extract
31 | }
32 | } :
33 | Prop;
34 |
35 | type JsonPathWithNever = Omit & {
36 | properties: {
37 | [Key in keyof ObjectRuleProps as ExtractKey extends Key ? Key : never ]: ObjectRuleProps[Key];
38 | } & {
39 | [Key in keyof ObjectRuleProps as ExtractKey extends Key ? never : ExtractKey]: Extract[Key]>;
40 | }
41 | }
42 |
43 | export type AppliedJsonPath =
44 | Rule extends ObjectRule ?
45 | (
46 | keyof PickNever['properties']> extends never ?
47 | JsonPathWithNever :
48 | never
49 | ) :
50 | never;
51 |
52 |
--------------------------------------------------------------------------------
/src/extractors/youtube-new/parser/types/objectProps.ts:
--------------------------------------------------------------------------------
1 | import type { ObjectRule, PropertyRule, ObjectRuleProps } from "./common";
2 | import type { IndexType } from "./appliedRule";
3 | import type { PropDefaultSet } from "./default";
4 | import type { RuleStrictMode } from "./strict";
5 | import type { PropRequired } from "./required";
6 |
7 | type PropNonNullable = PropDefaultSet<
8 | Prop,
9 | KeyType,
10 | RuleStrictMode<
11 | Rule,
12 | PropRequired<
13 | Prop,
14 | KeyType,
15 | never
16 | >,
17 | never
18 | >
19 | >;
20 |
21 | type PropNullable = PropDefaultSet<
22 | Prop,
23 | never,
24 | RuleStrictMode<
25 | Rule,
26 | PropRequired<
27 | Prop,
28 | never,
29 | KeyType
30 | >,
31 | KeyType
32 | >
33 | >;
34 |
35 | type NonNullableProps = {
36 | [Key in keyof ObjectRuleProps as PropNonNullable[Key], Key>]: IndexType[Key]>;
37 | };
38 |
39 | type NullableProps = {
40 | [Key in keyof ObjectRuleProps as PropNullable[Key], Key>]?: IndexType[Key]>;
41 | };
42 |
43 | export type ObjectProps =
44 | Rule extends ObjectRule ?
45 | NonNullableProps & NullableProps :
46 | never
47 | ;
--------------------------------------------------------------------------------
/src/extractors/youtube-new/parser/types/remap.ts:
--------------------------------------------------------------------------------
1 | import type { ObjectRule, ObjectRuleProps } from "./common";
2 |
3 | type RuleKeymap = Rule['keymap'];
4 |
5 | type RuleRemap =
6 | Omit &
7 | {
8 | properties: {
9 | [
10 | Key in keyof ObjectRuleProps as
11 | Key extends keyof RuleKeymap ?
12 | RuleKeymap[Key] extends string ?
13 | RuleKeymap[Key] :
14 | Key :
15 | Key
16 | ]: ObjectRuleProps[Key] extends ObjectRule ? // Remap done recursively for flat to work correctly
17 | RuleRemap[Key]> :
18 | ObjectRuleProps[Key];
19 | }
20 | }
21 |
22 | export type RuleKeyRemap =
23 | Rule extends ObjectRule ?
24 | RuleRemap :
25 | never;
--------------------------------------------------------------------------------
/src/extractors/youtube-new/parser/types/required.ts:
--------------------------------------------------------------------------------
1 | import type { PropertyRule } from "./common";
2 |
3 | export type PropRequired =
4 | Prop extends { required: false } ?
5 | Second :
6 | First;
7 |
--------------------------------------------------------------------------------
/src/extractors/youtube-new/parser/types/strict.ts:
--------------------------------------------------------------------------------
1 | import type { ObjectRule } from "./common";
2 |
3 |
4 | export type RuleStrictMode =
5 | Rule extends { strict: false } ?
6 | Second :
7 | First;
8 |
--------------------------------------------------------------------------------
/src/extractors/youtube-new/parser/types/utils.ts:
--------------------------------------------------------------------------------
1 | export type PickNever = {
2 | [Key in keyof T as T[Key] extends never ? Key: never]: T[Key];
3 | }
4 |
5 | // Util type from https://github.com/sindresorhus/type-fest/blob/main/source/union-to-intersection.d.ts
6 |
7 | export type UnionToIntersection = (
8 | Union extends unknown
9 | ? (distributedUnion: Union) => void
10 | : never
11 | ) extends ((mergedIntersection: infer Intersection) => void)
12 | ? Intersection
13 | : never;
--------------------------------------------------------------------------------
/src/extractors/youtube-new/proto/__tests__/protoTest.test.ts:
--------------------------------------------------------------------------------
1 | import proto from "../index";
2 | import * as cases from "./testCases.json";
3 | import { searchFilter } from "../types";
4 | import { commentOptions } from "../types";
5 |
6 | describe("protobuf parsing tests", () => {
7 | describe("if encodeVisitorData works", () => {
8 | test.each(cases.encodeVisitorData)(
9 | "%s and %s should be %s",
10 | (input1, input2, expectResult) => {
11 | const visitorData = proto.encodeVisitorData(
12 | input1 as string,
13 | input2 as number
14 | );
15 | expect(visitorData).toBe(expectResult);
16 | }
17 | );
18 | });
19 | describe("if encodeSearchFilter works", () => {
20 | test.each(
21 | cases.encodeSearchFilter as unknown as [Partial, string]
22 | )("%s should be %s ", (input1, expectResult) => {
23 | const searchFilter = proto.encodeSearchFilter(input1 as searchFilter);
24 | expect(searchFilter).toBe(expectResult);
25 | });
26 | });
27 | describe("if encodeCommentOptions works", () => {
28 | test.each(cases.encodeCommentOptions)(
29 | "video %s with options %s should be %s",
30 | (video_id, options, expectResult) => {
31 | const commentOptions = proto.encodeCommentOptions(
32 | video_id as string,
33 | (options as commentOptions) || undefined
34 | );
35 | expect(commentOptions).toBe(expectResult);
36 | }
37 | );
38 | });
39 | });
40 |
--------------------------------------------------------------------------------
/src/extractors/youtube-new/proto/__tests__/testCases.json:
--------------------------------------------------------------------------------
1 | {
2 | "encodeVisitorData": [
3 | ["abcd", 100, "CgRhYmNkKGQ%3D"],
4 | ["efgh", 200, "CgRlZmdoKMgB"],
5 | ["fgee-Eads1Q", 1640995200, "CgtmZ2VlLUVhZHMxUSiAs76OBg%3D%3D"]
6 | ],
7 | "encodeSearchFilter": [
8 | [
9 | {
10 | "uploadDate": "year",
11 | "order": "rating",
12 | "type": "video",
13 | "duration": "medium"
14 | },
15 | "CAESBggFEAEYAw%3D%3D"
16 | ],
17 | [
18 | {
19 | "type": "channel"
20 | },
21 | "EgIQAg%3D%3D"
22 | ],
23 | [
24 | {
25 | "features": ["hd", "subtitles"]
26 | },
27 | "EgQgASgB"
28 | ],
29 | [
30 | {
31 | "features": ["live", "location"]
32 | },
33 | "EgVAAbgBAQ%3D%3D"
34 | ],
35 | [
36 | {
37 | "uploadDate": "week",
38 | "type": "video",
39 | "duration": "long",
40 | "features": ["video3d"],
41 | "order": "uploadDate"
42 | },
43 | "CAISCAgDEAEYAjgB"
44 | ]
45 | ],
46 | "encodeCommentOptions": [
47 | [
48 | "abcd",
49 | { "sortBy": "newestFirst" },
50 | "EgYSBGFiY2QYBjIeIgoiBGFiY2QwAXgCQhBjb21tZW50cy1zZWN0aW9u"
51 | ],
52 | [
53 | "efgh",
54 | { "sortBy": "topComments" },
55 | "EgYSBGVmZ2gYBjIeIgoiBGVmZ2gwAHgCQhBjb21tZW50cy1zZWN0aW9u"
56 | ],
57 | [
58 | "dQw4w9WgXcQ",
59 | { "sortBy": "topComments" },
60 | "Eg0SC2RRdzR3OVdnWGNRGAYyJSIRIgtkUXc0dzlXZ1hjUTAAeAJCEGNvbW1lbnRzLXNlY3Rpb24%3D"
61 | ]
62 | ]
63 | }
64 |
--------------------------------------------------------------------------------
/src/extractors/youtube-new/proto/index.ts:
--------------------------------------------------------------------------------
1 | import { Root, Type, loadSync } from 'protobufjs';
2 | import path from 'path';
3 |
4 | import type { searchProto, protoFilters, commentOptions } from './types';
5 | import { SearchFilter } from '../utils/types';
6 | import {
7 | SEARCH_UPLOAD_DATE_OPTIONS,
8 | SEARCH_TYPE_OPTIONS,
9 | SEARCH_DURATION_OPTIONS,
10 | SEARCH_ORDER_OPTIONS,
11 | FEATURE_BY_SEARCH_FEATURE,
12 | COMMENT_SORT_OPTIONS,
13 | } from '../utils/constants';
14 |
15 | class Proto {
16 | private protoRoot: Root;
17 | constructor() {
18 | this.protoRoot = loadSync(path.join(__dirname, 'youtube.proto'));
19 | }
20 |
21 | /**
22 | * encodes Visitor Data to protobuf format
23 | *
24 | * @param {string} id - visitor id. Should be an 11 character long random string
25 | * @param {number} timestamp - timestamp of initialization
26 | *
27 | * @returns {string} encoded visitor data
28 | */
29 | encodeVisitorData(id: string, timestamp: number): string {
30 | const visitorData: Type = this.protoRoot.lookupType('youtube.VisitorData');
31 | const buf: Uint8Array = visitorData.encode({ id, timestamp }).finish();
32 | return encodeURIComponent(Buffer.from(buf).toString('base64').replace(/\+/g, '-').replace(/\//g, '_'));
33 | }
34 |
35 | /**
36 | * encodes search filter to protobuf format
37 | * @param {Partial} filters - search filters
38 | * @returns {string} encoded search filter
39 | */
40 | encodeSearchFilter(filters: Partial): string {
41 | if (filters?.uploadDate && filters?.type !== 'video') {
42 | throw new Error(JSON.stringify(filters) + '\n' + 'Search filter type must be video');
43 | }
44 | const data: searchProto = filters ? { filters: {} } : { noFilter: 0 };
45 | if (data.filters) {
46 | data.filters = {
47 | ...data.filters,
48 | ...(filters.uploadDate && { param_0: SEARCH_UPLOAD_DATE_OPTIONS[filters.uploadDate] }),
49 | ...(filters.type && { param_1: SEARCH_TYPE_OPTIONS[filters.type] }),
50 | ...(filters.duration && { param_2: SEARCH_DURATION_OPTIONS[filters.duration] }),
51 | };
52 | if (filters.order) data.sort = SEARCH_ORDER_OPTIONS[filters.order];
53 | if (filters.features) {
54 | for (const feature of filters.features) {
55 | data.filters[FEATURE_BY_SEARCH_FEATURE[feature] as keyof protoFilters] = 1;
56 | }
57 | }
58 | }
59 | const SearchFilter: Type = this.protoRoot.lookupType('youtube.SearchFilter');
60 | const buf: Uint8Array = SearchFilter.encode(data).finish();
61 | return encodeURIComponent(Buffer.from(buf).toString('base64'));
62 | }
63 |
64 | /**
65 | * encodes comment options to protobuf format
66 | * @param {string} videoId - video id
67 | * @param options
68 | * @returns {string} encoded comment options
69 | */
70 | encodeCommentOptions(videoId: string, options: commentOptions = {}): string {
71 | const commentOptions: Type = this.protoRoot.lookupType('youtube.CommentsSection');
72 | const data = {
73 | ctx: { videoId },
74 | unkParam: 6,
75 | params: {
76 | opts: {
77 | videoId,
78 | sortBy: COMMENT_SORT_OPTIONS[options.sortBy || 'topComments'],
79 | type: options.type || 2,
80 | },
81 | target: 'comments-section',
82 | },
83 | };
84 | const buf: Uint8Array = commentOptions.encode(data).finish();
85 | return encodeURIComponent(Buffer.from(buf).toString('base64'));
86 | }
87 | }
88 |
89 | const singletonProto = new Proto();
90 |
91 | Object.freeze(singletonProto);
92 |
93 | export default singletonProto;
94 |
--------------------------------------------------------------------------------
/src/extractors/youtube-new/proto/types.ts:
--------------------------------------------------------------------------------
1 | /**
2 | * Use `SearchFeatures` in src\extractors\youtube-new\utils\types.ts
3 | * @deprecated
4 | */
5 | export type searchFeatures =
6 | | 'live'
7 | | 'video4k'
8 | | 'hd'
9 | | 'subtitles'
10 | | 'cc'
11 | | 'video360'
12 | | 'vr180'
13 | | 'video3d'
14 | | 'hdr'
15 | | 'location'
16 | | 'purchased';
17 |
18 | /**
19 | * Use `SearchFilter` in src\extractors\youtube-new\utils\types.ts
20 | * @deprecated
21 | */
22 | interface searchFilter {
23 | uploadDate: 'hour' | 'day' | 'week' | 'month' | 'year' | 'all';
24 | order: 'relevance' | 'viewCount' | 'rating' | 'uploadDate';
25 | type: 'video' | 'playlist' | 'channel' | 'all';
26 | duration: 'short' | 'medium' | 'long' | 'all';
27 | features: Array;
28 | }
29 |
30 | interface searchProto {
31 | sort?: number | null;
32 | noFilter?: number | null;
33 | noCorrection?: number | null;
34 | filters?: protoFilters;
35 | }
36 |
37 | interface protoFilters {
38 | param_0?: number | null;
39 | param_1?: number | null;
40 | param_2?: number | null;
41 | featuresHd?: number | null;
42 | features4k?: number | null;
43 | featuresVr180?: number | null;
44 | featuresSubtitles?: number | null;
45 | featuresCreativeCommons?: number | null;
46 | features360?: number | null;
47 | features3d?: number | null;
48 | featuresHdr?: number | null;
49 | featuresLocation?: number | null;
50 | featuresPurchased?: number | null;
51 | featuresLive?: number | null;
52 | }
53 |
54 | interface commentOptions {
55 | sortBy?: 'topComments' | 'newestFirst';
56 | type?: number;
57 | }
58 |
59 | export { searchProto, protoFilters, commentOptions, searchFilter };
60 |
--------------------------------------------------------------------------------
/src/extractors/youtube-new/proto/youtube.proto:
--------------------------------------------------------------------------------
1 | package youtube;
2 |
3 | message VisitorData {
4 | required string id = 1;
5 | required int32 timestamp = 5;
6 | }
7 |
8 | message SearchFilter {
9 | optional int32 sort = 1; // sort by
10 | optional int32 noCorrection = 8;
11 | optional int32 noFilter = 19;
12 |
13 | message Filters {
14 | optional int32 param_0 = 1; // upload date
15 | optional int32 param_1 = 2; // type
16 | optional int32 param_2 = 3; // duration
17 | // type filters
18 | optional int32 featuresHd = 4; // hd filter
19 | optional int32 featuresSubtitles = 5; // subtitles filter
20 | optional int32 featuresCreativeCommons = 6; // creative commons filter
21 | optional int32 features3d = 7; // 3d filter
22 | optional int32 featuresLive = 8; // live filter
23 | optional int32 featuresPurchased = 9; // purchased filter
24 | optional int32 features4k = 14; // 4k filter
25 | optional int32 features360 = 15; // 360 view filter
26 | optional int32 featuresLocation = 23; // location filter
27 | optional int32 featuresHdr = 25; // hdr filter
28 | optional int32 featuresVr180 = 26; // vr180 filter
29 | }
30 |
31 | optional Filters filters = 2;
32 | }
33 |
34 | message CommentsSection {
35 | message Context {
36 | required string videoId = 2;
37 | }
38 |
39 | message Options {
40 | required string videoId = 4;
41 | required int32 sortBy = 6;
42 | required int32 type = 15;
43 | }
44 |
45 | message UnkOpts {
46 | required int32 unkParam = 1;
47 | }
48 |
49 | message RepliesOptions {
50 | required string comment_id = 2;
51 | required UnkOpts unkopts = 4;
52 | optional string channel_id = 5;
53 | required string videoId = 6;
54 | required int32 unkParam1 = 8;
55 | required int32 unkParam2 = 9;
56 | }
57 |
58 | message Params {
59 |
60 | optional string unk_token = 1;
61 |
62 | optional Options opts = 4;
63 | optional RepliesOptions replies_opts = 3;
64 |
65 | optional int32 page = 5;
66 | required string target = 8;
67 | }
68 |
69 | required Context ctx = 2;
70 | required int32 unk_param = 3;
71 | required Params params = 6;
72 | }
--------------------------------------------------------------------------------
/src/extractors/youtube-new/requester/__tests__/controllerTest.test.ts:
--------------------------------------------------------------------------------
1 | import 'isomorphic-fetch';
2 | import { Config } from '../initializer/config';
3 | import { HomePageController } from '../controllers/homepage.controller';
4 |
5 | describe('Controller Tests', () => {
6 | let validConfig: Config;
7 | beforeAll(async () => {
8 | validConfig = new Config();
9 | await validConfig.getInnertubeConfig();
10 | });
11 |
12 | describe('Home Page Tests', () => {
13 | let homePageController: HomePageController;
14 | beforeAll(() => {
15 | homePageController = new HomePageController();
16 | });
17 | test('Should get a valid request option', () => {
18 | const requestOptions = homePageController.buildRequestOptions(validConfig);
19 | expect(typeof requestOptions).toBe('object');
20 | expect(typeof requestOptions.option).toBe('object');
21 | expect(typeof requestOptions.option.url).toBe('string');
22 | expect(typeof requestOptions.option.data).toBe('object');
23 | expect(typeof requestOptions.option.params).toBe('object');
24 | expect(typeof requestOptions.option.data.continuation).toBe('undefined');
25 | const requestOptionsWithContinuation = homePageController.buildRequestOptions(validConfig, 'continuation');
26 | expect(requestOptionsWithContinuation.option.data.continuation).toBe('continuation');
27 | });
28 | });
29 | });
30 |
--------------------------------------------------------------------------------
/src/extractors/youtube-new/requester/__tests__/initializerTest.test.ts:
--------------------------------------------------------------------------------
1 | import 'isomorphic-fetch';
2 | import Errors from '../../../../utils/errors';
3 | import { DeviceType } from '../../utils/types';
4 | import { Config } from '../initializer/config';
5 | import { Context } from '../initializer/context';
6 |
7 | describe('Initializer Tests', () => {
8 | let validConfig: Config;
9 |
10 | beforeAll(async () => {
11 | validConfig = new Config();
12 | await validConfig.getInnertubeConfig();
13 | });
14 |
15 | describe('Config Tests', () => {
16 | test('Should get a valid API key and client object', async () => {
17 | expect(typeof validConfig.apiKey).toBe('string');
18 | expect(typeof validConfig.client).toBe('object');
19 | expect(typeof validConfig.client.gl).toBe('string');
20 | expect(typeof validConfig.client.hl).toBe('string');
21 | expect(typeof validConfig.client.clientVersion).toBe('string');
22 | expect(typeof validConfig.client.remoteHost).toBe('string');
23 | expect(typeof validConfig.client.visitorData).toBe('string');
24 | });
25 |
26 | describe('getInnertubeConfig error handling', () => {
27 | test('Should throw an error if an invalid API URL is provided', async () => {
28 | const config = new Config();
29 | const wrongDataURL = 'http://invalid';
30 | await expect(config.getInnertubeConfig(wrongDataURL)).rejects.toThrow(Errors.VueTubeExtractorError);
31 | });
32 |
33 | test('Should throw an error if an invalid data structure is returned', async () => {
34 | const config = new Config();
35 | const wrongDataURL = 'http://example.com'; // returns 200 OK for all routes
36 | await expect(config.getInnertubeConfig(wrongDataURL)).rejects.toThrow(Errors.VueTubeExtractorError);
37 | });
38 | });
39 | });
40 |
41 | describe('Context Tests', () => {
42 | test('Should get a valid context object for any valid device type', async () => {
43 | const validDeviceTypes: DeviceType[] = ['MOBILE_WEB', 'MOBILE_APP', 'DESKTOP_WEB'];
44 | for (const deviceType of validDeviceTypes) {
45 | const context = new Context(deviceType).getContext(validConfig.client);
46 | expect(typeof context).toBe('object');
47 | expect(typeof context.client).toBe('object');
48 | expect(typeof context.client.gl).toBe('string');
49 | expect(typeof context.client.hl).toBe('string');
50 | expect(typeof context.client.deviceMake).toBe('string');
51 | expect(typeof context.client.deviceModel).toBe('string');
52 | expect(typeof context.client.userAgent).toBe('string');
53 | expect(['ANDROID', 'IOS', 'WEB', 'MWEB']).toContain(context.client.clientName);
54 | expect(typeof context.client.clientVersion).toBe('string');
55 | expect(['Android', 'iOS', 'Windows', 'Macintosh', 'Linux', 'MWeb']).toContain(context.client.osName);
56 | expect(['MOBILE', 'DESKTOP']).toContain(context.client.platform);
57 | expect(typeof context.client.remoteHost).toBe('string');
58 | expect(typeof context.client.clientFormFactor).toBe('string');
59 | expect(typeof context.client.visitorData).toBe('string');
60 | }
61 | });
62 |
63 | describe('getInnertubeConfig error handling', () => {
64 | test('Should throw an error if an invalid device type is provided', async () => {
65 | expect(() => new Context('WRONG' as DeviceType)).toThrow(Errors.VueTubeExtractorError);
66 | });
67 | });
68 | });
69 | });
70 |
--------------------------------------------------------------------------------
/src/extractors/youtube-new/requester/controllers/base.controller.model.ts:
--------------------------------------------------------------------------------
1 | import { pageSegment } from '@types';
2 | import { HttpOptions } from '@vuetubeapp/http';
3 | import { Config } from '../initializer/config';
4 |
5 | export interface BaseControllerModel {
6 | getRequest(config: Config, continuation?: string): Promise>;
7 | buildRequestOptions(config: Config, continuation?: string): RequestOptions;
8 | parseRawResponse(data: Record): object;
9 | postProcessResponse(data: Record): T;
10 | parseData(data: Record): Promise;
11 | throwErrors(data: Record): void;
12 | }
13 |
14 | export interface RequestOptions {
15 | option: HttpOptions;
16 | key?: string;
17 | }
18 |
19 | export interface GenericPage {
20 | /**
21 | * TODO: update and move to `types.ts`
22 | */
23 | segments: pageSegment[];
24 | chips?: string[];
25 | continue?: () => Promise;
26 | }
27 |
28 | // list all valid endpoints
29 | export enum endpoints {
30 | browse = '/browse',
31 | search = '/search',
32 | player = '/player',
33 | next = '/next',
34 | qoe = 'stats/qoe',
35 | }
36 |
37 | // function to select the url to use for the request. Endpoints follow the same pattern: https://www.youtube.com/youtubei/v1/[endpoint]
38 | export function retrieveEndpoint(endpoint: endpoints): string {
39 | return `https://www.youtube.com/youtubei/v1${endpoint}`;
40 | }
41 |
--------------------------------------------------------------------------------
/src/extractors/youtube-new/requester/controllers/browse.ts:
--------------------------------------------------------------------------------
1 | // TODO
--------------------------------------------------------------------------------
/src/extractors/youtube-new/requester/controllers/homepage.controller.ts:
--------------------------------------------------------------------------------
1 | import { Http, HttpOptions } from '@vuetubeapp/http';
2 | import { BaseControllerModel, GenericPage, RequestOptions, endpoints, retrieveEndpoint } from './base.controller.model';
3 | import { Config } from '../initializer/config';
4 | import { Context } from '../initializer/context';
5 | import { utilityErrors, ErrorMessages } from '@utils';
6 |
7 | export class HomePageController implements BaseControllerModel {
8 | //TODO - move to base controller
9 | getRequest(config: Config, continuation?: string): Promise> {
10 | const requestOptions = this.buildRequestOptions(config, continuation);
11 | return Http.post(requestOptions.option);
12 | }
13 |
14 | buildRequestOptions(config: Config, continuation?: string): RequestOptions {
15 | const context = new Context('MOBILE_APP').getContext(config.client);
16 | const requestOptions: RequestOptions = {
17 | option: {
18 | url: retrieveEndpoint(endpoints.browse),
19 | data: {
20 | // Include the context in the data payload.
21 | ...context,
22 | // The continuation token is only included in the data payload if it is truthy.
23 | ...(continuation && { continuation }),
24 | },
25 | params: { key: config.apiKey },
26 | },
27 | };
28 | return requestOptions;
29 | }
30 |
31 | parseRawResponse(data: Record): object {
32 | throw new Error('Method not implemented.');
33 | }
34 | postProcessResponse(data: Record): GenericPage {
35 | throw new Error('Method not implemented.');
36 | }
37 | parseData(data: Record): Promise {
38 | throw new Error('Method not implemented.');
39 | }
40 | throwErrors(data: Record): void {
41 | throw new Error('Method not implemented.');
42 | }
43 | }
44 |
--------------------------------------------------------------------------------
/src/extractors/youtube-new/requester/index.ts:
--------------------------------------------------------------------------------
1 | // TODO
--------------------------------------------------------------------------------
/src/extractors/youtube-new/requester/initializer/config.ts:
--------------------------------------------------------------------------------
1 | import { Http, HttpResponse } from '@vuetubeapp/http';
2 | import { YTClient } from 'extractors/youtube-new/utils/types';
3 | import { YT_CONSTANTS } from '../../utils/constants';
4 | import Errors from '../../../../utils/errors';
5 |
6 | export class Config {
7 | private client_: Partial;
8 | private apiKey_: string;
9 |
10 | get client(): Partial {
11 | return this.client_;
12 | }
13 |
14 | get apiKey(): string {
15 | return this.apiKey_;
16 | }
17 |
18 | parseJSDataResponse(response: HttpResponse) {
19 | try {
20 | const rawData: Array = JSON.parse(response.data.replace(")]}'", ''));
21 | const data = rawData[0][2];
22 | this.apiKey_ = data[1];
23 | this.client_ = {
24 | gl: data[0][0][1],
25 | hl: data[0][0][0],
26 | clientVersion: data[0][0][16],
27 | remoteHost: data[0][0][3],
28 | visitorData: data[6],
29 | };
30 | } catch (e) {
31 | throw new Errors.VueTubeExtractorError('Invalid data structure returned by YouTube API');
32 | }
33 | }
34 |
35 | /**
36 | * Fetches the innertube key from the YouTube API.
37 | * @returns {Promise} Class containing API key and initial client object
38 | */
39 | async getInnertubeConfig(baseURL: string = YT_CONSTANTS.URL.YT_MOBILE): Promise {
40 | const response: void | HttpResponse = await Http.get({
41 | url: `${baseURL}/sw.js_data`,
42 | responseType: 'text',
43 | }).catch(err => {
44 | if (typeof err === 'string') {
45 | throw new Errors.VueTubeExtractorError(err.toUpperCase());
46 | } else if (err instanceof Error) {
47 | throw new Errors.VueTubeExtractorError(err.message);
48 | }
49 | });
50 | if (!response || !response.data) {
51 | throw new Errors.VueTubeExtractorError('No response from YouTube API');
52 | }
53 | this.parseJSDataResponse(response);
54 | return this;
55 | }
56 | }
57 |
--------------------------------------------------------------------------------
/src/extractors/youtube-new/requester/initializer/consts/context.const.ts:
--------------------------------------------------------------------------------
1 | import { DEVICE_TYPE, YT_CONSTANTS, OS_NAME, PLATFORM, CLIENT_FORM_FACTOR } from '../../../utils/constants';
2 | import { DeviceType, Device } from '../../../utils/types';
3 |
4 | export const DEVICE_CONFIG: Record = {
5 | [DEVICE_TYPE.DESKTOP_WEB]: {
6 | baseURL: YT_CONSTANTS.URL.YT_URL,
7 | clientName: YT_CONSTANTS.YTAPIVAL.CLIENT_WEB_Desktop,
8 | clientVersion: YT_CONSTANTS.YTAPIVAL.VERSION_WEB,
9 | clientFormFactor: CLIENT_FORM_FACTOR.UNKNOWN,
10 | osName: OS_NAME.MAC,
11 | platform: PLATFORM.DESKTOP,
12 | },
13 | [DEVICE_TYPE.MOBILE_WEB]: {
14 | baseURL: YT_CONSTANTS.URL.YT_MOBILE,
15 | clientName: YT_CONSTANTS.YTAPIVAL.CLIENT_WEB_Mobile,
16 | clientVersion: YT_CONSTANTS.YTAPIVAL.VERSION_WEB,
17 | clientFormFactor: CLIENT_FORM_FACTOR.SMALL,
18 | osName: OS_NAME.ANDROID,
19 | platform: PLATFORM.MOBILE,
20 | },
21 | [DEVICE_TYPE.MOBILE_APP]: {
22 | baseURL: YT_CONSTANTS.URL.YT_MOBILE,
23 | clientName: YT_CONSTANTS.YTAPIVAL.CLIENTNAME,
24 | clientVersion: YT_CONSTANTS.YTAPIVAL.VERSION,
25 | clientFormFactor: CLIENT_FORM_FACTOR.SMALL,
26 | osName: OS_NAME.ANDROID,
27 | platform: PLATFORM.MOBILE,
28 | },
29 | } as const;
30 |
--------------------------------------------------------------------------------
/src/extractors/youtube-new/requester/initializer/context.ts:
--------------------------------------------------------------------------------
1 | import { Device, DeviceType, YTClient, YTContext } from 'extractors/youtube-new/utils/types';
2 | import UserAgent from 'user-agents';
3 | import Errors from '../../../../utils/errors';
4 | import { DEVICE_CONFIG } from './consts/context.const';
5 |
6 | export class Context {
7 | private device: Device;
8 |
9 | constructor(deviceType: DeviceType) {
10 | if (!deviceType || Object.keys(DEVICE_CONFIG).indexOf(deviceType) === -1) {
11 | throw new Errors.VueTubeExtractorError('Invalid device type used when generating context');
12 | }
13 | this.device = DEVICE_CONFIG[deviceType];
14 | }
15 |
16 | getContext(configClient: Partial): YTContext {
17 | const { clientName, osName, platform, clientFormFactor } = this.device;
18 |
19 | const userAgentData = new UserAgent({ deviceCategory: platform.toLowerCase() }).data;
20 |
21 | const { vendor: deviceMake, platform: deviceModel, userAgent } = userAgentData;
22 |
23 | return {
24 | client: {
25 | deviceMake,
26 | deviceModel,
27 | userAgent,
28 | clientName,
29 | osName,
30 | platform,
31 | clientFormFactor,
32 | ...configClient,
33 | } as YTClient,
34 | user: { lockedSafetyMode: false },
35 | request: { useSsl: true },
36 | } as YTContext;
37 | }
38 | }
39 |
--------------------------------------------------------------------------------
/src/extractors/youtube-new/requester/initializer/index.ts:
--------------------------------------------------------------------------------
1 | export { Initializer } from './initializer';
--------------------------------------------------------------------------------
/src/extractors/youtube-new/requester/initializer/initializer.ts:
--------------------------------------------------------------------------------
1 | import { YT_CONSTANTS, DEVICE_TYPE } from 'extractors/youtube-new/utils/constants';
2 | import { YTContext } from 'extractors/youtube-new/utils/types';
3 | import { Config } from './config';
4 | import { Context } from './context';
5 |
6 | export class Initializer {
7 | private _mWebContext: YTContext;
8 | private _appContext: YTContext;
9 |
10 | async init(): Promise {
11 | // Request API key from YouTube's Innertube API (as well as client object)
12 | const { client } = await new Config().getInnertubeConfig(YT_CONSTANTS.URL.YT_MOBILE);
13 |
14 | // Generate context(s) for future requests
15 | this._mWebContext = new Context(DEVICE_TYPE.MOBILE_WEB).getContext(client);
16 | this._appContext = new Context(DEVICE_TYPE.MOBILE_APP).getContext(client);
17 | }
18 |
19 | get mWebContext(): YTContext {
20 | return this._mWebContext;
21 | }
22 |
23 | get appContext(): YTContext {
24 | return this._appContext;
25 | }
26 | }
27 |
--------------------------------------------------------------------------------
/src/extractors/youtube-new/requester/initializer/options.ts:
--------------------------------------------------------------------------------
1 | // TODO
--------------------------------------------------------------------------------
/src/extractors/youtube-new/rules/__tests__/ruleFactory.test.ts:
--------------------------------------------------------------------------------
1 | import ruleFactory from '../index';
2 | import { Rule } from '../../parser/types';
3 |
4 | describe('ruleFactory tests', () => {
5 | let factory: ruleFactory;
6 | beforeEach(() => {
7 | // create a new rule factory before each test
8 | factory = new ruleFactory();
9 | });
10 | test('ruleFactory should be able to create a rule', () => {
11 | const rule: Rule = {
12 | type: 'array',
13 | name: 'test',
14 | items: {
15 | name: 'test',
16 | type: 'object',
17 | properties: {
18 | test: {
19 | type: 'string',
20 | },
21 | },
22 | },
23 | };
24 | factory.createRule(rule);
25 | expect(factory.getRule('test')).toBe(rule);
26 | });
27 | test('ruleFactory should be able to create a rule with aliases', () => {
28 | const rule: Rule = {
29 | type: 'array',
30 | name: 'test',
31 | aliases: ['test2'],
32 | items: {
33 | name: 'test',
34 | type: 'object',
35 | properties: {
36 | test: {
37 | type: 'string',
38 | },
39 | },
40 | },
41 | };
42 | factory.createRule(rule);
43 | expect(factory.getRule('test')).toBe(rule);
44 | expect(factory.getRule('test2')).toBe(rule);
45 | });
46 | test('ruleFactory should be able to populate a rule with sub-rules', () => {
47 | const subRule: Rule = {
48 | type: 'object',
49 | name: 'testSubrule',
50 | properties: {
51 | foo: {
52 | type: 'string',
53 | },
54 | },
55 | };
56 | const rule: Rule = {
57 | type: 'object',
58 | name: 'test',
59 | properties: {
60 | bar: {
61 | type: 'string',
62 | },
63 | },
64 | };
65 | const expectedRule: Rule = rule;
66 | expectedRule.properties['testSubrule'] = subRule;
67 | const responseObject = {
68 | bar: 'test',
69 | testSubrule: {
70 | foo: 'test',
71 | },
72 | };
73 | factory.createRule(subRule);
74 | const actualRule = factory.getSubRules(rule, responseObject);
75 | expect(actualRule).toEqual(expectedRule);
76 | });
77 | });
78 |
--------------------------------------------------------------------------------
/src/extractors/youtube-new/rules/continuations.const.ts:
--------------------------------------------------------------------------------
1 | import { Rule } from '../parser/types';
2 |
3 | export const CONTINUATIONS: Rule = {
4 | type: 'array',
5 | name: 'continuations',
6 | items: {
7 | name: 'continuationArray',
8 | type: 'object',
9 | properties: {
10 | nextContinuationData: {
11 | type: 'rule',
12 | rule: {
13 | type: 'object',
14 | name: 'continuation',
15 | properties: {
16 | continuation: {
17 | type: 'string',
18 | },
19 | },
20 | },
21 | },
22 | reloadContinuationData: {
23 | type: 'rule',
24 | rule: {
25 | type: 'object',
26 | name: 'continuation',
27 | properties: {
28 | continuation: {
29 | type: 'string',
30 | },
31 | },
32 | },
33 | },
34 | },
35 | },
36 | };
37 |
--------------------------------------------------------------------------------
/src/extractors/youtube-new/rules/index.ts:
--------------------------------------------------------------------------------
1 | //imports here
2 | import { objectRule, Rule } from '../parser/types';
3 | import { utilityErrors, ErrorMessages } from '@utils';
4 | import { CONTINUATIONS } from './continuations.const';
5 | //Put all imports into this array
6 |
7 | const rulesImport: Rule[] = [CONTINUATIONS];
8 |
9 | export default class RuleFactory {
10 | private rules: { [key: string]: Rule };
11 |
12 | /**
13 | * Create a new rule factory
14 | * @param importDefault If true, the default rules will be imported. Only recommended for testing.
15 | */
16 | constructor(importDefault = true) {
17 | this.rules = {};
18 | if (importDefault) rulesImport.forEach(rule => this.createRule(rule));
19 | }
20 |
21 | /**
22 | * Add a new rule to the factory. If the rule has aliases, they will be added as well.
23 | *
24 | * Not recommended for usage outside of testing unless you know what you are doing.
25 | * @param rule The rule to add
26 | */
27 | createRule(rule: Rule) {
28 | // check if the rule should be added
29 | if (rule.isDiscoverable === false) return;
30 | // find the name of the rule
31 | const name: string = rule.name;
32 | const names = [name, ...(rule.aliases || [])];
33 | names.forEach(alias => {
34 | this.namespaceCheck(alias);
35 | this.rules[alias] = rule;
36 | });
37 | }
38 |
39 | /**
40 | * Detects if a name is already in use. If it is, an error will be thrown.
41 | * @param name The name to check
42 | */
43 | private namespaceCheck(name: string) {
44 | if (this.rules[name]) {
45 | throw new utilityErrors.VueTubeExtractorError(ErrorMessages.nameConflict(name));
46 | }
47 | }
48 |
49 | /**
50 | * Retrieves a rule if it exists.
51 | * @param name The name of the rule to get
52 | * @returns The rule with the given name
53 | */
54 | getRule(name: string) {
55 | return this.rules[name];
56 | }
57 |
58 | /**
59 | * Loops through a given object and adds valid sub-rules to a given rule.
60 | * @param rule The rule to add sub-rules to
61 | * @param response The object to loop through
62 | */
63 | getSubRules(rule: objectRule, response: { [key: string]: any }) {
64 | const result = rule;
65 | for (const key in response) {
66 | const subRule = this.getRule(key);
67 | if (!(subRule && !rule.properties[key])) continue;
68 | result.properties[key] = subRule;
69 | }
70 | return result;
71 | }
72 | }
73 |
--------------------------------------------------------------------------------
/src/extractors/youtube-new/utils/constants.ts:
--------------------------------------------------------------------------------
1 | import { SearchDuration, SearchOrder, SearchType, SearchUploadDate, Features, SearchFeatures } from './types';
2 |
3 | export const PLATFORM = {
4 | MOBILE: 'MOBILE',
5 | DESKTOP: 'DESKTOP',
6 | } as const;
7 |
8 | export const DEVICE_TYPE = {
9 | DESKTOP_WEB: 'DESKTOP_WEB',
10 | MOBILE_WEB: 'MOBILE_WEB',
11 | MOBILE_APP: 'MOBILE_APP',
12 | } as const;
13 |
14 | export const CLIENT_FORM_FACTOR = {
15 | UNKNOWN: 'UNKNOWN_FORM_FACTOR',
16 | SMALL: 'SMALL_FORM_FACTOR',
17 | } as const;
18 |
19 | export const CLIENT_NAME = {
20 | ANDROID: 'ANDROID',
21 | IOS: 'IOS',
22 | WEB: 'WEB',
23 | MWEB: 'MWEB',
24 | } as const;
25 |
26 | export const OS_NAME = {
27 | ANDROID: 'Android',
28 | IOS: 'iOS',
29 | WINDOWS: 'Windows',
30 | MAC: 'Macintosh',
31 | LINUX: 'Linux',
32 | MWEB: 'MWeb',
33 | } as const;
34 |
35 | export const PARSE_TYPE = {
36 | VIDEO_DETAIL: 'videoDetail',
37 | HOMEPAGE: 'homePage',
38 | SEARCH_SUGGESTIONS: 'searchSuggestions',
39 | SEARCH_RESULT: 'searchResult',
40 | } as const;
41 |
42 | export const SEARCH_FEATURE = {
43 | LIVE: 'live',
44 | VIDEO4K: 'video4k',
45 | HD: 'hd',
46 | SUBTITLES: 'subtitles',
47 | CC: 'cc',
48 | VIDEO360: 'video360',
49 | VR180: 'vr180',
50 | VIDEO3D: 'video3d',
51 | HDR: 'hdr',
52 | LOCATION: 'location',
53 | PURCHASED: 'purchased',
54 | } as const;
55 |
56 | export const FEATURE = {
57 | LIVE: 'featuresLive',
58 | VIDEO4K: 'features4k',
59 | HD: 'featuresHd',
60 | SUBTITLES: 'featuresSubtitles',
61 | CC: 'featuresCreativeCommons',
62 | VIDEO360: 'features360',
63 | VR180: 'featuresVr180',
64 | VIDEO3D: 'features3d',
65 | HDR: 'featuresHdr',
66 | LOCATION: 'featuresLocation',
67 | PURCHASED: 'featuresPurchased',
68 | } as const;
69 |
70 | export const SEARCH_UPLOAD_DATE = {
71 | HOUR: 'hour',
72 | DAY: 'day',
73 | WEEK: 'week',
74 | MONTH: 'month',
75 | YEAR: 'year',
76 | ALL: 'all',
77 | } as const;
78 |
79 | export const SEARCH_ORDER = {
80 | RELEVANCE: 'relevance',
81 | VIEW_COUNT: 'viewCount',
82 | RATING: 'rating',
83 | UPLOAD_DATE: 'uploadDate',
84 | } as const;
85 |
86 | export const SEARCH_TYPE = {
87 | VIDEO: 'video',
88 | PLAYLIST: 'playlist',
89 | CHANNEL: 'channel',
90 | ALL: 'all',
91 | } as const;
92 |
93 | export const SEARCH_DURATION = {
94 | SHORT: 'short',
95 | MEDIUM: 'medium',
96 | LONG: 'long',
97 | ALL: 'all',
98 | } as const;
99 |
100 | export const SEARCH_DURATION_OPTIONS: { [key in SearchDuration]: number | null } = {
101 | [SEARCH_DURATION.ALL]: null,
102 | [SEARCH_DURATION.SHORT]: 1,
103 | [SEARCH_DURATION.LONG]: 2,
104 | [SEARCH_DURATION.MEDIUM]: 3,
105 | } as const;
106 |
107 | export const SEARCH_ORDER_OPTIONS: { [key in SearchOrder]: number | null } = {
108 | [SEARCH_ORDER.RELEVANCE]: 0,
109 | [SEARCH_ORDER.RATING]: 1,
110 | [SEARCH_ORDER.UPLOAD_DATE]: 2,
111 | [SEARCH_ORDER.VIEW_COUNT]: 3,
112 | } as const;
113 |
114 | export const SEARCH_TYPE_OPTIONS: { [key in SearchType]: number | null } = {
115 | [SEARCH_TYPE.ALL]: null,
116 | [SEARCH_TYPE.VIDEO]: 1,
117 | [SEARCH_TYPE.CHANNEL]: 2,
118 | [SEARCH_TYPE.PLAYLIST]: 3,
119 | } as const;
120 |
121 | export const SEARCH_UPLOAD_DATE_OPTIONS: { [key in SearchUploadDate]: number | null } = {
122 | [SEARCH_UPLOAD_DATE.ALL]: null,
123 | [SEARCH_UPLOAD_DATE.HOUR]: 1,
124 | [SEARCH_UPLOAD_DATE.DAY]: 2,
125 | [SEARCH_UPLOAD_DATE.WEEK]: 3,
126 | [SEARCH_UPLOAD_DATE.MONTH]: 4,
127 | [SEARCH_UPLOAD_DATE.YEAR]: 5,
128 | } as const;
129 |
130 | export const SEARCH_FEATURE_BY_FEATURE: { [key in Features]: SearchFeatures } = {
131 | [FEATURE.HD]: SEARCH_FEATURE.HD,
132 | [FEATURE.VIDEO4K]: SEARCH_FEATURE.VIDEO4K,
133 | [FEATURE.VR180]: SEARCH_FEATURE.VR180,
134 | [FEATURE.SUBTITLES]: SEARCH_FEATURE.SUBTITLES,
135 | [FEATURE.CC]: SEARCH_FEATURE.CC,
136 | [FEATURE.VIDEO360]: SEARCH_FEATURE.VIDEO360,
137 | [FEATURE.VIDEO3D]: SEARCH_FEATURE.VIDEO3D,
138 | [FEATURE.HDR]: SEARCH_FEATURE.HDR,
139 | [FEATURE.LOCATION]: SEARCH_FEATURE.LOCATION,
140 | [FEATURE.PURCHASED]: SEARCH_FEATURE.PURCHASED,
141 | [FEATURE.LIVE]: SEARCH_FEATURE.LIVE,
142 | } as const;
143 |
144 | export const FEATURE_BY_SEARCH_FEATURE: { [key in SearchFeatures]: Features } = {
145 | [SEARCH_FEATURE.HD]: FEATURE.HD,
146 | [SEARCH_FEATURE.VIDEO4K]: FEATURE.VIDEO4K,
147 | [SEARCH_FEATURE.VR180]: FEATURE.VR180,
148 | [SEARCH_FEATURE.SUBTITLES]: FEATURE.SUBTITLES,
149 | [SEARCH_FEATURE.CC]: FEATURE.CC,
150 | [SEARCH_FEATURE.VIDEO360]: FEATURE.VIDEO360,
151 | [SEARCH_FEATURE.VIDEO3D]: FEATURE.VIDEO3D,
152 | [SEARCH_FEATURE.HDR]: FEATURE.HDR,
153 | [SEARCH_FEATURE.LOCATION]: FEATURE.LOCATION,
154 | [SEARCH_FEATURE.PURCHASED]: FEATURE.PURCHASED,
155 | [SEARCH_FEATURE.LIVE]: FEATURE.LIVE,
156 | } as const;
157 |
158 | export const COMMENT_SORT_OPTIONS = {
159 | topComments: 0,
160 | newestFirst: 1,
161 | } as const;
162 |
163 | export const YT_CONSTANTS = {
164 | URL: {
165 | YT_URL: 'https://www.youtube.com',
166 | YT_MOBILE: 'https://m.youtube.com',
167 | YT_MUSIC_URL: 'https://music.youtube.com',
168 | YT_BASE_API: 'https://www.youtube.com/youtubei/v1',
169 | YT_SUGGESTION_API: 'https://suggestqueries.google.com/complete/search',
170 | },
171 | YTAPIVAL: {
172 | VERSION: '17.20',
173 | CLIENTNAME: CLIENT_NAME.ANDROID,
174 | VERSION_WEB: '2.20230206.06.00',
175 | CLIENT_WEB_Mobile: CLIENT_NAME.MWEB,
176 | CLIENT_WEB_Desktop: CLIENT_NAME.WEB,
177 | },
178 | } as const;
179 |
--------------------------------------------------------------------------------
/src/extractors/youtube-new/utils/types.ts:
--------------------------------------------------------------------------------
1 | // import { imageData as ImageData, audioFormat as AudioFormat, videoFormat as VideoFormat } from "@types";
2 |
3 | import {
4 | CLIENT_FORM_FACTOR,
5 | CLIENT_NAME,
6 | DEVICE_TYPE,
7 | FEATURE,
8 | OS_NAME,
9 | PARSE_TYPE,
10 | PLATFORM,
11 | SEARCH_DURATION,
12 | SEARCH_FEATURE,
13 | SEARCH_ORDER,
14 | SEARCH_TYPE,
15 | SEARCH_UPLOAD_DATE,
16 | } from './constants';
17 |
18 | // export {
19 | // video as Video,
20 | // videoCard as VideoCard,
21 | // channelCard as ChannelCard,
22 | // playlist as Playlist,
23 | // pageSegment as PageSegment,
24 | // genericPage as GenericPage,
25 | // pageSegmentTypes as PageSegmentTypes,
26 | // pageDivider as PageDivider,
27 | // shelfSegment as ShelfSegment,
28 | // searchResult as SearchResult,
29 | // searchSuggestion as SearchSuggestion,
30 | // pageElements as PageElements,
31 | // thumbnail as Thumbnail,
32 | // } from "@types";
33 |
34 | // export interface PlayerResponse {
35 | // playabilityStatus: {
36 | // status: string;
37 | // playableInEmbed?: boolean;
38 | // reason?: string;
39 | // contextParams: string;
40 | // [x: string | number | symbol]: unknown;
41 | // };
42 | // streamingData: {
43 | // expiresInSeconds: number;
44 | // formats: Array;
45 | // adaptiveFormats: Array;
46 | // };
47 | // playbackTracking: {
48 | // videostatsPlaybackUrl: { baseUrl: string };
49 | // videostatsDelayplayUrl: { baseUrl: string };
50 | // videostatsWatchtimeUrl: { baseUrl: string };
51 | // [x: string | number | symbol]: unknown;
52 | // };
53 | // captions?: {
54 | // playerCaptionsTracklistRenderer: {
55 | // captionTracks: [
56 | // {
57 | // baseUrl: string;
58 | // name: object;
59 | // vssId: string;
60 | // languageCode: string;
61 | // isTranslatable: boolean;
62 | // }
63 | // ];
64 | // translationLanguages: [
65 | // {
66 | // languageCode: string;
67 | // languageName: object;
68 | // }
69 | // ];
70 | // audioTracks: Array;
71 | // defaultAudioTrackIndex: number;
72 | // };
73 | // };
74 | // videoDetails: {
75 | // videoId: string;
76 | // title: string;
77 | // lengthSeconds: number;
78 | // keywords: Array;
79 | // channelId: string;
80 | // isOwnerViewing: boolean;
81 | // shortDescription: string;
82 | // isCrawlable: boolean;
83 | // thumbnail: {
84 | // thumbnails: Array;
85 | // };
86 | // allowRatings: boolean;
87 | // viewCount: number;
88 | // author: string;
89 | // isPrivate: boolean;
90 | // isUnpluggedCorpus: boolean;
91 | // isLiveContent: boolean;
92 | // [x: string | number | symbol]: unknown;
93 | // };
94 | // microformat: {
95 | // playerMicroformatRenderer: {
96 | // lengthSeconds: number;
97 | // ownerProfileUrl: string;
98 | // externalChannelId: string;
99 | // isFamilySafe: boolean;
100 | // availableCountries: Array;
101 | // isUnlisted: boolean;
102 | // viewCount: number;
103 | // category?: string;
104 | // publishDate: string;
105 | // ownerChannelName: string;
106 | // uploadDate: string;
107 | // liveBroadcastDetails?: {
108 | // isLiveNow: boolean;
109 | // startTimestamp: string;
110 | // };
111 | // [x: string | number | symbol]: unknown;
112 | // };
113 | // };
114 | // }
115 |
116 | // interface RequesterConfig {
117 | // data?: any;
118 | // params?: any;
119 | // }
120 |
121 | // export interface BrowseConfig extends RequesterConfig {
122 | // isContinuation?: boolean;
123 | // }
124 |
125 | export interface SearchFilter {
126 | uploadDate: SearchUploadDate;
127 | order: SearchOrder;
128 | type: SearchType;
129 | duration: SearchDuration;
130 | features: SearchFeatures[];
131 | }
132 |
133 | // export type UserConfig = {
134 | // hl?: string;
135 | // gl?: string;
136 | // maxRetryCount?: number;
137 | // };
138 |
139 | export type ParseTypes = (typeof PARSE_TYPE)[keyof typeof PARSE_TYPE];
140 | export type ClientName = (typeof CLIENT_NAME)[keyof typeof CLIENT_NAME];
141 | export type ClientFormFactor = (typeof CLIENT_FORM_FACTOR)[keyof typeof CLIENT_FORM_FACTOR];
142 | export type OsName = (typeof OS_NAME)[keyof typeof OS_NAME];
143 | export type Platform = (typeof PLATFORM)[keyof typeof PLATFORM];
144 | export type SearchFeatures = (typeof SEARCH_FEATURE)[keyof typeof SEARCH_FEATURE];
145 | export type Features = (typeof FEATURE)[keyof typeof FEATURE];
146 |
147 | export type SearchUploadDate = (typeof SEARCH_UPLOAD_DATE)[keyof typeof SEARCH_UPLOAD_DATE];
148 | export type SearchOrder = (typeof SEARCH_ORDER)[keyof typeof SEARCH_ORDER];
149 | export type SearchType = (typeof SEARCH_TYPE)[keyof typeof SEARCH_TYPE];
150 | export type SearchDuration = (typeof SEARCH_DURATION)[keyof typeof SEARCH_DURATION];
151 |
152 | export type DeviceType = (typeof DEVICE_TYPE)[keyof typeof DEVICE_TYPE];
153 |
154 | export type Device = {
155 | baseURL: string;
156 | clientName: ClientName;
157 | clientVersion: string;
158 | clientFormFactor: ClientFormFactor;
159 | osName: string;
160 | platform: Platform;
161 | };
162 |
163 | export type YTClient = {
164 | gl: string;
165 | hl: string;
166 | deviceMake?: string;
167 | deviceModel?: string;
168 | userAgent: string;
169 | clientName: ClientName;
170 | clientVersion: string;
171 | osName: OsName;
172 | osVersion?: number;
173 | platform: Platform;
174 | remoteHost?: string;
175 | clientFormFactor?: string;
176 | visitorData?: string;
177 | };
178 |
179 | export type YTContext = {
180 | client: YTClient;
181 | user: { lockedSafetyMode: boolean };
182 | request: { useSsl: boolean };
183 | };
184 |
185 | export type HTTPMetadata = {
186 | apiKey: string;
187 | context: YTContext;
188 | };
189 |
190 | // // -- parsers -- //
191 |
192 | // export type YTVideoData = {
193 | // player: PlayerResponse;
194 | // next: object;
195 | // };
196 |
197 | // export interface Continuation {
198 | // nextContinuationData: string;
199 | // reloadContinuationData: string;
200 | // }
201 |
202 | // export interface YTPageParseResults {
203 | // page: T;
204 | // Continuation?: Continuation;
205 | // }
206 |
--------------------------------------------------------------------------------
/src/extractors/youtube/README.md:
--------------------------------------------------------------------------------
1 | # YouTube
2 |
3 | **⚠️This directory is deprecated and only exists for future reference. Please refer to the [youtube-new](../youtube-new)
4 | directory instead. ⚠️**
--------------------------------------------------------------------------------
/src/extractors/youtube/controllers/basicController.ts:
--------------------------------------------------------------------------------
1 | import type YouTube from "..";
2 | import type { browseConfig } from "../types";
3 | import { YouTubeHTTPOptions } from "../utils";
4 | import type { HttpOptions } from "@vuetubeapp/http";
5 | import { Http } from "@vuetubeapp/http";
6 | import { ytErrors } from "../utils";
7 |
8 | export default abstract class basicController {
9 | protected config: browseConfig;
10 | protected session: YouTube;
11 | protected baseHttpOptions: YouTubeHTTPOptions;
12 | protected androidHttpOptions: YouTubeHTTPOptions;
13 |
14 | constructor(session: YouTube, config: browseConfig = {}) {
15 | this.config = config;
16 | this.session = session;
17 | this.baseHttpOptions = session.getBaseHttpOptions("web");
18 | this.androidHttpOptions = session.getBaseHttpOptions("android");
19 | }
20 |
21 | protected abstract buildRequestOptions(): Array<{
22 | option: HttpOptions;
23 | key?: string;
24 | }>;
25 |
26 | public async getRequest(): Promise<
27 | HttpOptions | { [key: string]: HttpOptions }
28 | > {
29 | const options = this.buildRequestOptions();
30 | const responseObject: { [key: string]: HttpOptions } = {};
31 | for (const { option, key } of options) {
32 | const response = await Http.post(option);
33 | if (response.status === 200) {
34 | responseObject[key || option.url] = response.data;
35 | }
36 | }
37 | let toReturn;
38 | if (Object.keys(responseObject).length === 1) {
39 | toReturn = responseObject[Object.keys(responseObject)[0]];
40 | } else if (!responseObject) {
41 | throw new ytErrors.ParserError("No data returned");
42 | } else {
43 | toReturn = responseObject;
44 | }
45 | this.throwErrors(toReturn);
46 | return toReturn;
47 | }
48 |
49 | protected abstract parseRawResponse(data: { [key: string]: any }): object;
50 |
51 | protected abstract postProcessResponse(data: { [key: string]: any }): V;
52 |
53 | protected throwErrors(data: { [key: string]: any }): void {}
54 |
55 | public async parseData(data: { [key: string]: any }): Promise {
56 | const parsedData = this.parseRawResponse(data);
57 | const postProcessed = this.postProcessResponse(parsedData);
58 | return postProcessed;
59 | }
60 | }
61 |
--------------------------------------------------------------------------------
/src/extractors/youtube/controllers/commentController.ts:
--------------------------------------------------------------------------------
1 | import basicController from "./basicController";
2 | import type YouTube from "..";
3 | import type { genericPage, browseConfig } from "../types";
4 | import { ytErrors } from "../utils";
5 | import type { HttpOptions } from "@vuetubeapp/http";
6 | import type { commentOptions } from "../proto/types";
7 | import proto from "../proto";
8 |
9 |
10 |
11 | export default class commentController extends basicController {
12 | id: string;
13 | sortBy: commentOptions;
14 |
15 |
16 | constructor(
17 | id: string,
18 | sortBy: commentOptions,
19 | session: YouTube,
20 | config: browseConfig = {}
21 | ) {
22 | super(session, config);
23 | this.id = id;
24 | this.sortBy = sortBy;
25 | }
26 |
27 | protected buildRequestOptions(): Array<{
28 | option: HttpOptions;
29 | key?: string;
30 | }> {
31 | if (!this.id) {
32 | throw new ytErrors.ParserError("No id provided");
33 | }
34 | let continuation_id
35 | if (this.config.isContinuation) {
36 | continuation_id = this.id;
37 | }
38 | else {
39 | continuation_id = proto.encodeCommentOptions(this.id, this.sortBy);
40 | }
41 | const requestOptions = {
42 | data: {
43 | ...this.config.data,
44 | ...{ continuation: this.id },
45 | },
46 | params: { continuation_id },
47 | };
48 | return [
49 | {
50 | option: this.androidHttpOptions.getOptions(requestOptions, "/next"),
51 | key: "next",
52 | },
53 | ];
54 | }
55 |
56 | protected parseRawResponse(data: { [key: string]: any; }): object {
57 | return data; // TODO: parse comment data
58 | }
59 |
60 | protected postProcessResponse(data: { [key: string]: any; }): genericPage {
61 | return data as unknown as genericPage; // TODO: complete this
62 | }
63 |
64 |
65 |
66 | }
--------------------------------------------------------------------------------
/src/extractors/youtube/controllers/homePageController.ts:
--------------------------------------------------------------------------------
1 | import basicController from "./basicController";
2 | import type YouTube from "..";
3 | import type { genericPage, ytPageParseResults, browseConfig } from "../types";
4 | import type { HttpOptions } from "@vuetubeapp/http";
5 | import Parser from "../parsers";
6 | import { ytErrors } from "../utils";
7 |
8 | export default class homePageController extends basicController {
9 | id: string;
10 |
11 | constructor(
12 | id = "FEwhat_to_watch",
13 | session: YouTube,
14 | config: browseConfig = {}
15 | ) {
16 | super(session, config);
17 | this.id = id;
18 | }
19 |
20 | protected buildRequestOptions(): Array<{
21 | option: HttpOptions;
22 | key?: string;
23 | }> {
24 | const requestOptions = {
25 | data: {
26 | ...this.config.data,
27 | ...(this.config.isContinuation
28 | ? { continuation: this.id }
29 | : { browse_id: this.id }),
30 | },
31 | params: { ...this.config.params },
32 | };
33 | return [
34 | {
35 | option: this.androidHttpOptions.getOptions(requestOptions, "/browse"),
36 | key: "browse",
37 | },
38 | ];
39 | }
40 |
41 | protected parseRawResponse(data: {
42 | [key: string]: any;
43 | }): ytPageParseResults {
44 | return new Parser(
45 | "homePage",
46 | data
47 | ).parse() as ytPageParseResults;
48 | }
49 |
50 | protected postProcessResponse(
51 | data: ytPageParseResults
52 | ): genericPage {
53 | const continueMethod = async (): Promise => {
54 | if (!data.Continuation?.nextContinuationData) {
55 | throw new ytErrors.EndOfPageError("No continuation data");
56 | }
57 | const continueController = new homePageController(
58 | data.Continuation?.nextContinuationData,
59 | this.session,
60 | { ...this.config, ...{ isContinuation: true } }
61 | );
62 | const requestedData = await continueController.getRequest();
63 | return continueController.parseData(requestedData);
64 | };
65 | return {
66 | ...data.page,
67 | continue: continueMethod,
68 | };
69 | }
70 | }
71 |
--------------------------------------------------------------------------------
/src/extractors/youtube/controllers/searchController.ts:
--------------------------------------------------------------------------------
1 | import basicController from "./basicController";
2 | import type YouTube from "..";
3 | import type {
4 | searchResult,
5 | ytPageParseResults,
6 | browseConfig,
7 | searchFilter,
8 | } from "../types";
9 | import type { HttpOptions } from "@vuetubeapp/http";
10 | import Parser from "../parsers";
11 | import proto from "../proto";
12 | import { ytErrors } from "../utils";
13 |
14 | export default class searchPageController extends basicController {
15 | id: string;
16 | filters: Partial;
17 |
18 | constructor(
19 | id: string,
20 | filters: Partial = {},
21 | session: YouTube,
22 | config: browseConfig = {}
23 | ) {
24 | super(session, config);
25 | this.id = id;
26 | this.filters = filters;
27 | }
28 |
29 | protected buildRequestOptions(): Array<{
30 | option: HttpOptions;
31 | key?: string;
32 | }> {
33 | this.filters.features = [...new Set(this.filters.features)]; // enforce unique values
34 | const params = proto.encodeSearchFilter(this.filters);
35 | const requestOptions = {
36 | data: {
37 | ...this.config.data,
38 | ...{params},
39 | ...(this.config.isContinuation
40 | ? { continuation: this.id }
41 | : { query: this.id }),
42 | },
43 | params: { ...this.config.params },
44 | };
45 | return [
46 | {
47 | option: this.androidHttpOptions.getOptions(requestOptions, "/search"),
48 | key: "search",
49 | },
50 | ];
51 | }
52 |
53 | protected parseRawResponse(data: {
54 | [key: string]: any;
55 | }): ytPageParseResults {
56 | return new Parser(
57 | "searchResult",
58 | data
59 | ).parse() as ytPageParseResults;
60 | }
61 |
62 | protected postProcessResponse(
63 | data: ytPageParseResults
64 | ): searchResult {
65 | const continueMethod = async (): Promise => {
66 | if (!data.Continuation?.nextContinuationData) {
67 | throw new ytErrors.EndOfPageError("No continuation data");
68 | }
69 | const continueController = new searchPageController(
70 | data.Continuation?.nextContinuationData,
71 | this.filters,
72 | this.session,
73 | { ...this.config, ...{ isContinuation: true } }
74 | );
75 | const requestedData = await continueController.getRequest();
76 | return continueController.parseData(requestedData);
77 | };
78 | return {
79 | ...data.page,
80 | continue: continueMethod,
81 | };
82 | }
83 | }
84 |
--------------------------------------------------------------------------------
/src/extractors/youtube/controllers/videoPageController.ts:
--------------------------------------------------------------------------------
1 | import basicController from "./basicController";
2 | import type YouTube from "..";
3 | import type { browseConfig, video } from "../types";
4 | import type { HttpOptions } from "@vuetubeapp/http";
5 | import Parser from "../parsers";
6 | import { ytErrors } from "../utils";
7 |
8 | export default class videoPageController extends basicController {
9 | id: string;
10 |
11 | constructor(id: string, session: YouTube, config: browseConfig = {}) {
12 | super(session, config);
13 | this.id = id;
14 | }
15 |
16 | protected buildRequestOptions(): Array<{
17 | option: HttpOptions;
18 | key?: string;
19 | }> {
20 | const requestOptions = {
21 | data: {
22 | ...this.config.data,
23 | ...{
24 | videoId: this.id,
25 | },
26 | },
27 | params: { ...this.config.params },
28 | };
29 | return [
30 | {
31 | option: this.baseHttpOptions.getOptions(requestOptions, "/player"),
32 | key: "player",
33 | },
34 | // {
35 | // option: this.androidHttpOptions.getOptions(requestOptions, "/next"),
36 | // key: "next",
37 | // },
38 | ];
39 | }
40 |
41 | protected parseRawResponse(data: { [key: string]: any }): video {
42 | return new Parser("videoDetail", data).parse() as video;
43 | }
44 |
45 | protected postProcessResponse(data: video): video {
46 | return {
47 | ...data,
48 | };
49 | }
50 |
51 | protected throwErrors(videoInfo: { [key: string]: any }): void {
52 | if (videoInfo.playabilityStatus.status == "ERROR") {
53 | throw new ytErrors.VideoNotFoundError(
54 | this.id,
55 | videoInfo.playabilityStatus.reason || "UNKNOWN",
56 | videoInfo
57 | );
58 | } else if (videoInfo.playabilityStatus.status == "UNPLAYABLE") {
59 | throw new ytErrors.VideoNotAvailableError(
60 | this.id,
61 | videoInfo.playabilityStatus.reason || "UNKNOWN",
62 | videoInfo
63 | );
64 | } else if (videoInfo.playabilityStatus.status == "LOGIN_REQUIRED") {
65 | throw new ytErrors.LoginRequiredError(
66 | videoInfo.playabilityStatus.reason || "UNKNOWN",
67 | videoInfo
68 | );
69 | }
70 | }
71 | }
72 |
--------------------------------------------------------------------------------
/src/extractors/youtube/core/initializer.ts:
--------------------------------------------------------------------------------
1 | import {Http, HttpResponse} from "@vuetubeapp/http";
2 | import {YouTubeHTTPOptions, ytConstants, ytErrors, YtUtils} from "../utils";
3 | import {clientName, userConfig, ytContext} from "../types";
4 |
5 | export default class initialization {
6 | private config: userConfig;
7 |
8 | private innertubeKey: string;
9 | private context: ytContext;
10 | private androidContext: ytContext;
11 |
12 | private baseHttpOptions: YouTubeHTTPOptions;
13 | private androidHttpOptions: YouTubeHTTPOptions;
14 | private INNERTUBE_CONTEXT: object;
15 |
16 | /**
17 | * ```typescript
18 | * const initial = new initialization(config).buildAsync();
19 | * ```
20 | * @param {userConfig} config
21 | */
22 | constructor(config: userConfig) {
23 | this.config = config;
24 | }
25 |
26 | /**
27 | * Must be run before any other methods. Changes readiness to true on success.
28 | *
29 | * @returns {Promise}
30 | */
31 | async buildAsync(): Promise {
32 | let data: Array = await this.getDefaultConfig();
33 | data = data[0][2];
34 | this.innertubeKey = data[1];
35 | this.INNERTUBE_CONTEXT = {
36 | gl: this.config.gl || data[0][0][1],
37 | hl: this.config.hl || data[0][0][0],
38 | clientVersion: data[0][0][16],
39 | clientName: ytConstants.YTAPIVAL.CLIENT_WEB_Mobile as clientName,
40 | remoteHost: data[0][0][3],
41 | visitorData: data[6],
42 | };
43 |
44 | this.context = this.buildContext();
45 | this.androidContext = this.buildAndroidContext();
46 |
47 | this.buildBaseHttpOptions();
48 |
49 | return this;
50 | }
51 |
52 | /**
53 | * Returns context information for the YouTube extractor.
54 | * @returns {ytContext}
55 | */
56 | private buildContext(): ytContext {
57 | const userAgent = YtUtils.randomMobileUserAgent();
58 | return {
59 | client: {
60 | ...{
61 | gl: "US",
62 | hl: "en",
63 | deviceMake: userAgent.vendor,
64 | deviceModel: userAgent.platform,
65 | userAgent: userAgent.userAgent,
66 | clientName: ytConstants.YTAPIVAL.CLIENT_WEB_Mobile as clientName,
67 | clientVersion: ytConstants.YTAPIVAL.VERSION_WEB,
68 | osName: "Android",
69 | platform: "MOBILE",
70 | },
71 | ...this.INNERTUBE_CONTEXT,
72 | },
73 | user: {lockedSafetyMode: false},
74 | request: {useSsl: true},
75 | };
76 | }
77 |
78 | private buildAndroidContext(): ytContext {
79 | const androidContext = JSON.parse(JSON.stringify(this.context));
80 | androidContext.client.clientName = ytConstants.YTAPIVAL
81 | .CLIENTNAME as clientName;
82 | androidContext.client.clientVersion = ytConstants.YTAPIVAL.VERSION;
83 | androidContext.client.clientFormFactor = "SMALL_FORM_FACTOR";
84 | return androidContext;
85 | }
86 |
87 | /**
88 | * builds and sets the base HTTP options for the YouTube extractor.
89 | * @returns {void}
90 | */
91 | private buildBaseHttpOptions(): void {
92 | this.baseHttpOptions = new YouTubeHTTPOptions({
93 | apiKey: this.innertubeKey,
94 | context: this.context,
95 | });
96 | this.androidHttpOptions = new YouTubeHTTPOptions({
97 | apiKey: this.innertubeKey,
98 | context: this.androidContext,
99 | });
100 | }
101 |
102 | /**
103 | * Fetches the innertube key from the YouTube API.
104 | * @returns {Promise}
105 | */
106 | private async getDefaultConfig(): Promise> {
107 | const response: void | HttpResponse = await Http.get({
108 | url: `${ytConstants.URL.YT_MOBILE}/sw.js_data`,
109 | responseType: "text",
110 | }).catch((err) => {
111 | if (typeof err === "string") {
112 | throw new ytErrors.InitializationError(err.toUpperCase());
113 | } else if (err instanceof Error) {
114 | throw new ytErrors.InitializationError(err.message);
115 | }
116 | });
117 | if (!response || !response.data) {
118 | throw new ytErrors.InitializationError("No response from YouTube API");
119 | }
120 | return JSON.parse(response.data.replace(")]}'", ""));
121 | }
122 |
123 | getBaseHttpOptions(
124 | optionType: "android" | "web" = "web"
125 | ): YouTubeHTTPOptions {
126 | if (optionType === "android") return this.androidHttpOptions;
127 | return this.baseHttpOptions;
128 | }
129 | }
130 |
--------------------------------------------------------------------------------
/src/extractors/youtube/core/requester.ts:
--------------------------------------------------------------------------------
1 | import { YouTubeHTTPOptions } from "../utils";
2 | import YouTube from "..";
3 | import { Http, HttpResponse, HttpOptions } from "@vuetubeapp/http";
4 | import { ytVideoData, browseConfig, searchFilter } from "../types";
5 | import proto from "../proto";
6 | import constants from "../utils/constants";
7 |
8 | export default class youtubeRequester {
9 | private session: YouTube;
10 | private baseHttpOptions: YouTubeHTTPOptions;
11 | private androidHttpOptions: YouTubeHTTPOptions;
12 |
13 | /**
14 | * the requester for all YouTube requests. Do not use directly.
15 | * @param session The session to use
16 | */
17 | constructor(session: YouTube) {
18 | this.session = session;
19 | this.baseHttpOptions = session.getBaseHttpOptions("web");
20 | this.androidHttpOptions = session.getBaseHttpOptions("android");
21 | }
22 |
23 | /**
24 | * retrieves a video from YouTube.
25 | *
26 | * @param {string} videoId The video id to get the details for
27 | * @param {boolean} includeRecommendations Whether to include recommendations or not
28 | *
29 | * @returns {Promise}
30 | */
31 | async getVideoInfo(videoId: string): Promise {
32 | const httpOptionsPlayer = this.baseHttpOptions.getOptions(
33 | { data: { videoId: videoId } },
34 | "/player"
35 | );
36 | const responsePlayer = await Http.post(httpOptionsPlayer);
37 | const responseNext = await this.getNext({ videoId: videoId });
38 | return { player: responsePlayer.data, next: responseNext };
39 | }
40 |
41 | /**
42 | *
43 | * Calls the YouTube browse endpoint.
44 | * This is most commonly used for getting
45 | * the home page.
46 | *
47 | * @param {string} id The id to call the browse endpoint with
48 | * @param {Record} args The arguments to pass to the browse endpoint
49 | * @returns {Promise} The response from the browse endpoint
50 | */
51 | async browse(id: string, args: browseConfig = {}): Promise {
52 | const data: HttpOptions["data"] = args.data || {};
53 | (args.isContinuation && (data.continuation = id)) || (data.browseId = id);
54 | const httpOptionsNext = this.androidHttpOptions.getOptions(
55 | { data: data || {}, params: args.params || {} },
56 | `/browse`
57 | );
58 | const response = await Http.post(httpOptionsNext);
59 | return response.data;
60 | }
61 |
62 | /**
63 | * Calls YouTube's next endpoint. This is mostly used for pagination, however it can also be used to get page data
64 | *
65 | * @param {Record} args The arguments to pass to the next endpoint
66 | *
67 | * @returns {Promise} The response from the next endpoint
68 | *
69 | */
70 | async getNext(args: Record): Promise {
71 | const httpOptionsNext = this.androidHttpOptions.getOptions(
72 | { data: args },
73 | "/next"
74 | );
75 | const response = await Http.post(httpOptionsNext);
76 | return response.data;
77 | }
78 |
79 | /**
80 | * Calls the YouTube search endpoint.
81 | * @param {string} query - The query to search for
82 | * @param options
83 | * @param continuation
84 | *
85 | * @returns {Promise} The response from the search endpoint
86 | */
87 | async search(
88 | query: string,
89 | options: { filters: Partial; isContinuation: boolean },
90 | continuation?: string
91 | ): Promise {
92 | options.filters.features = [...new Set(options.filters.features)]; // enforce unique values
93 | const params = proto.encodeSearchFilter(options.filters);
94 | const data: HttpOptions["data"] = { params };
95 | (options.isContinuation && (data.continuation = continuation)) ||
96 | (data.query = query);
97 | const httpOptions = this.androidHttpOptions.getOptions({ data }, "/search");
98 | const response = await Http.post(httpOptions);
99 | return response.data;
100 | }
101 |
102 | /**
103 | * Gets YouTube suggestions for a query
104 | * @param {string} query - The query to get suggestions for
105 | * @returns {Promise} The response from the suggestions endpoint
106 | */
107 | async getSuggestions(query: string) {
108 | const params = this.baseHttpOptions.getOptions({}, "/search");
109 | const hl = params.data.context.client.hl;
110 | const gl = params.data.context.client.gl;
111 | const response = await Http.get({
112 | url: constants.URL.YT_SUGGESTION_API,
113 | params: {
114 | q: query,
115 | ds: "yt",
116 | client: "youtube",
117 | xssi: "t",
118 | oe: "utf8",
119 | gl: gl,
120 | hl: hl,
121 | },
122 | responseType: "text",
123 | });
124 | return response.data;
125 | }
126 | }
127 |
--------------------------------------------------------------------------------
/src/extractors/youtube/index.ts:
--------------------------------------------------------------------------------
1 | import initialization from "./core/initializer";
2 | import {genericPage, searchResult, searchSuggestion, userConfig, video,} from "./types";
3 | import {YouTubeHTTPOptions, ytErrors} from "./utils";
4 | import youtubeRequester from "./core/requester";
5 | import Parser from "./parsers";
6 | import homePageController from "./controllers/homePageController";
7 | import searchPageController from "./controllers/searchController";
8 | import videoPageController from "./controllers/videoPageController";
9 |
10 | export default class YouTube {
11 | private config: userConfig;
12 | private baseData: initialization;
13 | private requester: youtubeRequester;
14 | private ready = false;
15 |
16 | /**
17 | * extractor for YouTube
18 | * ```typescript
19 | * import { YouTube } from 'vuetube-extractor';
20 | * const yt = await new YouTube().init();
21 | * ```
22 | *
23 | * @param {userConfig} [config] The config parameter is optional.
24 | */
25 | constructor(config?: userConfig) {
26 | this.config = config || {};
27 | }
28 |
29 | /**
30 | * Initializes the extractor. This is required before any other method can be called.
31 | * @returns {Promise}
32 | */
33 | async init(): Promise {
34 | let initError;
35 | let retry_count = 0;
36 | while (retry_count <= (this.config.maxRetryCount || 5)) {
37 | try {
38 | this.baseData = await new initialization(this.config).buildAsync();
39 | this.requester = new youtubeRequester(this);
40 | this.ready = true;
41 | return this;
42 | } catch (err) {
43 | retry_count++;
44 | console.warn("Failed, retrying...", retry_count);
45 | console.warn(err);
46 | initError = err;
47 | }
48 | }
49 | let errorDetails = { info: "maxRetryCount reached" };
50 | let errorMsg = "UNKNOWN";
51 | if (initError instanceof Error) {
52 | errorMsg = initError.message;
53 | if (
54 | initError instanceof ytErrors.YoutubeError &&
55 | initError.details instanceof Object
56 | ) {
57 | errorDetails = { ...errorDetails, ...initError.details };
58 | }
59 | }
60 | throw new ytErrors.InitializationError(errorMsg, errorDetails, retry_count);
61 | }
62 |
63 | /**
64 | *
65 | * Retrieves the video details for a given video id.
66 | *
67 | * @param {string} videoId video id to get the details for
68 | * @param {boolean} includeRecommendations whether to include recommendations or not
69 | * @returns {Promise}
70 | */
71 | async getVideoDetails(
72 | videoId: string,
73 | includeRecommendations = true
74 | ): Promise {
75 | this.checkReady();
76 | const videoController = new videoPageController(videoId, this);
77 | const videoPage = await videoController.getRequest();
78 | return videoController.parseData(videoPage);
79 | }
80 |
81 | /**
82 | * Retrieves home page data.
83 | * @returns {Promise}
84 | */
85 | async getHomePage(): Promise {
86 | this.checkReady();
87 | const homeController = new homePageController(undefined, this, {
88 | isContinuation: false,
89 | });
90 | const homePage = await homeController.getRequest();
91 | return homeController.parseData(homePage);
92 | }
93 |
94 | /**
95 | * Searches youtube for a given query.
96 | *
97 | * @param {string} query - The query to search for.
98 | * @param {object} filters - The filters to apply to the search.
99 | * @returns {Promise}
100 | */
101 | async getSearchPage(
102 | query: string,
103 | filters: object = []
104 | ): Promise {
105 | this.checkReady();
106 | const searchController = new searchPageController(query, filters, this, {
107 | isContinuation: false,
108 | });
109 | const searchPage = await searchController.getRequest();
110 | return searchController.parseData(searchPage);
111 | }
112 |
113 | /**
114 | * Gets search suggestions for a given query.
115 | * @param {string} query - The query to search for.
116 | * @returns {Promise<{query: string, results:Array}>}
117 | */
118 | async getSearchSuggestions(query: string): Promise {
119 | this.checkReady();
120 | const searchResponse = await this.requester.getSuggestions(query);
121 | return new Parser(
122 | "searchSuggestions",
123 | JSON.parse(searchResponse.replace(")]}'", ""))
124 | ).parse() as searchSuggestion;
125 | }
126 |
127 | private checkReady() {
128 | if (!this.ready) {
129 | throw new ytErrors.ExtractorNotReadyError(
130 | "Extractor is not ready. Please call init() first."
131 | );
132 | }
133 | }
134 |
135 | getBaseHttpOptions(optionType?: "android" | "web"): YouTubeHTTPOptions {
136 | return this.baseData.getBaseHttpOptions(optionType);
137 | }
138 | }
139 |
--------------------------------------------------------------------------------
/src/extractors/youtube/parsers/__tests__/parserTest.test.ts:
--------------------------------------------------------------------------------
1 | import parser from "../index";
2 | import { parseTypes } from "../../types";
3 |
4 | describe("parser error handling", () => {
5 | const cases: Array<{ parser: parseTypes; errorCode: string }> = [
6 | { parser: "videoDetail", errorCode: "No player data found" },
7 | { parser: "homePage", errorCode: "No section contents" },
8 | { parser: "bad" as any, errorCode: "Parser not found" },
9 | ];
10 | describe.each(cases)("$parser", (caseData) => {
11 | const parserInstance = new parser(caseData.parser, {});
12 | test("if parser throws error on bad input", () => {
13 | expect(() => {
14 | parserInstance.parse();
15 | }).toThrowError(caseData.errorCode);
16 | });
17 | });
18 | });
19 |
20 | describe("parser handling", () => {
21 | test("searchSuggestions", () => {
22 | const testData = {
23 | data: [
24 | "LTT",
25 | [
26 | ["ltt", 0, [512, 433]],
27 | ["ltt intel extreme upgrade", 0, [512]],
28 | ["ltt game nerf war", 0, [512, 433]],
29 | ["ltt april fools", 0, [512, 433]],
30 | ["ltt keymouse", 0, [512]],
31 | ],
32 | { k: 1, q: "PNWEva_a9eulPp0wIyXVaUqAThs" },
33 | ],
34 | expected: {
35 | query: "LTT",
36 | results: [
37 | "ltt",
38 | "ltt intel extreme upgrade",
39 | "ltt game nerf war",
40 | "ltt april fools",
41 | "ltt keymouse",
42 | ],
43 | },
44 | };
45 | const searchSuggestionsParse = new parser(
46 | "searchSuggestions",
47 | testData.data
48 | ).parse();
49 | expect(searchSuggestionsParse).toStrictEqual(testData.expected);
50 | });
51 | });
52 |
--------------------------------------------------------------------------------
/src/extractors/youtube/parsers/abstractParser.ts:
--------------------------------------------------------------------------------
1 | /**
2 | * The abstract parser class. Extend this class to create a new parser.
3 | */
4 | export default abstract class YouTubeParser {
5 | abstract parse(data: any): object | undefined;
6 | }
7 |
8 | export interface parsersList {
9 | [key: string]: Pick &
10 | (new () => YouTubeParser);
11 | }
12 |
--------------------------------------------------------------------------------
/src/extractors/youtube/parsers/index.ts:
--------------------------------------------------------------------------------
1 | import {abstractParser, homePage, parsersList, searchPage, searchSuggestions, ytVideo,} from "./strats";
2 | import {ytErrors} from "../utils";
3 | import {parseTypes, playerResponse} from "../types";
4 |
5 | export default class Parser {
6 | private readonly toParse: parseTypes;
7 | private readonly data: playerResponse | object;
8 |
9 | constructor(toParse: parseTypes, data: object) {
10 | this.toParse = toParse;
11 | this.data = data;
12 | }
13 |
14 | parse(): object | void {
15 | try {
16 | const parser = this.getParser();
17 | if (!parser)
18 | throw new ytErrors.ParserError("Parser not found", {
19 | toParse: this.toParse,
20 | });
21 | return parser.parse(this.data);
22 | } catch (error) {
23 | this.handleError(error);
24 | }
25 | }
26 |
27 | private handleError(error: unknown): void {
28 | if (error instanceof ytErrors.YoutubeError) {
29 | throw error;
30 | } else if (error instanceof Error) {
31 | throw new ytErrors.ParserError(error.message, {
32 | toParse: this.toParse,
33 | });
34 | } else {
35 | throw new ytErrors.ParserError("Unknown error", {
36 | toParse: this.toParse,
37 | });
38 | }
39 | }
40 |
41 | getParser = (): abstractParser | void => {
42 | const parserStrats: parsersList = {
43 | homePage: homePage,
44 | videoDetail: ytVideo,
45 | searchSuggestions,
46 | searchResult: searchPage,
47 | };
48 | const parser = parserStrats[this.toParse];
49 | if (!parser) {
50 | return undefined;
51 | }
52 | return new parser();
53 | };
54 | }
55 |
--------------------------------------------------------------------------------
/src/extractors/youtube/parsers/mixins/section.ts:
--------------------------------------------------------------------------------
1 | import { MixinConstructor } from "@types";
2 | import { pageSegment, pageElements } from "@types";
3 | import {
4 | toplevelIdentifierFinder,
5 | elementRendererIdentifierFinder,
6 | itemSectionRendererFinder,
7 | shelfRenderer,
8 | shelfSegmentMaker,
9 | pageSegmentMaker,
10 | } from "../stratIdentifiers";
11 |
12 | export default function section(Base: TBase) {
13 | return class extends Base {
14 | _getSectionElements(itemSection: {
15 | [key: string]: any;
16 | }): pageSegment | void {
17 | const segments = [];
18 | const sectionList = this._findSectionList(itemSection);
19 | for (const itemElement of sectionList) {
20 | const identifier = this._findIdentifiers(itemElement);
21 | const parsedElement = this._callParsers(
22 | identifier,
23 | itemElement
24 | ) as unknown as pageElements;
25 | if (parsedElement) segments.push(parsedElement);
26 | }
27 | return this._makePageSegment(segments, itemSection);
28 | }
29 |
30 | _callParsers(
31 | identifier: string,
32 | itemElement: {
33 | [key: string]: any;
34 | }
35 | ): void { }
36 |
37 | _makePageSegment(
38 | itemElement: Array,
39 | contextElement: { [key: string]: any }
40 | ): pageSegment | void {
41 | const makers = [new pageSegmentMaker(), new shelfSegmentMaker()];
42 | for (const maker of makers) {
43 | const segment = maker.make(itemElement, contextElement);
44 | if (segment) return segment;
45 | }
46 | return undefined;
47 | }
48 |
49 | _findIdentifiers(itemElement: { [key: string]: any }): string {
50 | const finders = [
51 | new elementRendererIdentifierFinder(),
52 | new toplevelIdentifierFinder(),
53 | ];
54 | for (const finder of finders) {
55 | const identifier = finder.find(itemElement);
56 | if (identifier) return identifier;
57 | }
58 | return "";
59 | }
60 |
61 | _findSectionList(itemSection: {
62 | [key: string]: any;
63 | }): Array {
64 | const finders = [new itemSectionRendererFinder(), new shelfRenderer()];
65 | for (const finder of finders) {
66 | const sectionList = finder.find(itemSection);
67 | if (sectionList) return sectionList;
68 | }
69 | return [];
70 | }
71 | };
72 | }
--------------------------------------------------------------------------------
/src/extractors/youtube/parsers/modelParsers/CellDividerParser.ts:
--------------------------------------------------------------------------------
1 | import abstractParser from "../abstractParser";
2 | import {pageDivider} from "@types";
3 |
4 | export default class VideoContextParser extends abstractParser {
5 | parse(data: { [key: string]: any }): pageDivider {
6 | return {
7 | type: "divider"
8 | };
9 | }
10 | }
11 |
--------------------------------------------------------------------------------
/src/extractors/youtube/parsers/modelParsers/ChannelRendererParser.ts:
--------------------------------------------------------------------------------
1 | import abstractParser from "../abstractParser";
2 | import {channelCard} from "@types";
3 | import {YtUtils} from "../../utils";
4 |
5 | export default class VideoContextParser extends abstractParser {
6 | parse(data: { [key: string]: any }): channelCard {
7 | data = data.compactChannelRenderer
8 | return {
9 | channelId: data.channelId,
10 | channelName: YtUtils.getStringFromRuns(data.displayName?.runs),
11 | videoCountText: YtUtils.getStringFromRuns(data.videoCountText?.runs),
12 | thumbnail: data.thumbnail?.thumbnails,
13 | banner: data.tvBanner?.thumbnails,
14 | navigationEndpoint: {
15 | browseId: data.navigationEndpoint?.browseEndpoint?.browseId,
16 | canonicalBaseUrl: data.navigationEndpoint?.browseEndpoint?.canonicalBaseUrl,
17 | },
18 | subscriberCountText: YtUtils.getStringFromRuns(data.subscriberCountText.runs),
19 | type: "channel"
20 | };
21 | }
22 | }
23 |
--------------------------------------------------------------------------------
/src/extractors/youtube/parsers/modelParsers/Continuations.ts:
--------------------------------------------------------------------------------
1 | import abstractParser from "../abstractParser";
2 | import { continuation } from "../../types"
3 |
4 | export default class Continuations extends abstractParser {
5 | parse(data: [{ [key: string]: any }]): continuation {
6 | const nextContinuationData = data.find((item) => item.nextContinuationData)
7 | ?.nextContinuationData.continuation;
8 | const reloadContinuationData = data.find(
9 | (item) => item.reloadContinuationData
10 | )?.reloadContinuationData.continuation;
11 | return {
12 | nextContinuationData,
13 | reloadContinuationData,
14 | };
15 | }
16 | }
--------------------------------------------------------------------------------
/src/extractors/youtube/parsers/modelParsers/Thumbnail.ts:
--------------------------------------------------------------------------------
1 | import { thumbnail } from "@types";
2 |
3 | export default class Thumbnail implements thumbnail {
4 | private data: { [key: string]: any };
5 | private ignoreKeys: Array = [
6 | "isAndroid",
7 | "isVideoWithContext",
8 | "maxOverlayWidth",
9 | ];
10 | constructor(data: { [key: string]: any }) {
11 | this.data = data;
12 | Object.keys(data).forEach((key) => {
13 | if (this.ignoreKeys.includes(key)) {
14 | delete data[key];
15 | }
16 | });
17 | Object.assign(this, data);
18 | }
19 | get thumbnails() {
20 | return this.data.image.sources;
21 | }
22 | }
--------------------------------------------------------------------------------
/src/extractors/youtube/parsers/modelParsers/VideoContextParser.ts:
--------------------------------------------------------------------------------
1 | import abstractParser from "../abstractParser";
2 | import Thumbnail from "./Thumbnail";
3 | import { videoCard, playlist } from "@types";
4 | import { ytErrors } from "../../utils";
5 |
6 | export default class VideoContextParser extends abstractParser {
7 | protected data: { [key: string]: any };
8 | protected contextData: { [key: string]: any };
9 | protected metadata: { [key: string]: any };
10 | protected channelId: string;
11 | protected channelAvatar: { [key: string]: any };
12 | protected innertubeCommand: {[key: string]: any};
13 |
14 | parse(data: { [key: string]: any }): videoCard | playlist | undefined {
15 | this.data = data;
16 | this.getAliases();
17 | if (this.checkIfAd()) return undefined;
18 | if (this.isPlaylist()) {
19 | return this.parsePlaylist();
20 | } else {
21 | return this.parseVideo();
22 | }
23 | }
24 |
25 | private parseVideo(): videoCard {
26 | return {
27 | title: this.metadata.title,
28 | details: this.metadata.metadataDetails,
29 | thumbnails: new Thumbnail(this.contextData.videoData.thumbnail),
30 | videoId: this.contextData.onTap.innertubeCommand.watchEndpoint.videoId,
31 | channelData: {
32 | channelId: this.channelId,
33 | channelUrl: `/channel/${this.channelId}`,
34 | channelThumbnails: this.channelAvatar?.image.sources,
35 | },
36 | type: "video",
37 | };
38 | }
39 |
40 | private parsePlaylist(): playlist {
41 | return new PlaylistParser().parse(this.data);
42 | }
43 |
44 | private isPlaylist(): boolean {
45 | return !!this.metadata?.isPlaylistMix;
46 | }
47 |
48 | protected getAliases() {
49 | const componentType =
50 | this.data.elementRenderer.newElement.type.componentType.model;
51 | let videoWithContextModel;
52 |
53 | if (componentType.videoWithContextModel) {
54 | videoWithContextModel = componentType.videoWithContextModel;
55 | } else if (componentType.videoWithContextSlotsModel) {
56 | videoWithContextModel = componentType.videoWithContextSlotsModel;
57 | } else {
58 | throw new ytErrors.ParserError("No videoWithContextModel found");
59 | }
60 |
61 | this.contextData = videoWithContextModel.videoWithContextData;
62 |
63 | if (this.contextData?.videoData) {
64 | try {
65 | this.metadata = this.contextData?.videoData?.metadata
66 | } catch (err) {
67 | console.log(this.data.elementRenderer.newElement.type);
68 | throw new ytErrors.ParserError("No metadata found");
69 | }
70 | this.channelAvatar = (
71 | this.contextData.videoData.decoratedAvatar || this.contextData.videoData
72 | ).avatar;
73 |
74 | this.channelId =
75 | this.contextData.videoData.channelId ||
76 | this.channelAvatar?.endpoint?.innertubeCommand.browseEndpoint?.browseId;
77 |
78 | this.innertubeCommand = this.contextData.onTap.innertubeCommand;
79 | } else {
80 | this.metadata = videoWithContextModel.videoDisplayAd;
81 | }
82 | }
83 |
84 | protected checkIfAd(): boolean {
85 | return this.metadata?.adBadge;
86 | }
87 | }
88 | class PlaylistParser extends VideoContextParser {
89 | parse(data: { [key: string]: any }): playlist {
90 | this.data = data;
91 | this.getAliases();
92 | return {
93 | title: this.metadata.title,
94 | byline: this.metadata.byline,
95 | details: this.metadata.metadataDetails,
96 | playlistId:
97 | this.innertubeCommand?.browseEndpoint?.browseId?.slice(2) ||
98 | this.innertubeCommand?.watchEndpoint?.playlistId,
99 | videoId: this.innertubeCommand?.watchEndpoint?.videoId || undefined,
100 | thumbnails: new Thumbnail(this.contextData.videoData.thumbnail),
101 | type: "playlist",
102 | };
103 | }
104 | }
--------------------------------------------------------------------------------
/src/extractors/youtube/parsers/modelParsers/index.ts:
--------------------------------------------------------------------------------
1 | import type abstractParser from "../abstractParser";
2 | import type { parsersList } from "../abstractParser";
3 | import VideoContextParser from "./VideoContextParser";
4 | import compactChannelRenderer from "./ChannelRendererParser";
5 | import cellDividerModel from "./CellDividerParser";
6 | import continuations from "./Continuations";
7 |
8 | const parserStrats: parsersList = {
9 | videoWithContextModel: VideoContextParser,
10 | videoWithContextSlotsModel: VideoContextParser,
11 | compactChannelRenderer,
12 | cellDividerModel,
13 | continuations,
14 | };
15 |
16 | const parserFactory = (model: string): abstractParser | void => {
17 | const parser = parserStrats[model];
18 | if (!parser) {
19 | return undefined;
20 | }
21 | return new parser();
22 | };
23 |
24 | export default parserFactory;
25 |
--------------------------------------------------------------------------------
/src/extractors/youtube/parsers/pageParser.ts:
--------------------------------------------------------------------------------
1 | import modelParsers from "./modelParsers";
2 | import abstractParser from "./abstractParser";
3 | import type {pageElements} from "@types";
4 | import {applyMixins} from "@utils";
5 | import sectionMixin from "./mixins/section";
6 |
7 | const ytPageParserBase = applyMixins(abstractParser, sectionMixin);
8 | export default abstract class YoutubePageParsers extends ytPageParserBase {
9 | protected _callParsers(
10 | identifier: string,
11 | itemElement: unknown
12 | ): pageElements | false {
13 | const parser = modelParsers(identifier);
14 | if (!parser) return false;
15 | return parser.parse(itemElement) as pageElements;
16 | }
17 | }
18 |
--------------------------------------------------------------------------------
/src/extractors/youtube/parsers/stratIdentifiers.ts:
--------------------------------------------------------------------------------
1 | import { pageSegment, shelfSegment, pageElements } from "@types";
2 | import { YtUtils } from "../utils";
3 |
4 | interface sectionListFinder {
5 | find(itemSection: { [key: string]: any }): Array | void;
6 | }
7 |
8 | interface identityFinder {
9 | find(itemElement: { [key: string]: any }): string | void;
10 | }
11 |
12 | interface pageSegmentMaker {
13 | make(
14 | itemSection: { [key: string]: any },
15 | contextSection: { [key: string]: any }
16 | ): pageSegment | void;
17 | }
18 |
19 | // RENDERER IDENTIFIERS
20 |
21 | class elementRendererIdentifierFinder implements identityFinder {
22 | find(itemElement: { [key: string]: any }): string | void {
23 | const model = itemElement?.elementRenderer?.newElement?.type?.componentType?.model
24 | if (model) return Object.keys(model)[0]; else return undefined;
25 | }
26 | }
27 |
28 | class toplevelIdentifierFinder implements identityFinder {
29 | find(itemElement: { [key: string]: any }) {
30 | const identifier: string = Object.keys(itemElement)[0];
31 | return identifier;
32 | }
33 | }
34 |
35 | // SECTION LIST FINDERS
36 | class itemSectionRendererFinder implements sectionListFinder {
37 | find(itemSection: { [key: string]: any }) {
38 | return itemSection.itemSectionRenderer?.contents;
39 | }
40 | }
41 |
42 | class shelfRenderer implements sectionListFinder {
43 | find(itemSection: { [key: string]: any }) {
44 | return itemSection.shelfRenderer?.content?.verticalListRenderer?.items;
45 | }
46 | }
47 |
48 | // PAGE SEGMENT MAKERS
49 | class pageSegmentMaker implements pageSegmentMaker {
50 | make(
51 | itemSection: Array,
52 | contextSection: { [key: string]: any }
53 | ): pageSegment | void {
54 | if (new itemSectionRendererFinder().find(contextSection)) {
55 | return {
56 | type: "genericSegment",
57 | contents: itemSection,
58 | };
59 | }
60 | return undefined;
61 | }
62 | }
63 |
64 | class shelfSegmentMaker implements pageSegmentMaker {
65 | make(
66 | itemSection: Array,
67 | contextSection: { [key: string]: any }
68 | ): shelfSegment | void {
69 | if (new shelfRenderer().find(contextSection)) {
70 | return {
71 | type: "shelf",
72 | contents: itemSection,
73 | header:
74 | contextSection.shelfRenderer?.headerRenderer?.elementRenderer
75 | ?.newElement?.type?.componentType?.model
76 | ?.shelfHeaderModel?.shelfHeaderData?.title,
77 | collapseCount:
78 | contextSection.shelfRenderer?.content?.verticalListRenderer
79 | ?.collapsedItemCount || undefined,
80 | collapseText: YtUtils.getStringFromRuns(contextSection.shelfRenderer?.content?.verticalListRenderer?.collapsedStateButtonText
81 | ?.runs),
82 | };
83 | }
84 | return undefined;
85 | }
86 | }
87 |
88 | export {
89 | elementRendererIdentifierFinder,
90 | toplevelIdentifierFinder,
91 | itemSectionRendererFinder,
92 | shelfRenderer,
93 | pageSegmentMaker,
94 | shelfSegmentMaker,
95 | };
96 |
--------------------------------------------------------------------------------
/src/extractors/youtube/parsers/strats/homePage.ts:
--------------------------------------------------------------------------------
1 | import pageParser from "../pageParser";
2 | import { ytPageParseResults, continuation, genericPage } from "../../types";
3 | import { ytErrors } from "../../utils";
4 |
5 | export default class homePage extends pageParser {
6 | parse(data: { [key: string]: any }): ytPageParseResults {
7 | const sectionRoot =
8 | data.continuationContents?.sectionListContinuation ||
9 | data.contents?.singleColumnBrowseResultsRenderer?.tabs[0]?.tabRenderer
10 | ?.content.sectionListRenderer;
11 | const sectionContents = sectionRoot?.contents;
12 | if (!sectionContents)
13 | throw new ytErrors.ParserError("No section contents", {
14 | receivedObject: data,
15 | });
16 |
17 | const response: ytPageParseResults = {
18 | page: { segments: [], chips: [] },
19 | };
20 |
21 | for (const itemSection of sectionContents) {
22 | const nextSection = this._getSectionElements(itemSection);
23 | if (nextSection)
24 | response.page.segments = response.page.segments.concat(nextSection);
25 | }
26 |
27 | const continuationsData = this._callParsers(
28 | "continuations",
29 | sectionRoot.continuations
30 | ) as unknown as continuation;
31 |
32 | if (continuationsData) response.Continuation = continuationsData;
33 | return response;
34 | }
35 | }
36 |
--------------------------------------------------------------------------------
/src/extractors/youtube/parsers/strats/index.ts:
--------------------------------------------------------------------------------
1 | export { default as abstractParser } from "../abstractParser";
2 | export { parsersList } from "../abstractParser";
3 | export { default as ytVideo } from "./ytVideo";
4 | export { default as homePage } from "./homePage";
5 | export { default as searchSuggestions } from "./searchSuggestions";
6 | export { default as searchPage } from "./searchPage";
7 |
--------------------------------------------------------------------------------
/src/extractors/youtube/parsers/strats/searchPage.ts:
--------------------------------------------------------------------------------
1 | import pageParser from "../pageParser";
2 | import { ytPageParseResults, continuation, searchResult } from "../../types";
3 | import { ytErrors } from "../../utils";
4 |
5 | export default class homePage extends pageParser {
6 | parse(data: { [key: string]: any }): ytPageParseResults {
7 | const sectionRoot =
8 | data.continuationContents?.sectionListContinuation ||
9 | data.contents?.sectionListRenderer;
10 |
11 | const sectionContents = sectionRoot?.contents;
12 | if (!sectionContents)
13 | throw new ytErrors.ParserError("No section contents", {
14 | receivedObject: data,
15 | });
16 |
17 | const response: ytPageParseResults = {
18 | page: {
19 | segments: [],
20 | chips: [],
21 | searchRefinements: data.refinements,
22 | resultCount: data.estimatedResults,
23 | },
24 | };
25 |
26 | for (const itemSection of sectionContents) {
27 | const nextSection = super._getSectionElements(itemSection);
28 | if (nextSection) response.page.segments.push(nextSection);
29 | }
30 |
31 | const continuationsData = this._callParsers(
32 | "continuations",
33 | sectionRoot.continuations
34 | ) as unknown as continuation;
35 |
36 | if (continuationsData) response.Continuation = continuationsData;
37 |
38 | return response;
39 | }
40 | }
41 |
--------------------------------------------------------------------------------
/src/extractors/youtube/parsers/strats/searchSuggestions.ts:
--------------------------------------------------------------------------------
1 | import abstractParser from "../abstractParser";
2 | import { searchSuggestion } from "@types";
3 |
4 | export default class searchSuggestions extends abstractParser {
5 | parse(data: [string, Array<[any]>]): searchSuggestion {
6 | return {
7 | query: data[0],
8 | results: data[1].map((res) => res[0]) as Array,
9 | };
10 | }
11 | }
12 |
--------------------------------------------------------------------------------
/src/extractors/youtube/parsers/strats/ytVideo.ts:
--------------------------------------------------------------------------------
1 | import abstractParser from "../abstractParser";
2 | import {playerResponse, video} from "../../types";
3 | import {ytErrors} from "../../utils";
4 |
5 | /**
6 | * ```typescript
7 | * import {ytVideo} from './parsers';
8 | * const video = new ytVideo.parse(data);
9 | * ```
10 | */
11 | export default class ytVideo extends abstractParser {
12 | /**
13 | * Main parse function.
14 | * @param data The data to parse
15 | * @returns {video}
16 | */
17 | parse(data: playerResponse): video {
18 | this.checkValidity(data);
19 | // Play endpoint data
20 | const videoDetails = data.videoDetails;
21 | const microformat = data.microformat.playerMicroformatRenderer;
22 | // const playbackTracking = data.player.playbackTracking;
23 | // const captions = data.player.captions?.playerCaptionsTracklistRenderer;
24 | const streamingData = data.streamingData;
25 |
26 | // Next endpoint data
27 | // const engagementPanels = data.next.engagementPanels;
28 |
29 | return {
30 | id: videoDetails.videoId,
31 | title: videoDetails.title,
32 | descriptionText: videoDetails.shortDescription,
33 | thumbnails: videoDetails.thumbnail.thumbnails,
34 | metadata: {
35 | lengthSeconds: Number(videoDetails.lengthSeconds),
36 | views: videoDetails.viewCount,
37 | isLive: videoDetails.isLiveContent,
38 | isFamilyFriendly: microformat.isFamilySafe,
39 | isUnlisted: microformat.isUnlisted,
40 | isPrivate: videoDetails.isPrivate,
41 | category: microformat.category,
42 | publishedAt: microformat.publishDate,
43 | uploadedAt: microformat.uploadDate,
44 | tags: videoDetails.keywords,
45 | playbackEndpoints: streamingData.adaptiveFormats.concat(
46 | streamingData.formats
47 | ),
48 | },
49 | };
50 | }
51 |
52 | private checkValidity(data: playerResponse): void {
53 | if (Object.keys(data).length === 0) {
54 | throw new ytErrors.ParserError("No player data found", {
55 | received: data,
56 | });
57 | }
58 | }
59 | }
60 |
--------------------------------------------------------------------------------
/src/extractors/youtube/proto/__tests__/protoTest.test.ts:
--------------------------------------------------------------------------------
1 | import proto from "../index";
2 | import * as cases from "./testCases.json";
3 | import { searchFilter } from "../../types";
4 | import { commentOptions } from "../types";
5 |
6 | describe("protobuf parsing tests", () => {
7 | describe("if encodeVisitorData works", () => {
8 | test.each(cases.encodeVisitorData)(
9 | "%s and %s should be %s",
10 | (input1, input2, expectResult) => {
11 | const visitorData = proto.encodeVisitorData(
12 | input1 as string,
13 | input2 as number
14 | );
15 | expect(visitorData).toBe(expectResult);
16 | }
17 | );
18 | });
19 | describe("if encodeSearchFilter works", () => {
20 | test.each(
21 | cases.encodeSearchFilter as unknown as [Partial, string]
22 | )("%s should be %s ", (input1, expectResult) => {
23 | const searchFilter = proto.encodeSearchFilter(input1 as searchFilter);
24 | expect(searchFilter).toBe(expectResult);
25 | });
26 | });
27 | describe("if encodeCommentOptions works", () => {
28 | test.each(cases.encodeCommentOptions)(
29 | "video %s with options %s should be %s",
30 | (video_id, options, expectResult) => {
31 | const commentOptions = proto.encodeCommentOptions(
32 | video_id as string,
33 | (options as commentOptions) || undefined
34 | );
35 | expect(commentOptions).toBe(expectResult);
36 | }
37 | );
38 | });
39 | });
40 |
--------------------------------------------------------------------------------
/src/extractors/youtube/proto/__tests__/testCases.json:
--------------------------------------------------------------------------------
1 | {
2 | "encodeVisitorData": [
3 | ["abcd", 100, "CgRhYmNkKGQ%3D"],
4 | ["efgh", 200, "CgRlZmdoKMgB"],
5 | ["fgee-Eads1Q", 1640995200, "CgtmZ2VlLUVhZHMxUSiAs76OBg%3D%3D"]
6 | ],
7 | "encodeSearchFilter": [
8 | [
9 | {
10 | "uploadDate": "year",
11 | "order": "rating",
12 | "type": "video",
13 | "duration": "medium"
14 | },
15 | "CAESBggFEAEYAw%3D%3D"
16 | ],
17 | [
18 | {
19 | "type": "channel"
20 | },
21 | "EgIQAg%3D%3D"
22 | ],
23 | [
24 | {
25 | "features": ["hd", "subtitles"]
26 | },
27 | "EgQgASgB"
28 | ],
29 | [
30 | {
31 | "features": ["live", "location"]
32 | },
33 | "EgVAAbgBAQ%3D%3D"
34 | ],
35 | [
36 | {
37 | "uploadDate": "week",
38 | "type": "video",
39 | "duration": "long",
40 | "features": ["video3d"],
41 | "order": "uploadDate"
42 | },
43 | "CAISCAgDEAEYAjgB"
44 | ]
45 | ],
46 | "encodeCommentOptions": [
47 | [
48 | "abcd",
49 | { "sortBy": "newestFirst" },
50 | "EgYSBGFiY2QYBjIeIgoiBGFiY2QwAXgCQhBjb21tZW50cy1zZWN0aW9u"
51 | ],
52 | [
53 | "efgh",
54 | { "sortBy": "topComments" },
55 | "EgYSBGVmZ2gYBjIeIgoiBGVmZ2gwAHgCQhBjb21tZW50cy1zZWN0aW9u"
56 | ],
57 | [
58 | "dQw4w9WgXcQ",
59 | { "sortBy": "topComments" },
60 | "Eg0SC2RRdzR3OVdnWGNRGAYyJSIRIgtkUXc0dzlXZ1hjUTAAeAJCEGNvbW1lbnRzLXNlY3Rpb24%3D"
61 | ]
62 | ]
63 | }
64 |
--------------------------------------------------------------------------------
/src/extractors/youtube/proto/conversion.ts:
--------------------------------------------------------------------------------
1 | import { searchFilter, searchFeatures } from "../types";
2 |
3 | const duration: { [key in searchFilter["duration"]]: number | null } = {
4 | all: null,
5 | short: 1,
6 | long: 2,
7 | medium: 3,
8 | };
9 |
10 | const order: { [key in searchFilter["order"]]: number | null } = {
11 | relevance: 0,
12 | rating: 1,
13 | uploadDate: 2,
14 | viewCount: 3,
15 | };
16 |
17 | const type: { [key in searchFilter["type"]]: number | null } = {
18 | all: null,
19 | video: 1,
20 | channel: 2,
21 | playlist: 3,
22 | };
23 |
24 | const uploadDate: { [key in searchFilter["uploadDate"]]: number | null } = {
25 | all: null,
26 | hour: 1,
27 | day: 2,
28 | week: 3,
29 | month: 4,
30 | year: 5,
31 | };
32 |
33 | const features: { [key in searchFeatures]: string } = {
34 | hd: "featuresHd",
35 | video4k: "features4k",
36 | vr180: "featuresVr180",
37 | subtitles: "featuresSubtitles",
38 | cc: "featuresCreativeCommons",
39 | video360: "features360",
40 | video3d: "features3d",
41 | hdr: "featuresHdr",
42 | location: "featuresLocation",
43 | purchased: "featuresPurchased",
44 | live: "featuresLive",
45 | };
46 |
47 | const commentSortOptions = { topComments: 0, newestFirst: 1 };
48 |
49 | export { duration, order, type, uploadDate, commentSortOptions, features };
50 |
--------------------------------------------------------------------------------
/src/extractors/youtube/proto/index.ts:
--------------------------------------------------------------------------------
1 | import { Root, Type, loadSync } from "protobufjs";
2 | import path from "path";
3 | import type { searchFilter } from "../types";
4 | import {
5 | duration,
6 | order,
7 | type,
8 | uploadDate,
9 | commentSortOptions,
10 | features,
11 | } from "./conversion";
12 | import type { searchProto, protoFilters, commentOptions } from "./types";
13 | import { ytErrors } from "../utils";
14 |
15 | class Proto {
16 | private protoRoot: Root;
17 | constructor() {
18 | this.protoRoot = loadSync(path.join(__dirname, "youtube.proto"));
19 | }
20 |
21 | /**
22 | * encodes Visitor Data to protobuf format
23 | *
24 | * @param {string} id - visitor id. Should be an 11 character long random string
25 | * @param {number} timestamp - timestamp of initialization
26 | *
27 | * @returns {string} encoded visitor data
28 | */
29 | encodeVisitorData(id: string, timestamp: number): string {
30 | const visitorData: Type = this.protoRoot.lookupType("youtube.VisitorData");
31 | const buf: Uint8Array = visitorData.encode({ id, timestamp }).finish();
32 | return encodeURIComponent(Buffer.from(buf).toString("base64"));
33 | }
34 |
35 | /**
36 | * encodes search filter to protobuf format
37 | * @param {Partial} filters - search filters
38 | * @returns {string} encoded search filter
39 | */
40 | encodeSearchFilter(filters: Partial): string {
41 | if (filters?.uploadDate && filters?.type !== "video") {
42 | throw new ytErrors.SearchError(
43 | JSON.stringify(filters),
44 | "Search filter type must be video"
45 | );
46 | }
47 | const data: searchProto = filters ? { filters: {} } : { noFilter: 0 };
48 | if (data.filters) {
49 | data.filters = {
50 | ...data.filters,
51 | ...(filters.uploadDate && { param_0: uploadDate[filters.uploadDate] }),
52 | ...(filters.type && { param_1: type[filters.type] }),
53 | ...(filters.duration && { param_2: duration[filters.duration] }),
54 | };
55 | if (filters.order) data.sort = order[filters.order];
56 | if (filters.features) {
57 | for (const feature of filters.features) {
58 | data.filters[features[feature] as keyof protoFilters] = 1;
59 | }
60 | }
61 | }
62 | const searchFilter: Type = this.protoRoot.lookupType(
63 | "youtube.SearchFilter"
64 | );
65 | const buf: Uint8Array = searchFilter.encode(data).finish();
66 | return encodeURIComponent(Buffer.from(buf).toString("base64"));
67 | }
68 |
69 | /**
70 | * encodes comment options to protobuf format
71 | * @param {string} videoId - video id
72 | * @param options
73 | * @returns {string} encoded comment options
74 | */
75 | encodeCommentOptions(videoId: string, options: commentOptions = {}): string {
76 | const commentOptions: Type = this.protoRoot.lookupType(
77 | "youtube.CommentsSection"
78 | );
79 | const data = {
80 | ctx: { videoId },
81 | unkParam: 6,
82 | params: {
83 | opts: {
84 | videoId,
85 | sortBy: commentSortOptions[options.sortBy || "topComments"],
86 | type: options.type || 2,
87 | },
88 | target: "comments-section",
89 | },
90 | };
91 | const buf: Uint8Array = commentOptions.encode(data).finish();
92 | return encodeURIComponent(Buffer.from(buf).toString("base64"));
93 | }
94 | }
95 |
96 | const singletonProto = new Proto();
97 |
98 | Object.freeze(singletonProto);
99 |
100 | export default singletonProto;
101 |
--------------------------------------------------------------------------------
/src/extractors/youtube/proto/types.ts:
--------------------------------------------------------------------------------
1 | interface searchProto {
2 | sort?: number | null;
3 | noFilter?: number | null;
4 | noCorrection?: number | null;
5 | filters?: protoFilters;
6 | }
7 |
8 | interface protoFilters {
9 | param_0?: number | null;
10 | param_1?: number | null;
11 | param_2?: number | null;
12 | featuresHd?: number | null;
13 | features4k?: number | null;
14 | featuresVr180?: number | null;
15 | featuresSubtitles?: number | null;
16 | featuresCreativeCommons?: number | null;
17 | features360?: number | null;
18 | features3d?: number | null;
19 | featuresHdr?: number | null;
20 | featuresLocation?: number | null;
21 | featuresPurchased?: number | null;
22 | featuresLive?: number | null;
23 | }
24 |
25 | interface commentOptions {
26 | sortBy?: "topComments" | "newestFirst";
27 | type?: number;
28 | }
29 |
30 | export { searchProto, protoFilters, commentOptions };
31 |
--------------------------------------------------------------------------------
/src/extractors/youtube/proto/youtube.proto:
--------------------------------------------------------------------------------
1 | package youtube;
2 |
3 | message VisitorData {
4 | required string id = 1;
5 | required int32 timestamp = 5;
6 | }
7 |
8 | message SearchFilter {
9 | optional int32 sort = 1; // sort by
10 | optional int32 noCorrection = 8;
11 | optional int32 noFilter = 19;
12 |
13 | message Filters {
14 | optional int32 param_0 = 1; // upload date
15 | optional int32 param_1 = 2; // type
16 | optional int32 param_2 = 3; // duration
17 | // type filters
18 | optional int32 featuresHd = 4; // hd filter
19 | optional int32 featuresSubtitles = 5; // subtitles filter
20 | optional int32 featuresCreativeCommons = 6; // creative commons filter
21 | optional int32 features3d = 7; // 3d filter
22 | optional int32 featuresLive = 8; // live filter
23 | optional int32 featuresPurchased = 9; // purchased filter
24 | optional int32 features4k = 14; // 4k filter
25 | optional int32 features360 = 15; // 360 view filter
26 | optional int32 featuresLocation = 23; // location filter
27 | optional int32 featuresHdr = 25; // hdr filter
28 | optional int32 featuresVr180 = 26; // vr180 filter
29 | }
30 |
31 | optional Filters filters = 2;
32 | }
33 |
34 | message CommentsSection {
35 | message Context {
36 | required string videoId = 2;
37 | }
38 |
39 | message Options {
40 | required string videoId = 4;
41 | required int32 sortBy = 6;
42 | required int32 type = 15;
43 | }
44 |
45 | message UnkOpts {
46 | required int32 unkParam = 1;
47 | }
48 |
49 | message RepliesOptions {
50 | required string comment_id = 2;
51 | required UnkOpts unkopts = 4;
52 | optional string channel_id = 5;
53 | required string videoId = 6;
54 | required int32 unkParam1 = 8;
55 | required int32 unkParam2 = 9;
56 | }
57 |
58 | message Params {
59 |
60 | optional string unk_token = 1;
61 |
62 | optional Options opts = 4;
63 | optional RepliesOptions replies_opts = 3;
64 |
65 | optional int32 page = 5;
66 | required string target = 8;
67 | }
68 |
69 | required Context ctx = 2;
70 | required int32 unk_param = 3;
71 | required Params params = 6;
72 | }
--------------------------------------------------------------------------------
/src/extractors/youtube/types.ts:
--------------------------------------------------------------------------------
1 | import { imageData, audioFormat, videoFormat } from "@types";
2 |
3 | export {
4 | video,
5 | videoCard,
6 | channelCard,
7 | playlist,
8 | pageSegment,
9 | genericPage,
10 | pageSegmentTypes,
11 | pageDivider,
12 | shelfSegment,
13 | searchResult,
14 | searchSuggestion,
15 | pageElements,
16 | thumbnail,
17 | } from "@types";
18 |
19 | export interface playerResponse {
20 | playabilityStatus: {
21 | status: string;
22 | playableInEmbed?: boolean;
23 | reason?: string;
24 | contextParams: string;
25 | [x: string | number | symbol]: unknown;
26 | };
27 | streamingData: {
28 | expiresInSeconds: number;
29 | formats: Array;
30 | adaptiveFormats: Array;
31 | };
32 | playbackTracking: {
33 | videostatsPlaybackUrl: { baseUrl: string };
34 | videostatsDelayplayUrl: { baseUrl: string };
35 | videostatsWatchtimeUrl: { baseUrl: string };
36 | [x: string | number | symbol]: unknown;
37 | };
38 | captions?: {
39 | playerCaptionsTracklistRenderer: {
40 | captionTracks: [
41 | {
42 | baseUrl: string;
43 | name: object;
44 | vssId: string;
45 | languageCode: string;
46 | isTranslatable: boolean;
47 | }
48 | ];
49 | translationLanguages: [
50 | {
51 | languageCode: string;
52 | languageName: object;
53 | }
54 | ];
55 | audioTracks: Array;
56 | defaultAudioTrackIndex: number;
57 | };
58 | };
59 | videoDetails: {
60 | videoId: string;
61 | title: string;
62 | lengthSeconds: number;
63 | keywords: Array;
64 | channelId: string;
65 | isOwnerViewing: boolean;
66 | shortDescription: string;
67 | isCrawlable: boolean;
68 | thumbnail: {
69 | thumbnails: Array;
70 | };
71 | allowRatings: boolean;
72 | viewCount: number;
73 | author: string;
74 | isPrivate: boolean;
75 | isUnpluggedCorpus: boolean;
76 | isLiveContent: boolean;
77 | [x: string | number | symbol]: unknown;
78 | };
79 | microformat: {
80 | playerMicroformatRenderer: {
81 | lengthSeconds: number;
82 | ownerProfileUrl: string;
83 | externalChannelId: string;
84 | isFamilySafe: boolean;
85 | availableCountries: Array;
86 | isUnlisted: boolean;
87 | viewCount: number;
88 | category?: string;
89 | publishDate: string;
90 | ownerChannelName: string;
91 | uploadDate: string;
92 | liveBroadcastDetails?: {
93 | isLiveNow: boolean;
94 | startTimestamp: string;
95 | };
96 | [x: string | number | symbol]: unknown;
97 | };
98 | };
99 | }
100 |
101 | interface requesterConfig {
102 | data?: any;
103 | params?: any;
104 | }
105 |
106 | export interface browseConfig extends requesterConfig {
107 | isContinuation?: boolean;
108 | }
109 |
110 | export type searchFeatures =
111 | | "live"
112 | | "video4k"
113 | | "hd"
114 | | "subtitles"
115 | | "cc"
116 | | "video360"
117 | | "vr180"
118 | | "video3d"
119 | | "hdr"
120 | | "location"
121 | | "purchased";
122 | export interface searchFilter {
123 | uploadDate: "hour" | "day" | "week" | "month" | "year" | "all";
124 | order: "relevance" | "viewCount" | "rating" | "uploadDate";
125 | type: "video" | "playlist" | "channel" | "all";
126 | duration: "short" | "medium" | "long" | "all";
127 | features: Array;
128 | }
129 |
130 | export type parseTypes =
131 | | "videoDetail"
132 | | "homePage"
133 | | "searchSuggestions"
134 | | "searchResult";
135 |
136 | export type userConfig = {
137 | hl?: string;
138 | gl?: string;
139 | maxRetryCount?: number;
140 | };
141 |
142 | export type clientName = "ANDROID" | "IOS" | "WEB" | "MWEB";
143 |
144 | export type ytClient = {
145 | gl: string;
146 | hl: string;
147 | deviceMake?: string;
148 | deviceModel?: string;
149 | userAgent: string;
150 | clientName: clientName;
151 | clientVersion: string;
152 | osName: "Android" | "iOS" | "Windows" | "MacOS" | "Linux" | "MWeb";
153 | osVersion?: number;
154 | platform: "MOBILE" | "DESKTOP";
155 | remoteHost?: string;
156 | clientFormFactor?: string;
157 | visitorData?: string;
158 | };
159 |
160 | export type ytContext = {
161 | client: ytClient;
162 | user: { lockedSafetyMode: boolean };
163 | request: { useSsl: boolean };
164 | };
165 |
166 | export type httpMetadata = {
167 | apiKey: string;
168 | context: ytContext;
169 | };
170 |
171 | // -- parsers -- //
172 |
173 | export type ytVideoData = {
174 | player: playerResponse;
175 | next: object;
176 | };
177 |
178 | export interface continuation {
179 | nextContinuationData: string;
180 | reloadContinuationData: string;
181 | }
182 |
183 | export interface ytPageParseResults {
184 | page: T;
185 | Continuation?: continuation;
186 | }
187 |
--------------------------------------------------------------------------------
/src/extractors/youtube/utils/constants.ts:
--------------------------------------------------------------------------------
1 | export default {
2 | URL: {
3 | YT_URL: "https://www.youtube.com",
4 | YT_MOBILE: "https://m.youtube.com",
5 | YT_MUSIC_URL: "https://music.youtube.com",
6 | YT_BASE_API: "https://www.youtube.com/youtubei/v1",
7 | YT_SUGGESTION_API: "https://suggestqueries.google.com/complete/search",
8 | },
9 | YTAPIVAL: {
10 | VERSION: "17.20",
11 | CLIENTNAME: "ANDROID",
12 | VERSION_WEB: "2.20220411.09.00",
13 | CLIENT_WEB_Mobile: "MWEB",
14 | CLIENT_WEB_Desktop: "WEB",
15 | },
16 | };
17 |
--------------------------------------------------------------------------------
/src/extractors/youtube/utils/errors.ts:
--------------------------------------------------------------------------------
1 | import { utilityErrors } from "@utils";
2 |
3 | /**
4 | * @class General error class for the YouTube extractor. It is recommended to extend this class for more detailed error names.
5 | */
6 | class YoutubeError extends utilityErrors.VueTubeExtractorError { }
7 |
8 | /**
9 | * @abstract Errors for http requests.
10 | *
11 | * @param {string} id - the id of the video
12 | */
13 | abstract class PageError extends YoutubeError {
14 | id: string;
15 | constructor(id: string, message: string, details?: unknown) {
16 | super(message, details);
17 | // Object.setPrototypeOf(this, PageError.prototype);
18 | this.id = id;
19 | }
20 | }
21 |
22 | abstract class VideoError extends PageError { }
23 |
24 | class AgeRestrictionError extends VideoError { }
25 | class VideoNotFoundError extends VideoError { }
26 | class VideoNotAvailableError extends VideoError { }
27 | class NoStreamingDataError extends VideoError { }
28 |
29 | abstract class PlaylistError extends PageError { }
30 | class PlaylistNotFoundError extends PlaylistError { }
31 | class PlaylistNotAvailableError extends PlaylistError { }
32 |
33 | abstract class ChannelError extends PageError { }
34 | class ChannelNotFoundError extends ChannelError { }
35 | class ChannelNotAvailableError extends ChannelError { }
36 |
37 | class ExtractorNotReadyError extends utilityErrors.ExtractorNotReadyError { }
38 |
39 | class ParserError extends YoutubeError { }
40 | class LoginRequiredError extends YoutubeError { }
41 |
42 | class EndOfPageError extends YoutubeError { }
43 | /**
44 | * @class Errors for search requests.
45 | *
46 | * @param {string} query - the query of the search
47 | */
48 | class SearchError extends YoutubeError {
49 | query: string;
50 | constructor(query: string, message: string, details?: unknown) {
51 | super(message, details);
52 | // Object.setPrototypeOf(this, SearchError.prototype);
53 | this.query = query;
54 | }
55 | }
56 |
57 | /**
58 | * @class Error for initializing the extractor.
59 | *
60 | * @param {number} trials - the number of trials
61 | */
62 | class InitializationError extends YoutubeError {
63 | trials?: number;
64 | constructor(message: string, details?: unknown, trials?: number) {
65 | super(message, details);
66 | Object.setPrototypeOf(this, InitializationError.prototype);
67 | this.trials = trials;
68 | }
69 | }
70 |
71 | export default {
72 | YoutubeError,
73 | AgeRestrictionError,
74 | VideoNotFoundError,
75 | VideoNotAvailableError,
76 | NoStreamingDataError,
77 | PlaylistNotFoundError,
78 | PlaylistNotAvailableError,
79 | ChannelNotFoundError,
80 | ChannelNotAvailableError,
81 | SearchError,
82 | InitializationError,
83 | ExtractorNotReadyError,
84 | ParserError,
85 | LoginRequiredError,
86 | EndOfPageError,
87 | };
88 |
--------------------------------------------------------------------------------
/src/extractors/youtube/utils/index.ts:
--------------------------------------------------------------------------------
1 | import { UtilsBase } from "@utils";
2 |
3 | export { applyMixins } from "@utils";
4 |
5 | export { default as ytConstants } from "./constants";
6 |
7 | export { default as YouTubeHTTPOptions } from "./youtubeHTTPOptions";
8 |
9 | export { default as ytErrors } from "./errors";
10 |
11 | /**
12 | * @class the helper class for the YouTube extractor
13 | *
14 | * @extends UtilsBase
15 | */
16 | export class YtUtils extends UtilsBase {
17 | /**
18 | * Generates the CPN, which is a random string of length 16 characters containing of only alphanumeric characters as well as the following special characters: _ -
19 | * @returns {string} a random CPN
20 | */
21 |
22 | static randomCPN(): string {
23 | return super.randomString(16);
24 | }
25 |
26 | /**
27 | * Get string from runs array
28 | * @param {Array 
4 |
5 |
6 | 
22 | 
23 | 
24 | 
25 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2022 The VueTube Team
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | 
13 | 
45 |
46 |
47 | 
50 |