├── .dockerignore ├── .github └── workflows │ ├── ci.yml │ ├── docker-build-test.yml │ ├── docker-build.yml │ ├── docker-migrations-build-test.yml │ └── fat-build.yml ├── .gitignore ├── Dockerfile ├── Dockerfile.azul ├── Dockerfile.azul.ci ├── Dockerfile.ci ├── Dockerfile.graalvm-jvm ├── Dockerfile.graalvm-jvm.ci ├── Dockerfile.openj9 ├── Dockerfile.openj9.ci ├── LICENSE ├── README.md ├── build.gradle ├── config.properties ├── docker-compose.yml ├── docker-healthcheck.sh ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── hotspot-entrypoint.sh ├── renovate.json ├── settings.gradle ├── src └── main │ ├── java │ └── me │ │ └── kavin │ │ └── piped │ │ ├── Main.java │ │ ├── consts │ │ └── Constants.java │ │ ├── server │ │ ├── ServerLauncher.java │ │ └── handlers │ │ │ ├── ChannelHandlers.java │ │ │ ├── GenericHandlers.java │ │ │ ├── PlaylistHandlers.java │ │ │ ├── PubSubHandlers.java │ │ │ ├── SearchHandlers.java │ │ │ ├── StreamHandlers.java │ │ │ ├── TrendingHandlers.java │ │ │ └── auth │ │ │ ├── AuthPlaylistHandlers.java │ │ │ ├── FeedHandlers.java │ │ │ ├── StorageHandlers.java │ │ │ └── UserHandlers.java │ │ └── utils │ │ ├── Alea.java │ │ ├── BgPoTokenProvider.java │ │ ├── CaptchaSolver.java │ │ ├── ChannelHelpers.java │ │ ├── CollectionUtils.java │ │ ├── CustomServletDecorator.java │ │ ├── DatabaseHelper.java │ │ ├── DatabaseSessionFactory.java │ │ ├── DownloaderImpl.java │ │ ├── ErrorResponse.java │ │ ├── ExceptionHandler.java │ │ ├── FeedHelpers.java │ │ ├── GeoRestrictionBypassHelper.java │ │ ├── IStatusCode.java │ │ ├── LbryHelper.java │ │ ├── LiquibaseHelper.java │ │ ├── MpdBuilder.java │ │ ├── Multithreading.java │ │ ├── PageMixin.java │ │ ├── PlaylistHelpers.java │ │ ├── PubSubHelper.java │ │ ├── RequestUtils.java │ │ ├── RydHelper.java │ │ ├── SponsorBlockUtils.java │ │ ├── SponsorCategories.java │ │ ├── URLUtils.java │ │ ├── VideoHelpers.java │ │ ├── WaitingListener.java │ │ ├── matrix │ │ └── SyncRunner.java │ │ ├── obj │ │ ├── Channel.java │ │ ├── ChannelItem.java │ │ ├── ChannelTab.java │ │ ├── ChannelTabData.java │ │ ├── ChapterSegment.java │ │ ├── Comment.java │ │ ├── CommentsPage.java │ │ ├── ContentItem.java │ │ ├── MatrixHelper.java │ │ ├── MetaInfo.java │ │ ├── PipedStream.java │ │ ├── Playlist.java │ │ ├── PlaylistItem.java │ │ ├── PreviewFrames.java │ │ ├── SearchResults.java │ │ ├── SolvedCaptcha.java │ │ ├── StreamItem.java │ │ ├── Streams.java │ │ ├── StreamsPage.java │ │ ├── SubscriptionChannel.java │ │ ├── Subtitle.java │ │ ├── db │ │ │ ├── Channel.java │ │ │ ├── Playlist.java │ │ │ ├── PlaylistVideo.java │ │ │ ├── PubSub.java │ │ │ ├── UnauthenticatedSubscription.java │ │ │ ├── User.java │ │ │ └── Video.java │ │ └── federation │ │ │ ├── FederatedChannelInfo.java │ │ │ ├── FederatedGeoBypassRequest.java │ │ │ ├── FederatedGeoBypassResponse.java │ │ │ └── FederatedVideoInfo.java │ │ └── resp │ │ ├── AcceptedResponse.java │ │ ├── AlreadyRegisteredResponse.java │ │ ├── AuthenticationFailureResponse.java │ │ ├── CompromisedPasswordResponse.java │ │ ├── DeleteUserRequest.java │ │ ├── DeleteUserResponse.java │ │ ├── DisabledRegistrationResponse.java │ │ ├── IncorrectCredentialsResponse.java │ │ ├── InvalidRequestResponse.java │ │ ├── ListLinkHandlerMixin.java │ │ ├── LoginRequest.java │ │ ├── LoginResponse.java │ │ ├── SimpleErrorMessage.java │ │ ├── StackTraceResponse.java │ │ ├── SubscribeStatusResponse.java │ │ ├── SubscriptionUpdateRequest.java │ │ └── VideoResolvedResponse.java │ └── resources │ ├── changelog │ ├── db.changelog-master.xml │ └── version │ │ ├── 0-0-init-yb.sql │ │ ├── 0-1-init-crdb.sql │ │ ├── 0-1-init-hsqldb.sql │ │ ├── 0-1-init-pg.sql │ │ ├── 0-1-init.sql │ │ ├── 0-init.xml │ │ ├── 1-fix-subs.xml │ │ └── 2-fix-playlist-reordering-in-postgresql.xml │ └── hibernate.cfg.xml └── testing ├── api-test.sh ├── config.cockroachdb.properties ├── config.hsqldb.properties ├── config.yugabytedb.properties ├── docker-compose.cockroachdb.yml ├── docker-compose.hsqldb.yml └── docker-compose.yugabytedb.yml /.dockerignore: -------------------------------------------------------------------------------- 1 | .* 2 | build 3 | bin 4 | Dockerfile* 5 | docker-compose.yml 6 | *.bat 7 | LICENSE 8 | *.md 9 | config.properties 10 | data/ 11 | testing/ 12 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: 8 | 9 | jobs: 10 | build-and-test: 11 | runs-on: ubuntu-latest 12 | strategy: 13 | matrix: 14 | java: [ 21 ] 15 | steps: 16 | - uses: actions/checkout@v4 17 | 18 | - name: set up JDK ${{ matrix.java }} 19 | uses: actions/setup-java@v4 20 | with: 21 | java-version: ${{ matrix.java }} 22 | distribution: zulu 23 | cache: "gradle" 24 | - name: Run Build 25 | run: ./gradlew build 26 | -------------------------------------------------------------------------------- /.github/workflows/docker-build-test.yml: -------------------------------------------------------------------------------- 1 | name: Docker-Compose Build and Test 2 | 3 | on: 4 | push: 5 | paths-ignore: 6 | - "**.md" 7 | branches: 8 | - master 9 | pull_request: 10 | 11 | jobs: 12 | build-jdk: 13 | uses: ./.github/workflows/fat-build.yml 14 | 15 | build-test: 16 | runs-on: ubuntu-latest 17 | needs: build-jdk 18 | strategy: 19 | matrix: 20 | docker-compose-file: 21 | - docker-compose.yml 22 | - testing/docker-compose.hsqldb.yml 23 | - testing/docker-compose.cockroachdb.yml 24 | - testing/docker-compose.yugabytedb.yml 25 | dockerfile: 26 | - Dockerfile.ci 27 | - Dockerfile.azul.ci 28 | #- Dockerfile.openj9.ci 29 | - Dockerfile.graalvm-jvm.ci 30 | include: 31 | - sleep: 20 32 | - docker-compose-file: testing/docker-compose.cockroachdb.yml 33 | sleep: 30 34 | - docker-compose-file: testing/docker-compose.yugabytedb.yml 35 | sleep: 120 36 | fail-fast: false 37 | steps: 38 | - uses: actions/checkout@v4 39 | - uses: actions/download-artifact@v4 40 | with: 41 | name: piped.jar 42 | - name: Create Version File 43 | run: echo $(git log -1 --date=short --pretty=format:%cd)-$(git rev-parse --short HEAD) > VERSION 44 | - name: Build Image Locally 45 | uses: docker/build-push-action@v6 46 | with: 47 | context: . 48 | load: true 49 | file: ${{ matrix.dockerfile }} 50 | tags: 1337kavin/piped:latest 51 | - name: Start Docker-Compose services 52 | run: docker compose -f ${{ matrix.docker-compose-file }} up -d && sleep ${{ matrix.sleep }} 53 | - name: Run tests 54 | run: ./testing/api-test.sh 55 | - name: Collect services logs 56 | if: failure() 57 | run: docker compose -f ${{ matrix.docker-compose-file }} logs 58 | -------------------------------------------------------------------------------- /.github/workflows/docker-build.yml: -------------------------------------------------------------------------------- 1 | name: Docker Multi-Architecture Build 2 | 3 | on: 4 | push: 5 | paths-ignore: 6 | - "**.md" 7 | branches: 8 | - master 9 | 10 | jobs: 11 | build-jdk: 12 | uses: ./.github/workflows/fat-build.yml 13 | 14 | build-docker: 15 | needs: build-jdk 16 | runs-on: ubuntu-latest 17 | strategy: 18 | matrix: 19 | include: 20 | # - image: 1337kavin/piped:openj9 21 | # dockerfile: ./Dockerfile.openj9.ci 22 | - image: 1337kavin/piped:hotspot 23 | dockerfile: ./Dockerfile.ci 24 | - image: 1337kavin/piped:latest,1337kavin/piped:azul-zulu 25 | dockerfile: ./Dockerfile.azul.ci 26 | - image: 1337kavin/piped:graalvm-jvm 27 | dockerfile: ./Dockerfile.graalvm-jvm.ci 28 | steps: 29 | - uses: actions/checkout@v4 30 | - uses: actions/download-artifact@v4 31 | with: 32 | name: piped.jar 33 | - name: Create Version File 34 | run: echo $(git log -1 --date=short --pretty=format:%cd)-$(git rev-parse --short HEAD) > VERSION 35 | - name: Set up QEMU 36 | uses: docker/setup-qemu-action@v3 37 | with: 38 | platforms: all 39 | image: tonistiigi/binfmt:qemu-v7.0.0 40 | - name: Set up Docker Buildx 41 | id: buildx 42 | uses: docker/setup-buildx-action@v3 43 | with: 44 | version: latest 45 | - name: Login to DockerHub 46 | uses: docker/login-action@v3 47 | with: 48 | username: ${{ secrets.DOCKER_USERNAME }} 49 | password: ${{ secrets.DOCKER_PASSWORD }} 50 | - name: Build and push 51 | uses: docker/build-push-action@v6 52 | with: 53 | context: . 54 | file: ${{ matrix.dockerfile }} 55 | platforms: linux/amd64,linux/arm64 56 | push: true 57 | tags: ${{ matrix.image }} 58 | cache-from: type=gha 59 | cache-to: type=gha,mode=max 60 | -------------------------------------------------------------------------------- /.github/workflows/docker-migrations-build-test.yml: -------------------------------------------------------------------------------- 1 | name: Docker-Compose Build and Test Migration 2 | 3 | on: 4 | pull_request: 5 | paths: 6 | - "src/main/resources/changelog/**" 7 | - "src/main/java/me/kavin/piped/utils/obj/db/**" 8 | 9 | jobs: 10 | build-new: 11 | uses: ./.github/workflows/fat-build.yml 12 | build-old: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: actions/checkout@v4 16 | with: 17 | ref: ${{ github.event.pull_request.base.sha }} 18 | - name: set up JDK 21 19 | uses: actions/setup-java@v4 20 | with: 21 | java-version: 21 22 | distribution: zulu 23 | cache: "gradle" 24 | - name: Run Build 25 | run: ./gradlew shadowJar 26 | - run: mv build/libs/piped-*-all.jar piped.jar 27 | - uses: actions/upload-artifact@v4 28 | with: 29 | name: piped-old.jar 30 | path: piped.jar 31 | 32 | docker-build-test: 33 | needs: [ build-new, build-old ] 34 | runs-on: ubuntu-latest 35 | strategy: 36 | matrix: 37 | docker-compose-file: 38 | - docker-compose.yml 39 | - testing/docker-compose.cockroachdb.yml 40 | - testing/docker-compose.yugabytedb.yml 41 | dockerfile: 42 | - Dockerfile.azul.ci 43 | include: 44 | - sleep: 20 45 | - docker-compose-file: testing/docker-compose.cockroachdb.yml 46 | sleep: 30 47 | - docker-compose-file: testing/docker-compose.yugabytedb.yml 48 | sleep: 120 49 | fail-fast: false 50 | steps: 51 | - uses: actions/checkout@v4 52 | - run: echo "unknown" > VERSION 53 | - uses: actions/download-artifact@v4 54 | with: 55 | name: piped-old.jar 56 | - name: Build Old Image Locally 57 | uses: docker/build-push-action@v6 58 | with: 59 | context: . 60 | load: true 61 | file: ${{ matrix.dockerfile }} 62 | tags: 1337kavin/piped:latest 63 | - name: Start Docker-Compose services 64 | run: docker compose -f ${{ matrix.docker-compose-file }} up -d && sleep ${{ matrix.sleep }} 65 | - run: rm piped.jar 66 | - uses: actions/download-artifact@v4 67 | with: 68 | name: piped.jar 69 | - name: Build New Image Locally 70 | uses: docker/build-push-action@v6 71 | with: 72 | context: . 73 | load: true 74 | file: ${{ matrix.dockerfile }} 75 | tags: 1337kavin/piped:latest 76 | - name: Start Docker-Compose services 77 | run: docker compose -f ${{ matrix.docker-compose-file }} up -d && sleep ${{ matrix.sleep }} 78 | - name: Run tests 79 | run: ./testing/api-test.sh 80 | - name: Collect services logs 81 | if: failure() 82 | run: docker compose -f ${{ matrix.docker-compose-file }} logs 83 | -------------------------------------------------------------------------------- /.github/workflows/fat-build.yml: -------------------------------------------------------------------------------- 1 | name: Fat JAR Build 2 | 3 | on: 4 | workflow_call: 5 | 6 | jobs: 7 | build-and-test: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@v4 11 | 12 | - name: set up JDK 21 13 | uses: actions/setup-java@v4 14 | with: 15 | java-version: 21 16 | distribution: zulu 17 | cache: "gradle" 18 | - name: Run Build 19 | run: ./gradlew shadowJar 20 | - run: mv build/libs/piped-*-all.jar piped.jar 21 | - uses: actions/upload-artifact@v4 22 | with: 23 | name: piped.jar 24 | path: piped.jar 25 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled class file 2 | *.class 3 | bin/ 4 | 5 | .* 6 | 7 | !.gitignore 8 | !.dockerignore 9 | 10 | !.github/ 11 | 12 | # Log file 13 | *.log 14 | 15 | ### Gradle ### 16 | /build/ 17 | 18 | # Database Data 19 | /data/ 20 | 21 | # TxT File 22 | *.txt 23 | 24 | # Jar files 25 | *.jar 26 | 27 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 28 | hs_err_pid* 29 | 30 | # Version File 31 | VERSION -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM eclipse-temurin:21-jdk AS build 2 | 3 | WORKDIR /app/ 4 | 5 | COPY . /app/ 6 | 7 | RUN --mount=type=cache,target=/root/.gradle/caches/ \ 8 | ./gradlew shadowJar 9 | 10 | FROM eclipse-temurin:21-jre 11 | 12 | RUN --mount=type=cache,target=/var/cache/apt/ \ 13 | apt-get update && \ 14 | apt-get install -y --no-install-recommends \ 15 | curl \ 16 | && \ 17 | apt-get clean && \ 18 | rm -rf /var/lib/apt/lists/* 19 | 20 | WORKDIR /app/ 21 | 22 | COPY hotspot-entrypoint.sh docker-healthcheck.sh / 23 | 24 | COPY --from=build /app/build/libs/piped-1.0-all.jar /app/piped.jar 25 | 26 | COPY VERSION . 27 | 28 | EXPOSE 8080 29 | 30 | HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 CMD /docker-healthcheck.sh 31 | ENTRYPOINT ["/hotspot-entrypoint.sh"] 32 | -------------------------------------------------------------------------------- /Dockerfile.azul: -------------------------------------------------------------------------------- 1 | FROM azul/zulu-openjdk:21-latest AS build 2 | 3 | WORKDIR /app/ 4 | 5 | COPY . /app/ 6 | 7 | RUN --mount=type=cache,target=/root/.gradle/caches/ \ 8 | ./gradlew shadowJar 9 | 10 | FROM azul/zulu-openjdk:21-jre-headless-latest 11 | 12 | RUN --mount=type=cache,target=/var/cache/apt/ \ 13 | apt-get update && \ 14 | apt-get install -y --no-install-recommends \ 15 | curl \ 16 | && \ 17 | apt-get clean && \ 18 | rm -rf /var/lib/apt/lists/* 19 | 20 | WORKDIR /app/ 21 | 22 | COPY hotspot-entrypoint.sh docker-healthcheck.sh / 23 | 24 | COPY --from=build /app/build/libs/piped-1.0-all.jar /app/piped.jar 25 | 26 | COPY VERSION . 27 | 28 | EXPOSE 8080 29 | 30 | HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 CMD /docker-healthcheck.sh 31 | ENTRYPOINT ["/hotspot-entrypoint.sh"] 32 | -------------------------------------------------------------------------------- /Dockerfile.azul.ci: -------------------------------------------------------------------------------- 1 | FROM azul/zulu-openjdk:21-jre-headless-latest 2 | 3 | RUN --mount=type=cache,target=/var/cache/apt/ \ 4 | apt-get update && \ 5 | apt-get install -y --no-install-recommends \ 6 | curl \ 7 | && \ 8 | apt-get clean && \ 9 | rm -rf /var/lib/apt/lists/* 10 | 11 | WORKDIR /app/ 12 | 13 | COPY hotspot-entrypoint.sh docker-healthcheck.sh / 14 | 15 | COPY ./piped.jar /app/piped.jar 16 | 17 | COPY VERSION . 18 | 19 | EXPOSE 8080 20 | 21 | HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 CMD /docker-healthcheck.sh 22 | ENTRYPOINT ["/hotspot-entrypoint.sh"] 23 | -------------------------------------------------------------------------------- /Dockerfile.ci: -------------------------------------------------------------------------------- 1 | FROM eclipse-temurin:21-jre 2 | 3 | RUN --mount=type=cache,target=/var/cache/apt/ \ 4 | apt-get update && \ 5 | apt-get install -y --no-install-recommends \ 6 | curl \ 7 | && \ 8 | apt-get clean && \ 9 | rm -rf /var/lib/apt/lists/* 10 | 11 | WORKDIR /app/ 12 | 13 | COPY hotspot-entrypoint.sh docker-healthcheck.sh / 14 | 15 | COPY ./piped.jar /app/piped.jar 16 | 17 | COPY VERSION . 18 | 19 | EXPOSE 8080 20 | 21 | HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 CMD /docker-healthcheck.sh 22 | ENTRYPOINT ["/hotspot-entrypoint.sh"] 23 | -------------------------------------------------------------------------------- /Dockerfile.graalvm-jvm: -------------------------------------------------------------------------------- 1 | FROM container-registry.oracle.com/graalvm/native-image:latest as build 2 | 3 | WORKDIR /app/ 4 | 5 | COPY . /app/ 6 | 7 | RUN --mount=type=cache,target=/root/.gradle/caches/ \ 8 | ./gradlew shadowJar 9 | 10 | RUN jlink \ 11 | --add-modules java.base,java.logging,java.sql,java.management,java.xml,java.naming,java.desktop,jdk.crypto.ec \ 12 | --strip-debug \ 13 | --no-man-pages \ 14 | --no-header-files \ 15 | --compress=2 \ 16 | --output /javaruntime 17 | 18 | FROM debian:stable-slim 19 | 20 | RUN --mount=type=cache,target=/var/cache/apt/ \ 21 | apt-get update && \ 22 | apt-get install -y --no-install-recommends \ 23 | curl \ 24 | && \ 25 | apt-get clean && \ 26 | rm -rf /var/lib/apt/lists/* 27 | 28 | ENV JAVA_HOME=/opt/java/openjdk 29 | ENV PATH "${JAVA_HOME}/bin:${PATH}" 30 | COPY --from=build /javaruntime $JAVA_HOME 31 | 32 | WORKDIR /app/ 33 | 34 | COPY docker-healthcheck.sh / 35 | 36 | COPY --from=build /app/build/libs/piped-1.0-all.jar /app/piped.jar 37 | 38 | COPY VERSION . 39 | 40 | EXPOSE 8080 41 | 42 | HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 CMD /docker-healthcheck.sh 43 | CMD java -jar /app/piped.jar 44 | -------------------------------------------------------------------------------- /Dockerfile.graalvm-jvm.ci: -------------------------------------------------------------------------------- 1 | FROM container-registry.oracle.com/graalvm/native-image:latest as build 2 | 3 | RUN jlink \ 4 | --add-modules java.base,java.logging,java.sql,java.management,java.xml,java.naming,java.desktop,jdk.crypto.ec \ 5 | --strip-debug \ 6 | --no-man-pages \ 7 | --no-header-files \ 8 | --compress=2 \ 9 | --output /javaruntime 10 | 11 | FROM debian:stable-slim 12 | 13 | RUN --mount=type=cache,target=/var/cache/apt/ \ 14 | apt-get update && \ 15 | apt-get install -y --no-install-recommends \ 16 | curl \ 17 | && \ 18 | apt-get clean && \ 19 | rm -rf /var/lib/apt/lists/* 20 | 21 | ENV JAVA_HOME=/opt/java/openjdk 22 | ENV PATH "${JAVA_HOME}/bin:${PATH}" 23 | COPY --from=build /javaruntime $JAVA_HOME 24 | 25 | WORKDIR /app/ 26 | 27 | COPY docker-healthcheck.sh / 28 | 29 | COPY ./piped.jar /app/piped.jar 30 | 31 | COPY VERSION . 32 | 33 | EXPOSE 8080 34 | 35 | HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 CMD /docker-healthcheck.sh 36 | CMD java -jar /app/piped.jar 37 | -------------------------------------------------------------------------------- /Dockerfile.openj9: -------------------------------------------------------------------------------- 1 | FROM ibm-semeru-runtimes:open-17-jdk AS build 2 | 3 | WORKDIR /app/ 4 | 5 | COPY . /app/ 6 | 7 | RUN --mount=type=cache,target=/root/.gradle/caches/ \ 8 | ./gradlew shadowJar 9 | 10 | FROM ibm-semeru-runtimes:open-17-jre 11 | 12 | WORKDIR /app/ 13 | 14 | COPY --from=build /app/build/libs/piped-1.0-all.jar /app/piped.jar 15 | 16 | COPY VERSION . 17 | 18 | EXPOSE 8080 19 | 20 | CMD java -server -Xmx1G -Xaggressive -XX:+UnlockExperimentalVMOptions -XX:+OptimizeStringConcat -XX:+UseStringDeduplication -XX:+UseCompressedOops -XX:+UseNUMA -XX:+IdleTuningGcOnIdle -Xgcpolicy:gencon -Xshareclasses:allowClasspaths -Xtune:virtualized -Xcompactgc -jar /app/piped.jar 21 | -------------------------------------------------------------------------------- /Dockerfile.openj9.ci: -------------------------------------------------------------------------------- 1 | FROM ibm-semeru-runtimes:open-17-jre 2 | 3 | WORKDIR /app/ 4 | 5 | COPY ./piped.jar /app/piped.jar 6 | 7 | COPY VERSION . 8 | 9 | EXPOSE 8080 10 | 11 | CMD java -server -Xmx1G -Xaggressive -XX:+UnlockExperimentalVMOptions -XX:+OptimizeStringConcat -XX:+UseStringDeduplication -XX:+UseCompressedOops -XX:+UseNUMA -XX:+IdleTuningGcOnIdle -Xgcpolicy:gencon -Xshareclasses:allowClasspaths -Xtune:virtualized -Xcompactgc -jar /app/piped.jar 12 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Piped-Backend 2 | 3 | An advanced open-source privacy friendly alternative to YouTube, crafted with the help of [NewPipeExtractor](https://github.com/TeamNewPipe/NewPipeExtractor). 4 | 5 | ## Official Frontend 6 | 7 | - VueJS frontend - [Piped](https://github.com/TeamPiped/Piped) 8 | 9 | ## Community Projects 10 | 11 | - See https://github.com/TeamPiped/Piped#made-with-piped 12 | -------------------------------------------------------------------------------- /build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id "com.github.johnrengelman.shadow" version "8.1.1" 3 | id "java" 4 | id "eclipse" 5 | } 6 | 7 | repositories { 8 | mavenCentral() 9 | maven { url 'https://jitpack.io' } 10 | } 11 | 12 | dependencies { 13 | implementation 'org.apache.commons:commons-lang3:3.14.0' 14 | implementation 'org.apache.commons:commons-text:1.12.0' 15 | implementation 'commons-io:commons-io:2.16.1' 16 | implementation 'it.unimi.dsi:fastutil-core:8.5.13' 17 | implementation 'commons-codec:commons-codec:1.17.0' 18 | implementation 'org.bouncycastle:bcprov-jdk18on:1.78.1' 19 | implementation 'com.github.FireMasterK.NewPipeExtractor:NewPipeExtractor:aa40823e8b2f87c32ed244889c7a5542b9d820ec' 20 | implementation 'com.github.FireMasterK:nanojson:9f4af3b739cc13f3d0d9d4b758bbe2b2ae7119d7' 21 | implementation 'com.fasterxml.jackson.core:jackson-core:2.17.2' 22 | implementation 'com.fasterxml.jackson.core:jackson-annotations:2.17.2' 23 | implementation 'com.fasterxml.jackson.core:jackson-databind:2.17.2' 24 | implementation 'com.rometools:rome:2.1.0' 25 | implementation 'com.rometools:rome-modules:2.1.0' 26 | implementation 'org.jsoup:jsoup:1.18.1' 27 | implementation 'io.activej:activej-common:5.5' 28 | implementation 'io.activej:activej-http:5.5' 29 | implementation 'io.activej:activej-boot:5.5' 30 | implementation 'io.activej:activej-specializer:5.5' 31 | implementation 'io.activej:activej-launchers-http:5.5' 32 | implementation 'org.hsqldb:hsqldb:2.7.3' 33 | implementation 'org.postgresql:postgresql:42.7.3' 34 | implementation 'org.hibernate:hibernate-core:6.4.1.Final' 35 | implementation 'org.hibernate:hibernate-hikaricp:6.4.1.Final' 36 | implementation 'org.liquibase:liquibase-core:4.28.0' 37 | implementation('org.liquibase.ext:liquibase-yugabytedb:4.28.0') { exclude group: 'org.liquibase' } 38 | implementation 'com.zaxxer:HikariCP:5.1.0' 39 | implementation 'org.springframework.security:spring-security-crypto:6.3.1' 40 | implementation 'commons-logging:commons-logging:1.3.3' 41 | implementation(platform("com.squareup.okhttp3:okhttp-bom:4.12.0")) 42 | implementation 'com.squareup.okhttp3:okhttp' 43 | implementation 'com.squareup.okhttp3:okhttp-brotli' 44 | implementation 'io.sentry:sentry:7.11.0' 45 | implementation 'rocks.kavin:reqwest4j:1.0.14' 46 | implementation 'io.minio:minio:8.5.11' 47 | compileOnly 'org.projectlombok:lombok:1.18.34' 48 | annotationProcessor 'org.projectlombok:lombok:1.18.34' 49 | } 50 | 51 | shadowJar { 52 | // minimize() 53 | } 54 | 55 | jar { 56 | manifest { 57 | attributes( 58 | 'Main-Class': 'me.kavin.piped.Main' 59 | ) 60 | } 61 | } 62 | 63 | java { 64 | sourceCompatibility = JavaVersion.VERSION_21 65 | targetCompatibility = JavaVersion.VERSION_21 66 | } 67 | 68 | group = 'me.kavin.piped' 69 | version = '1.0' 70 | -------------------------------------------------------------------------------- /config.properties: -------------------------------------------------------------------------------- 1 | # The port to Listen on. 2 | PORT:8080 3 | # The number of workers to use for the server 4 | HTTP_WORKERS:2 5 | 6 | # Proxy 7 | PROXY_PART:https://pipedproxy-cdg.kavin.rocks 8 | 9 | # Proxy Hash Secret 10 | #PROXY_HASH_SECRET:INSERT_HERE 11 | 12 | # Outgoing proxy to be used by reqwest4j - eg: socks5://127.0.0.1:1080 13 | #REQWEST_PROXY: socks5://127.0.0.1:1080 14 | # Optional proxy username and password 15 | #REQWEST_PROXY_USER: username 16 | #REQWEST_PROXY_PASS: password 17 | 18 | # Captcha Parameters 19 | CAPTCHA_BASE_URL:https://api.capmonster.cloud/ 20 | CAPTCHA_API_KEY:INSERT_HERE 21 | 22 | # Public API URL 23 | API_URL:https://pipedapi.kavin.rocks 24 | 25 | # Public Frontend URL 26 | FRONTEND_URL:https://piped.video 27 | 28 | # Enable haveibeenpwned compromised password API 29 | COMPROMISED_PASSWORD_CHECK:true 30 | 31 | # Disable Registration 32 | DISABLE_REGISTRATION:false 33 | 34 | # Feed Retention Time in Days 35 | FEED_RETENTION:30 36 | 37 | # Disable CPU expensive timers (for nodes with low CPU, at least one node should have this disabled) 38 | DISABLE_TIMERS:false 39 | 40 | # RYD Proxy URL (see https://github.com/TeamPiped/RYD-Proxy) 41 | RYD_PROXY_URL:https://ryd-proxy.kavin.rocks 42 | 43 | # SponsorBlock Servers(s) 44 | # Comma separated list of SponsorBlock Servers to use 45 | SPONSORBLOCK_SERVERS:https://sponsor.ajay.app,https://sponsorblock.kavin.rocks 46 | 47 | # Disable the usage of RYD 48 | DISABLE_RYD:false 49 | 50 | # Disable API server (node just runs timers if enabled) 51 | DISABLE_SERVER:false 52 | 53 | # Disable the inclusion of LBRY streams 54 | DISABLE_LBRY:false 55 | 56 | # How long should unauthenticated subscriptions last for 57 | SUBSCRIPTIONS_EXPIRY:30 58 | 59 | # Send consent accepted cookie 60 | # This is required for certain features to work in some countries 61 | CONSENT_COOKIE:true 62 | 63 | # Sentry DSN 64 | # Use Sentry to log errors and trace performance 65 | #SENTRY_DSN:INSERT_HERE 66 | 67 | # Matrix Client Server URL 68 | MATRIX_SERVER:https://matrix-client.matrix.org 69 | # Matrix Access Token 70 | # If not present, will work in anon mode 71 | #MATRIX_TOKEN:INSERT_HERE 72 | 73 | # Geo Restriction Checker for federated bypassing of Geo Restrictions 74 | #GEO_RESTRICTION_CHECKER_URL:INSERT_HERE 75 | 76 | # BG Helper URL for supplying PoTokens 77 | #BG_HELPER_URL:INSERT_HERE 78 | 79 | # S3 Configuration Data (compatible with any provider that offers an S3 compatible API) 80 | #S3_ENDPOINT:INSERT_HERE 81 | #S3_ACCESS_KEY:INSERT_HERE 82 | #S3_SECRET_KEY:INSERT_HERE 83 | #S3_BUCKET:INSERT_HERE 84 | 85 | # Hibernate properties 86 | hibernate.connection.url:jdbc:postgresql://postgres:5432/piped 87 | hibernate.connection.driver_class:org.postgresql.Driver 88 | hibernate.dialect:org.hibernate.dialect.PostgreSQLDialect 89 | hibernate.connection.username:piped 90 | hibernate.connection.password:changeme 91 | 92 | # Frontend configuration 93 | #frontend.statusPageUrl:https://kavin.rocks 94 | #frontend.donationUrl:https://kavin.rocks 95 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | piped: 3 | image: 1337kavin/piped:latest 4 | restart: unless-stopped 5 | ports: 6 | - "127.0.0.1:8080:8080" 7 | volumes: 8 | - ./config.properties:/app/config.properties 9 | depends_on: 10 | - postgres 11 | postgres: 12 | image: postgres:16-alpine 13 | restart: unless-stopped 14 | volumes: 15 | - ./data/db:/var/lib/postgresql/data 16 | environment: 17 | - POSTGRES_DB=piped 18 | - POSTGRES_USER=piped 19 | - POSTGRES_PASSWORD=changeme 20 | -------------------------------------------------------------------------------- /docker-healthcheck.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | # If PORT env var is set, use it, otherwise default to 8080 4 | PORT=${PORT:-8080} 5 | 6 | curl -f http://localhost:$PORT/healthcheck || exit 1 7 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TeamPiped/Piped-Backend/dc834a374713305fc8d1b9e058c3538b6e47ded9/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://downloads.gradle.org/distributions/gradle-8.9-bin.zip 4 | networkTimeout=10000 5 | validateDistributionUrl=true 6 | zipStoreBase=GRADLE_USER_HOME 7 | zipStorePath=wrapper/dists 8 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | @rem SPDX-License-Identifier: Apache-2.0 17 | @rem 18 | 19 | @if "%DEBUG%"=="" @echo off 20 | @rem ########################################################################## 21 | @rem 22 | @rem Gradle startup script for Windows 23 | @rem 24 | @rem ########################################################################## 25 | 26 | @rem Set local scope for the variables with windows NT shell 27 | if "%OS%"=="Windows_NT" setlocal 28 | 29 | set DIRNAME=%~dp0 30 | if "%DIRNAME%"=="" set DIRNAME=. 31 | @rem This is normally unused 32 | set APP_BASE_NAME=%~n0 33 | set APP_HOME=%DIRNAME% 34 | 35 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 36 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 37 | 38 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 39 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 40 | 41 | @rem Find java.exe 42 | if defined JAVA_HOME goto findJavaFromJavaHome 43 | 44 | set JAVA_EXE=java.exe 45 | %JAVA_EXE% -version >NUL 2>&1 46 | if %ERRORLEVEL% equ 0 goto execute 47 | 48 | echo. 1>&2 49 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 50 | echo. 1>&2 51 | echo Please set the JAVA_HOME variable in your environment to match the 1>&2 52 | echo location of your Java installation. 1>&2 53 | 54 | goto fail 55 | 56 | :findJavaFromJavaHome 57 | set JAVA_HOME=%JAVA_HOME:"=% 58 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 59 | 60 | if exist "%JAVA_EXE%" goto execute 61 | 62 | echo. 1>&2 63 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 64 | echo. 1>&2 65 | echo Please set the JAVA_HOME variable in your environment to match the 1>&2 66 | echo location of your Java installation. 1>&2 67 | 68 | goto fail 69 | 70 | :execute 71 | @rem Setup the command line 72 | 73 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 74 | 75 | 76 | @rem Execute Gradle 77 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* 78 | 79 | :end 80 | @rem End local scope for the variables with windows NT shell 81 | if %ERRORLEVEL% equ 0 goto mainEnd 82 | 83 | :fail 84 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 85 | rem the _cmd.exe /c_ return code! 86 | set EXIT_CODE=%ERRORLEVEL% 87 | if %EXIT_CODE% equ 0 set EXIT_CODE=1 88 | if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% 89 | exit /b %EXIT_CODE% 90 | 91 | :mainEnd 92 | if "%OS%"=="Windows_NT" endlocal 93 | 94 | :omega 95 | -------------------------------------------------------------------------------- /hotspot-entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | MAX_MEMORY=${MAX_MEMORY:-1G} 4 | 5 | java -server -Xmx"$MAX_MEMORY" -XX:+UnlockExperimentalVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:+OptimizeStringConcat -XX:+UseStringDeduplication -XX:+UseCompressedOops -XX:+UseNUMA -XX:+UseG1GC -jar /app/piped.jar 6 | -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json", 3 | "extends": [ 4 | "config:base", 5 | "group:recommended" 6 | ], 7 | "ignorePresets": [ 8 | ":prHourlyLimit2" 9 | ], 10 | "packageRules": [ 11 | { 12 | "matchPackagePrefixes": [ 13 | "io.activej:" 14 | ], 15 | "groupName": "activej" 16 | }, 17 | { 18 | "matchPackagePrefixes": [ 19 | "com.fasterxml.jackson" 20 | ], 21 | "groupName": "jackson" 22 | }, 23 | { 24 | "matchPackagePrefixes": [ 25 | "org.liquibase" 26 | ], 27 | "groupName": "liquibase" 28 | }, 29 | { 30 | "matchPackagePrefixes": [ 31 | "com.github.firemasterk." 32 | ], 33 | "groupName": "Personal Forks", 34 | "enabled": false 35 | } 36 | ], 37 | "ignoreDeps": [ 38 | "com.github.FireMasterK.NewPipeExtractor:NewPipeExtractor" 39 | ] 40 | } 41 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | 2 | rootProject.name = 'piped' 3 | -------------------------------------------------------------------------------- /src/main/java/me/kavin/piped/server/handlers/GenericHandlers.java: -------------------------------------------------------------------------------- 1 | package me.kavin.piped.server.handlers; 2 | 3 | import me.kavin.piped.consts.Constants; 4 | import me.kavin.piped.utils.DatabaseSessionFactory; 5 | import org.hibernate.StatelessSession; 6 | 7 | import static me.kavin.piped.consts.Constants.mapper; 8 | 9 | public class GenericHandlers { 10 | 11 | public static byte[] configResponse() throws Exception { 12 | return mapper.writeValueAsBytes(Constants.frontendProperties); 13 | } 14 | 15 | public static String registeredBadgeRedirect() { 16 | try (StatelessSession s = DatabaseSessionFactory.createStatelessSession()) { 17 | long registered = s.createQuery("select count(*) from User", Long.class).uniqueResult(); 18 | 19 | return String.format("https://img.shields.io/badge/Registered%%20Users-%s-blue", registered); 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/me/kavin/piped/server/handlers/PlaylistHandlers.java: -------------------------------------------------------------------------------- 1 | package me.kavin.piped.server.handlers; 2 | 3 | import com.rometools.rome.feed.synd.SyndEntry; 4 | import com.rometools.rome.feed.synd.SyndEntryImpl; 5 | import com.rometools.rome.feed.synd.SyndFeed; 6 | import com.rometools.rome.feed.synd.SyndFeedImpl; 7 | import com.rometools.rome.io.FeedException; 8 | import com.rometools.rome.io.SyndFeedOutput; 9 | import io.sentry.Sentry; 10 | import it.unimi.dsi.fastutil.objects.ObjectArrayList; 11 | import me.kavin.piped.consts.Constants; 12 | import me.kavin.piped.server.handlers.auth.AuthPlaylistHandlers; 13 | import me.kavin.piped.utils.ExceptionHandler; 14 | import me.kavin.piped.utils.obj.ContentItem; 15 | import me.kavin.piped.utils.obj.Playlist; 16 | import me.kavin.piped.utils.obj.StreamsPage; 17 | import me.kavin.piped.utils.resp.InvalidRequestResponse; 18 | import org.apache.commons.lang3.StringUtils; 19 | import org.schabi.newpipe.extractor.ListExtractor; 20 | import org.schabi.newpipe.extractor.Page; 21 | import org.schabi.newpipe.extractor.exceptions.ExtractionException; 22 | import org.schabi.newpipe.extractor.playlist.PlaylistInfo; 23 | import org.schabi.newpipe.extractor.stream.StreamInfoItem; 24 | 25 | import java.io.IOException; 26 | import java.util.Date; 27 | import java.util.List; 28 | 29 | import static java.nio.charset.StandardCharsets.UTF_8; 30 | import static me.kavin.piped.consts.Constants.YOUTUBE_SERVICE; 31 | import static me.kavin.piped.consts.Constants.mapper; 32 | import static me.kavin.piped.utils.CollectionUtils.collectRelatedItems; 33 | import static me.kavin.piped.utils.URLUtils.getLastThumbnail; 34 | import static me.kavin.piped.utils.URLUtils.substringYouTube; 35 | 36 | public class PlaylistHandlers { 37 | public static byte[] playlistResponse(String playlistId) throws Exception { 38 | 39 | if (StringUtils.isBlank(playlistId)) 40 | ExceptionHandler.throwErrorResponse(new InvalidRequestResponse("playlistId is a required parameter")); 41 | 42 | if (playlistId.matches("[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}")) 43 | return AuthPlaylistHandlers.playlistPipedResponse(playlistId); 44 | 45 | return playlistYouTubeResponse(playlistId); 46 | } 47 | 48 | private static byte[] playlistYouTubeResponse(String playlistId) 49 | throws IOException, ExtractionException { 50 | 51 | Sentry.setExtra("playlistId", playlistId); 52 | 53 | final PlaylistInfo info = PlaylistInfo.getInfo("https://www.youtube.com/playlist?list=" + playlistId); 54 | 55 | final List relatedStreams = collectRelatedItems(info.getRelatedItems()); 56 | 57 | String nextpage = null; 58 | if (info.hasNextPage()) { 59 | Page page = info.getNextPage(); 60 | nextpage = mapper.writeValueAsString(page); 61 | } 62 | 63 | final Playlist playlist = new Playlist(info.getName(), getLastThumbnail(info.getThumbnails()), 64 | info.getDescription().getContent(), getLastThumbnail(info.getBanners()), nextpage, 65 | info.getUploaderName().isEmpty() ? null : info.getUploaderName(), 66 | substringYouTube(info.getUploaderUrl()), getLastThumbnail(info.getUploaderAvatars()), 67 | (int) info.getStreamCount(), relatedStreams); 68 | 69 | return mapper.writeValueAsBytes(playlist); 70 | 71 | } 72 | 73 | public static byte[] playlistPageResponse(String playlistId, String prevpageStr) 74 | throws IOException, ExtractionException { 75 | 76 | if (StringUtils.isEmpty(prevpageStr)) 77 | ExceptionHandler.throwErrorResponse(new InvalidRequestResponse("nextpage is a required parameter")); 78 | 79 | Page prevpage = mapper.readValue(prevpageStr, Page.class); 80 | 81 | if (prevpage == null) 82 | ExceptionHandler.throwErrorResponse(new InvalidRequestResponse("nextpage is a required parameter")); 83 | 84 | ListExtractor.InfoItemsPage info = PlaylistInfo.getMoreItems(YOUTUBE_SERVICE, 85 | "https://www.youtube.com/playlist?list=" + playlistId, prevpage); 86 | 87 | final List relatedStreams = collectRelatedItems(info.getItems()); 88 | 89 | String nextpage = null; 90 | if (info.hasNextPage()) { 91 | Page page = info.getNextPage(); 92 | nextpage = mapper.writeValueAsString(page); 93 | } 94 | 95 | final StreamsPage streamspage = new StreamsPage(nextpage, relatedStreams); 96 | 97 | return mapper.writeValueAsBytes(streamspage); 98 | 99 | } 100 | 101 | public static byte[] playlistRSSResponse(String playlistId) throws Exception { 102 | 103 | if (StringUtils.isBlank(playlistId)) 104 | ExceptionHandler.throwErrorResponse(new InvalidRequestResponse("playlistId is a required parameter")); 105 | 106 | if (playlistId.matches("[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}")) 107 | return AuthPlaylistHandlers.playlistPipedRSSResponse(playlistId); 108 | 109 | return playlistYouTubeRSSResponse(playlistId); 110 | } 111 | 112 | private static byte[] playlistYouTubeRSSResponse(String playlistId) 113 | throws IOException, ExtractionException, FeedException { 114 | 115 | final PlaylistInfo info = PlaylistInfo.getInfo("https://www.youtube.com/playlist?list=" + playlistId); 116 | 117 | final List entries = new ObjectArrayList<>(); 118 | 119 | SyndFeed feed = new SyndFeedImpl(); 120 | feed.setFeedType("rss_2.0"); 121 | feed.setTitle(info.getName()); 122 | feed.setAuthor(info.getUploaderName()); 123 | feed.setDescription(String.format("%s - Piped", info.getName())); 124 | feed.setLink(Constants.FRONTEND_URL + substringYouTube(info.getUrl())); 125 | feed.setPublishedDate(new Date()); 126 | 127 | info.getRelatedItems().forEach(item -> { 128 | SyndEntry entry = new SyndEntryImpl(); 129 | entry.setAuthor(item.getUploaderName()); 130 | entry.setLink(item.getUrl()); 131 | entry.setUri(item.getUrl()); 132 | entry.setTitle(item.getName()); 133 | entries.add(entry); 134 | }); 135 | 136 | feed.setEntries(entries); 137 | 138 | return new SyndFeedOutput().outputString(feed).getBytes(UTF_8); 139 | 140 | } 141 | 142 | } 143 | -------------------------------------------------------------------------------- /src/main/java/me/kavin/piped/server/handlers/PubSubHandlers.java: -------------------------------------------------------------------------------- 1 | package me.kavin.piped.server.handlers; 2 | 3 | import com.rometools.rome.feed.synd.SyndFeed; 4 | import com.rometools.rome.io.SyndFeedInput; 5 | import io.sentry.Sentry; 6 | import me.kavin.piped.consts.Constants; 7 | import me.kavin.piped.utils.*; 8 | import me.kavin.piped.utils.obj.MatrixHelper; 9 | import me.kavin.piped.utils.obj.federation.FederatedVideoInfo; 10 | import org.apache.commons.lang3.StringUtils; 11 | import org.hibernate.StatelessSession; 12 | import org.schabi.newpipe.extractor.exceptions.ParsingException; 13 | import org.schabi.newpipe.extractor.localization.DateWrapper; 14 | import org.xml.sax.InputSource; 15 | 16 | import java.io.ByteArrayInputStream; 17 | import java.util.concurrent.LinkedBlockingQueue; 18 | import java.util.concurrent.TimeUnit; 19 | 20 | import static me.kavin.piped.consts.Constants.YOUTUBE_SERVICE; 21 | 22 | public class PubSubHandlers { 23 | 24 | private static final LinkedBlockingQueue pubSubQueue = new LinkedBlockingQueue<>(); 25 | 26 | public static void handlePubSub(byte[] body) throws Exception { 27 | SyndFeed feed = new SyndFeedInput().build(new InputSource(new ByteArrayInputStream(body))); 28 | 29 | 30 | for (var entry : feed.getEntries()) { 31 | String url = entry.getLinks().get(0).getHref(); 32 | String videoId = StringUtils.substring(url, -11); 33 | 34 | long publishedDate = entry.getPublishedDate().getTime(); 35 | 36 | String str = videoId + ":" + publishedDate; 37 | 38 | if (pubSubQueue.contains(str)) 39 | continue; 40 | 41 | pubSubQueue.put(str); 42 | } 43 | } 44 | 45 | static { 46 | for (int i = 0; i < Runtime.getRuntime().availableProcessors(); i++) { 47 | new Thread(() -> { 48 | try { 49 | while (true) { 50 | String str = pubSubQueue.take(); 51 | 52 | String videoId = StringUtils.substringBefore(str, ":"); 53 | long publishedDate = Long.parseLong(StringUtils.substringAfter(str, ":")); 54 | 55 | try (StatelessSession s = DatabaseSessionFactory.createStatelessSession()) { 56 | if (DatabaseHelper.doesVideoExist(s, videoId)) 57 | continue; 58 | } 59 | 60 | try { 61 | Sentry.setExtra("videoId", videoId); 62 | var extractor = YOUTUBE_SERVICE.getStreamExtractor("https://youtube.com/watch?v=" + videoId); 63 | extractor.fetchPage(); 64 | 65 | Multithreading.runAsync(() -> { 66 | 67 | DateWrapper uploadDate; 68 | 69 | try { 70 | uploadDate = extractor.getUploadDate(); 71 | } catch (ParsingException e) { 72 | throw new RuntimeException(e); 73 | } 74 | 75 | if (uploadDate != null && System.currentTimeMillis() - uploadDate.offsetDateTime().toInstant().toEpochMilli() < TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION)) { 76 | try { 77 | MatrixHelper.sendEvent("video.piped.stream.info", new FederatedVideoInfo( 78 | StringUtils.substring(extractor.getUrl(), -11), StringUtils.substring(extractor.getUploaderUrl(), -24), 79 | extractor.getName(), 80 | extractor.getLength(), extractor.getViewCount()) 81 | ); 82 | } catch (Exception e) { 83 | ExceptionHandler.handle(e); 84 | } 85 | } 86 | }); 87 | 88 | VideoHelpers.handleNewVideo(extractor, publishedDate, null); 89 | } catch (Exception e) { 90 | ExceptionHandler.handle(e); 91 | } 92 | } 93 | } catch (Exception e) { 94 | ExceptionHandler.handle(e); 95 | } 96 | }, "PubSub-Worker-" + i).start(); 97 | } 98 | } 99 | 100 | } 101 | -------------------------------------------------------------------------------- /src/main/java/me/kavin/piped/server/handlers/SearchHandlers.java: -------------------------------------------------------------------------------- 1 | package me.kavin.piped.server.handlers; 2 | 3 | import io.sentry.Sentry; 4 | import me.kavin.piped.utils.ExceptionHandler; 5 | import me.kavin.piped.utils.obj.ContentItem; 6 | import me.kavin.piped.utils.obj.SearchResults; 7 | import me.kavin.piped.utils.resp.InvalidRequestResponse; 8 | import org.apache.commons.lang3.StringUtils; 9 | import org.schabi.newpipe.extractor.InfoItem; 10 | import org.schabi.newpipe.extractor.ListExtractor; 11 | import org.schabi.newpipe.extractor.Page; 12 | import org.schabi.newpipe.extractor.exceptions.ExtractionException; 13 | import org.schabi.newpipe.extractor.search.SearchInfo; 14 | 15 | import java.io.IOException; 16 | import java.util.Arrays; 17 | import java.util.Collections; 18 | import java.util.List; 19 | 20 | import static me.kavin.piped.consts.Constants.YOUTUBE_SERVICE; 21 | import static me.kavin.piped.consts.Constants.mapper; 22 | import static me.kavin.piped.utils.CollectionUtils.collectRelatedItems; 23 | 24 | public class SearchHandlers { 25 | public static byte[] suggestionsResponse(String query) 26 | throws IOException, ExtractionException { 27 | 28 | if (StringUtils.isEmpty(query)) 29 | ExceptionHandler.throwErrorResponse(new InvalidRequestResponse("query is a required parameter")); 30 | 31 | if (query.length() > 100) 32 | ExceptionHandler.throwErrorResponse(new InvalidRequestResponse("query is too long")); 33 | 34 | return mapper.writeValueAsBytes(YOUTUBE_SERVICE.getSuggestionExtractor().suggestionList(query)); 35 | 36 | } 37 | 38 | public static byte[] opensearchSuggestionsResponse(String query) 39 | throws IOException, ExtractionException { 40 | 41 | if (StringUtils.isEmpty(query)) 42 | ExceptionHandler.throwErrorResponse(new InvalidRequestResponse("query is a required parameter")); 43 | 44 | if (query.length() > 100) 45 | ExceptionHandler.throwErrorResponse(new InvalidRequestResponse("query is too long")); 46 | 47 | return mapper.writeValueAsBytes(Arrays.asList( 48 | query, 49 | YOUTUBE_SERVICE.getSuggestionExtractor().suggestionList(query) 50 | )); 51 | 52 | } 53 | 54 | public static byte[] searchResponse(String q, String filter) 55 | throws IOException, ExtractionException { 56 | 57 | if (StringUtils.isEmpty(q) || StringUtils.isEmpty(filter)) 58 | ExceptionHandler.throwErrorResponse(new InvalidRequestResponse("query and filter are required parameters")); 59 | 60 | Sentry.setExtra("query", q); 61 | 62 | final SearchInfo info = SearchInfo.getInfo(YOUTUBE_SERVICE, 63 | YOUTUBE_SERVICE.getSearchQHFactory().fromQuery(q, Collections.singletonList(filter), null)); 64 | 65 | List items = collectRelatedItems(info.getRelatedItems()); 66 | 67 | Page nextpage = info.getNextPage(); 68 | 69 | return mapper.writeValueAsBytes(new SearchResults(items, 70 | mapper.writeValueAsString(nextpage), info.getSearchSuggestion(), info.isCorrectedSearch())); 71 | 72 | } 73 | 74 | public static byte[] searchPageResponse(String q, String filter, String prevpageStr) 75 | throws IOException, ExtractionException { 76 | 77 | if (StringUtils.isEmpty(q) || StringUtils.isEmpty(prevpageStr)) 78 | ExceptionHandler.throwErrorResponse(new InvalidRequestResponse("query and nextpage are required parameter")); 79 | 80 | Page prevpage = mapper.readValue(prevpageStr, Page.class); 81 | 82 | if (prevpage == null) 83 | ExceptionHandler.throwErrorResponse(new InvalidRequestResponse("nextpage is a required parameter")); 84 | 85 | ListExtractor.InfoItemsPage pages = SearchInfo.getMoreItems(YOUTUBE_SERVICE, 86 | YOUTUBE_SERVICE.getSearchQHFactory().fromQuery(q, Collections.singletonList(filter), null), prevpage); 87 | 88 | List items = collectRelatedItems(pages.getItems()); 89 | 90 | Page nextpage = pages.getNextPage(); 91 | 92 | return mapper 93 | .writeValueAsBytes(new SearchResults(items, mapper.writeValueAsString(nextpage))); 94 | 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /src/main/java/me/kavin/piped/server/handlers/TrendingHandlers.java: -------------------------------------------------------------------------------- 1 | package me.kavin.piped.server.handlers; 2 | 3 | import me.kavin.piped.utils.ExceptionHandler; 4 | import me.kavin.piped.utils.obj.ContentItem; 5 | import me.kavin.piped.utils.resp.InvalidRequestResponse; 6 | import org.schabi.newpipe.extractor.exceptions.ExtractionException; 7 | import org.schabi.newpipe.extractor.kiosk.KioskExtractor; 8 | import org.schabi.newpipe.extractor.kiosk.KioskInfo; 9 | import org.schabi.newpipe.extractor.kiosk.KioskList; 10 | import org.schabi.newpipe.extractor.localization.ContentCountry; 11 | 12 | import java.io.IOException; 13 | import java.util.List; 14 | 15 | import static me.kavin.piped.consts.Constants.YOUTUBE_SERVICE; 16 | import static me.kavin.piped.consts.Constants.mapper; 17 | import static me.kavin.piped.utils.CollectionUtils.collectRelatedItems; 18 | 19 | public class TrendingHandlers { 20 | public static byte[] trendingResponse(String region) 21 | throws ExtractionException, IOException { 22 | 23 | if (region == null) 24 | ExceptionHandler.throwErrorResponse(new InvalidRequestResponse("region is a required parameter")); 25 | 26 | KioskList kioskList = YOUTUBE_SERVICE.getKioskList(); 27 | kioskList.forceContentCountry(new ContentCountry(region)); 28 | KioskExtractor extractor = kioskList.getDefaultKioskExtractor(); 29 | extractor.fetchPage(); 30 | KioskInfo info = KioskInfo.getInfo(extractor); 31 | 32 | final List relatedStreams = collectRelatedItems(info.getRelatedItems()); 33 | 34 | return mapper.writeValueAsBytes(relatedStreams); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/me/kavin/piped/server/handlers/auth/StorageHandlers.java: -------------------------------------------------------------------------------- 1 | package me.kavin.piped.server.handlers.auth; 2 | 3 | import io.minio.GetObjectArgs; 4 | import io.minio.PutObjectArgs; 5 | import io.minio.StatObjectArgs; 6 | import io.minio.errors.ErrorResponseException; 7 | import me.kavin.piped.consts.Constants; 8 | import me.kavin.piped.utils.DatabaseHelper; 9 | import me.kavin.piped.utils.ExceptionHandler; 10 | import me.kavin.piped.utils.obj.db.User; 11 | import me.kavin.piped.utils.resp.SimpleErrorMessage; 12 | import org.apache.commons.io.IOUtils; 13 | import org.apache.commons.lang3.StringUtils; 14 | import org.apache.commons.lang3.exception.ExceptionUtils; 15 | 16 | import java.io.ByteArrayInputStream; 17 | 18 | import static me.kavin.piped.consts.Constants.mapper; 19 | 20 | public class StorageHandlers { 21 | 22 | public static byte[] statFile(String session, String name) throws Exception { 23 | 24 | if (Constants.S3_CLIENT == null) 25 | ExceptionHandler.throwErrorResponse(new SimpleErrorMessage("Storage is not configured on this instance!")); 26 | 27 | if (!StringUtils.isAlphanumeric(name) || name.length() > 32) 28 | ExceptionHandler.throwErrorResponse(new SimpleErrorMessage("Invalid path provided!")); 29 | 30 | User user = DatabaseHelper.getUserFromSession(session); 31 | 32 | if (user == null) 33 | ExceptionHandler.throwErrorResponse(new SimpleErrorMessage("Invalid session provided!")); 34 | 35 | try { 36 | var statData = Constants.S3_CLIENT.statObject( 37 | StatObjectArgs.builder() 38 | .bucket(Constants.S3_BUCKET) 39 | .object(user.getId() + "/" + name) 40 | .build() 41 | ); 42 | 43 | return mapper.writeValueAsBytes( 44 | mapper.createObjectNode() 45 | .put("status", "exists") 46 | .put("etag", statData.etag()) 47 | .put("date", statData.lastModified().toInstant().toEpochMilli()) 48 | ); 49 | } catch (ErrorResponseException e) { 50 | if (e.errorResponse().code().equals("NoSuchKey")) 51 | return mapper.writeValueAsBytes( 52 | mapper.createObjectNode() 53 | .put("status", "not_exists") 54 | ); 55 | else 56 | throw e; 57 | } 58 | } 59 | 60 | public static byte[] putFile(String session, String name, String etag, byte[] content) throws Exception { 61 | 62 | if (Constants.S3_CLIENT == null) 63 | ExceptionHandler.throwErrorResponse(new SimpleErrorMessage("Storage is not configured on this instance!")); 64 | 65 | if (!StringUtils.isAlphanumeric(name) || name.length() > 32) 66 | ExceptionHandler.throwErrorResponse(new SimpleErrorMessage("Invalid path provided!")); 67 | 68 | User user = DatabaseHelper.getUserFromSession(session); 69 | 70 | if (user == null) 71 | ExceptionHandler.throwErrorResponse(new SimpleErrorMessage("Invalid session provided!")); 72 | 73 | // check if file size is greater than 500kb 74 | if (content.length > 500 * 1024) 75 | ExceptionHandler.throwErrorResponse(new SimpleErrorMessage("File size is too large!")); 76 | 77 | // check if file already exists, if it does, check if the etag matches 78 | try { 79 | var statData = Constants.S3_CLIENT.statObject( 80 | StatObjectArgs.builder() 81 | .bucket(Constants.S3_BUCKET) 82 | .object(user.getId() + "/" + name) 83 | .build() 84 | ); 85 | 86 | if (!statData.etag().equals(etag)) 87 | ExceptionHandler.throwErrorResponse(new SimpleErrorMessage("Invalid etag provided! (File uploaded by another client?)")); 88 | 89 | } catch (ErrorResponseException e) { 90 | if (!e.errorResponse().code().equals("NoSuchKey")) 91 | ExceptionUtils.rethrow(e); 92 | } 93 | 94 | var stream = new ByteArrayInputStream(content); 95 | 96 | Constants.S3_CLIENT.putObject( 97 | PutObjectArgs.builder() 98 | .bucket(Constants.S3_BUCKET) 99 | .object(user.getId() + "/" + name) 100 | .stream(stream, content.length, -1) 101 | .build() 102 | ); 103 | 104 | return mapper.writeValueAsBytes( 105 | mapper.createObjectNode() 106 | .put("status", "ok") 107 | ); 108 | } 109 | 110 | public static byte[] getFile(String session, String name) throws Exception { 111 | 112 | if (Constants.S3_CLIENT == null) 113 | ExceptionHandler.throwErrorResponse(new SimpleErrorMessage("Storage is not configured on this instance!")); 114 | 115 | if (!StringUtils.isAlphanumeric(name) || name.length() > 32) 116 | ExceptionHandler.throwErrorResponse(new SimpleErrorMessage("Invalid path provided!")); 117 | 118 | User user = DatabaseHelper.getUserFromSession(session); 119 | 120 | if (user == null) 121 | ExceptionHandler.throwErrorResponse(new SimpleErrorMessage("Invalid session provided!")); 122 | 123 | try (var stream = Constants.S3_CLIENT.getObject(GetObjectArgs.builder() 124 | .bucket(Constants.S3_BUCKET) 125 | .object(user.getId() + "/" + name) 126 | .build())) { 127 | return IOUtils.toByteArray(stream); 128 | } 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /src/main/java/me/kavin/piped/server/handlers/auth/UserHandlers.java: -------------------------------------------------------------------------------- 1 | package me.kavin.piped.server.handlers.auth; 2 | 3 | import com.fasterxml.jackson.core.JsonProcessingException; 4 | import jakarta.persistence.criteria.CriteriaBuilder; 5 | import jakarta.persistence.criteria.CriteriaQuery; 6 | import jakarta.persistence.criteria.Root; 7 | import me.kavin.piped.consts.Constants; 8 | import me.kavin.piped.utils.DatabaseHelper; 9 | import me.kavin.piped.utils.DatabaseSessionFactory; 10 | import me.kavin.piped.utils.ExceptionHandler; 11 | import me.kavin.piped.utils.RequestUtils; 12 | import me.kavin.piped.utils.obj.db.User; 13 | import me.kavin.piped.utils.resp.*; 14 | import org.apache.commons.codec.digest.DigestUtils; 15 | import org.apache.commons.lang3.StringUtils; 16 | import org.hibernate.Session; 17 | import org.hibernate.StatelessSession; 18 | import org.springframework.security.crypto.argon2.Argon2PasswordEncoder; 19 | import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; 20 | 21 | import java.io.IOException; 22 | import java.util.Set; 23 | import java.util.UUID; 24 | 25 | import static me.kavin.piped.consts.Constants.mapper; 26 | 27 | public class UserHandlers { 28 | private static final Argon2PasswordEncoder argon2PasswordEncoder = Argon2PasswordEncoder.defaultsForSpringSecurity_v5_8(); 29 | private static final BCryptPasswordEncoder bcryptPasswordEncoder = new BCryptPasswordEncoder(); 30 | 31 | public static byte[] registerResponse(String user, String pass) throws Exception { 32 | 33 | if (Constants.DISABLE_REGISTRATION) 34 | ExceptionHandler.throwErrorResponse(new DisabledRegistrationResponse()); 35 | 36 | if (StringUtils.isBlank(user) || StringUtils.isBlank(pass)) 37 | ExceptionHandler.throwErrorResponse(new InvalidRequestResponse()); 38 | 39 | if (user.length() > 24) 40 | ExceptionHandler.throwErrorResponse(new InvalidRequestResponse("The username must be less than 24 characters")); 41 | 42 | user = user.toLowerCase(); 43 | 44 | try (Session s = DatabaseSessionFactory.createSession()) { 45 | CriteriaBuilder cb = s.getCriteriaBuilder(); 46 | CriteriaQuery cr = cb.createQuery(User.class); 47 | Root root = cr.from(User.class); 48 | cr.select(root).where(cb.equal(root.get("username"), user)); 49 | boolean registered = s.createQuery(cr).uniqueResult() != null; 50 | 51 | if (registered) 52 | ExceptionHandler.throwErrorResponse(new AlreadyRegisteredResponse()); 53 | 54 | if (Constants.COMPROMISED_PASSWORD_CHECK) { 55 | String sha1Hash = DigestUtils.sha1Hex(pass).toUpperCase(); 56 | String prefix = sha1Hash.substring(0, 5); 57 | String suffix = sha1Hash.substring(5); 58 | String[] entries = RequestUtils 59 | .sendGet("https://api.pwnedpasswords.com/range/" + prefix, "github.com/TeamPiped/Piped-Backend") 60 | .thenApplyAsync(str -> str.split("\n")) 61 | .get(); 62 | for (String entry : entries) 63 | if (StringUtils.substringBefore(entry, ":").equals(suffix)) 64 | ExceptionHandler.throwErrorResponse(new CompromisedPasswordResponse()); 65 | } 66 | 67 | User newuser = new User(user, argon2PasswordEncoder.encode(pass), Set.of()); 68 | 69 | var tr = s.beginTransaction(); 70 | s.persist(newuser); 71 | tr.commit(); 72 | 73 | 74 | return mapper.writeValueAsBytes(new LoginResponse(newuser.getSessionId())); 75 | } 76 | } 77 | 78 | private static boolean hashMatch(String hash, String pass) { 79 | return hash.startsWith("$argon2") ? 80 | argon2PasswordEncoder.matches(pass, hash) : 81 | bcryptPasswordEncoder.matches(pass, hash); 82 | } 83 | 84 | public static byte[] loginResponse(String user, String pass) 85 | throws IOException { 86 | 87 | if (user == null || pass == null) 88 | ExceptionHandler.throwErrorResponse(new InvalidRequestResponse("username and password are required parameters")); 89 | 90 | user = user.toLowerCase(); 91 | 92 | try (StatelessSession s = DatabaseSessionFactory.createStatelessSession()) { 93 | CriteriaBuilder cb = s.getCriteriaBuilder(); 94 | CriteriaQuery cr = cb.createQuery(User.class); 95 | Root root = cr.from(User.class); 96 | cr.select(root).where(root.get("username").in(user)); 97 | 98 | User dbuser = s.createQuery(cr).uniqueResult(); 99 | 100 | if (dbuser != null) { 101 | String hash = dbuser.getPassword(); 102 | if (hashMatch(hash, pass)) { 103 | return mapper.writeValueAsBytes(new LoginResponse(dbuser.getSessionId())); 104 | } 105 | } 106 | 107 | ExceptionHandler.throwErrorResponse(new IncorrectCredentialsResponse()); 108 | return null; 109 | } 110 | } 111 | 112 | public static byte[] deleteUserResponse(String session, String pass) throws IOException { 113 | 114 | if (StringUtils.isBlank(session) || StringUtils.isBlank(pass)) 115 | ExceptionHandler.throwErrorResponse(new InvalidRequestResponse("session and password are required parameters")); 116 | 117 | try (Session s = DatabaseSessionFactory.createSession()) { 118 | User user = DatabaseHelper.getUserFromSession(session); 119 | 120 | if (user == null) 121 | ExceptionHandler.throwErrorResponse(new AuthenticationFailureResponse()); 122 | 123 | String hash = user.getPassword(); 124 | 125 | if (!hashMatch(hash, pass)) 126 | ExceptionHandler.throwErrorResponse(new IncorrectCredentialsResponse()); 127 | 128 | var tr = s.beginTransaction(); 129 | s.remove(user); 130 | tr.commit(); 131 | 132 | return mapper.writeValueAsBytes(new DeleteUserResponse(user.getUsername())); 133 | } 134 | } 135 | 136 | public static byte[] logoutResponse(String session) throws JsonProcessingException { 137 | 138 | if (StringUtils.isBlank(session)) 139 | ExceptionHandler.throwErrorResponse(new InvalidRequestResponse("session is a required parameter")); 140 | 141 | try (StatelessSession s = DatabaseSessionFactory.createStatelessSession()) { 142 | var tr = s.beginTransaction(); 143 | if (s.createMutationQuery("UPDATE User user SET user.sessionId = :newSessionId where user.sessionId = :sessionId") 144 | .setParameter("sessionId", session).setParameter("newSessionId", String.valueOf(UUID.randomUUID())) 145 | .executeUpdate() > 0) { 146 | tr.commit(); 147 | return Constants.mapper.writeValueAsBytes(new AcceptedResponse()); 148 | } else 149 | tr.rollback(); 150 | } 151 | 152 | return Constants.mapper.writeValueAsBytes(new AuthenticationFailureResponse()); 153 | } 154 | } 155 | -------------------------------------------------------------------------------- /src/main/java/me/kavin/piped/utils/Alea.java: -------------------------------------------------------------------------------- 1 | package me.kavin.piped.utils; 2 | 3 | public class Alea { 4 | 5 | private static final double NORM32 = 2.3283064365386963e-10; // 2^-32 6 | private double s0, s1, s2; 7 | private int c = 1; 8 | 9 | public double next() { 10 | double t = 2091639.0 * s0 + c * NORM32; // 2^-32 11 | s0 = s1; 12 | s1 = s2; 13 | return s2 = t - (c = (int) t); 14 | } 15 | 16 | public Alea(String seed) { 17 | s0 = mash(" "); 18 | s1 = mash(" "); 19 | s2 = mash(" "); 20 | 21 | s0 -= mash(seed); 22 | 23 | if (s0 < 0) 24 | s0 += 1; 25 | s1 -= mash(seed); 26 | if (s1 < 0) 27 | s1 += 1; 28 | s2 -= mash(seed); 29 | if (s2 < 0) 30 | s2 += 1; 31 | } 32 | 33 | private long n = 0xefc8249dL; 34 | 35 | public double mash(String x) { 36 | double h; 37 | 38 | for (char c : x.toCharArray()) { 39 | n += c; 40 | h = 0.02519603282416938 * n; 41 | n = (long) h; 42 | h -= n; 43 | h *= n; 44 | n = (long) h; 45 | h -= n; 46 | n += h * 0x100000000L; 47 | } 48 | return n * 2.3283064365386963e-10; // 2^-32 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/me/kavin/piped/utils/BgPoTokenProvider.java: -------------------------------------------------------------------------------- 1 | package me.kavin.piped.utils; 2 | 3 | import lombok.RequiredArgsConstructor; 4 | import org.jetbrains.annotations.Nullable; 5 | import org.schabi.newpipe.extractor.services.youtube.PoTokenProvider; 6 | import org.schabi.newpipe.extractor.services.youtube.PoTokenResult; 7 | import rocks.kavin.reqwest4j.ReqwestUtils; 8 | 9 | import java.util.Map; 10 | import java.util.Queue; 11 | import java.util.concurrent.*; 12 | import java.util.regex.Pattern; 13 | 14 | import static me.kavin.piped.consts.Constants.mapper; 15 | 16 | @RequiredArgsConstructor 17 | public class BgPoTokenProvider implements PoTokenProvider { 18 | 19 | private final String bgHelperUrl; 20 | 21 | private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1); 22 | 23 | private String getWebVisitorData() throws Exception { 24 | var html = RequestUtils.sendGet("https://www.youtube.com").get(); 25 | var matcher = Pattern.compile("visitorData\":\"([\\w%-]+)\"").matcher(html); 26 | 27 | if (matcher.find()) { 28 | return matcher.group(1); 29 | } 30 | 31 | throw new RuntimeException("Failed to get visitor data"); 32 | } 33 | 34 | private final Queue validPoTokens = new ConcurrentLinkedQueue<>(); 35 | 36 | private PoTokenResult getPoTokenPooled() throws Exception { 37 | PoTokenResult poToken = validPoTokens.poll(); 38 | 39 | if (poToken == null) { 40 | poToken = createWebClientPoToken(); 41 | } 42 | 43 | // if still null, return null 44 | if (poToken == null) { 45 | return null; 46 | } 47 | 48 | // timer to insert back into queue after 10 + random seconds 49 | int delay = 10_000 + ThreadLocalRandom.current().nextInt(5000); 50 | PoTokenResult finalPoToken = poToken; 51 | scheduler.schedule(() -> validPoTokens.offer(finalPoToken), delay, TimeUnit.MILLISECONDS); 52 | 53 | return poToken; 54 | } 55 | 56 | private PoTokenResult createWebClientPoToken() throws Exception { 57 | String visitorDate = getWebVisitorData(); 58 | 59 | String poToken = ReqwestUtils.fetch(bgHelperUrl + "/generate", "POST", mapper.writeValueAsBytes(mapper.createObjectNode().put( 60 | "visitorData", visitorDate 61 | )), Map.of( 62 | "Content-Type", "application/json" 63 | )).thenApply(response -> { 64 | try { 65 | return mapper.readTree(response.body()).get("poToken").asText(); 66 | } catch (Exception e) { 67 | return null; 68 | } 69 | }).join(); 70 | 71 | if (poToken != null) { 72 | return new PoTokenResult(visitorDate, poToken, null); 73 | } 74 | 75 | return null; 76 | } 77 | 78 | @Override 79 | public @Nullable PoTokenResult getWebClientPoToken(String videoId) { 80 | try { 81 | return getPoTokenPooled(); 82 | } catch (Exception e) { 83 | e.printStackTrace(); 84 | } 85 | return null; 86 | } 87 | 88 | @Override 89 | public @Nullable PoTokenResult getWebEmbedClientPoToken(String videoId) { 90 | return null; 91 | } 92 | 93 | @Override 94 | public @Nullable PoTokenResult getAndroidClientPoToken(String videoId) { 95 | return null; 96 | } 97 | 98 | @Override 99 | public @Nullable PoTokenResult getIosClientPoToken(String videoId) { 100 | return null; 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /src/main/java/me/kavin/piped/utils/CaptchaSolver.java: -------------------------------------------------------------------------------- 1 | package me.kavin.piped.utils; 2 | 3 | import com.grack.nanojson.JsonObject; 4 | import com.grack.nanojson.JsonParser; 5 | import com.grack.nanojson.JsonParserException; 6 | import com.grack.nanojson.JsonWriter; 7 | import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap; 8 | import me.kavin.piped.consts.Constants; 9 | import me.kavin.piped.utils.obj.SolvedCaptcha; 10 | import okhttp3.MediaType; 11 | import okhttp3.Request; 12 | import okhttp3.RequestBody; 13 | 14 | import java.io.IOException; 15 | import java.util.Map; 16 | 17 | public class CaptchaSolver { 18 | 19 | public static SolvedCaptcha solve(String url, String sitekey, String data_s) 20 | throws JsonParserException, IOException, InterruptedException { 21 | 22 | int taskId = createTask(url, sitekey, data_s); 23 | 24 | return waitForSolve(taskId); 25 | 26 | } 27 | 28 | private static int createTask(String url, String sitekey, String data_s) 29 | throws JsonParserException, IOException { 30 | 31 | JsonObject jObject = new JsonObject(); 32 | jObject.put("clientKey", Constants.CAPTCHA_API_KEY); 33 | { 34 | JsonObject task = new JsonObject(); 35 | task.put("type", "NoCaptchaTaskProxyless"); 36 | task.put("websiteURL", url); 37 | task.put("websiteKey", sitekey); 38 | task.put("recaptchaDataSValue", data_s); 39 | jObject.put("task", task); 40 | } 41 | 42 | var builder = new Request.Builder().url(Constants.CAPTCHA_BASE_URL + "createTask") 43 | .post(RequestBody.create(JsonWriter.string(jObject), MediaType.get("application/json"))); 44 | var resp = Constants.h2client.newCall(builder.build()).execute(); 45 | 46 | JsonObject taskObj = JsonParser.object() 47 | .from(resp.body().byteStream()); 48 | 49 | resp.close(); 50 | 51 | return taskObj.getInt("taskId"); 52 | } 53 | 54 | private static SolvedCaptcha waitForSolve(int taskId) 55 | throws JsonParserException, IOException, InterruptedException { 56 | 57 | String body = JsonWriter.string( 58 | JsonObject.builder().value("clientKey", Constants.CAPTCHA_API_KEY).value("taskId", taskId).done()); 59 | 60 | SolvedCaptcha solved = null; 61 | 62 | while (true) { 63 | var builder = new Request.Builder() 64 | .url(Constants.CAPTCHA_BASE_URL + "getTaskResult") 65 | .post(RequestBody.create(body, MediaType.get("application/json"))); 66 | 67 | builder.header("Content-Type", "application/json"); 68 | var resp = Constants.h2client.newCall(builder.build()).execute(); 69 | 70 | 71 | JsonObject captchaObj = JsonParser.object() 72 | .from(resp.body().byteStream()); 73 | 74 | resp.close(); 75 | 76 | if (captchaObj.getInt("errorId") != 0) 77 | break; 78 | 79 | if (captchaObj.has("solution")) { 80 | JsonObject solution = captchaObj.getObject("solution"); 81 | String captchaResp = solution.getString("gRecaptchaResponse"); 82 | JsonObject cookieObj = solution.getObject("cookies"); 83 | Map cookies = new Object2ObjectOpenHashMap<>(); 84 | 85 | if (captchaResp != null) { 86 | 87 | cookieObj.keySet().forEach(cookie -> { 88 | cookies.put(cookie, cookieObj.getString(cookie)); 89 | }); 90 | 91 | solved = new SolvedCaptcha(cookies, captchaResp); 92 | break; 93 | } 94 | } 95 | 96 | Thread.sleep(1000); 97 | } 98 | 99 | return solved; 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /src/main/java/me/kavin/piped/utils/ChannelHelpers.java: -------------------------------------------------------------------------------- 1 | package me.kavin.piped.utils; 2 | 3 | import com.rometools.modules.mediarss.MediaEntryModuleImpl; 4 | import com.rometools.modules.mediarss.types.MediaContent; 5 | import com.rometools.modules.mediarss.types.Metadata; 6 | import com.rometools.modules.mediarss.types.PlayerReference; 7 | import com.rometools.modules.mediarss.types.Thumbnail; 8 | import com.rometools.rome.feed.synd.*; 9 | import me.kavin.piped.consts.Constants; 10 | import me.kavin.piped.utils.obj.db.Channel; 11 | import me.kavin.piped.utils.obj.db.Video; 12 | import okhttp3.Request; 13 | import org.apache.commons.lang3.StringUtils; 14 | import org.apache.commons.lang3.time.DurationFormatUtils; 15 | import org.apache.commons.text.StringEscapeUtils; 16 | import org.hibernate.StatelessSession; 17 | 18 | import java.io.IOException; 19 | import java.net.MalformedURLException; 20 | import java.net.URI; 21 | import java.net.URL; 22 | import java.util.Collections; 23 | import java.util.Date; 24 | import java.util.List; 25 | 26 | import static me.kavin.piped.utils.URLUtils.rewriteURL; 27 | 28 | public class ChannelHelpers { 29 | 30 | public static boolean isValidId(String id) { 31 | return !StringUtils.isBlank(id) && id.matches("UC[a-zA-Z\\d_-]{22}"); 32 | } 33 | 34 | public static void updateChannel(StatelessSession s, Channel channel, String name, String avatarUrl, boolean uploaderVerified) { 35 | 36 | boolean changed = false; 37 | 38 | if (name != null && !name.equals(channel.getUploader())) { 39 | channel.setUploader(name); 40 | changed = true; 41 | } 42 | 43 | if (avatarUrl != null && !avatarUrl.equals(channel.getUploaderAvatar())) { 44 | 45 | URL url; 46 | try { 47 | url = new URL(avatarUrl); 48 | final var host = url.getHost(); 49 | if (!host.endsWith(".ggpht.com") && !host.endsWith(".googleusercontent.com")) 50 | return; 51 | } catch (MalformedURLException e) { 52 | throw new RuntimeException(e); 53 | } 54 | 55 | try (var resp = Constants.h2client.newCall(new Request.Builder().url(url).head().build()).execute()) { 56 | 57 | if (resp.isSuccessful()) 58 | channel.setUploaderAvatar(avatarUrl); 59 | 60 | changed = true; 61 | } catch (IOException e) { 62 | return; 63 | } 64 | } 65 | 66 | if (uploaderVerified != channel.isVerified()) { 67 | channel.setVerified(uploaderVerified); 68 | changed = true; 69 | } 70 | 71 | if (changed) { 72 | var tr = s.beginTransaction(); 73 | s.update(channel); 74 | tr.commit(); 75 | } 76 | } 77 | 78 | public static SyndEntry createEntry(Video video, Channel channel) { 79 | SyndEntry entry = new SyndEntryImpl(); 80 | SyndPerson person = new SyndPersonImpl(); 81 | SyndContent content = new SyndContentImpl(); 82 | SyndContent thumbnail = new SyndContentImpl(); 83 | 84 | person.setName(channel.getUploader()); 85 | person.setUri(Constants.FRONTEND_URL + "/channel/" + channel.getUploaderId()); 86 | entry.setAuthors(Collections.singletonList(person)); 87 | entry.setLink(Constants.FRONTEND_URL + "/watch?v=" + video.getId()); 88 | entry.setUri(Constants.FRONTEND_URL + "/watch?v=" + video.getId()); 89 | 90 | entry.setTitle(video.getTitle()); 91 | entry.setPublishedDate(new Date(video.getUploaded())); 92 | 93 | String contentText = String.format("Title: %s\nViews: %d\nId: %s\nDuration: %s\nIs YT Shorts: %b", video.getTitle(), video.getViews(), video.getId(), DurationFormatUtils.formatDuration(video.getDuration() * 1000, "[HH]':'mm':'ss"), video.isShort()); 94 | content.setValue(contentText); 95 | 96 | String thumbnailContent = 97 | String.format("
", 98 | Constants.FRONTEND_URL + "/watch?v=" + video.getId(), 99 | StringEscapeUtils.escapeXml11(rewriteURL(video.getThumbnail())) 100 | ); 101 | thumbnail.setType("xhtml"); 102 | thumbnail.setValue(thumbnailContent); 103 | 104 | entry.setContents(List.of(thumbnail, content)); 105 | 106 | // the Media RSS content for embedding videos starts here 107 | // see https://www.rssboard.org/media-rss#media-content 108 | 109 | String playerUrl = Constants.FRONTEND_URL + "/embed/" + video.getId(); 110 | MediaContent media = new MediaContent(new PlayerReference(URI.create(playerUrl))); 111 | media.setDuration(video.getDuration()); 112 | 113 | Metadata metadata = new Metadata(); 114 | metadata.setTitle(video.getTitle()); 115 | Thumbnail metadataThumbnail = new Thumbnail(URI.create(video.getThumbnail())); 116 | metadata.setThumbnail(new Thumbnail[]{ metadataThumbnail }); 117 | media.setMetadata(metadata); 118 | 119 | MediaEntryModuleImpl mediaModule = new MediaEntryModuleImpl(); 120 | mediaModule.setMediaContents(new MediaContent[]{ media }); 121 | entry.getModules().add(mediaModule); 122 | 123 | return entry; 124 | } 125 | 126 | public static void addChannelInformation(SyndFeed feed, Channel channel) { 127 | feed.setTitle("Piped - " + channel.getUploader()); 128 | SyndImage channelIcon = new SyndImageImpl(); 129 | channelIcon.setLink(Constants.FRONTEND_URL + "/channel/" + channel.getUploaderId()); 130 | channelIcon.setTitle(channel.getUploader()); 131 | channelIcon.setUrl(rewriteURL(channel.getUploaderAvatar())); 132 | feed.setIcon(channelIcon); 133 | feed.setImage(channelIcon); 134 | } 135 | } 136 | -------------------------------------------------------------------------------- /src/main/java/me/kavin/piped/utils/CollectionUtils.java: -------------------------------------------------------------------------------- 1 | package me.kavin.piped.utils; 2 | 3 | import it.unimi.dsi.fastutil.objects.ObjectArrayList; 4 | import me.kavin.piped.utils.obj.*; 5 | import org.schabi.newpipe.extractor.InfoItem; 6 | import org.schabi.newpipe.extractor.channel.ChannelInfoItem; 7 | import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler; 8 | import org.schabi.newpipe.extractor.linkhandler.ReadyChannelTabListLinkHandler; 9 | import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem; 10 | import org.schabi.newpipe.extractor.stream.StreamInfo; 11 | import org.schabi.newpipe.extractor.stream.StreamInfoItem; 12 | import org.schabi.newpipe.extractor.stream.StreamType; 13 | 14 | import java.util.List; 15 | import java.util.Locale; 16 | import java.util.Map; 17 | import java.util.Optional; 18 | 19 | import static me.kavin.piped.utils.URLUtils.*; 20 | 21 | public class CollectionUtils { 22 | 23 | public static Streams collectStreamInfo(StreamInfo info) { 24 | final List subtitles = new ObjectArrayList<>(); 25 | final List chapters = new ObjectArrayList<>(); 26 | 27 | info.getStreamSegments().forEach(segment -> chapters.add(new ChapterSegment(segment.getTitle(), rewriteURL(segment.getPreviewUrl()), 28 | segment.getStartTimeSeconds()))); 29 | 30 | final List previewFrames = new ObjectArrayList<>(); 31 | 32 | info.getPreviewFrames().forEach(frame -> previewFrames.add(new PreviewFrames(frame.getUrls().stream().map(URLUtils::rewriteURL).toList(), frame.getFrameWidth(), 33 | frame.getFrameHeight(), frame.getTotalCount(), frame.getDurationPerFrame(), frame.getFramesPerPageX(), 34 | frame.getFramesPerPageY()))); 35 | 36 | info.getSubtitles() 37 | .forEach(subtitle -> subtitles.add(new Subtitle(rewriteURL(subtitle.getContent()), 38 | subtitle.getFormat().getMimeType(), subtitle.getDisplayLanguageName(), 39 | subtitle.getLanguageTag(), subtitle.isAutoGenerated()))); 40 | 41 | final List videoStreams = new ObjectArrayList<>(); 42 | final List audioStreams = new ObjectArrayList<>(); 43 | 44 | boolean livestream = info.getStreamType() == StreamType.LIVE_STREAM; 45 | 46 | final Map extraParams = Map.of( 47 | // "ump", "1", 48 | // "srfvp", "1" 49 | ); 50 | 51 | if (!livestream) { 52 | info.getVideoOnlyStreams().forEach(stream -> videoStreams.add(new PipedStream(stream.getItag(), rewriteVideoURL(stream.getContent(), extraParams), 53 | String.valueOf(stream.getFormat()), stream.getResolution(), stream.getFormat().getMimeType(), true, 54 | stream.getBitrate(), stream.getInitStart(), stream.getInitEnd(), stream.getIndexStart(), 55 | stream.getIndexEnd(), stream.getCodec(), stream.getWidth(), stream.getHeight(), stream.getFps(), stream.getItagItem().getContentLength()))); 56 | info.getVideoStreams() 57 | .forEach(stream -> videoStreams 58 | .add(new PipedStream(stream.getItag(), rewriteVideoURL(stream.getContent(), Map.of()), String.valueOf(stream.getFormat()), 59 | stream.getResolution(), stream.getFormat().getMimeType(), false, stream.getItagItem().getContentLength()))); 60 | 61 | info.getAudioStreams() 62 | .forEach(stream -> audioStreams.add(new PipedStream(stream.getItag(), rewriteVideoURL(stream.getContent(), extraParams), 63 | String.valueOf(stream.getFormat()), stream.getAverageBitrate() + " kbps", 64 | stream.getFormat().getMimeType(), false, stream.getBitrate(), stream.getInitStart(), 65 | stream.getInitEnd(), stream.getIndexStart(), stream.getIndexEnd(), stream.getItagItem().getContentLength(), stream.getCodec(), stream.getAudioTrackId(), 66 | stream.getAudioTrackName(), Optional.ofNullable(stream.getAudioTrackType()).map(Enum::name).orElse(null), 67 | Optional.ofNullable(stream.getAudioLocale()).map(Locale::toLanguageTag).orElse(null) 68 | ))); 69 | } 70 | 71 | final List relatedStreams = collectRelatedItems(info.getRelatedItems()); 72 | 73 | final List metaInfo = new ObjectArrayList<>(); 74 | info.getMetaInfo().forEach(metaInfoItem -> metaInfo.add(new MetaInfo(metaInfoItem.getTitle(), metaInfoItem.getContent().getContent(), 75 | metaInfoItem.getUrls(), metaInfoItem.getUrlTexts() 76 | ))); 77 | 78 | return new Streams(info.getName(), info.getDescription().getContent(), 79 | info.getTextualUploadDate(), info.getUploadDate() != null ? info.getUploadDate().offsetDateTime().toInstant().toEpochMilli() : -1, 80 | info.getUploaderName(), substringYouTube(info.getUploaderUrl()), getLastThumbnail(info.getUploaderAvatars()), 81 | getLastThumbnail(info.getThumbnails()), info.getDuration(), info.getViewCount(), info.getLikeCount(), info.getDislikeCount(), 82 | info.getUploaderSubscriberCount(), info.isUploaderVerified(), 83 | audioStreams, videoStreams, relatedStreams, subtitles, livestream, rewriteVideoURL(info.getHlsUrl(), Map.of()), 84 | rewriteVideoURL(info.getDashMpdUrl(), Map.of()), null, info.getCategory(), info.getLicence(), 85 | info.getPrivacy().name().toLowerCase(), info.getTags(), metaInfo, chapters, previewFrames); 86 | } 87 | 88 | public static List collectRelatedItems(List items) { 89 | return items 90 | .stream() 91 | .parallel() 92 | .map(item -> { 93 | if (item instanceof StreamInfoItem) { 94 | return collectRelatedStream(item); 95 | } else if (item instanceof PlaylistInfoItem) { 96 | return collectRelatedPlaylist(item); 97 | } else if (item instanceof ChannelInfoItem) { 98 | return collectRelatedChannel(item); 99 | } else { 100 | throw new RuntimeException( 101 | "Unknown item type: " + item.getClass().getName()); 102 | } 103 | }).toList(); 104 | } 105 | 106 | private static StreamItem collectRelatedStream(Object o) { 107 | 108 | StreamInfoItem item = (StreamInfoItem) o; 109 | 110 | return new StreamItem(substringYouTube(item.getUrl()), item.getName(), 111 | getLastThumbnail(item.getThumbnails()), 112 | item.getUploaderName(), substringYouTube(item.getUploaderUrl()), 113 | getLastThumbnail(item.getUploaderAvatars()), item.getTextualUploadDate(), 114 | item.getShortDescription(), item.getDuration(), 115 | item.getViewCount(), item.getUploadDate() != null ? 116 | item.getUploadDate().offsetDateTime().toInstant().toEpochMilli() : -1, 117 | item.isUploaderVerified(), item.isShortFormContent()); 118 | } 119 | 120 | private static PlaylistItem collectRelatedPlaylist(Object o) { 121 | 122 | PlaylistInfoItem item = (PlaylistInfoItem) o; 123 | 124 | return new PlaylistItem(substringYouTube(item.getUrl()), item.getName(), 125 | getLastThumbnail(item.getThumbnails()), 126 | item.getUploaderName(), substringYouTube(item.getUploaderUrl()), 127 | item.isUploaderVerified(), 128 | item.getPlaylistType().name(), item.getStreamCount()); 129 | } 130 | 131 | private static ChannelItem collectRelatedChannel(Object o) { 132 | 133 | ChannelInfoItem item = (ChannelInfoItem) o; 134 | 135 | return new ChannelItem(substringYouTube(item.getUrl()), item.getName(), 136 | getLastThumbnail(item.getThumbnails()), 137 | item.getDescription(), item.getSubscriberCount(), item.getStreamCount(), 138 | item.isVerified()); 139 | } 140 | 141 | public static List collectPreloadedTabs(List tabs) { 142 | return tabs 143 | .stream() 144 | .filter(ReadyChannelTabListLinkHandler.class::isInstance) 145 | .map(ReadyChannelTabListLinkHandler.class::cast) 146 | .toList(); 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /src/main/java/me/kavin/piped/utils/CustomServletDecorator.java: -------------------------------------------------------------------------------- 1 | package me.kavin.piped.utils; 2 | 3 | import io.activej.http.*; 4 | import io.activej.promise.Promisable; 5 | import org.jetbrains.annotations.NotNull; 6 | 7 | import static io.activej.http.HttpHeaders.*; 8 | 9 | public class CustomServletDecorator implements AsyncServlet { 10 | 11 | private static final HttpHeader HEADER = HttpHeaders.of("Server-Timing"); 12 | 13 | private final AsyncServlet servlet; 14 | 15 | public CustomServletDecorator(AsyncServlet servlet) { 16 | this.servlet = servlet; 17 | } 18 | 19 | @Override 20 | public @NotNull Promisable serve(@NotNull HttpRequest request) throws Exception { 21 | long before = System.nanoTime(); 22 | return servlet.serve(request).promise().map(response -> { 23 | 24 | HttpHeaderValue headerValue = HttpHeaderValue.of("app;dur=" + (System.nanoTime() - before) / 1000000.0); 25 | 26 | return response.withHeader(HEADER, headerValue) 27 | .withHeader(ACCESS_CONTROL_ALLOW_ORIGIN, "*") 28 | .withHeader(ACCESS_CONTROL_ALLOW_HEADERS, "*, Authorization") 29 | .withHeader(ACCESS_CONTROL_ALLOW_METHODS, "*"); 30 | 31 | }); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/me/kavin/piped/utils/DatabaseSessionFactory.java: -------------------------------------------------------------------------------- 1 | package me.kavin.piped.utils; 2 | 3 | import me.kavin.piped.consts.Constants; 4 | import me.kavin.piped.utils.obj.db.*; 5 | import org.hibernate.Session; 6 | import org.hibernate.SessionFactory; 7 | import org.hibernate.StatelessSession; 8 | import org.hibernate.cfg.Configuration; 9 | 10 | public class DatabaseSessionFactory { 11 | 12 | private static final SessionFactory sessionFactory; 13 | 14 | static { 15 | try { 16 | final Configuration configuration = new Configuration(); 17 | 18 | Constants.hibernateProperties.forEach(configuration::setProperty); 19 | configuration.configure(); 20 | 21 | sessionFactory = configuration.addAnnotatedClass(User.class).addAnnotatedClass(Channel.class) 22 | .addAnnotatedClass(Video.class).addAnnotatedClass(PubSub.class).addAnnotatedClass(Playlist.class) 23 | .addAnnotatedClass(PlaylistVideo.class).addAnnotatedClass(UnauthenticatedSubscription.class).buildSessionFactory(); 24 | } catch (Exception e) { 25 | throw new RuntimeException(e); 26 | } 27 | } 28 | 29 | public static Session createSession() { 30 | return sessionFactory.openSession(); 31 | } 32 | 33 | public static StatelessSession createStatelessSession() { 34 | return sessionFactory.openStatelessSession(); 35 | } 36 | 37 | public static void close() { 38 | sessionFactory.close(); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/me/kavin/piped/utils/DownloaderImpl.java: -------------------------------------------------------------------------------- 1 | package me.kavin.piped.utils; 2 | 3 | import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap; 4 | import org.schabi.newpipe.extractor.downloader.Downloader; 5 | import org.schabi.newpipe.extractor.downloader.Request; 6 | import org.schabi.newpipe.extractor.downloader.Response; 7 | import org.schabi.newpipe.extractor.exceptions.ReCaptchaException; 8 | import rocks.kavin.reqwest4j.ReqwestUtils; 9 | 10 | import java.io.IOException; 11 | import java.net.HttpCookie; 12 | import java.util.List; 13 | import java.util.Map; 14 | import java.util.concurrent.ExecutionException; 15 | import java.util.concurrent.TimeUnit; 16 | import java.util.concurrent.TimeoutException; 17 | 18 | public class DownloaderImpl extends Downloader { 19 | 20 | private static HttpCookie saved_cookie; 21 | private static long cookie_received; 22 | private static final Object cookie_lock = new Object(); 23 | 24 | /** 25 | * Executes a request with HTTP/2. 26 | */ 27 | @Override 28 | public Response execute(Request request) throws IOException, ReCaptchaException { 29 | 30 | // TODO: HTTP/3 aka QUIC 31 | var bytes = request.dataToSend(); 32 | Map headers = new Object2ObjectOpenHashMap<>(); 33 | 34 | if (saved_cookie != null && !saved_cookie.hasExpired()) 35 | headers.put("Cookie", saved_cookie.getName() + "=" + saved_cookie.getValue()); 36 | 37 | request.headers().forEach((name, values) -> values.forEach(value -> headers.put(name, value))); 38 | 39 | var future = ReqwestUtils.fetch(request.url(), request.httpMethod(), bytes, headers); 40 | 41 | // Recaptcha solver code 42 | // Commented out, as it hasn't been ported to reqwest4j yet 43 | // Also, this was last seen a long time back 44 | 45 | // future.thenAcceptAsync(resp -> { 46 | // if (resp.status() == 429) { 47 | // synchronized (cookie_lock) { 48 | // 49 | // if (saved_cookie != null && saved_cookie.hasExpired() 50 | // || (System.currentTimeMillis() - cookie_received > TimeUnit.MINUTES.toMillis(30))) 51 | // saved_cookie = null; 52 | // 53 | // String redir_url = String.valueOf(resp.finalUrl()); 54 | // 55 | // if (saved_cookie == null && redir_url.startsWith("https://www.google.com/sorry")) { 56 | // 57 | // var formBuilder = new FormBody.Builder(); 58 | // String sitekey = null, data_s = null; 59 | // 60 | // for (Element el : Jsoup.parse(new String(resp.body())).selectFirst("form").children()) { 61 | // String name; 62 | // if (!(name = el.tagName()).equals("script")) { 63 | // if (name.equals("input")) 64 | // formBuilder.add(el.attr("name"), el.attr("value")); 65 | // else if (name.equals("div") && el.attr("id").equals("recaptcha")) { 66 | // sitekey = el.attr("data-sitekey"); 67 | // data_s = el.attr("data-s"); 68 | // } 69 | // } 70 | // } 71 | // 72 | // if (StringUtils.isEmpty(sitekey) || StringUtils.isEmpty(data_s)) 73 | // ExceptionHandler.handle(new ReCaptchaException("Could not get recaptcha", redir_url)); 74 | // 75 | // SolvedCaptcha solved = null; 76 | // 77 | // try { 78 | // solved = CaptchaSolver.solve(redir_url, sitekey, data_s); 79 | // } catch (JsonParserException | InterruptedException | IOException e) { 80 | // e.printStackTrace(); 81 | // } 82 | // 83 | // formBuilder.add("g-recaptcha-response", solved.getRecaptchaResponse()); 84 | // 85 | // var formReqBuilder = new okhttp3.Request.Builder() 86 | // .url("https://www.google.com/sorry/index") 87 | // .header("User-Agent", Constants.USER_AGENT) 88 | // .post(formBuilder.build()); 89 | // 90 | // okhttp3.Response formResponse; 91 | // try { 92 | // formResponse = Constants.h2_no_redir_client.newCall(formReqBuilder.build()).execute(); 93 | // } catch (IOException e) { 94 | // throw new RuntimeException(e); 95 | // } 96 | // 97 | // saved_cookie = HttpCookie.parse(URLUtils.silentDecode(StringUtils 98 | // .substringAfter(formResponse.headers().get("Location"), "google_abuse="))) 99 | // .get(0); 100 | // cookie_received = System.currentTimeMillis(); 101 | // } 102 | // } 103 | // } 104 | // }, Multithreading.getCachedExecutor()); 105 | 106 | var responseFuture = future.thenApplyAsync(resp -> { 107 | Map> headerMap = resp.headers().entrySet().stream() 108 | .collect(Object2ObjectOpenHashMap::new, (m, e) -> m.put(e.getKey(), List.of(e.getValue())), Map::putAll); 109 | 110 | return new Response(resp.status(), null, headerMap, new String(resp.body()), 111 | resp.finalUrl()); 112 | }, Multithreading.getCachedExecutor()); 113 | 114 | try { 115 | return responseFuture.get(10, TimeUnit.SECONDS); 116 | } catch (InterruptedException | ExecutionException | TimeoutException e) { 117 | throw new IOException(e); 118 | } 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /src/main/java/me/kavin/piped/utils/ErrorResponse.java: -------------------------------------------------------------------------------- 1 | package me.kavin.piped.utils; 2 | 3 | import com.fasterxml.jackson.core.JsonProcessingException; 4 | 5 | import java.io.Serial; 6 | 7 | import static me.kavin.piped.consts.Constants.mapper; 8 | 9 | public class ErrorResponse extends Exception { 10 | 11 | @Serial 12 | private static final long serialVersionUID = 1L; 13 | 14 | private final int code; 15 | 16 | private final byte[] content; 17 | 18 | public ErrorResponse(int code, byte[] content) { 19 | this.code = code; 20 | this.content = content; 21 | } 22 | 23 | public ErrorResponse(IStatusCode statusObj) throws JsonProcessingException { 24 | this.code = statusObj.getStatusCode(); 25 | this.content = mapper.writeValueAsBytes(statusObj); 26 | } 27 | 28 | public ErrorResponse(IStatusCode statusObj, Throwable throwable) throws JsonProcessingException { 29 | super(throwable); 30 | this.code = statusObj.getStatusCode(); 31 | this.content = mapper.writeValueAsBytes(statusObj); 32 | } 33 | 34 | public ErrorResponse(int code, Object content) throws JsonProcessingException { 35 | this.code = code; 36 | this.content = mapper.writeValueAsBytes(content); 37 | } 38 | 39 | public ErrorResponse(int code, Object content, Throwable throwable) throws JsonProcessingException { 40 | super(throwable); 41 | this.code = code; 42 | this.content = mapper.writeValueAsBytes(content); 43 | } 44 | 45 | public int getCode() { 46 | return code; 47 | } 48 | 49 | public byte[] getContent() { 50 | return content; 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/main/java/me/kavin/piped/utils/ExceptionHandler.java: -------------------------------------------------------------------------------- 1 | package me.kavin.piped.utils; 2 | 3 | import com.fasterxml.jackson.core.JsonProcessingException; 4 | import io.sentry.Sentry; 5 | import me.kavin.piped.consts.Constants; 6 | import me.kavin.piped.utils.resp.InvalidRequestResponse; 7 | import org.apache.commons.lang3.exception.ExceptionUtils; 8 | import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; 9 | import org.schabi.newpipe.extractor.exceptions.ExtractionException; 10 | 11 | import java.util.concurrent.CompletionException; 12 | import java.util.concurrent.ExecutionException; 13 | 14 | public class ExceptionHandler { 15 | 16 | public static Exception handle(Exception e) { 17 | return handle(e, null); 18 | } 19 | 20 | public static Exception handle(Exception e, String path) { 21 | 22 | if (e.getCause() != null && (e instanceof ExecutionException || e instanceof CompletionException)) 23 | e = (Exception) e.getCause(); 24 | 25 | if (e instanceof ContentNotAvailableException || e instanceof ErrorResponse) 26 | return e; 27 | 28 | if ((e instanceof ExtractionException extractionException && extractionException.getMessage().contains("No service can handle the url"))) 29 | try { 30 | return new ErrorResponse(new InvalidRequestResponse("Invalid parameter provided, unknown service"), extractionException); 31 | } catch (JsonProcessingException jsonProcessingException) { 32 | throw new RuntimeException(jsonProcessingException); 33 | } 34 | 35 | Sentry.captureException(e); 36 | if (Constants.SENTRY_DSN.isEmpty()) { 37 | if (path != null) 38 | System.err.println("An error occoured in the path: " + path); 39 | e.printStackTrace(); 40 | } 41 | 42 | return e; 43 | } 44 | 45 | public static void throwErrorResponse(IStatusCode statusObj) { 46 | try { 47 | ExceptionUtils.rethrow(new ErrorResponse(statusObj)); 48 | } catch (JsonProcessingException e) { 49 | throw new RuntimeException(e); 50 | } 51 | } 52 | 53 | public static void throwErrorResponse(int code, Object content) { 54 | try { 55 | ExceptionUtils.rethrow(new ErrorResponse(code, content)); 56 | } catch (JsonProcessingException e) { 57 | throw new RuntimeException(e); 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/main/java/me/kavin/piped/utils/FeedHelpers.java: -------------------------------------------------------------------------------- 1 | package me.kavin.piped.utils; 2 | 3 | import com.rometools.rome.feed.synd.SyndFeed; 4 | import com.rometools.rome.feed.synd.SyndFeedImpl; 5 | import jakarta.persistence.criteria.CriteriaBuilder; 6 | import jakarta.persistence.criteria.CriteriaQuery; 7 | import jakarta.persistence.criteria.JoinType; 8 | import me.kavin.piped.consts.Constants; 9 | import me.kavin.piped.utils.obj.SubscriptionChannel; 10 | import me.kavin.piped.utils.obj.db.Channel; 11 | import me.kavin.piped.utils.obj.db.User; 12 | import me.kavin.piped.utils.obj.db.Video; 13 | import org.hibernate.StatelessSession; 14 | 15 | import javax.annotation.Nullable; 16 | import java.util.Comparator; 17 | import java.util.Date; 18 | import java.util.Set; 19 | import java.util.function.Predicate; 20 | import java.util.stream.Stream; 21 | 22 | import static me.kavin.piped.utils.URLUtils.rewriteURL; 23 | 24 | public class FeedHelpers { 25 | public static Stream