├── .dockerignore ├── .github ├── FUNDING.yml ├── dependabot.yml └── workflows │ ├── ci.yml │ ├── nightly.yml │ └── release.yml ├── .gitignore ├── .golangci.yml ├── .goreleaser.yml ├── .vscode └── launch.json ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── cloud_formation.yml ├── cmd └── podsync │ ├── config.go │ ├── config_test.go │ └── main.go ├── config.toml.example ├── docker-compose.yml ├── docs ├── cron.md ├── how_to_get_vimeo_token.md ├── how_to_get_youtube_api_key.md ├── how_to_setup_podsync_on_qnap_nas.md ├── how_to_setup_podsync_on_synology_nas.md └── img │ ├── logo.png │ ├── vimeo_access_token.png │ ├── vimeo_create_app.png │ ├── vimeo_token.png │ ├── youtube_copy_token.png │ ├── youtube_create_api_key.png │ ├── youtube_dashboard.png │ ├── youtube_data_api_enable.png │ ├── youtube_data_api_v3.png │ ├── youtube_new_project.png │ └── youtube_select_project.png ├── go.mod ├── go.sum ├── pkg ├── builder │ ├── builder.go │ ├── soundcloud.go │ ├── soundcloud_test.go │ ├── url.go │ ├── url_test.go │ ├── vimeo.go │ ├── vimeo_test.go │ ├── youtube.go │ └── youtube_test.go ├── db │ ├── badger.go │ ├── badger_test.go │ ├── config.go │ └── storage.go ├── feed │ ├── config.go │ ├── deps.go │ ├── deps_mock_test.go │ ├── key.go │ ├── key_test.go │ ├── opml.go │ ├── opml_test.go │ ├── xml.go │ └── xml_test.go ├── fs │ ├── local.go │ ├── local_test.go │ ├── s3.go │ ├── s3_test.go │ └── storage.go ├── model │ ├── defaults.go │ ├── errors.go │ ├── feed.go │ └── link.go └── ytdl │ ├── temp_file.go │ ├── ytdl.go │ └── ytdl_test.go └── services ├── update ├── matcher.go └── updater.go └── web └── server.go /.dockerignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | node_modules/ 3 | vendor/ 4 | bin/ -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: mxpv 4 | patreon: podsync 5 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "gomod" 4 | directory: "/" 5 | schedule: 6 | interval: "daily" 7 | 8 | - package-ecosystem: "github-actions" 9 | directory: "/" 10 | schedule: 11 | interval: "daily" 12 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | on: 3 | push: 4 | branches: 5 | - main 6 | pull_request: 7 | branches: 8 | - main 9 | 10 | jobs: 11 | build: 12 | name: Build 13 | runs-on: ${{ matrix.os }} 14 | timeout-minutes: 10 15 | 16 | strategy: 17 | matrix: 18 | os: [ubuntu-latest, windows-latest, macos-latest] 19 | 20 | steps: 21 | - uses: actions/setup-go@v5 22 | with: 23 | go-version: 1.18 24 | - uses: actions/checkout@v4 25 | - run: make build 26 | - uses: actions/upload-artifact@v4 27 | with: 28 | name: podsync-${{ matrix.os }} 29 | path: bin/ 30 | 31 | test: 32 | name: Test 33 | runs-on: ubuntu-latest 34 | timeout-minutes: 10 35 | 36 | steps: 37 | - uses: actions/setup-go@v5 38 | with: 39 | go-version: 1.18 40 | - uses: actions/checkout@v4 41 | - env: 42 | VIMEO_TEST_API_KEY: ${{ secrets.VIMEO_ACCESS_TOKEN }} 43 | YOUTUBE_TEST_API_KEY: ${{ secrets.YOUTUBE_API_KEY }} 44 | run: make test 45 | 46 | checks: 47 | name: Checks 48 | runs-on: ubuntu-latest 49 | timeout-minutes: 10 50 | 51 | steps: 52 | - uses: actions/setup-go@v5 53 | - uses: actions/checkout@v4 54 | - uses: golangci/golangci-lint-action@v6 55 | with: 56 | version: v1.57.2 57 | 58 | - name: Go mod 59 | env: 60 | DIFF_PATH: "go.mod go.sum" 61 | run: | 62 | go mod tidy 63 | DIFF=$(git status --porcelain -- $DIFF_PATH) 64 | if [ "$DIFF" ]; then 65 | echo 66 | echo "These files were modified:" 67 | echo 68 | echo "$DIFF" 69 | echo 70 | exit 1 71 | fi 72 | -------------------------------------------------------------------------------- /.github/workflows/nightly.yml: -------------------------------------------------------------------------------- 1 | name: Nightly 2 | 3 | on: 4 | schedule: 5 | - cron: "0 0 * * *" # Every day at midnight 6 | push: 7 | paths: 8 | - ".github/workflows/nightly.yml" 9 | 10 | env: 11 | REGISTRY: ghcr.io 12 | IMAGE_NAME: ${{ github.repository }} 13 | 14 | jobs: 15 | publish: 16 | name: Nightly 17 | runs-on: ubuntu-latest 18 | timeout-minutes: 20 19 | 20 | permissions: 21 | contents: read 22 | packages: write 23 | 24 | steps: 25 | - name: 📦 Checkout repository 26 | uses: actions/checkout@v4 27 | 28 | - name: 🧪 Set up Docker Buildx 29 | uses: docker/setup-buildx-action@v3 30 | 31 | - name: 🔒 Log in to the Container registry 32 | uses: docker/login-action@v3 33 | with: 34 | registry: ${{ env.REGISTRY }} 35 | username: ${{ github.actor }} 36 | password: ${{ secrets.GITHUB_TOKEN }} 37 | 38 | - name: 🏗️ Build and push 39 | uses: docker/build-push-action@v6 40 | env: 41 | TAG: nightly 42 | COMMIT: ${{ github.sha }} 43 | with: 44 | cache-from: type=gha 45 | cache-to: type=gha,mode=max 46 | platforms: linux/amd64,linux/arm64 47 | push: true 48 | tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:nightly 49 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'v*' 7 | 8 | env: 9 | REGISTRY: ghcr.io 10 | IMAGE_NAME: ${{ github.repository }} 11 | 12 | jobs: 13 | publish: 14 | name: Publish 15 | runs-on: ubuntu-latest 16 | timeout-minutes: 20 17 | 18 | permissions: 19 | contents: write 20 | packages: write 21 | 22 | steps: 23 | - name: 📦 Checkout repository 24 | uses: actions/checkout@v4 25 | with: 26 | fetch-depth: 0 27 | 28 | - name: 🧪 Set up Docker Buildx 29 | uses: docker/setup-buildx-action@v3 30 | 31 | - name: 🔒 Log in to the Container registry 32 | uses: docker/login-action@v3 33 | with: 34 | registry: ${{ env.REGISTRY }} 35 | username: ${{ github.actor }} 36 | password: ${{ secrets.GITHUB_TOKEN }} 37 | 38 | - name: 🏗️ Build container and push 39 | uses: docker/build-push-action@v6 40 | env: 41 | TAG: ${{ github.ref_name }} 42 | COMMIT: ${{ github.sha }} 43 | with: 44 | cache-from: type=gha 45 | cache-to: type=gha,mode=max 46 | platforms: linux/amd64,linux/arm64 47 | push: true 48 | tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest, ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.ref_name }} 49 | 50 | - name: 🚧️ Make release 51 | uses: goreleaser/goreleaser-action@v6 52 | if: startsWith(github.ref, 'refs/tags/') 53 | env: 54 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 55 | with: 56 | version: latest 57 | args: release --clean 58 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | vendor/ 10 | 11 | # Architecture specific extensions/prefixes 12 | *.[568vq] 13 | [568vq].out 14 | 15 | *.cgo1.go 16 | *.cgo2.c 17 | _cgo_defun.c 18 | _cgo_gotypes.go 19 | _cgo_export.* 20 | 21 | _testmain.go 22 | 23 | *.test 24 | *.prof 25 | 26 | .idea/ 27 | bin/ 28 | node_modules/ 29 | dist/ 30 | venv/ 31 | 32 | .DS_Store 33 | /podsync 34 | podsync.log 35 | 36 | db 37 | config.toml -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | linters: 2 | enable: 3 | - structcheck 4 | - varcheck 5 | - staticcheck 6 | - unconvert 7 | - gofmt 8 | - goimports 9 | - golint 10 | - ineffassign 11 | - vet 12 | - unused 13 | - misspell 14 | - bodyclose 15 | - interfacer 16 | - unconvert 17 | - maligned 18 | # - depguard 19 | - nakedret 20 | - prealloc 21 | - whitespace 22 | disable: 23 | - errcheck 24 | 25 | run: 26 | deadline: 3m 27 | skip-dirs: 28 | - bin 29 | - docs 30 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | project_name: "Podsync" 2 | 3 | before: 4 | hooks: 5 | - go mod tidy 6 | 7 | builds: 8 | - main: ./cmd/podsync/ 9 | binary: podsync 10 | env: 11 | - CGO_ENABLED=0 12 | goos: 13 | - linux 14 | - darwin 15 | - windows 16 | goarch: 17 | - 386 18 | - amd64 19 | - arm 20 | - arm64 21 | 22 | archives: 23 | - id: arc 24 | name_template: >- 25 | {{- .ProjectName }}_{{.Version}}_ 26 | {{- title .Os }}_ 27 | {{- if eq .Arch "amd64" }}x86_64 28 | {{- else if eq .Arch "386" }}i386 29 | {{- else }}{{ .Arch }}{{ end }} 30 | {{- if .Arm }}v{{ .Arm }}{{ end -}} 31 | format_overrides: 32 | - goos: windows 33 | format: zip 34 | 35 | checksum: 36 | name_template: 'checksums.txt' 37 | 38 | snapshot: 39 | name_template: '{{ .Tag }}-next' 40 | 41 | changelog: 42 | sort: asc 43 | filters: 44 | exclude: 45 | - '^docs:' 46 | - '^test:' 47 | - Merge pull request 48 | - Merge branch 49 | 50 | release: 51 | # We publish Docker image manually, 52 | # include links to the release notes. 53 | footer: | 54 | # Docker images 55 | ``` 56 | docker pull ghcr.io/mxpv/podsync:{{ .Tag }} 57 | docker pull ghcr.io/mxpv/podsync:latest 58 | ``` 59 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.2.0", 3 | "configurations": [ 4 | { 5 | // from https://github.com/vscode-debug-specs/go#debugging-executable-file 6 | "name": "Debug Podsync", 7 | "type": "go", 8 | "request": "launch", 9 | "mode": "debug", 10 | "program": "${workspaceFolder}/cmd/podsync", 11 | "cwd": "${workspaceFolder}", 12 | "args": ["--config", "config.toml"] 13 | } 14 | ] 15 | } -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.20 as builder 2 | 3 | ENV TAG="nightly" 4 | ENV COMMIT="" 5 | 6 | WORKDIR /build 7 | 8 | COPY . . 9 | 10 | RUN make build 11 | 12 | # Download youtube-dl 13 | RUN wget -O /usr/bin/yt-dlp https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp && \ 14 | chmod +x /usr/bin/yt-dlp 15 | 16 | # Alpine 3.21 will go EOL on 2026-11-01 17 | FROM alpine:3.21 18 | 19 | WORKDIR /app 20 | 21 | RUN apk --no-cache add ca-certificates python3 py3-pip ffmpeg tzdata \ 22 | # https://github.com/golang/go/issues/59305 23 | libc6-compat && ln -s /lib/libc.so.6 /usr/lib/libresolv.so.2 24 | 25 | COPY --from=builder /usr/bin/yt-dlp /usr/bin/youtube-dl 26 | COPY --from=builder /build/bin/podsync /app/podsync 27 | 28 | ENTRYPOINT ["/app/podsync"] 29 | CMD ["--no-banner"] 30 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Maksym Pavlenko 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | BINPATH := $(abspath ./bin) 2 | 3 | .PHONY: all 4 | all: build test 5 | 6 | # 7 | # Build Podsync CLI binary 8 | # Example: 9 | # $ GOOS=amd64 make build 10 | # 11 | 12 | GOARCH ?= $(shell go env GOARCH) 13 | GOOS ?= $(shell go env GOOS) 14 | 15 | TAG ?= $(shell git tag --points-at HEAD) 16 | COMMIT ?= $(shell git rev-parse --short HEAD) 17 | DATE := $(shell date) 18 | 19 | LDFLAGS := "-X 'main.version=${TAG}' -X 'main.commit=${COMMIT}' -X 'main.date=${DATE}' -X 'main.arch=${GOARCH}'" 20 | 21 | .PHONY: build 22 | build: 23 | go build -ldflags ${LDFLAGS} -o bin/podsync ./cmd/podsync 24 | 25 | # 26 | # Build a local Docker image 27 | # Example: 28 | # $ make docker 29 | # $ docker run -it --rm localhost/podsync:latest 30 | # 31 | IMAGE_TAG ?= localhost/podsync 32 | .PHONY: docker 33 | docker: 34 | docker buildx build -t $(IMAGE_TAG) . 35 | 36 | # 37 | # Run unit tests 38 | # 39 | .PHONY: test 40 | test: 41 | go test -v ./... 42 | 43 | # 44 | # Clean 45 | # 46 | .PHONY: clean 47 | clean: 48 | - rm -rf $(BINPATH) 49 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Podsync 2 | 3 | ![Podsync](docs/img/logo.png) 4 | 5 | [![](https://github.com/mxpv/podsync/workflows/CI/badge.svg)](https://github.com/mxpv/podsync/actions?query=workflow%3ACI) 6 | [![Nightly](https://github.com/mxpv/podsync/actions/workflows/nightly.yml/badge.svg)](https://github.com/mxpv/podsync/actions/workflows/nightly.yml) 7 | [![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/mxpv/podsync)](https://github.com/mxpv/podsync/releases) 8 | [![Go Report Card](https://goreportcard.com/badge/github.com/mxpv/podsync)](https://goreportcard.com/report/github.com/mxpv/podsync) 9 | [![GitHub Sponsors](https://img.shields.io/github/sponsors/mxpv)](https://github.com/sponsors/mxpv) 10 | [![Patreon](https://img.shields.io/badge/support-patreon-E6461A.svg)](https://www.patreon.com/podsync) 11 | [![Twitter Follow](https://img.shields.io/twitter/follow/pod_sync?style=social)](https://twitter.com/pod_sync) 12 | 13 | Podsync - is a simple, free service that lets you listen to any YouTube / Vimeo channels, playlists or user videos in 14 | podcast format. 15 | 16 | Podcast applications have a rich functionality for content delivery - automatic download of new episodes, 17 | remembering last played position, sync between devices and offline listening. This functionality is not available 18 | on YouTube and Vimeo. So the aim of Podsync is to make your life easier and enable you to view/listen to content on 19 | any device in podcast client. 20 | 21 | ## Features 22 | 23 | - Works with YouTube and Vimeo. 24 | - Supports feeds configuration: video/audio, high/low quality, max video height, etc. 25 | - mp3 encoding 26 | - Update scheduler supports cron expressions 27 | - Episodes filtering (match by title, duration). 28 | - Feeds customizations (custom artwork, category, language, etc). 29 | - OPML export. 30 | - Supports episodes cleanup (keep last X episodes). 31 | - One-click deployment for AWS. 32 | - Runs on Windows, Mac OS, Linux, and Docker. 33 | - Supports ARM. 34 | - Automatic youtube-dl self update. 35 | - Supports API keys rotation. 36 | 37 | ## Dependencies 38 | 39 | If you're running the CLI as binary (e.g. not via Docker), you need to make sure that dependencies are available on 40 | your system. Currently, Podsync depends on `youtube-dl` , `ffmpeg`, and `go`. 41 | 42 | On Mac you can install those with `brew`: 43 | ``` 44 | brew install youtube-dl ffmpeg go 45 | ``` 46 | 47 | ## Documentation 48 | 49 | - [How to get Vimeo API token](./docs/how_to_get_vimeo_token.md) 50 | - [How to get YouTube API Key](./docs/how_to_get_youtube_api_key.md) 51 | - [Podsync on QNAP NAS Guide](./docs/how_to_setup_podsync_on_qnap_nas.md) 52 | - [Schedule updates with cron](./docs/cron.md) 53 | 54 | ## Nightly builds 55 | 56 | Nightly builds uploaded every midnight from the `main` branch and available for testing: 57 | 58 | ```bash 59 | $ docker run -it --rm ghcr.io/mxpv/podsync:nightly 60 | ``` 61 | 62 | ### Access tokens 63 | 64 | In order to query YouTube or Vimeo API you have to obtain an API token first. 65 | 66 | - [How to get YouTube API key](https://elfsight.com/blog/2016/12/how-to-get-youtube-api-key-tutorial/) 67 | - [Generate an access token for Vimeo](https://developer.vimeo.com/api/guides/start#generate-access-token) 68 | 69 | ## Configuration 70 | 71 | You need to create a configuration file (for instance `config.toml`) and specify the list of feeds that you're going to host. 72 | See [config.toml.example](./config.toml.example) for all possible configuration keys available in Podsync. 73 | 74 | Minimal configuration would look like this: 75 | 76 | ```toml 77 | [server] 78 | port = 8080 79 | 80 | [storage] 81 | [storage.local] 82 | # Don't change if you run podsync via docker 83 | data_dir = "/app/data/" 84 | 85 | [tokens] 86 | youtube = "PASTE YOUR API KEY HERE" 87 | 88 | [feeds] 89 | [feeds.ID1] 90 | url = "https://www.youtube.com/channel/UCxC5Ls6DwqV0e-CYcAKkExQ" 91 | ``` 92 | 93 | If you want to hide Podsync behind reverse proxy like nginx, you can use `hostname` field: 94 | 95 | ```toml 96 | [server] 97 | port = 8080 98 | hostname = "https://my.test.host:4443" 99 | 100 | [feeds] 101 | [feeds.ID1] 102 | ... 103 | ``` 104 | 105 | Server will be accessible from `http://localhost:8080`, but episode links will point to `https://my.test.host:4443/ID1/...` 106 | 107 | ## One click deployment 108 | 109 | [![Deploy to AWS](https://s3.amazonaws.com/cloudformation-examples/cloudformation-launch-stack.png)](https://console.aws.amazon.com/cloudformation/home?region=us-west-1#/stacks/new?stackName=Podsync&templateURL=https://podsync-cf.s3.amazonaws.com/cloud_formation.yml) 110 | 111 | ## How to run 112 | 113 | 114 | ### Build and run as binary: 115 | 116 | Make sure you have created the file `config.toml`. Also note the location of the `data_dir`. Depending on the operating system, you may have to choose a different location since `/app/data` might be not writable. 117 | 118 | ``` 119 | $ git clone https://github.com/mxpv/podsync 120 | $ cd podsync 121 | $ make 122 | $ ./bin/podsync --config config.toml 123 | ``` 124 | 125 | ### How to debug 126 | 127 | Use the editor [Visual Studio Code](https://code.visualstudio.com/) and install the official [Go](https://marketplace.visualstudio.com/items?itemName=golang.go) extension. Afterwards you can execute "Run & Debug" ▶︎ "Debug Podsync" to debug the application. The required configuration is already prepared (see `.vscode/launch.json`). 128 | 129 | 130 | ### Run via Docker: 131 | ``` 132 | $ docker pull ghcr.io/mxpv/podsync:latest 133 | $ docker run \ 134 | -p 8080:8080 \ 135 | -v $(pwd)/data:/app/data/ \ 136 | -v $(pwd)/config.toml:/app/config.toml \ 137 | ghcr.io/mxpv/podsync:latest 138 | ``` 139 | 140 | ### Run via Docker Compose: 141 | ``` 142 | $ cat docker-compose.yml 143 | services: 144 | podsync: 145 | image: ghcr.io/mxpv/podsync 146 | container_name: podsync 147 | volumes: 148 | - ./data:/app/data/ 149 | - ./config.toml:/app/config.toml 150 | ports: 151 | - 8080:8080 152 | 153 | $ docker compose up 154 | ``` 155 | 156 | ## How to make a release 157 | 158 | Just push a git tag. CI will do the rest. 159 | 160 | -------------------------------------------------------------------------------- /cloud_formation.yml: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: '2010-09-09' 2 | 3 | Parameters: 4 | InstanceType: 5 | Type: String 6 | Default: t3.micro 7 | Description: EC2 machine instance size (see https://aws.amazon.com/ec2/instance-types/) 8 | 9 | AmiId: 10 | Type: AWS::SSM::Parameter::Value 11 | Default: '/aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2' 12 | Description: Amazon Linux 2 image ID (leave this as is) 13 | 14 | VolumeSize: 15 | Type: Number 16 | Default: 64 17 | MinValue: 16 18 | Description: Disk size in Gb to allocate for storing downloaded episodes 19 | 20 | PodsyncVersion: 21 | Type: String 22 | Default: latest 23 | Description: Podsync version to use (see https://github.com/mxpv/podsync/releases) 24 | 25 | PodsyncPort: 26 | Type: Number 27 | Default: 8080 28 | MaxValue: 65535 29 | Description: Server port to use 30 | 31 | YouTubeApiKey: 32 | Type: String 33 | AllowedPattern: '.+' # Required 34 | Description: | 35 | Key to use for YouTube API access (see https://github.com/mxpv/podsync/blob/master/docs/how_to_get_youtube_api_key.md) 36 | 37 | VimeoAccessToken: 38 | Type: String 39 | AllowedPattern: '.+' # Required 40 | Description: | 41 | Key to use for Vimeo API access (see https://github.com/mxpv/podsync/blob/master/docs/how_to_get_vimeo_token.md) 42 | 43 | FeedId: 44 | Type: String 45 | Default: 'ID1' 46 | AllowedPattern: '.+' # Required 47 | Description: | 48 | ID to use for the feed (you'll access it as http://localhost/ID1.xml) 49 | 50 | FeedUrl: 51 | Type: String 52 | AllowedPattern: '.+' 53 | Description: | 54 | YouTube or Vimeo URL to host with Podsync (for example: https://www.youtube.com/user/XYZ) 55 | 56 | PageSize: 57 | Type: Number 58 | Default: 50 59 | MinValue: 5 60 | Description: | 61 | The number of episodes to query each time 62 | 63 | Format: 64 | Type: String 65 | AllowedValues: 66 | - 'audio' 67 | - 'video' 68 | Default: 'video' 69 | Description: Feed format (audio or video) 70 | 71 | Quality: 72 | Type: String 73 | AllowedValues: 74 | - 'high' 75 | - 'low' 76 | Default: 'high' 77 | Description: Feed quality (high or low) 78 | 79 | Metadata: 80 | AWS::CloudFormation::Interface: 81 | ParameterGroups: 82 | - Label: 83 | default: 'VM configuration' 84 | Parameters: 85 | - InstanceType 86 | - KeyName 87 | - AmiId 88 | - VolumeSize 89 | - Label: 90 | default: 'Podsync configuration' 91 | Parameters: 92 | - PodsyncVersion 93 | - PodsyncPort 94 | - YouTubeApiKey 95 | - VimeoAccessToken 96 | - Label: 97 | default: 'Feed configuration' 98 | Parameters: 99 | - FeedId 100 | - FeedUrl 101 | - PageSize 102 | - Format 103 | - Quality 104 | 105 | ParameterLabels: 106 | InstanceType: 107 | default: 'Instance type' 108 | AmiId: 109 | default: 'AMI ID' 110 | VolumeSize: 111 | default: 'Volume size' 112 | PodsyncVersion: 113 | default: 'Version' 114 | PodsyncPort: 115 | default: 'Server port' 116 | YouTubeApiKey: 117 | default: 'YouTube API Key' 118 | VimeoAccessToken: 119 | default: 'Vimeo Token' 120 | FeedId: 121 | default: 'Feed ID' 122 | FeedUrl: 123 | default: 'Feed URL' 124 | PageSize: 125 | default: 'Page size' 126 | 127 | Resources: 128 | NewKeyPair: 129 | Type: AWS::EC2::KeyPair 130 | Properties: 131 | KeyName: !Sub "${AWS::StackName}" 132 | Ec2Instance: 133 | Type: AWS::EC2::Instance 134 | CreationPolicy: 135 | ResourceSignal: 136 | Count: 1 137 | Properties: 138 | InstanceType: !Ref InstanceType 139 | KeyName: !Ref NewKeyPair 140 | ImageId: !Ref AmiId 141 | SecurityGroups: 142 | - !Ref AccessSecurityGroup 143 | EbsOptimized: true 144 | BlockDeviceMappings: 145 | - DeviceName: /dev/xvda 146 | Ebs: 147 | VolumeSize: !Ref VolumeSize 148 | IamInstanceProfile: !Ref SsmInstanceProfile 149 | Tags: 150 | - Key: 'Name' 151 | Value: !Sub "${AWS::StackName}" 152 | UserData: 153 | Fn::Base64: !Sub | 154 | #!/usr/bin/env bash 155 | set -ex 156 | trap '/opt/aws/bin/cfn-signal --exit-code 1 --resource Ec2Instance --region ${AWS::Region} --stack ${AWS::StackName}' ERR 157 | 158 | # Install Docker 159 | yum update -y 160 | amazon-linux-extras install docker 161 | systemctl start docker 162 | usermod -a -G docker ec2-user 163 | 164 | export publichost=$(ec2-metadata --public-hostname | cut -d ' ' -f2) 165 | # Create configuration file 166 | mkdir -p /home/ec2-user/podsync/data 167 | tee /home/ec2-user/podsync/config.toml < %s (update '%s')", cronFeed.ID, cronFeed.CronSchedule) 215 | // Perform initial update after CLI restart 216 | updates <- cronFeed 217 | } 218 | 219 | c.Start() 220 | 221 | for { 222 | <-ctx.Done() 223 | 224 | log.Info("shutting down cron") 225 | c.Stop() 226 | 227 | return ctx.Err() 228 | } 229 | }) 230 | 231 | if cfg.Storage.Type == "s3" { 232 | return // S3 content is hosted externally 233 | } 234 | 235 | // Run web server 236 | srv := web.New(cfg.Server, storage) 237 | 238 | group.Go(func() error { 239 | log.Infof("running listener at %s", srv.Addr) 240 | if cfg.Server.TLS { 241 | return srv.ListenAndServeTLS(cfg.Server.CertificatePath, cfg.Server.KeyFilePath) 242 | } else { 243 | return srv.ListenAndServe() 244 | } 245 | }) 246 | 247 | group.Go(func() error { 248 | // Shutdown web server 249 | defer func() { 250 | ctxShutDown, cancel := context.WithTimeout(context.Background(), 5*time.Second) 251 | defer func() { 252 | cancel() 253 | }() 254 | log.Info("shutting down web server") 255 | if err := srv.Shutdown(ctxShutDown); err != nil { 256 | log.WithError(err).Error("server shutdown failed") 257 | } 258 | }() 259 | 260 | for { 261 | select { 262 | case <-ctx.Done(): 263 | return ctx.Err() 264 | case <-stop: 265 | cancel() 266 | return nil 267 | } 268 | } 269 | }) 270 | } 271 | -------------------------------------------------------------------------------- /config.toml.example: -------------------------------------------------------------------------------- 1 | # This is an example of TOML configuration file for Podsync. 2 | 3 | # Web server related configuration. 4 | [server] 5 | # HTTP server port. 6 | port = 8080 7 | # Optional. If you want to hide Podsync behind reverse proxy like nginx, you can use hostname field. 8 | # Server will be accessible from http://localhost:8080, but episode links will point to https://my.test.host:4443/ID1/XYZ 9 | hostname = "https://my.test.host:4443" 10 | # Bind a specific IP addresses for server ,"*": bind all IP addresses which is default option, localhost or 127.0.0.1 bind a single IPv4 address 11 | bind_address = "172.20.10.2" 12 | # Specify path for reverse proxy and only [A-Za-z0-9] 13 | path = "test" 14 | # Optional. If you want to use TLS you must set the TLS flag and path to the certificate file and private key file. 15 | tls = true 16 | certificate_path = "/var/www/cert.pem" 17 | key_file_path = "/var/www/priv.pem" 18 | 19 | # Configure where to store the episode data 20 | [storage] 21 | # Could be "local" (default) for the local file system, or "s3" for a S3-compatible storage provider (e.g. AWS S3) 22 | type = "local" 23 | 24 | [storage.local] 25 | data_dir = "/app/data" # Don't change if you run podsync via docker 26 | 27 | # To configure for a S3 provider, set key and secret in environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`, respectively; 28 | # then fillout the API endpoint, region, and bucket below. 29 | [storage.s3] 30 | endpoint_url = "https://s3.us-west-2.amazonaws.com" 31 | region = "us-west-2" 32 | bucket = "example-bucket-name" 33 | # If you use prefix, you may need to add a path to `server.hostname` setting 34 | # e.g. https://example-bucket-name.s3.us-west-2.amazonaws.com/example/prefix/ 35 | prefix = "example/prefix" 36 | 37 | # API keys to be used to access Youtube and Vimeo. 38 | # These can be either specified as string parameter or array of string (so those will be rotated). 39 | [tokens] 40 | youtube = "YOUTUBE_API_TOKEN" # YouTube API Key. See https://developers.google.com/youtube/registering_an_application 41 | vimeo = [ # Multiple keys will be rotated. 42 | "VIMEO_API_KEY_1", # Vimeo developer keys. See https://developer.vimeo.com/api/guides/start#generate-access-token 43 | "VIMEO_API_KEY_2" 44 | ] 45 | 46 | # The list of data sources to be hosted by Podsync. 47 | # These are channels, users, playlists, etc. 48 | [feeds] 49 | # Each channel must have a unique identifier (in this example "ID1"). 50 | [feeds.ID1] 51 | # URL address of a channel, group, user, or playlist. 52 | url = "https://www.youtube.com/channel/CHANNEL_NAME_TO_HOST" 53 | 54 | # The number of episodes to query each update (keep in mind, that this might drain API token) 55 | page_size = 50 56 | 57 | # How often query for updates, examples: "60m", "4h", "2h45m" 58 | update_period = "12h" 59 | 60 | quality = "high" # "high" or "low" 61 | format = "video" # "audio", "video" or "custom" 62 | # When format = "custom" 63 | # YouTubeDL format parameter and result file extension 64 | custom_format = { youtube_dl_format = "bestaudio[ext=m4a]", extension = "m4a" } 65 | 66 | playlist_sort = "asc" # or "desc", which will fetch playlist items from the end 67 | 68 | # Optional maximal height of video, example: 720, 1080, 1440, 2160, ... 69 | max_height = 720 70 | 71 | # Optionally include this feed in OPML file (default value: false) 72 | opml = true 73 | 74 | # Optional cron expression format for more precise update schedule. 75 | # If set then overwrite 'update_period'. 76 | cron_schedule = "@every 12h" 77 | 78 | # Whether to cleanup old episodes. 79 | # Keep last 10 episodes (order desc by PubDate) 80 | clean = { keep_last = 10 } 81 | 82 | # Optional Golang regexp format. 83 | # If set, then only download matching episodes. 84 | # Duration filters are in seconds. 85 | # max_age filter is in days. 86 | filters = { title = "regex for title here", not_title = "regex for negative title match", description = "...", not_description = "...", min_duration = 0, max_duration = 86400, max_age = 365 } 87 | 88 | # Optional extra arguments passed to youtube-dl when downloading videos from this feed. 89 | # This example would embed available English closed captions in the videos. 90 | # Note that setting '--audio-format' for audio format feeds, or '--format' or '--output' for any format may cause 91 | # unexpected behaviour. You should only use this if you know what you are doing, and have read up on youtube-dl's options! 92 | youtube_dl_args = ["--write-sub", "--embed-subs", "--sub-lang", "en,en-US,en-GB"] 93 | 94 | # When set to true, podcasts indexers such as iTunes or Google Podcasts will not index this podcast 95 | private_feed = true 96 | 97 | # Optional feed customizations 98 | [feeds.ID1.custom] 99 | title = "Level1News" 100 | description = "News sections of Level1Techs, in a podcast feed!" 101 | author = "Level1Tech" 102 | cover_art = "{IMAGE_URL}" 103 | cover_art_quality = "high" 104 | category = "TV" 105 | subcategories = ["Documentary", "Tech News"] 106 | explicit = true 107 | lang = "en" 108 | author = "Mrs. Smith (mrs@smith.org)" 109 | ownerName = "Mrs. Smith" 110 | ownerEmail = "mrs@smith.org" 111 | # optional: this will override the default link (usually the URL address) in the generated RSS feed with another link 112 | link = "https://example.org" 113 | 114 | # Podsync uses local database to store feeds and episodes metadata. 115 | # This section is optional and usually not needed to configure unless some very specific corner cases. 116 | # Refer to https://dgraph.io/docs/badger/get-started/#memory-usage for documentation. 117 | [database] 118 | badger = { truncate = true, file_io = true } 119 | 120 | # Youtube-dl specific configuration. 121 | [downloader] 122 | # Optional, auto update youtube-dl every 24 hours 123 | self_update = true 124 | # Download timeout in minutes. 125 | timeout = 15 126 | 127 | # Optional log config. If not specified logs to the stdout 128 | [log] 129 | filename = "podsync.log" 130 | max_size = 50 # MB 131 | max_age = 30 # days 132 | max_backups = 7 133 | compress = true 134 | debug = false 135 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2.2' 2 | 3 | services: 4 | podsync: 5 | container_name: podsync 6 | image: mxpv/podsync:latest 7 | restart: always 8 | ports: 9 | - 80:80 10 | volumes: 11 | - ./data:/app/data/ 12 | - ./config.toml:/app/config.toml 13 | -------------------------------------------------------------------------------- /docs/cron.md: -------------------------------------------------------------------------------- 1 | # Schedule via cron expression 2 | 3 | You can use `cron_schedule` field to build more precise update checks schedule. 4 | A cron expression represents a set of times, using 5 space-separated fields. 5 | 6 | | Field name | Mandatory? | Allowed values | Allowed special characters | 7 | | ------------ | ---------- | --------------- | -------------------------- | 8 | | Minutes | Yes | 0-59 | * / , - | 9 | | Hours | Yes | 0-23 | * / , - | 10 | | Day of month | Yes | 1-31 | * / , - ? | 11 | | Month | Yes | 1-12 or JAN-DEC | * / , - | 12 | | Day of week | Yes | 0-6 or SUN-SAT | * / , - ? | 13 | 14 | Month and Day-of-week field values are case insensitive. `SUN`, `Sun`, and `sun` are equally accepted. 15 | The specific interpretation of the format is based on the Cron Wikipedia page: https://en.wikipedia.org/wiki/Cron 16 | 17 | #### Predefined schedules 18 | 19 | You may use one of several pre-defined schedules in place of a cron expression. 20 | 21 | | Entry | Description | Equivalent to | 22 | | ----------------------- | -------------------------------------------| ------------- | 23 | | `@monthly` | Run once a month, midnight, first of month | `0 0 1 * *` | 24 | | `@weekly` | Run once a week, midnight between Sat/Sun | `0 0 * * 0` | 25 | | `@daily (or @midnight)` | Run once a day, midnight | `0 0 * * *` | 26 | | `@hourly` | Run once an hour, beginning of hour | `0 * * * *` | 27 | 28 | #### Intervals 29 | 30 | You may also schedule a job to execute at fixed intervals, starting at the time it's added 31 | or cron is run. This is supported by formatting the cron spec like this: 32 | 33 | @every 34 | 35 | where "duration" is a string accepted by [time.ParseDuration](http://golang.org/pkg/time/#ParseDuration). 36 | 37 | For example, `@every 1h30m10s` would indicate a schedule that activates after 1 hour, 30 minutes, 10 seconds, and then every interval after that. 38 | -------------------------------------------------------------------------------- /docs/how_to_get_vimeo_token.md: -------------------------------------------------------------------------------- 1 | # How to get Vimeo API token 2 | 3 | 1. Create an account on https://vimeo.com 4 | 2. Navigate to https://developer.vimeo.com 5 | 3. Click `New app` button. 6 | ![Create a new app](img/vimeo_create_app.png) 7 | 4. Click `Create App`. 8 | 5. Navigate to [Generate an access token](https://developer.vimeo.com/apps/160740#generate_access_token) section. 9 | ![Generate an access token](img/vimeo_access_token.png) 10 | 6. Click `Generate`. 11 | ![Tokens](img/vimeo_token.png) 12 | 7. Copy a token to your CLI's configuration file. 13 | ```toml 14 | [tokens] 15 | vimeo = "ecd4d34b07bcb9509ABCD" 16 | ``` 17 | -------------------------------------------------------------------------------- /docs/how_to_get_youtube_api_key.md: -------------------------------------------------------------------------------- 1 | # How to get YouTube API Key 2 | 3 | 1. Navigate to https://console.developers.google.com 4 | 2. Click `Select a project`. 5 | ![Select project](img/youtube_select_project.png) 6 | 3. Click `New project`. 7 | ![New project](img/youtube_new_project.png) 8 | 4. Give it a name and click `Create` button. 9 | ![Dashboard](img/youtube_dashboard.png) 10 | 5. Click `Library`, find and click on `YouTube Data API v3` box. 11 | ![YouTube Data API](img/youtube_data_api_v3.png) 12 | 6. Click `Enable`. 13 | ![YouTube Enable](img/youtube_data_api_enable.png) 14 | 5. Click `Credentials`. 15 | 6. Click `Create credentials`. 16 | 7. Select `API key`. 17 | ![Create API key](img/youtube_create_api_key.png) 18 | 8. Copy token to your CLI's configuration file. 19 | ![Copy token](img/youtube_copy_token.png) 20 | ```toml 21 | [tokens] 22 | youtube = "AIzaSyD4w2s-k79YNR98ABC" 23 | ``` -------------------------------------------------------------------------------- /docs/how_to_setup_podsync_on_qnap_nas.md: -------------------------------------------------------------------------------- 1 | # Podsync on QNAP NAS Guide 2 | 3 | *Written by [@Rumik](https://github.com/Rumik)* 4 | 5 | 1. Install Container Station from App Center. 6 | 2. Create a shared folder on your QNAP for where you want Podsync to store its config file and data, 7 | e.g. `/share/CACHEDEV1_DATA/appdata/podsync` 8 | 3. Create a `config.toml` file in Notepad or whatever editor you want to use and copy it into the above folder. 9 | Here you will configure your specific settings. Here's mine as an example: 10 | 11 | ```toml 12 | [server] 13 | port = 6969 14 | data_dir = "/share/CACHEDEV1_DATA/appdata/podsync" 15 | hostname = "http://my.customhostname.com:6969" 16 | 17 | [tokens] 18 | youtube = "INSERTYOUTUBEAPI" # Tokens from `Access tokens` section 19 | 20 | [feeds] 21 | [feeds.KFGD] # Kinda Funny Games Daily 22 | url = "youtube.com/playlist?list=PLy3mMHt2i7RIl9pkdvrA98kN-RD4yoRhv" 23 | page_size = 3 24 | update_period = "60m" 25 | quality = "high" 26 | format = "video" 27 | cover_art = "http://i1.sndcdn.com/avatars-000319281278-0merek-original.jpg" 28 | ``` 29 | 30 | Note that I'm not using port `8080` because I already have another app on my QNAP using that port. 31 | I'm using port `6969` specifically because `Bill & Ted!`. 32 | Also, I'm using my own hostname so I can download the podcasts to my podcast app from outside my network, 33 | but you don't need to do this. To make that work, make sure you forward port `6969` to your QNAP. 34 | 35 | 4. By now, Container Station should have finished installing and should now be running. 36 | Now you need to SSH into the QNAP using an app like Putty (on Windows - just google for an app). 37 | 38 | 5. Copy and paste the following command: 39 | 40 | ```bash 41 | docker pull mxpv/podsync:latest 42 | ``` 43 | 44 | Docker will download the latest version of Podsync. 45 | 46 | 6. Copy and paste the following command: 47 | 48 | ```bash 49 | docker run \ 50 | -p 6969:6969 \ 51 | -v /share/CACHEDEV1_DATA/appdata/podsync:/app/data/ \ 52 | -v /share/CACHEDEV1_DATA/appdata/podsync/config.toml:/app/config.toml \ 53 | mxpv/podsync:latest 54 | ``` 55 | 56 | This will install a container in Container Station and run it. Podsync will load and read your config.toml file and start downloading episodes. 57 | 58 | 7. I recommend you go into the container's settings in Container Station and set it to Auto Start. 59 | 60 | 8. Once the downloads have finished for each of your feeds, you will then have an XML feed for each feed 61 | that you should be able to access at `http://ipaddressorhostname:6969/`. Paste them into your podcast app of choice, 62 | and you're good to go! 63 | -------------------------------------------------------------------------------- /docs/how_to_setup_podsync_on_synology_nas.md: -------------------------------------------------------------------------------- 1 | # Podsync on Synology NAS Guide 2 | 3 | *Written by [@lucasjanin](https://github.com/lucasjanin)* 4 | 5 | This installs `podsync` on a Synology NAS with SSL and port 443 6 | It requires to have a domain with ddns and an SSL Certificate 7 | I'm using a ddns from Synolgy with a SSL Certificate. By chance, my provider doesn't block ports 80 and 443. 8 | 9 | 10 | 1. Open "Package Center" and install "Apache HTTP Server 2.4" 11 | 2. In the "Web Station", select the default server, click edit and active "Enable personal website" 12 | 3. Create a folder "podsync" in web share using "File Station", the path will be like "/volume1/web/podsync" (where the files will be saved) 13 | 4. Create a folder "podsync" in another share using "File Station", the path will be like "/volume1/docker/podsync" (where the config will be saved) 14 | 5. Create a `config.toml` file in Notepad (or any other editor) and copy it into the above folder. 15 | Here you will configure your specific settings. Here's mine as an example: 16 | 17 | ```toml 18 | [server] 19 | port = 9090 20 | hostname = "https://xxxxxxxx.xxx" 21 | 22 | [storage] 23 | [storage.local] 24 | data_dir = "/app/data" 25 | 26 | [tokens] 27 | youtube = "xxxxxxx" 28 | 29 | [feeds] 30 | [feeds.ID1] 31 | url = "https://www.youtube.com/channel/UCJldRgT_D7Am-ErRHQZ90uw" 32 | update_period = "1h" 33 | quality = "high" # "high" or "low" 34 | format = "audio" # "audio", "video" or "custom" 35 | filters = { title = "Yann Marguet" } 36 | opml = true 37 | clean = { keep_last = 20 } 38 | private_feed = true 39 | [feeds.ID1.custom] 40 | title = "Yann Marguet - Moi, ce que j'en dis..." 41 | description = "Yann Marguet sur France Inter" 42 | author = "Yann Marguet" 43 | cover_art = "https://www.radiofrance.fr/s3/cruiser-production/2023/01/834dd18e-a74c-4a65-afb0-519a5f7b11c1/1400x1400_moi-ce-que-j-en-dis-marguet.jpg" 44 | cover_art_quality = "high" 45 | category = "Comedy" 46 | subcategories = ["Stand-Up"] 47 | lang = "fr" 48 | ownerName = "xxxx xxxxx" 49 | ownerEmail = "xx@xxxx.xx" 50 | ``` 51 | 52 | Note that I'm not using port `8080` because I already have another app on my Synology using that port. 53 | Also, I'm using my own hostname so I can download the podcasts to my podcast app from outside my network, 54 | but you don't need to do this. 55 | 56 | 6. Now you need to SSH into Synology using an app like Putty (on Windows - just google for an app). 57 | 58 | 5. Copy and paste the following command: 59 | 60 | ```bash 61 | docker pull mxpv/podsync:latest 62 | ``` 63 | 64 | Docker will download the latest version of Podsync. 65 | 66 | 6. Copy and paste the following command: 67 | 68 | ```bash 69 | docker run \ 70 | -p 9090:9090 \ 71 | -v /volume1/web/podsync:/app/data/ \ 72 | -v /volume1/docker/podsync/podsync-config.toml:/app/config.toml \ 73 | mxpv/podsync:latest 74 | ``` 75 | 76 | This will install a container in Docker and run it. Podsync will load and read your config.toml file and start downloading episodes. 77 | 78 | 7. I recommend you go into the container's settings in Container Station and set it to Auto Start. 79 | 80 | 8. Once the downloads have finished for each of your feeds, you will then have an XML feed for each feed 81 | that you should be able to access at `https://xxxxxxxx.xxx/podsync/ID1.xml`. Paste them into your podcast app of choice, 82 | and you're good to go! 83 | 84 | Note: you can validate your XML using this website: 85 | https://www.castfeedvalidator.com/validate.php 86 | -------------------------------------------------------------------------------- /docs/img/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mxpv/podsync/62fa6bfe3057e29e6d6fe8cdce458580a619b422/docs/img/logo.png -------------------------------------------------------------------------------- /docs/img/vimeo_access_token.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mxpv/podsync/62fa6bfe3057e29e6d6fe8cdce458580a619b422/docs/img/vimeo_access_token.png -------------------------------------------------------------------------------- /docs/img/vimeo_create_app.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mxpv/podsync/62fa6bfe3057e29e6d6fe8cdce458580a619b422/docs/img/vimeo_create_app.png -------------------------------------------------------------------------------- /docs/img/vimeo_token.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mxpv/podsync/62fa6bfe3057e29e6d6fe8cdce458580a619b422/docs/img/vimeo_token.png -------------------------------------------------------------------------------- /docs/img/youtube_copy_token.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mxpv/podsync/62fa6bfe3057e29e6d6fe8cdce458580a619b422/docs/img/youtube_copy_token.png -------------------------------------------------------------------------------- /docs/img/youtube_create_api_key.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mxpv/podsync/62fa6bfe3057e29e6d6fe8cdce458580a619b422/docs/img/youtube_create_api_key.png -------------------------------------------------------------------------------- /docs/img/youtube_dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mxpv/podsync/62fa6bfe3057e29e6d6fe8cdce458580a619b422/docs/img/youtube_dashboard.png -------------------------------------------------------------------------------- /docs/img/youtube_data_api_enable.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mxpv/podsync/62fa6bfe3057e29e6d6fe8cdce458580a619b422/docs/img/youtube_data_api_enable.png -------------------------------------------------------------------------------- /docs/img/youtube_data_api_v3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mxpv/podsync/62fa6bfe3057e29e6d6fe8cdce458580a619b422/docs/img/youtube_data_api_v3.png -------------------------------------------------------------------------------- /docs/img/youtube_new_project.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mxpv/podsync/62fa6bfe3057e29e6d6fe8cdce458580a619b422/docs/img/youtube_new_project.png -------------------------------------------------------------------------------- /docs/img/youtube_select_project.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mxpv/podsync/62fa6bfe3057e29e6d6fe8cdce458580a619b422/docs/img/youtube_select_project.png -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/mxpv/podsync 2 | 3 | go 1.21 4 | 5 | require ( 6 | github.com/BrianHicks/finch v0.0.0-20140409222414-419bd73c29ec 7 | github.com/aws/aws-sdk-go v1.44.144 8 | github.com/dgraph-io/badger v1.6.2 9 | github.com/eduncan911/podcast v1.4.2 10 | github.com/gilliek/go-opml v1.0.0 11 | github.com/golang/mock v1.6.0 12 | github.com/hashicorp/go-multierror v1.1.1 13 | github.com/jessevdk/go-flags v1.6.1 14 | github.com/pelletier/go-toml v1.9.5 15 | github.com/pkg/errors v0.9.1 16 | github.com/robfig/cron/v3 v3.0.1 17 | github.com/silentsokolov/go-vimeo v0.0.0-20190116124215-06829264260c 18 | github.com/sirupsen/logrus v1.9.3 19 | github.com/stretchr/testify v1.10.0 20 | github.com/zackradisic/soundcloud-api v0.1.8 21 | golang.org/x/oauth2 v0.23.0 22 | golang.org/x/sync v0.10.0 23 | google.golang.org/api v0.0.0-20180718221112-efcb5f25ac56 24 | gopkg.in/natefinch/lumberjack.v2 v2.2.1 25 | ) 26 | 27 | require ( 28 | github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect 29 | github.com/cespare/xxhash v1.1.0 // indirect 30 | github.com/davecgh/go-spew v1.1.1 // indirect 31 | github.com/dgraph-io/ristretto v0.0.2 // indirect 32 | github.com/dustin/go-humanize v1.0.0 // indirect 33 | github.com/golang/protobuf v1.5.2 // indirect 34 | github.com/grafov/m3u8 v0.11.1 // indirect 35 | github.com/hashicorp/errwrap v1.0.0 // indirect 36 | github.com/jmespath/go-jmespath v0.4.0 // indirect 37 | github.com/pmezard/go-difflib v1.0.0 // indirect 38 | golang.org/x/net v0.23.0 // indirect 39 | golang.org/x/sys v0.21.0 // indirect 40 | google.golang.org/protobuf v1.33.0 // indirect 41 | gopkg.in/yaml.v3 v3.0.1 // indirect 42 | ) 43 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIoKjsnZuH8vjyaysT/ses3EvZeaV/1UkF2M= 2 | github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= 3 | github.com/BrianHicks/finch v0.0.0-20140409222414-419bd73c29ec h1:1VPruZMM1WQC7POhjxbZOWK564cuFz1hlpwYW6ocM4E= 4 | github.com/BrianHicks/finch v0.0.0-20140409222414-419bd73c29ec/go.mod h1:+hWo/MWgY8VtjZvdrYM2nPRMaK40zX2iPsH/qD0+Xs0= 5 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 6 | github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= 7 | github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= 8 | github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= 9 | github.com/aws/aws-sdk-go v1.44.144 h1:mMWdnYL8HZsobrQe1mwvQ18Xt8UbOVhWgipjuma5Mkg= 10 | github.com/aws/aws-sdk-go v1.44.144/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= 11 | github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= 12 | github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= 13 | github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= 14 | github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= 15 | github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= 16 | github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= 17 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 18 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 19 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 20 | github.com/dgraph-io/badger v1.6.2 h1:mNw0qs90GVgGGWylh0umH5iag1j6n/PeJtNvL6KY/x8= 21 | github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE= 22 | github.com/dgraph-io/ristretto v0.0.2 h1:a5WaUrDa0qm0YrAAS1tUykT5El3kt62KNZZeMxQn3po= 23 | github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= 24 | github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= 25 | github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= 26 | github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= 27 | github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= 28 | github.com/eduncan911/podcast v1.4.2 h1:S+fsUlbR2ULFou2Mc52G/MZI8JVJHedbxLQnoA+MY/w= 29 | github.com/eduncan911/podcast v1.4.2/go.mod h1:mSxiK1z5KeNO0YFaQ3ElJlUZbbDV9dA7R9c1coeeXkc= 30 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 31 | github.com/gilliek/go-opml v1.0.0 h1:X8xVjtySRXU/x6KvaiXkn7OV3a4DHqxY8Rpv6U/JvCY= 32 | github.com/gilliek/go-opml v1.0.0/go.mod h1:fOxmtlzyBvUjU6bjpdjyxCGlWz+pgtAHrHf/xRZl3lk= 33 | github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= 34 | github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= 35 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 36 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= 37 | github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= 38 | github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= 39 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 40 | github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= 41 | github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 42 | github.com/grafov/m3u8 v0.11.1 h1:igZ7EBIB2IAsPPazKwRKdbhxcoBKO3lO1UY57PZDeNA= 43 | github.com/grafov/m3u8 v0.11.1/go.mod h1:nqzOkfBiZJENr52zTVd/Dcl03yzphIMbJqkXGu+u080= 44 | github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= 45 | github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= 46 | github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= 47 | github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= 48 | github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= 49 | github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= 50 | github.com/jessevdk/go-flags v1.6.1 h1:Cvu5U8UGrLay1rZfv/zP7iLpSHGUZ/Ou68T0iX1bBK4= 51 | github.com/jessevdk/go-flags v1.6.1/go.mod h1:Mk8T1hIAWpOiJiHa9rJASDK2UGWji0EuPGBnNLMooyc= 52 | github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= 53 | github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= 54 | github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= 55 | github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= 56 | github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= 57 | github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 58 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 59 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 60 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 61 | github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= 62 | github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= 63 | github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= 64 | github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= 65 | github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= 66 | github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= 67 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 68 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 69 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 70 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 71 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 72 | github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= 73 | github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= 74 | github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= 75 | github.com/silentsokolov/go-vimeo v0.0.0-20190116124215-06829264260c h1:KhHx/Ta3c9C1gcSo5UhDeo/D4JnhnxJTrlcOEOFiMfY= 76 | github.com/silentsokolov/go-vimeo v0.0.0-20190116124215-06829264260c/go.mod h1:10FeaKUMy5t3KLsYfy54dFrq0rpwcfyKkKcF7vRGIRY= 77 | github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= 78 | github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= 79 | github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= 80 | github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= 81 | github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= 82 | github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= 83 | github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= 84 | github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= 85 | github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= 86 | github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 87 | github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= 88 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 89 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 90 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 91 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 92 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 93 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 94 | github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= 95 | github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= 96 | github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= 97 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= 98 | github.com/zackradisic/soundcloud-api v0.1.8 h1:Fc4IVbee8ggGZ/vyx26uyTwKeh6Vn3cCrPXdTbQypjI= 99 | github.com/zackradisic/soundcloud-api v0.1.8/go.mod h1:ycGIZFVZdUVC7B8pcfgze1bRBePPmjYlIGnRptKByQ0= 100 | golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 101 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 102 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 103 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 104 | golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 105 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= 106 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 107 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 108 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 109 | golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= 110 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 111 | golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= 112 | golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= 113 | golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= 114 | golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= 115 | golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= 116 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 117 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 118 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 119 | golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= 120 | golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= 121 | golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 122 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 123 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 124 | golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 125 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 126 | golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 127 | golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 128 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 129 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 130 | golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 131 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 132 | golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 133 | golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= 134 | golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 135 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 136 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 137 | golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 138 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 139 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 140 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 141 | golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 142 | golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= 143 | golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= 144 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 145 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 146 | golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= 147 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= 148 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 149 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 150 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 151 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 152 | google.golang.org/api v0.0.0-20180718221112-efcb5f25ac56 h1:iDRbkenn0VZEo05mHiCtN6/EfbZj7x1Rg+tPjB5HiQc= 153 | google.golang.org/api v0.0.0-20180718221112-efcb5f25ac56/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= 154 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 155 | google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= 156 | google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= 157 | google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= 158 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 159 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= 160 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 161 | gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= 162 | gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= 163 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 164 | gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= 165 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 166 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 167 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 168 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 169 | -------------------------------------------------------------------------------- /pkg/builder/builder.go: -------------------------------------------------------------------------------- 1 | package builder 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/mxpv/podsync/pkg/feed" 7 | "github.com/pkg/errors" 8 | 9 | "github.com/mxpv/podsync/pkg/model" 10 | ) 11 | 12 | type Builder interface { 13 | Build(ctx context.Context, cfg *feed.Config) (*model.Feed, error) 14 | } 15 | 16 | func New(ctx context.Context, provider model.Provider, key string) (Builder, error) { 17 | switch provider { 18 | case model.ProviderYoutube: 19 | return NewYouTubeBuilder(key) 20 | case model.ProviderVimeo: 21 | return NewVimeoBuilder(ctx, key) 22 | case model.ProviderSoundcloud: 23 | return NewSoundcloudBuilder() 24 | default: 25 | return nil, errors.Errorf("unsupported provider %q", provider) 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /pkg/builder/soundcloud.go: -------------------------------------------------------------------------------- 1 | package builder 2 | 3 | import ( 4 | "context" 5 | "strconv" 6 | "time" 7 | 8 | "github.com/mxpv/podsync/pkg/feed" 9 | "github.com/pkg/errors" 10 | soundcloudapi "github.com/zackradisic/soundcloud-api" 11 | 12 | "github.com/mxpv/podsync/pkg/model" 13 | ) 14 | 15 | type SoundCloudBuilder struct { 16 | client *soundcloudapi.API 17 | } 18 | 19 | func (s *SoundCloudBuilder) Build(_ctx context.Context, cfg *feed.Config) (*model.Feed, error) { 20 | info, err := ParseURL(cfg.URL) 21 | if err != nil { 22 | return nil, err 23 | } 24 | 25 | _feed := &model.Feed{ 26 | ItemID: info.ItemID, 27 | Provider: info.Provider, 28 | LinkType: info.LinkType, 29 | Format: cfg.Format, 30 | Quality: cfg.Quality, 31 | PageSize: cfg.PageSize, 32 | UpdatedAt: time.Now().UTC(), 33 | } 34 | 35 | if info.LinkType == model.TypePlaylist { 36 | if soundcloudapi.IsPlaylistURL(cfg.URL) { 37 | scplaylist, err := s.client.GetPlaylistInfo(cfg.URL) 38 | if err != nil { 39 | return nil, err 40 | } 41 | 42 | _feed.Title = scplaylist.Title 43 | _feed.Description = scplaylist.Description 44 | _feed.ItemURL = cfg.URL 45 | 46 | date, err := time.Parse(time.RFC3339, scplaylist.CreatedAt) 47 | if err == nil { 48 | _feed.PubDate = date 49 | } 50 | _feed.Author = scplaylist.User.Username 51 | _feed.CoverArt = scplaylist.ArtworkURL 52 | 53 | var added = 0 54 | for _, track := range scplaylist.Tracks { 55 | pubDate, _ := time.Parse(time.RFC3339, track.CreatedAt) 56 | var ( 57 | videoID = strconv.FormatInt(track.ID, 10) 58 | duration = track.DurationMS / 1000 59 | mediaURL = track.PermalinkURL 60 | trackSize = track.DurationMS * 15 // very rough estimate 61 | ) 62 | 63 | _feed.Episodes = append(_feed.Episodes, &model.Episode{ 64 | ID: videoID, 65 | Title: track.Title, 66 | Description: track.Description, 67 | Duration: duration, 68 | Size: trackSize, 69 | VideoURL: mediaURL, 70 | PubDate: pubDate, 71 | Thumbnail: track.ArtworkURL, 72 | Status: model.EpisodeNew, 73 | }) 74 | 75 | added++ 76 | 77 | if added >= _feed.PageSize { 78 | return _feed, nil 79 | } 80 | } 81 | 82 | return _feed, nil 83 | } 84 | } 85 | 86 | return nil, errors.New(("unsupported soundcloud feed type")) 87 | } 88 | 89 | func NewSoundcloudBuilder() (*SoundCloudBuilder, error) { 90 | sc, err := soundcloudapi.New(soundcloudapi.APIOptions{}) 91 | if err != nil { 92 | return nil, errors.Wrap(err, "failed to create soundcloud client") 93 | } 94 | 95 | return &SoundCloudBuilder{client: sc}, nil 96 | } 97 | -------------------------------------------------------------------------------- /pkg/builder/soundcloud_test.go: -------------------------------------------------------------------------------- 1 | package builder 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/mxpv/podsync/pkg/feed" 7 | "github.com/stretchr/testify/assert" 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func TestSoundCloud_BuildFeed(t *testing.T) { 12 | builder, err := NewSoundcloudBuilder() 13 | require.NoError(t, err) 14 | 15 | urls := []string{ 16 | "https://soundcloud.com/moby/sets/remixes", 17 | "https://soundcloud.com/npr/sets/soundscapes", 18 | } 19 | 20 | for _, addr := range urls { 21 | t.Run(addr, func(t *testing.T) { 22 | _feed, err := builder.Build(testCtx, &feed.Config{URL: addr}) 23 | require.NoError(t, err) 24 | 25 | assert.NotEmpty(t, _feed.Title) 26 | assert.NotEmpty(t, _feed.Description) 27 | assert.NotEmpty(t, _feed.Author) 28 | assert.NotEmpty(t, _feed.ItemURL) 29 | 30 | assert.NotZero(t, len(_feed.Episodes)) 31 | 32 | for _, item := range _feed.Episodes { 33 | assert.NotEmpty(t, item.Title) 34 | assert.NotEmpty(t, item.VideoURL) 35 | assert.NotZero(t, item.Duration) 36 | assert.NotEmpty(t, item.Title) 37 | assert.NotEmpty(t, item.Thumbnail) 38 | } 39 | }) 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /pkg/builder/url.go: -------------------------------------------------------------------------------- 1 | package builder 2 | 3 | import ( 4 | "net/url" 5 | "strings" 6 | 7 | "github.com/pkg/errors" 8 | 9 | "github.com/mxpv/podsync/pkg/model" 10 | ) 11 | 12 | func ParseURL(link string) (model.Info, error) { 13 | parsed, err := parseURL(link) 14 | if err != nil { 15 | return model.Info{}, err 16 | } 17 | 18 | info := model.Info{} 19 | 20 | if strings.HasSuffix(parsed.Host, "youtube.com") { 21 | kind, id, err := parseYoutubeURL(parsed) 22 | if err != nil { 23 | return model.Info{}, err 24 | } 25 | 26 | info.Provider = model.ProviderYoutube 27 | info.LinkType = kind 28 | info.ItemID = id 29 | 30 | return info, nil 31 | } 32 | 33 | if strings.HasSuffix(parsed.Host, "vimeo.com") { 34 | kind, id, err := parseVimeoURL(parsed) 35 | if err != nil { 36 | return model.Info{}, err 37 | } 38 | 39 | info.Provider = model.ProviderVimeo 40 | info.LinkType = kind 41 | info.ItemID = id 42 | 43 | return info, nil 44 | } 45 | 46 | if strings.HasSuffix(parsed.Host, "soundcloud.com") { 47 | kind, id, err := parseSoundcloudURL(parsed) 48 | if err != nil { 49 | return model.Info{}, err 50 | } 51 | 52 | info.Provider = model.ProviderSoundcloud 53 | info.LinkType = kind 54 | info.ItemID = id 55 | 56 | return info, nil 57 | } 58 | 59 | return model.Info{}, errors.New("unsupported URL host") 60 | } 61 | 62 | func parseURL(link string) (*url.URL, error) { 63 | if !strings.HasPrefix(link, "http") { 64 | link = "https://" + link 65 | } 66 | 67 | parsed, err := url.Parse(link) 68 | if err != nil { 69 | return nil, errors.Wrapf(err, "failed to parse url: %s", link) 70 | } 71 | 72 | return parsed, nil 73 | } 74 | 75 | func parseYoutubeURL(parsed *url.URL) (model.Type, string, error) { 76 | path := parsed.EscapedPath() 77 | 78 | // https://www.youtube.com/playlist?list=PLCB9F975ECF01953C 79 | // https://www.youtube.com/watch?v=rbCbho7aLYw&list=PLMpEfaKcGjpWEgNtdnsvLX6LzQL0UC0EM 80 | if strings.HasPrefix(path, "/playlist") || strings.HasPrefix(path, "/watch") { 81 | kind := model.TypePlaylist 82 | 83 | id := parsed.Query().Get("list") 84 | if id != "" { 85 | return kind, id, nil 86 | } 87 | 88 | return "", "", errors.New("invalid playlist link") 89 | } 90 | 91 | // - https://www.youtube.com/channel/UC5XPnUk8Vvv_pWslhwom6Og 92 | // - https://www.youtube.com/channel/UCrlakW-ewUT8sOod6Wmzyow/videos 93 | if strings.HasPrefix(path, "/channel") { 94 | kind := model.TypeChannel 95 | parts := strings.Split(parsed.EscapedPath(), "/") 96 | if len(parts) <= 2 { 97 | return "", "", errors.New("invalid youtube channel link") 98 | } 99 | 100 | id := parts[2] 101 | if id == "" { 102 | return "", "", errors.New("invalid id") 103 | } 104 | 105 | return kind, id, nil 106 | } 107 | 108 | // - https://www.youtube.com/user/fxigr1 109 | if strings.HasPrefix(path, "/user") { 110 | kind := model.TypeUser 111 | 112 | parts := strings.Split(parsed.EscapedPath(), "/") 113 | if len(parts) <= 2 { 114 | return "", "", errors.New("invalid user link") 115 | } 116 | 117 | id := parts[2] 118 | if id == "" { 119 | return "", "", errors.New("invalid id") 120 | } 121 | 122 | return kind, id, nil 123 | } 124 | 125 | return "", "", errors.New("unsupported link format") 126 | } 127 | 128 | func parseVimeoURL(parsed *url.URL) (model.Type, string, error) { 129 | parts := strings.Split(parsed.EscapedPath(), "/") 130 | if len(parts) <= 1 { 131 | return "", "", errors.New("invalid vimeo link path") 132 | } 133 | 134 | var kind model.Type 135 | switch parts[1] { 136 | case "groups": 137 | kind = model.TypeGroup 138 | case "channels": 139 | kind = model.TypeChannel 140 | default: 141 | kind = model.TypeUser 142 | } 143 | 144 | if kind == model.TypeGroup || kind == model.TypeChannel { 145 | if len(parts) <= 2 { 146 | return "", "", errors.New("invalid channel link") 147 | } 148 | 149 | id := parts[2] 150 | if id == "" { 151 | return "", "", errors.New("invalid id") 152 | } 153 | 154 | return kind, id, nil 155 | } 156 | 157 | if kind == model.TypeUser { 158 | id := parts[1] 159 | if id == "" { 160 | return "", "", errors.New("invalid id") 161 | } 162 | 163 | return kind, id, nil 164 | } 165 | 166 | return "", "", errors.New("unsupported link format") 167 | } 168 | 169 | func parseSoundcloudURL(parsed *url.URL) (model.Type, string, error) { 170 | parts := strings.Split(parsed.EscapedPath(), "/") 171 | if len(parts) <= 3 { 172 | return "", "", errors.New("invald soundcloud link path") 173 | } 174 | 175 | var kind model.Type 176 | 177 | // - https://soundcloud.com/user/sets/example-set 178 | switch parts[2] { 179 | case "sets": 180 | kind = model.TypePlaylist 181 | default: 182 | return "", "", errors.New("invalid soundcloud url, missing sets") 183 | } 184 | 185 | id := parts[3] 186 | 187 | return kind, id, nil 188 | } 189 | -------------------------------------------------------------------------------- /pkg/builder/url_test.go: -------------------------------------------------------------------------------- 1 | package builder 2 | 3 | import ( 4 | "net/url" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/require" 8 | 9 | "github.com/mxpv/podsync/pkg/model" 10 | ) 11 | 12 | func TestParseYoutubeURL_Playlist(t *testing.T) { 13 | link, _ := url.ParseRequestURI("https://www.youtube.com/playlist?list=PLCB9F975ECF01953C") 14 | kind, id, err := parseYoutubeURL(link) 15 | require.NoError(t, err) 16 | require.Equal(t, model.TypePlaylist, kind) 17 | require.Equal(t, "PLCB9F975ECF01953C", id) 18 | 19 | link, _ = url.ParseRequestURI("https://www.youtube.com/watch?v=rbCbho7aLYw&list=PLMpEfaKcGjpWEgNtdnsvLX6LzQL0UC0EM") 20 | kind, id, err = parseYoutubeURL(link) 21 | require.NoError(t, err) 22 | require.Equal(t, model.TypePlaylist, kind) 23 | require.Equal(t, "PLMpEfaKcGjpWEgNtdnsvLX6LzQL0UC0EM", id) 24 | } 25 | 26 | func TestParseYoutubeURL_Channel(t *testing.T) { 27 | link, _ := url.ParseRequestURI("https://www.youtube.com/channel/UC5XPnUk8Vvv_pWslhwom6Og") 28 | kind, id, err := parseYoutubeURL(link) 29 | require.NoError(t, err) 30 | require.Equal(t, model.TypeChannel, kind) 31 | require.Equal(t, "UC5XPnUk8Vvv_pWslhwom6Og", id) 32 | 33 | link, _ = url.ParseRequestURI("https://www.youtube.com/channel/UCrlakW-ewUT8sOod6Wmzyow/videos") 34 | kind, id, err = parseYoutubeURL(link) 35 | require.NoError(t, err) 36 | require.Equal(t, model.TypeChannel, kind) 37 | require.Equal(t, "UCrlakW-ewUT8sOod6Wmzyow", id) 38 | } 39 | 40 | func TestParseYoutubeURL_User(t *testing.T) { 41 | link, _ := url.ParseRequestURI("https://youtube.com/user/fxigr1") 42 | kind, id, err := parseYoutubeURL(link) 43 | require.NoError(t, err) 44 | require.Equal(t, model.TypeUser, kind) 45 | require.Equal(t, "fxigr1", id) 46 | } 47 | 48 | func TestParseYoutubeURL_InvalidLink(t *testing.T) { 49 | link, _ := url.ParseRequestURI("https://www.youtube.com/user///") 50 | _, _, err := parseYoutubeURL(link) 51 | require.Error(t, err) 52 | 53 | link, _ = url.ParseRequestURI("https://www.youtube.com/channel//videos") 54 | _, _, err = parseYoutubeURL(link) 55 | require.Error(t, err) 56 | } 57 | 58 | func TestParseVimeoURL_Group(t *testing.T) { 59 | link, _ := url.ParseRequestURI("https://vimeo.com/groups/109") 60 | kind, id, err := parseVimeoURL(link) 61 | require.NoError(t, err) 62 | require.Equal(t, model.TypeGroup, kind) 63 | require.Equal(t, "109", id) 64 | 65 | link, _ = url.ParseRequestURI("http://vimeo.com/groups/109") 66 | kind, id, err = parseVimeoURL(link) 67 | require.NoError(t, err) 68 | require.Equal(t, model.TypeGroup, kind) 69 | require.Equal(t, "109", id) 70 | 71 | link, _ = url.ParseRequestURI("http://www.vimeo.com/groups/109") 72 | kind, id, err = parseVimeoURL(link) 73 | require.NoError(t, err) 74 | require.Equal(t, model.TypeGroup, kind) 75 | require.Equal(t, "109", id) 76 | 77 | link, _ = url.ParseRequestURI("https://vimeo.com/groups/109/videos/") 78 | kind, id, err = parseVimeoURL(link) 79 | require.NoError(t, err) 80 | require.Equal(t, model.TypeGroup, kind) 81 | require.Equal(t, "109", id) 82 | } 83 | 84 | func TestParseVimeoURL_Channel(t *testing.T) { 85 | link, _ := url.ParseRequestURI("https://vimeo.com/channels/staffpicks") 86 | kind, id, err := parseVimeoURL(link) 87 | require.NoError(t, err) 88 | require.Equal(t, model.TypeChannel, kind) 89 | require.Equal(t, "staffpicks", id) 90 | 91 | link, _ = url.ParseRequestURI("http://vimeo.com/channels/staffpicks/146224925") 92 | kind, id, err = parseVimeoURL(link) 93 | require.NoError(t, err) 94 | require.Equal(t, model.TypeChannel, kind) 95 | require.Equal(t, "staffpicks", id) 96 | } 97 | 98 | func TestParseVimeoURL_User(t *testing.T) { 99 | link, _ := url.ParseRequestURI("https://vimeo.com/awhitelabelproduct") 100 | kind, id, err := parseVimeoURL(link) 101 | require.NoError(t, err) 102 | require.Equal(t, model.TypeUser, kind) 103 | require.Equal(t, "awhitelabelproduct", id) 104 | } 105 | 106 | func TestParseVimeoURL_InvalidLink(t *testing.T) { 107 | link, _ := url.ParseRequestURI("http://www.apple.com") 108 | _, _, err := parseVimeoURL(link) 109 | require.Error(t, err) 110 | 111 | link, _ = url.ParseRequestURI("http://www.vimeo.com") 112 | _, _, err = parseVimeoURL(link) 113 | require.Error(t, err) 114 | } 115 | -------------------------------------------------------------------------------- /pkg/builder/vimeo.go: -------------------------------------------------------------------------------- 1 | package builder 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | "strconv" 7 | "time" 8 | 9 | "github.com/mxpv/podsync/pkg/feed" 10 | "github.com/pkg/errors" 11 | "github.com/silentsokolov/go-vimeo/vimeo" 12 | "golang.org/x/oauth2" 13 | 14 | "github.com/mxpv/podsync/pkg/model" 15 | ) 16 | 17 | const ( 18 | vimeoDefaultPageSize = 50 19 | ) 20 | 21 | type VimeoBuilder struct { 22 | client *vimeo.Client 23 | } 24 | 25 | func (v *VimeoBuilder) selectImage(p *vimeo.Pictures, q model.Quality) string { 26 | if p == nil || len(p.Sizes) == 0 { 27 | return "" 28 | } 29 | 30 | if q == model.QualityLow { 31 | return p.Sizes[0].Link 32 | } 33 | 34 | return p.Sizes[len(p.Sizes)-1].Link 35 | } 36 | 37 | func (v *VimeoBuilder) queryChannel(feed *model.Feed) error { 38 | channelID := feed.ItemID 39 | 40 | ch, resp, err := v.client.Channels.Get(channelID) 41 | if err != nil { 42 | if resp != nil && resp.StatusCode == http.StatusNotFound { 43 | return model.ErrNotFound 44 | } 45 | 46 | return errors.Wrapf(err, "failed to query channel with id %q", channelID) 47 | } 48 | 49 | feed.Title = ch.Name 50 | feed.ItemURL = ch.Link 51 | feed.Description = ch.Description 52 | feed.CoverArt = v.selectImage(ch.Pictures, feed.Quality) 53 | feed.Author = ch.User.Name 54 | feed.PubDate = ch.CreatedTime 55 | feed.UpdatedAt = time.Now().UTC() 56 | 57 | return nil 58 | } 59 | 60 | func (v *VimeoBuilder) queryGroup(feed *model.Feed) error { 61 | groupID := feed.ItemID 62 | 63 | gr, resp, err := v.client.Groups.Get(groupID) 64 | if err != nil { 65 | if resp != nil && resp.StatusCode == http.StatusNotFound { 66 | return model.ErrNotFound 67 | } 68 | 69 | return errors.Wrapf(err, "failed to query group with id %q", groupID) 70 | } 71 | 72 | feed.Title = gr.Name 73 | feed.ItemURL = gr.Link 74 | feed.Description = gr.Description 75 | feed.CoverArt = v.selectImage(gr.Pictures, feed.Quality) 76 | feed.Author = gr.User.Name 77 | feed.PubDate = gr.CreatedTime 78 | feed.UpdatedAt = time.Now().UTC() 79 | 80 | return nil 81 | } 82 | 83 | func (v *VimeoBuilder) queryUser(feed *model.Feed) error { 84 | userID := feed.ItemID 85 | 86 | user, resp, err := v.client.Users.Get(userID) 87 | if err != nil { 88 | if resp != nil && resp.StatusCode == http.StatusNotFound { 89 | return model.ErrNotFound 90 | } 91 | 92 | return errors.Wrapf(err, "failed to query user with id %q", userID) 93 | } 94 | 95 | feed.Title = user.Name 96 | feed.ItemURL = user.Link 97 | feed.Description = user.Bio 98 | feed.CoverArt = v.selectImage(user.Pictures, feed.Quality) 99 | feed.Author = user.Name 100 | feed.PubDate = user.CreatedTime 101 | feed.UpdatedAt = time.Now().UTC() 102 | 103 | return nil 104 | } 105 | 106 | func (v *VimeoBuilder) getVideoSize(video *vimeo.Video) int64 { 107 | // Very approximate video file size 108 | return int64(float64(video.Duration*video.Width*video.Height) * 0.38848958333) 109 | } 110 | 111 | type getVideosFunc func(string, ...vimeo.CallOption) ([]*vimeo.Video, *vimeo.Response, error) 112 | 113 | func (v *VimeoBuilder) queryVideos(getVideos getVideosFunc, feed *model.Feed) error { 114 | var ( 115 | page = 1 116 | added = 0 117 | ) 118 | 119 | for { 120 | videos, response, err := getVideos(feed.ItemID, vimeo.OptPage(page), vimeo.OptPerPage(vimeoDefaultPageSize)) 121 | if err != nil { 122 | if response != nil { 123 | return errors.Wrapf(err, "failed to query videos (error %d %s)", response.StatusCode, response.Status) 124 | } 125 | 126 | return err 127 | } 128 | 129 | for _, video := range videos { 130 | var ( 131 | videoID = strconv.Itoa(video.GetID()) 132 | videoURL = video.Link 133 | duration = int64(video.Duration) 134 | size = v.getVideoSize(video) 135 | image = v.selectImage(video.Pictures, feed.Quality) 136 | ) 137 | 138 | feed.Episodes = append(feed.Episodes, &model.Episode{ 139 | ID: videoID, 140 | Title: video.Name, 141 | Description: video.Description, 142 | Duration: duration, 143 | Size: size, 144 | PubDate: video.CreatedTime, 145 | Thumbnail: image, 146 | VideoURL: videoURL, 147 | Status: model.EpisodeNew, 148 | }) 149 | 150 | added++ 151 | } 152 | 153 | if added >= feed.PageSize || response.NextPage == "" { 154 | return nil 155 | } 156 | 157 | page++ 158 | } 159 | } 160 | 161 | func (v *VimeoBuilder) Build(ctx context.Context, cfg *feed.Config) (*model.Feed, error) { 162 | info, err := ParseURL(cfg.URL) 163 | if err != nil { 164 | return nil, err 165 | } 166 | 167 | _feed := &model.Feed{ 168 | ItemID: info.ItemID, 169 | Provider: info.Provider, 170 | LinkType: info.LinkType, 171 | Format: cfg.Format, 172 | Quality: cfg.Quality, 173 | PageSize: cfg.PageSize, 174 | UpdatedAt: time.Now().UTC(), 175 | } 176 | 177 | if info.LinkType == model.TypeChannel { 178 | if err := v.queryChannel(_feed); err != nil { 179 | return nil, err 180 | } 181 | 182 | if err := v.queryVideos(v.client.Channels.ListVideo, _feed); err != nil { 183 | return nil, err 184 | } 185 | 186 | return _feed, nil 187 | } 188 | 189 | if info.LinkType == model.TypeGroup { 190 | if err := v.queryGroup(_feed); err != nil { 191 | return nil, err 192 | } 193 | 194 | if err := v.queryVideos(v.client.Groups.ListVideo, _feed); err != nil { 195 | return nil, err 196 | } 197 | 198 | return _feed, nil 199 | } 200 | 201 | if info.LinkType == model.TypeUser { 202 | if err := v.queryUser(_feed); err != nil { 203 | return nil, err 204 | } 205 | 206 | if err := v.queryVideos(v.client.Users.ListVideo, _feed); err != nil { 207 | return nil, err 208 | } 209 | 210 | return _feed, nil 211 | } 212 | 213 | return nil, errors.New("unsupported feed type") 214 | } 215 | 216 | func NewVimeoBuilder(ctx context.Context, token string) (*VimeoBuilder, error) { 217 | if token == "" { 218 | return nil, errors.New("empty Vimeo access token") 219 | } 220 | 221 | ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token}) 222 | tc := oauth2.NewClient(ctx, ts) 223 | 224 | client := vimeo.NewClient(tc, nil) 225 | return &VimeoBuilder{client}, nil 226 | } 227 | -------------------------------------------------------------------------------- /pkg/builder/vimeo_test.go: -------------------------------------------------------------------------------- 1 | package builder 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/require" 10 | 11 | "github.com/mxpv/podsync/pkg/model" 12 | ) 13 | 14 | var ( 15 | vimeoKey = os.Getenv("VIMEO_TEST_API_KEY") 16 | ) 17 | 18 | func TestQueryVimeoChannel(t *testing.T) { 19 | if vimeoKey == "" { 20 | t.Skip("Vimeo API key is not provided") 21 | } 22 | 23 | builder, err := NewVimeoBuilder(context.Background(), vimeoKey) 24 | require.NoError(t, err) 25 | 26 | podcast := &model.Feed{ItemID: "staffpicks", Quality: model.QualityHigh} 27 | err = builder.queryChannel(podcast) 28 | require.NoError(t, err) 29 | 30 | assert.Equal(t, "https://vimeo.com/channels/staffpicks", podcast.ItemURL) 31 | assert.Equal(t, "Vimeo Staff Picks", podcast.Title) 32 | assert.Equal(t, "Vimeo Curation", podcast.Author) 33 | assert.NotEmpty(t, podcast.Description) 34 | assert.NotEmpty(t, podcast.CoverArt) 35 | } 36 | 37 | func TestQueryVimeoGroup(t *testing.T) { 38 | if vimeoKey == "" { 39 | t.Skip("Vimeo API key is not provided") 40 | } 41 | 42 | builder, err := NewVimeoBuilder(context.Background(), vimeoKey) 43 | require.NoError(t, err) 44 | 45 | podcast := &model.Feed{ItemID: "motion", Quality: model.QualityHigh} 46 | err = builder.queryGroup(podcast) 47 | require.NoError(t, err) 48 | 49 | assert.Equal(t, "https://vimeo.com/groups/motion", podcast.ItemURL) 50 | assert.Equal(t, "Motion Graphic Artists", podcast.Title) 51 | assert.Equal(t, "Danny Garcia", podcast.Author) 52 | assert.NotEmpty(t, podcast.Description) 53 | assert.NotEmpty(t, podcast.CoverArt) 54 | } 55 | 56 | func TestQueryVimeoUser(t *testing.T) { 57 | if vimeoKey == "" { 58 | t.Skip("Vimeo API key is not provided") 59 | } 60 | 61 | builder, err := NewVimeoBuilder(context.Background(), vimeoKey) 62 | require.NoError(t, err) 63 | 64 | podcast := &model.Feed{ItemID: "motionarray", Quality: model.QualityHigh} 65 | err = builder.queryUser(podcast) 66 | require.NoError(t, err) 67 | 68 | require.Equal(t, "https://vimeo.com/motionarray", podcast.ItemURL) 69 | assert.NotEmpty(t, podcast.Title) 70 | assert.NotEmpty(t, podcast.Author) 71 | assert.NotEmpty(t, podcast.Description) 72 | } 73 | 74 | func TestQueryVimeoVideos(t *testing.T) { 75 | if vimeoKey == "" { 76 | t.Skip("Vimeo API key is not provided") 77 | } 78 | 79 | builder, err := NewVimeoBuilder(context.Background(), vimeoKey) 80 | require.NoError(t, err) 81 | 82 | feed := &model.Feed{ItemID: "staffpicks", Quality: model.QualityHigh} 83 | 84 | err = builder.queryVideos(builder.client.Channels.ListVideo, feed) 85 | require.NoError(t, err) 86 | 87 | require.Equal(t, vimeoDefaultPageSize, len(feed.Episodes)) 88 | 89 | for _, item := range feed.Episodes { 90 | require.NotEmpty(t, item.Title) 91 | require.NotEmpty(t, item.VideoURL) 92 | require.NotEmpty(t, item.ID) 93 | require.NotEmpty(t, item.Thumbnail) 94 | require.NotZero(t, item.Duration) 95 | require.NotZero(t, item.Size) 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /pkg/builder/youtube.go: -------------------------------------------------------------------------------- 1 | package builder 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "sort" 8 | "strconv" 9 | "strings" 10 | "time" 11 | 12 | "github.com/BrianHicks/finch/duration" 13 | "github.com/mxpv/podsync/pkg/feed" 14 | "github.com/pkg/errors" 15 | log "github.com/sirupsen/logrus" 16 | "google.golang.org/api/youtube/v3" 17 | 18 | "github.com/mxpv/podsync/pkg/model" 19 | ) 20 | 21 | const ( 22 | maxYoutubeResults = 50 23 | hdBytesPerSecond = 350000 24 | ldBytesPerSecond = 100000 25 | lowAudioBytesPerSecond = 48000 / 8 26 | highAudioBytesPerSecond = 128000 / 8 27 | ) 28 | 29 | type apiKey string 30 | 31 | func (key apiKey) Get() (string, string) { 32 | return "key", string(key) 33 | } 34 | 35 | type YouTubeBuilder struct { 36 | client *youtube.Service 37 | key apiKey 38 | } 39 | 40 | // Cost: 5 units (call method: 1, snippet: 2, contentDetails: 2) 41 | // See https://developers.google.com/youtube/v3/docs/channels/list#part 42 | func (yt *YouTubeBuilder) listChannels(ctx context.Context, linkType model.Type, id string, parts string) (*youtube.Channel, error) { 43 | req := yt.client.Channels.List(parts) 44 | 45 | switch linkType { 46 | case model.TypeChannel: 47 | req = req.Id(id) 48 | case model.TypeUser: 49 | req = req.ForUsername(id) 50 | default: 51 | return nil, errors.New("unsupported link type") 52 | } 53 | 54 | resp, err := req.Context(ctx).Do(yt.key) 55 | if err != nil { 56 | return nil, errors.Wrapf(err, "failed to query channel") 57 | } 58 | 59 | if len(resp.Items) == 0 { 60 | return nil, model.ErrNotFound 61 | } 62 | 63 | item := resp.Items[0] 64 | return item, nil 65 | } 66 | 67 | // Cost: 3 units (call method: 1, snippet: 2) 68 | // See https://developers.google.com/youtube/v3/docs/playlists/list#part 69 | func (yt *YouTubeBuilder) listPlaylists(ctx context.Context, id, channelID string, parts string) (*youtube.Playlist, error) { 70 | req := yt.client.Playlists.List(parts) 71 | 72 | if id != "" { 73 | req = req.Id(id) 74 | } else { 75 | req = req.ChannelId(channelID) 76 | } 77 | 78 | resp, err := req.Context(ctx).Do(yt.key) 79 | if err != nil { 80 | return nil, errors.Wrapf(err, "failed to query playlist") 81 | } 82 | 83 | if len(resp.Items) == 0 { 84 | return nil, model.ErrNotFound 85 | } 86 | 87 | item := resp.Items[0] 88 | return item, nil 89 | } 90 | 91 | // Cost: 3 units (call: 1, snippet: 2) 92 | // See https://developers.google.com/youtube/v3/docs/playlistItems/list#part 93 | func (yt *YouTubeBuilder) listPlaylistItems(ctx context.Context, feed *model.Feed, pageToken string) ([]*youtube.PlaylistItem, string, error) { 94 | count := maxYoutubeResults 95 | if count > feed.PageSize { 96 | // If we need less than 50 97 | count = feed.PageSize 98 | } 99 | 100 | req := yt.client.PlaylistItems.List("id,snippet").MaxResults(int64(count)).PlaylistId(feed.ItemID) 101 | if pageToken != "" { 102 | req = req.PageToken(pageToken) 103 | } 104 | 105 | resp, err := req.Context(ctx).Do(yt.key) 106 | if err != nil { 107 | return nil, "", errors.Wrap(err, "failed to query playlist items") 108 | } 109 | 110 | return resp.Items, resp.NextPageToken, nil 111 | } 112 | 113 | func (yt *YouTubeBuilder) parseDate(s string) (time.Time, error) { 114 | date, err := time.Parse(time.RFC3339, s) 115 | if err != nil { 116 | return time.Time{}, errors.Wrapf(err, "failed to parse date: %s", s) 117 | } 118 | 119 | return date, nil 120 | } 121 | 122 | func (yt *YouTubeBuilder) selectThumbnail(snippet *youtube.ThumbnailDetails, quality model.Quality, videoID string) string { 123 | if snippet == nil { 124 | if videoID != "" { 125 | return fmt.Sprintf("https://img.youtube.com/vi/%s/default.jpg", videoID) 126 | } 127 | 128 | // TODO: use Podsync's preview image if unable to retrieve from YouTube 129 | return "" 130 | } 131 | 132 | // Use high resolution thumbnails for high quality mode 133 | // https://github.com/mxpv/Podsync/issues/14 134 | if quality == model.QualityHigh { 135 | if snippet.Maxres != nil { 136 | return snippet.Maxres.Url 137 | } 138 | 139 | if snippet.High != nil { 140 | return snippet.High.Url 141 | } 142 | 143 | if snippet.Medium != nil { 144 | return snippet.Medium.Url 145 | } 146 | } 147 | 148 | return snippet.Default.Url 149 | } 150 | 151 | func (yt *YouTubeBuilder) GetVideoCount(ctx context.Context, info *model.Info) (uint64, error) { 152 | switch info.LinkType { 153 | case model.TypeChannel, model.TypeUser: 154 | // Cost: 3 units 155 | if channel, err := yt.listChannels(ctx, info.LinkType, info.ItemID, "id,statistics"); err != nil { 156 | return 0, err 157 | } else { // nolint:golint 158 | return channel.Statistics.VideoCount, nil 159 | } 160 | 161 | case model.TypePlaylist: 162 | // Cost: 3 units 163 | if playlist, err := yt.listPlaylists(ctx, info.ItemID, "", "id,contentDetails"); err != nil { 164 | return 0, err 165 | } else { // nolint:golint 166 | return uint64(playlist.ContentDetails.ItemCount), nil 167 | } 168 | 169 | default: 170 | return 0, errors.New("unsupported link format") 171 | } 172 | } 173 | 174 | func (yt *YouTubeBuilder) queryFeed(ctx context.Context, feed *model.Feed, info *model.Info) error { 175 | var ( 176 | thumbnails *youtube.ThumbnailDetails 177 | ) 178 | 179 | switch info.LinkType { 180 | case model.TypeChannel, model.TypeUser: 181 | // Cost: 5 units for channel or user 182 | channel, err := yt.listChannels(ctx, info.LinkType, info.ItemID, "id,snippet,contentDetails") 183 | if err != nil { 184 | return err 185 | } 186 | 187 | feed.Title = channel.Snippet.Title 188 | feed.Description = channel.Snippet.Description 189 | 190 | if channel.Kind == "youtube#channel" { 191 | feed.ItemURL = fmt.Sprintf("https://youtube.com/channel/%s", channel.Id) 192 | feed.Author = "" 193 | } else { 194 | feed.ItemURL = fmt.Sprintf("https://youtube.com/user/%s", channel.Snippet.CustomUrl) 195 | feed.Author = channel.Snippet.CustomUrl 196 | } 197 | 198 | feed.ItemID = channel.ContentDetails.RelatedPlaylists.Uploads 199 | 200 | if date, err := yt.parseDate(channel.Snippet.PublishedAt); err != nil { 201 | return err 202 | } else { // nolint:golint 203 | feed.PubDate = date 204 | } 205 | 206 | thumbnails = channel.Snippet.Thumbnails 207 | 208 | case model.TypePlaylist: 209 | // Cost: 3 units for playlist 210 | playlist, err := yt.listPlaylists(ctx, info.ItemID, "", "id,snippet") 211 | if err != nil { 212 | return err 213 | } 214 | 215 | feed.Title = fmt.Sprintf("%s: %s", playlist.Snippet.ChannelTitle, playlist.Snippet.Title) 216 | feed.Description = playlist.Snippet.Description 217 | 218 | feed.ItemURL = fmt.Sprintf("https://youtube.com/playlist?list=%s", playlist.Id) 219 | feed.ItemID = playlist.Id 220 | 221 | feed.Author = "" 222 | 223 | if date, err := yt.parseDate(playlist.Snippet.PublishedAt); err != nil { 224 | return err 225 | } else { // nolint:golint 226 | feed.PubDate = date 227 | } 228 | 229 | thumbnails = playlist.Snippet.Thumbnails 230 | 231 | default: 232 | return errors.New("unsupported link format") 233 | } 234 | 235 | if feed.Description == "" { 236 | feed.Description = fmt.Sprintf("%s (%s)", feed.Title, feed.PubDate) 237 | } 238 | 239 | feed.CoverArt = yt.selectThumbnail(thumbnails, feed.CoverArtQuality, "") 240 | 241 | return nil 242 | } 243 | 244 | // Video size information requires 1 additional call for each video (1 feed = 50 videos = 50 calls), 245 | // which is too expensive, so get approximated size depending on duration and definition params 246 | func (yt *YouTubeBuilder) getSize(duration int64, feed *model.Feed) int64 { 247 | if feed.Format == model.FormatAudio { 248 | if feed.Quality == model.QualityHigh { 249 | return highAudioBytesPerSecond * duration 250 | } 251 | 252 | return lowAudioBytesPerSecond * duration 253 | } 254 | 255 | // Video format 256 | 257 | if feed.Quality == model.QualityHigh { 258 | return duration * hdBytesPerSecond 259 | } 260 | 261 | return duration * ldBytesPerSecond 262 | } 263 | 264 | // Cost: 5 units (call: 1, snippet: 2, contentDetails: 2) 265 | // See https://developers.google.com/youtube/v3/docs/videos/list#part 266 | func (yt *YouTubeBuilder) queryVideoDescriptions(ctx context.Context, playlist map[string]*youtube.PlaylistItemSnippet, feed *model.Feed) error { 267 | // Make the list of video ids 268 | ids := make([]string, 0, len(playlist)) 269 | for _, s := range playlist { 270 | ids = append(ids, s.ResourceId.VideoId) 271 | } 272 | 273 | // Init a list that will contains the aggregated strings of videos IDs (capped at 50 IDs per API Calls) 274 | idsList := make([]string, 0, 1) 275 | 276 | // Chunk the list of IDs by slices limited to maxYoutubeResults 277 | for i := 0; i < len(ids); i += maxYoutubeResults { 278 | end := i + maxYoutubeResults 279 | if end > len(ids) { 280 | end = len(ids) 281 | } 282 | // Save each slice as comma-delimited string 283 | idsList = append(idsList, strings.Join(ids[i:end], ",")) 284 | } 285 | 286 | // Show how many API calls will be required 287 | log.Debugf("Expected to make %d API calls to get the descriptions for %d episode(s).", len(idsList), len(ids)) 288 | 289 | // Loop in each slices of 50 (or less) IDs and query their description 290 | for _, idsI := range idsList { 291 | req, err := yt.client.Videos.List("id,snippet,contentDetails").Id(idsI).Context(ctx).Do(yt.key) 292 | if err != nil { 293 | return errors.Wrap(err, "failed to query video descriptions") 294 | } 295 | 296 | for _, video := range req.Items { 297 | var ( 298 | snippet = video.Snippet 299 | videoID = video.Id 300 | videoURL = fmt.Sprintf("https://youtube.com/watch?v=%s", video.Id) 301 | image = yt.selectThumbnail(snippet.Thumbnails, feed.Quality, videoID) 302 | ) 303 | 304 | // Skip unreleased/airing Premiere videos 305 | if snippet.LiveBroadcastContent == "upcoming" || snippet.LiveBroadcastContent == "live" { 306 | continue 307 | } 308 | 309 | // Parse date added to playlist / publication date 310 | dateStr := "" 311 | playlistItem, ok := playlist[video.Id] 312 | if ok { 313 | dateStr = playlistItem.PublishedAt 314 | } else { 315 | dateStr = snippet.PublishedAt 316 | } 317 | 318 | pubDate, err := yt.parseDate(dateStr) 319 | if err != nil { 320 | return errors.Wrapf(err, "failed to parse video publish date: %s", dateStr) 321 | } 322 | 323 | // Sometimes YouTube retrun empty content defailt, use arbitrary one 324 | var seconds int64 = 1 325 | if video.ContentDetails != nil { 326 | // Parse duration 327 | d, err := duration.FromString(video.ContentDetails.Duration) 328 | if err != nil { 329 | return errors.Wrapf(err, "failed to parse duration %s", video.ContentDetails.Duration) 330 | } 331 | 332 | seconds = int64(d.ToDuration().Seconds()) 333 | } 334 | 335 | var ( 336 | order = strconv.FormatInt(playlistItem.Position, 10) 337 | size = yt.getSize(seconds, feed) 338 | ) 339 | 340 | feed.Episodes = append(feed.Episodes, &model.Episode{ 341 | ID: video.Id, 342 | Title: snippet.Title, 343 | Description: snippet.Description, 344 | Thumbnail: image, 345 | Duration: seconds, 346 | Size: size, 347 | VideoURL: videoURL, 348 | PubDate: pubDate, 349 | Order: order, 350 | Status: model.EpisodeNew, 351 | }) 352 | } 353 | } 354 | 355 | return nil 356 | } 357 | 358 | // Cost: 359 | // ASC mode = (3 units + 5 units) * X pages = 8 units per page 360 | // DESC mode = 3 units * (number of pages in the entire playlist) + 5 units 361 | func (yt *YouTubeBuilder) queryItems(ctx context.Context, feed *model.Feed) error { 362 | var ( 363 | token string 364 | count int 365 | allSnippets []*youtube.PlaylistItemSnippet 366 | ) 367 | 368 | for { 369 | items, pageToken, err := yt.listPlaylistItems(ctx, feed, token) 370 | if err != nil { 371 | return err 372 | } 373 | 374 | token = pageToken 375 | 376 | if len(items) == 0 { 377 | break 378 | } 379 | 380 | // Extract playlist snippets 381 | for _, item := range items { 382 | allSnippets = append(allSnippets, item.Snippet) 383 | count++ 384 | } 385 | 386 | if (feed.PlaylistSort != model.SortingDesc && count >= feed.PageSize) || token == "" { 387 | break 388 | } 389 | } 390 | 391 | if len(allSnippets) > feed.PageSize { 392 | if feed.PlaylistSort != model.SortingDesc { 393 | allSnippets = allSnippets[:feed.PageSize] 394 | } else { 395 | allSnippets = allSnippets[len(allSnippets)-feed.PageSize:] 396 | } 397 | } 398 | 399 | snippets := map[string]*youtube.PlaylistItemSnippet{} 400 | for _, snippet := range allSnippets { 401 | snippets[snippet.ResourceId.VideoId] = snippet 402 | } 403 | 404 | // Query video descriptions from the list of ids 405 | if err := yt.queryVideoDescriptions(ctx, snippets, feed); err != nil { 406 | return err 407 | } 408 | 409 | return nil 410 | } 411 | 412 | func (yt *YouTubeBuilder) Build(ctx context.Context, cfg *feed.Config) (*model.Feed, error) { 413 | info, err := ParseURL(cfg.URL) 414 | if err != nil { 415 | return nil, err 416 | } 417 | 418 | _feed := &model.Feed{ 419 | ItemID: info.ItemID, 420 | Provider: info.Provider, 421 | LinkType: info.LinkType, 422 | Format: cfg.Format, 423 | Quality: cfg.Quality, 424 | CoverArtQuality: cfg.Custom.CoverArtQuality, 425 | PageSize: cfg.PageSize, 426 | PlaylistSort: cfg.PlaylistSort, 427 | PrivateFeed: cfg.PrivateFeed, 428 | UpdatedAt: time.Now().UTC(), 429 | } 430 | 431 | if _feed.PageSize == 0 { 432 | _feed.PageSize = maxYoutubeResults 433 | } 434 | 435 | // Query general information about feed (title, description, lang, etc) 436 | if err := yt.queryFeed(ctx, _feed, &info); err != nil { 437 | return nil, err 438 | } 439 | 440 | if err := yt.queryItems(ctx, _feed); err != nil { 441 | return nil, err 442 | } 443 | 444 | // YT API client gets 50 episodes per query. 445 | // Round up to page size. 446 | if len(_feed.Episodes) > _feed.PageSize { 447 | _feed.Episodes = _feed.Episodes[:_feed.PageSize] 448 | } 449 | 450 | sort.Slice(_feed.Episodes, func(i, j int) bool { 451 | item1, _ := strconv.Atoi(_feed.Episodes[i].Order) 452 | item2, _ := strconv.Atoi(_feed.Episodes[j].Order) 453 | return item1 < item2 454 | }) 455 | 456 | return _feed, nil 457 | } 458 | 459 | func NewYouTubeBuilder(key string) (*YouTubeBuilder, error) { 460 | if key == "" { 461 | return nil, errors.New("empty YouTube API key") 462 | } 463 | 464 | yt, err := youtube.New(&http.Client{}) 465 | if err != nil { 466 | return nil, errors.Wrap(err, "failed to create youtube client") 467 | } 468 | 469 | return &YouTubeBuilder{client: yt, key: apiKey(key)}, nil 470 | } 471 | -------------------------------------------------------------------------------- /pkg/builder/youtube_test.go: -------------------------------------------------------------------------------- 1 | package builder 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "testing" 7 | 8 | "github.com/mxpv/podsync/pkg/feed" 9 | "github.com/stretchr/testify/assert" 10 | "github.com/stretchr/testify/require" 11 | 12 | "github.com/mxpv/podsync/pkg/model" 13 | ) 14 | 15 | var ( 16 | testCtx = context.Background() 17 | ytKey = os.Getenv("YOUTUBE_TEST_API_KEY") 18 | ) 19 | 20 | func TestYT_QueryChannel(t *testing.T) { 21 | if ytKey == "" { 22 | t.Skip("YouTube API key is not provided") 23 | } 24 | 25 | builder, err := NewYouTubeBuilder(ytKey) 26 | require.NoError(t, err) 27 | 28 | channel, err := builder.listChannels(testCtx, model.TypeChannel, "UC2yTVSttx7lxAOAzx1opjoA", "id") 29 | require.NoError(t, err) 30 | require.Equal(t, "UC2yTVSttx7lxAOAzx1opjoA", channel.Id) 31 | 32 | channel, err = builder.listChannels(testCtx, model.TypeUser, "fxigr1", "id") 33 | require.NoError(t, err) 34 | require.Equal(t, "UCr_fwF-n-2_olTYd-m3n32g", channel.Id) 35 | } 36 | 37 | func TestYT_BuildFeed(t *testing.T) { 38 | if ytKey == "" { 39 | t.Skip("YouTube API key is not provided") 40 | } 41 | 42 | builder, err := NewYouTubeBuilder(ytKey) 43 | require.NoError(t, err) 44 | 45 | urls := []string{ 46 | "https://www.youtube.com/channel/UCupvZG-5ko_eiXAupbDfxWw", 47 | "https://www.youtube.com/playlist?list=PLF7tUDhGkiCk_Ne30zu7SJ9gZF9R9ZruE", 48 | "https://www.youtube.com/channel/UCK9lZ2lHRBgx2LOcqPifukA", 49 | "https://youtube.com/user/WylsaLive", 50 | "https://www.youtube.com/playlist?list=PLUVl5pafUrBydT_gsCjRGeCy0hFHloec8", 51 | } 52 | 53 | for _, addr := range urls { 54 | t.Run(addr, func(t *testing.T) { 55 | _feed, err := builder.Build(testCtx, &feed.Config{URL: addr}) 56 | require.NoError(t, err) 57 | 58 | assert.NotEmpty(t, _feed.Title) 59 | assert.NotEmpty(t, _feed.Description) 60 | assert.NotEmpty(t, _feed.Author) 61 | assert.NotEmpty(t, _feed.ItemURL) 62 | 63 | assert.NotZero(t, len(_feed.Episodes)) 64 | 65 | for _, item := range _feed.Episodes { 66 | assert.NotEmpty(t, item.Title) 67 | assert.NotEmpty(t, item.VideoURL) 68 | assert.NotZero(t, item.Duration) 69 | 70 | assert.NotEmpty(t, item.Title) 71 | assert.NotEmpty(t, item.Thumbnail) 72 | } 73 | }) 74 | } 75 | } 76 | 77 | func TestYT_GetVideoCount(t *testing.T) { 78 | if ytKey == "" { 79 | t.Skip("YouTube API key is not provided") 80 | } 81 | 82 | builder, err := NewYouTubeBuilder(ytKey) 83 | require.NoError(t, err) 84 | 85 | feeds := []*model.Info{ 86 | {Provider: model.ProviderYoutube, LinkType: model.TypeUser, ItemID: "fxigr1"}, 87 | {Provider: model.ProviderYoutube, LinkType: model.TypeChannel, ItemID: "UCupvZG-5ko_eiXAupbDfxWw"}, 88 | {Provider: model.ProviderYoutube, LinkType: model.TypePlaylist, ItemID: "PLF7tUDhGkiCk_Ne30zu7SJ9gZF9R9ZruE"}, 89 | {Provider: model.ProviderYoutube, LinkType: model.TypeChannel, ItemID: "UCK9lZ2lHRBgx2LOcqPifukA"}, 90 | {Provider: model.ProviderYoutube, LinkType: model.TypeUser, ItemID: "WylsaLive"}, 91 | {Provider: model.ProviderYoutube, LinkType: model.TypePlaylist, ItemID: "PLUVl5pafUrBydT_gsCjRGeCy0hFHloec8"}, 92 | } 93 | 94 | for _, _feed := range feeds { 95 | feedCopy := _feed 96 | t.Run(_feed.ItemID, func(t *testing.T) { 97 | count, err := builder.GetVideoCount(testCtx, feedCopy) 98 | assert.NoError(t, err) 99 | assert.NotZero(t, count) 100 | }) 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /pkg/db/badger.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "os" 8 | 9 | "github.com/dgraph-io/badger" 10 | "github.com/dgraph-io/badger/options" 11 | "github.com/pkg/errors" 12 | log "github.com/sirupsen/logrus" 13 | 14 | "github.com/mxpv/podsync/pkg/model" 15 | ) 16 | 17 | const ( 18 | versionPath = "podsync/version" 19 | feedPrefix = "feed/" 20 | feedPath = "feed/%s" 21 | episodePrefix = "episode/%s/" 22 | episodePath = "episode/%s/%s" // FeedID + EpisodeID 23 | ) 24 | 25 | // BadgerConfig represents BadgerDB configuration parameters 26 | type BadgerConfig struct { 27 | Truncate bool `toml:"truncate"` 28 | FileIO bool `toml:"file_io"` 29 | } 30 | 31 | type Badger struct { 32 | db *badger.DB 33 | } 34 | 35 | var _ Storage = (*Badger)(nil) 36 | 37 | func NewBadger(config *Config) (*Badger, error) { 38 | var ( 39 | dir = config.Dir 40 | ) 41 | 42 | log.Infof("opening database %q", dir) 43 | 44 | // Make sure database directory exists 45 | if err := os.MkdirAll(dir, 0755); err != nil { 46 | return nil, errors.Wrap(err, "could not mkdir database dir") 47 | } 48 | 49 | opts := badger.DefaultOptions(dir). 50 | WithLogger(log.StandardLogger()). 51 | WithTruncate(true) 52 | 53 | if config.Badger != nil { 54 | opts.Truncate = config.Badger.Truncate 55 | if config.Badger.FileIO { 56 | opts.ValueLogLoadingMode = options.FileIO 57 | } 58 | } 59 | 60 | db, err := badger.Open(opts) 61 | if err != nil { 62 | return nil, errors.Wrap(err, "failed to open database") 63 | } 64 | 65 | storage := &Badger{db: db} 66 | 67 | if err := db.Update(func(txn *badger.Txn) error { 68 | if err := storage.setObj(txn, []byte(versionPath), CurrentVersion, false); err != nil && err != model.ErrAlreadyExists { 69 | return err 70 | } 71 | return nil 72 | }); err != nil { 73 | return nil, errors.Wrap(err, "failed to read database version") 74 | } 75 | 76 | return &Badger{db: db}, nil 77 | } 78 | 79 | func (b *Badger) Close() error { 80 | log.Debug("closing database") 81 | return b.db.Close() 82 | } 83 | 84 | func (b *Badger) Version() (int, error) { 85 | var ( 86 | version = -1 87 | ) 88 | 89 | err := b.db.View(func(txn *badger.Txn) error { 90 | return b.getObj(txn, []byte(versionPath), &version) 91 | }) 92 | 93 | return version, err 94 | } 95 | 96 | func (b *Badger) AddFeed(_ context.Context, feedID string, feed *model.Feed) error { 97 | return b.db.Update(func(txn *badger.Txn) error { 98 | // Insert or update feed info 99 | feedKey := b.getKey(feedPath, feedID) 100 | if err := b.setObj(txn, feedKey, feed, true); err != nil { 101 | return err 102 | } 103 | 104 | // Append new episodes 105 | for _, episode := range feed.Episodes { 106 | episodeKey := b.getKey(episodePath, feedID, episode.ID) 107 | err := b.setObj(txn, episodeKey, episode, false) 108 | if err == nil || err == model.ErrAlreadyExists { 109 | // Do nothing 110 | } else { 111 | return errors.Wrapf(err, "failed to save episode %q", feedID) 112 | } 113 | } 114 | 115 | return nil 116 | }) 117 | } 118 | 119 | func (b *Badger) GetFeed(_ context.Context, feedID string) (*model.Feed, error) { 120 | var ( 121 | feed = model.Feed{} 122 | feedKey = b.getKey(feedPath, feedID) 123 | ) 124 | 125 | if err := b.db.View(func(txn *badger.Txn) error { 126 | // Query feed 127 | if err := b.getObj(txn, feedKey, &feed); err != nil { 128 | return err 129 | } 130 | 131 | // Query episodes 132 | if err := b.walkEpisodes(txn, feedID, func(episode *model.Episode) error { 133 | feed.Episodes = append(feed.Episodes, episode) 134 | return nil 135 | }); err != nil { 136 | return err 137 | } 138 | 139 | return nil 140 | }); err != nil { 141 | return nil, err 142 | } 143 | 144 | return &feed, nil 145 | } 146 | 147 | func (b *Badger) WalkFeeds(_ context.Context, cb func(feed *model.Feed) error) error { 148 | return b.db.View(func(txn *badger.Txn) error { 149 | opts := badger.DefaultIteratorOptions 150 | opts.Prefix = b.getKey(feedPrefix) 151 | opts.PrefetchValues = true 152 | return b.iterator(txn, opts, func(item *badger.Item) error { 153 | feed := &model.Feed{} 154 | if err := b.unmarshalObj(item, feed); err != nil { 155 | return err 156 | } 157 | 158 | return cb(feed) 159 | }) 160 | }) 161 | } 162 | 163 | func (b *Badger) DeleteFeed(_ context.Context, feedID string) error { 164 | return b.db.Update(func(txn *badger.Txn) error { 165 | // Feed 166 | feedKey := b.getKey(feedPath, feedID) 167 | if err := txn.Delete(feedKey); err != nil { 168 | return errors.Wrapf(err, "failed to delete feed %q", feedID) 169 | } 170 | 171 | // Episodes 172 | opts := badger.DefaultIteratorOptions 173 | opts.Prefix = b.getKey(episodePrefix, feedID) 174 | opts.PrefetchValues = false 175 | if err := b.iterator(txn, opts, func(item *badger.Item) error { 176 | return txn.Delete(item.KeyCopy(nil)) 177 | }); err != nil { 178 | return errors.Wrapf(err, "failed to iterate episodes for feed %q", feedID) 179 | } 180 | 181 | return nil 182 | }) 183 | } 184 | 185 | func (b *Badger) GetEpisode(_ context.Context, feedID string, episodeID string) (*model.Episode, error) { 186 | var ( 187 | episode model.Episode 188 | err error 189 | key = b.getKey(episodePath, feedID, episodeID) 190 | ) 191 | 192 | err = b.db.View(func(txn *badger.Txn) error { 193 | return b.getObj(txn, key, &episode) 194 | }) 195 | 196 | return &episode, err 197 | } 198 | 199 | func (b *Badger) UpdateEpisode(feedID string, episodeID string, cb func(episode *model.Episode) error) error { 200 | var ( 201 | key = b.getKey(episodePath, feedID, episodeID) 202 | episode model.Episode 203 | ) 204 | 205 | return b.db.Update(func(txn *badger.Txn) error { 206 | if err := b.getObj(txn, key, &episode); err != nil { 207 | return err 208 | } 209 | 210 | if err := cb(&episode); err != nil { 211 | return err 212 | } 213 | 214 | if episode.ID != episodeID { 215 | return errors.New("can't change episode ID") 216 | } 217 | 218 | return b.setObj(txn, key, &episode, true) 219 | }) 220 | } 221 | 222 | func (b *Badger) DeleteEpisode(feedID, episodeID string) error { 223 | key := b.getKey(episodePath, feedID, episodeID) 224 | return b.db.Update(func(txn *badger.Txn) error { 225 | return txn.Delete(key) 226 | }) 227 | } 228 | 229 | func (b *Badger) WalkEpisodes(ctx context.Context, feedID string, cb func(episode *model.Episode) error) error { 230 | return b.db.View(func(txn *badger.Txn) error { 231 | return b.walkEpisodes(txn, feedID, cb) 232 | }) 233 | } 234 | 235 | func (b *Badger) walkEpisodes(txn *badger.Txn, feedID string, cb func(episode *model.Episode) error) error { 236 | opts := badger.DefaultIteratorOptions 237 | opts.Prefix = b.getKey(episodePrefix, feedID) 238 | opts.PrefetchValues = true 239 | return b.iterator(txn, opts, func(item *badger.Item) error { 240 | feed := &model.Episode{} 241 | if err := b.unmarshalObj(item, feed); err != nil { 242 | return err 243 | } 244 | 245 | return cb(feed) 246 | }) 247 | } 248 | 249 | func (b *Badger) iterator(txn *badger.Txn, opts badger.IteratorOptions, callback func(item *badger.Item) error) error { 250 | iter := txn.NewIterator(opts) 251 | defer iter.Close() 252 | 253 | for iter.Rewind(); iter.Valid(); iter.Next() { 254 | item := iter.Item() 255 | 256 | if err := callback(item); err != nil { 257 | return err 258 | } 259 | } 260 | 261 | return nil 262 | } 263 | 264 | func (b *Badger) getKey(format string, a ...interface{}) []byte { 265 | resourcePath := fmt.Sprintf(format, a...) 266 | fullPath := fmt.Sprintf("podsync/v%d/%s", CurrentVersion, resourcePath) 267 | 268 | return []byte(fullPath) 269 | } 270 | 271 | func (b *Badger) setObj(txn *badger.Txn, key []byte, obj interface{}, overwrite bool) error { 272 | if !overwrite { 273 | // Overwrites are not allowed, make sure there is no object with the given key 274 | _, err := txn.Get(key) 275 | if err == nil { 276 | return model.ErrAlreadyExists 277 | } else if err == badger.ErrKeyNotFound { 278 | // Key not found, do nothing 279 | } else { 280 | return errors.Wrap(err, "failed to check whether key exists") 281 | } 282 | } 283 | 284 | data, err := b.marshalObj(obj) 285 | if err != nil { 286 | return errors.Wrapf(err, "failed to serialize object for key %q", key) 287 | } 288 | 289 | return txn.Set(key, data) 290 | } 291 | 292 | func (b *Badger) getObj(txn *badger.Txn, key []byte, out interface{}) error { 293 | item, err := txn.Get(key) 294 | if err != nil { 295 | if err == badger.ErrKeyNotFound { 296 | return model.ErrNotFound 297 | } 298 | 299 | return err 300 | } 301 | 302 | return b.unmarshalObj(item, out) 303 | } 304 | 305 | func (b *Badger) marshalObj(obj interface{}) ([]byte, error) { 306 | return json.Marshal(obj) 307 | } 308 | 309 | func (b *Badger) unmarshalObj(item *badger.Item, out interface{}) error { 310 | return item.Value(func(val []byte) error { 311 | return json.Unmarshal(val, out) 312 | }) 313 | } 314 | -------------------------------------------------------------------------------- /pkg/db/badger_test.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | "time" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/require" 10 | 11 | "github.com/mxpv/podsync/pkg/model" 12 | ) 13 | 14 | var testCtx = context.TODO() 15 | 16 | func TestNewBadger(t *testing.T) { 17 | dir := t.TempDir() 18 | 19 | db, err := NewBadger(&Config{Dir: dir}) 20 | require.NoError(t, err) 21 | 22 | err = db.Close() 23 | assert.NoError(t, err) 24 | } 25 | 26 | func TestBadger_Version(t *testing.T) { 27 | dir := t.TempDir() 28 | 29 | db, err := NewBadger(&Config{Dir: dir}) 30 | require.NoError(t, err) 31 | defer db.Close() 32 | 33 | ver, err := db.Version() 34 | assert.NoError(t, err) 35 | assert.Equal(t, CurrentVersion, ver) 36 | } 37 | 38 | func TestBadger_AddFeed(t *testing.T) { 39 | dir := t.TempDir() 40 | 41 | db, err := NewBadger(&Config{Dir: dir}) 42 | require.NoError(t, err) 43 | defer db.Close() 44 | 45 | feed := getFeed() 46 | err = db.AddFeed(testCtx, feed.ID, feed) 47 | assert.NoError(t, err) 48 | } 49 | 50 | func TestBadger_GetFeed(t *testing.T) { 51 | dir := t.TempDir() 52 | 53 | db, err := NewBadger(&Config{Dir: dir}) 54 | require.NoError(t, err) 55 | defer db.Close() 56 | 57 | feed := getFeed() 58 | feed.Episodes = nil 59 | 60 | err = db.AddFeed(testCtx, feed.ID, feed) 61 | require.NoError(t, err) 62 | 63 | actual, err := db.GetFeed(testCtx, feed.ID) 64 | assert.NoError(t, err) 65 | assert.Equal(t, feed, actual) 66 | } 67 | 68 | func TestBadger_WalkFeeds(t *testing.T) { 69 | dir := t.TempDir() 70 | 71 | db, err := NewBadger(&Config{Dir: dir}) 72 | require.NoError(t, err) 73 | defer db.Close() 74 | 75 | feed := getFeed() 76 | feed.Episodes = nil // These are not serialized to database 77 | 78 | err = db.AddFeed(testCtx, feed.ID, feed) 79 | assert.NoError(t, err) 80 | 81 | called := 0 82 | err = db.WalkFeeds(testCtx, func(actual *model.Feed) error { 83 | assert.EqualValues(t, feed, actual) 84 | called++ 85 | return nil 86 | }) 87 | 88 | assert.NoError(t, err) 89 | assert.Equal(t, called, 1) 90 | } 91 | 92 | func TestBadger_DeleteFeed(t *testing.T) { 93 | dir := t.TempDir() 94 | 95 | db, err := NewBadger(&Config{Dir: dir}) 96 | require.NoError(t, err) 97 | defer db.Close() 98 | 99 | feed := getFeed() 100 | err = db.AddFeed(testCtx, feed.ID, feed) 101 | require.NoError(t, err) 102 | 103 | err = db.DeleteFeed(testCtx, feed.ID) 104 | assert.NoError(t, err) 105 | 106 | called := 0 107 | err = db.WalkFeeds(testCtx, func(feed *model.Feed) error { 108 | called++ 109 | return nil 110 | }) 111 | assert.NoError(t, err) 112 | assert.Equal(t, 0, called) 113 | } 114 | 115 | func TestBadger_UpdateEpisode(t *testing.T) { 116 | dir := t.TempDir() 117 | 118 | db, err := NewBadger(&Config{Dir: dir}) 119 | require.NoError(t, err) 120 | defer db.Close() 121 | 122 | feed := getFeed() 123 | err = db.AddFeed(testCtx, feed.ID, feed) 124 | assert.NoError(t, err) 125 | 126 | err = db.UpdateEpisode(feed.ID, feed.Episodes[0].ID, func(file *model.Episode) error { 127 | file.Size = 333 128 | file.Status = model.EpisodeDownloaded 129 | return nil 130 | }) 131 | assert.NoError(t, err) 132 | 133 | episode, err := db.GetEpisode(testCtx, feed.ID, feed.Episodes[0].ID) 134 | assert.NoError(t, err) 135 | 136 | assert.Equal(t, feed.Episodes[0].ID, episode.ID) 137 | assert.EqualValues(t, 333, episode.Size) 138 | assert.Equal(t, model.EpisodeDownloaded, episode.Status) 139 | 140 | assert.NoError(t, err) 141 | } 142 | 143 | func TestBadger_WalkEpisodes(t *testing.T) { 144 | dir := t.TempDir() 145 | 146 | db, err := NewBadger(&Config{Dir: dir}) 147 | require.NoError(t, err) 148 | defer db.Close() 149 | 150 | feed := getFeed() 151 | err = db.AddFeed(testCtx, feed.ID, feed) 152 | assert.NoError(t, err) 153 | 154 | called := 0 155 | err = db.WalkEpisodes(testCtx, feed.ID, func(actual *model.Episode) error { 156 | assert.EqualValues(t, feed.Episodes[called], actual) 157 | called++ 158 | return nil 159 | }) 160 | 161 | assert.NoError(t, err) 162 | assert.Equal(t, called, 2) 163 | } 164 | 165 | func getFeed() *model.Feed { 166 | return &model.Feed{ 167 | ID: "1", 168 | ItemID: "2", 169 | LinkType: model.TypeChannel, 170 | Provider: model.ProviderVimeo, 171 | CreatedAt: time.Now().UTC(), 172 | LastAccess: time.Now().UTC(), 173 | ExpirationTime: time.Now().UTC().Add(1 * time.Hour), 174 | Format: "video", 175 | Quality: "high", 176 | PageSize: 50, 177 | Title: "Test", 178 | Description: "Test", 179 | PubDate: time.Now().UTC(), 180 | Author: "", 181 | ItemURL: "https://vimeo.com", 182 | Episodes: []*model.Episode{ 183 | { 184 | ID: "1", 185 | Title: "Episode title 1", 186 | Description: "Episode description 1", 187 | Duration: 100, 188 | VideoURL: "https://vimeo.com/123", 189 | PubDate: time.Now().UTC(), 190 | Size: 1234, 191 | Order: "1", 192 | }, 193 | { 194 | ID: "2", 195 | Title: "Episode title 2", 196 | Description: "Episode description 2", 197 | Duration: 299, 198 | VideoURL: "https://vimeo.com/321", 199 | PubDate: time.Now().UTC(), 200 | Size: 4321, 201 | Order: "2", 202 | }, 203 | }, 204 | UpdatedAt: time.Now().UTC(), 205 | } 206 | } 207 | -------------------------------------------------------------------------------- /pkg/db/config.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | type Config struct { 4 | // Dir is a directory to keep database files 5 | Dir string `toml:"dir"` 6 | Badger *BadgerConfig `toml:"badger"` 7 | } 8 | -------------------------------------------------------------------------------- /pkg/db/storage.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/mxpv/podsync/pkg/model" 7 | ) 8 | 9 | type Version int 10 | 11 | const ( 12 | CurrentVersion = 1 13 | ) 14 | 15 | type Storage interface { 16 | Close() error 17 | Version() (int, error) 18 | 19 | // AddFeed will: 20 | // - Insert or update feed info 21 | // - Append new episodes to the existing list of episodes (existing episodes are not overwritten!) 22 | AddFeed(ctx context.Context, feedID string, feed *model.Feed) error 23 | 24 | // GetFeed gets a feed by ID 25 | GetFeed(ctx context.Context, feedID string) (*model.Feed, error) 26 | 27 | // WalkFeeds iterates over feeds saved to database 28 | WalkFeeds(ctx context.Context, cb func(feed *model.Feed) error) error 29 | 30 | // DeleteFeed deletes feed and all related data from database 31 | DeleteFeed(ctx context.Context, feedID string) error 32 | 33 | // GetEpisode gets episode by identifier 34 | GetEpisode(ctx context.Context, feedID string, episodeID string) (*model.Episode, error) 35 | 36 | // UpdateEpisode updates episode fields 37 | UpdateEpisode(feedID string, episodeID string, cb func(episode *model.Episode) error) error 38 | 39 | // DeleteEpisode deletes an episode 40 | DeleteEpisode(feedID string, episodeID string) error 41 | 42 | // WalkEpisodes iterates over episodes that belong to the given feed ID 43 | WalkEpisodes(ctx context.Context, feedID string, cb func(episode *model.Episode) error) error 44 | } 45 | -------------------------------------------------------------------------------- /pkg/feed/config.go: -------------------------------------------------------------------------------- 1 | package feed 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/mxpv/podsync/pkg/model" 7 | ) 8 | 9 | // Config is a configuration for a feed loaded from TOML 10 | type Config struct { 11 | ID string `toml:"-"` 12 | // URL is a full URL of the field 13 | URL string `toml:"url"` 14 | // PageSize is the number of pages to query from YouTube API. 15 | // NOTE: larger page sizes/often requests might drain your API token. 16 | PageSize int `toml:"page_size"` 17 | // UpdatePeriod is how often to check for updates. 18 | // Format is "300ms", "1.5h" or "2h45m". 19 | // Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". 20 | // NOTE: too often update check might drain your API token. 21 | UpdatePeriod time.Duration `toml:"update_period"` 22 | // Cron expression format is how often to check update 23 | // NOTE: too often update check might drain your API token. 24 | CronSchedule string `toml:"cron_schedule"` 25 | // Quality to use for this feed 26 | Quality model.Quality `toml:"quality"` 27 | // Maximum height of video 28 | MaxHeight int `toml:"max_height"` 29 | // Format to use for this feed 30 | Format model.Format `toml:"format"` 31 | // Custom format properties 32 | CustomFormat CustomFormat `toml:"custom_format"` 33 | // Only download episodes that match the filters (defaults to matching anything) 34 | Filters Filters `toml:"filters"` 35 | // Clean is a cleanup policy to use for this feed 36 | Clean Cleanup `toml:"clean"` 37 | // Custom is a list of feed customizations 38 | Custom Custom `toml:"custom"` 39 | // List of additional youtube-dl arguments passed at download time 40 | YouTubeDLArgs []string `toml:"youtube_dl_args"` 41 | // Included in OPML file 42 | OPML bool `toml:"opml"` 43 | // Private feed (not indexed by podcast aggregators) 44 | PrivateFeed bool `toml:"private_feed"` 45 | // Playlist sort 46 | PlaylistSort model.Sorting `toml:"playlist_sort"` 47 | } 48 | 49 | type CustomFormat struct { 50 | YouTubeDLFormat string `toml:"youtube_dl_format"` 51 | Extension string `toml:"extension"` 52 | } 53 | 54 | type Filters struct { 55 | Title string `toml:"title"` 56 | NotTitle string `toml:"not_title"` 57 | Description string `toml:"description"` 58 | NotDescription string `toml:"not_description"` 59 | MinDuration int64 `toml:"min_duration"` 60 | MaxDuration int64 `toml:"max_duration"` 61 | MaxAge int `toml:"max_age"` 62 | // More filters to be added here 63 | } 64 | 65 | type Custom struct { 66 | CoverArt string `toml:"cover_art"` 67 | CoverArtQuality model.Quality `toml:"cover_art_quality"` 68 | Category string `toml:"category"` 69 | Subcategories []string `toml:"subcategories"` 70 | Explicit bool `toml:"explicit"` 71 | Language string `toml:"lang"` 72 | Author string `toml:"author"` 73 | Title string `toml:"title"` 74 | Description string `toml:"description"` 75 | OwnerName string `toml:"ownerName"` 76 | OwnerEmail string `toml:"ownerEmail"` 77 | Link string `toml:"link"` 78 | } 79 | 80 | type Cleanup struct { 81 | // KeepLast defines how many episodes to keep 82 | KeepLast int `toml:"keep_last"` 83 | } 84 | -------------------------------------------------------------------------------- /pkg/feed/deps.go: -------------------------------------------------------------------------------- 1 | //go:generate mockgen -source=deps.go -destination=deps_mock_test.go -package=feed 2 | 3 | package feed 4 | 5 | import ( 6 | "context" 7 | 8 | "github.com/mxpv/podsync/pkg/model" 9 | ) 10 | 11 | type feedProvider interface { 12 | GetFeed(ctx context.Context, feedID string) (*model.Feed, error) 13 | } 14 | -------------------------------------------------------------------------------- /pkg/feed/deps_mock_test.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: deps.go 3 | 4 | // Package feed is a generated GoMock package. 5 | package feed 6 | 7 | import ( 8 | context "context" 9 | reflect "reflect" 10 | 11 | gomock "github.com/golang/mock/gomock" 12 | model "github.com/mxpv/podsync/pkg/model" 13 | ) 14 | 15 | // MockfeedProvider is a mock of feedProvider interface. 16 | type MockfeedProvider struct { 17 | ctrl *gomock.Controller 18 | recorder *MockfeedProviderMockRecorder 19 | } 20 | 21 | // MockfeedProviderMockRecorder is the mock recorder for MockfeedProvider. 22 | type MockfeedProviderMockRecorder struct { 23 | mock *MockfeedProvider 24 | } 25 | 26 | // NewMockfeedProvider creates a new mock instance. 27 | func NewMockfeedProvider(ctrl *gomock.Controller) *MockfeedProvider { 28 | mock := &MockfeedProvider{ctrl: ctrl} 29 | mock.recorder = &MockfeedProviderMockRecorder{mock} 30 | return mock 31 | } 32 | 33 | // EXPECT returns an object that allows the caller to indicate expected use. 34 | func (m *MockfeedProvider) EXPECT() *MockfeedProviderMockRecorder { 35 | return m.recorder 36 | } 37 | 38 | // GetFeed mocks base method. 39 | func (m *MockfeedProvider) GetFeed(ctx context.Context, feedID string) (*model.Feed, error) { 40 | m.ctrl.T.Helper() 41 | ret := m.ctrl.Call(m, "GetFeed", ctx, feedID) 42 | ret0, _ := ret[0].(*model.Feed) 43 | ret1, _ := ret[1].(error) 44 | return ret0, ret1 45 | } 46 | 47 | // GetFeed indicates an expected call of GetFeed. 48 | func (mr *MockfeedProviderMockRecorder) GetFeed(ctx, feedID interface{}) *gomock.Call { 49 | mr.mock.ctrl.T.Helper() 50 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetFeed", reflect.TypeOf((*MockfeedProvider)(nil).GetFeed), ctx, feedID) 51 | } 52 | -------------------------------------------------------------------------------- /pkg/feed/key.go: -------------------------------------------------------------------------------- 1 | package feed 2 | 3 | import ( 4 | "sync" 5 | 6 | "github.com/pkg/errors" 7 | ) 8 | 9 | type KeyProvider interface { 10 | Get() string 11 | } 12 | 13 | func NewKeyProvider(keys []string) (KeyProvider, error) { 14 | switch len(keys) { 15 | case 0: 16 | return nil, errors.New("no keys") 17 | case 1: 18 | return NewFixedKey(keys[0]) 19 | default: 20 | return NewRotatedKeys(keys) 21 | } 22 | } 23 | 24 | type FixedKeyProvider struct { 25 | key string 26 | } 27 | 28 | func NewFixedKey(key string) (KeyProvider, error) { 29 | if key == "" { 30 | return nil, errors.New("key can't be empty") 31 | } 32 | 33 | return &FixedKeyProvider{key: key}, nil 34 | } 35 | 36 | func (p FixedKeyProvider) Get() string { 37 | return p.key 38 | } 39 | 40 | type RotatedKeyProvider struct { 41 | keys []string 42 | lock sync.Mutex 43 | index int 44 | } 45 | 46 | func NewRotatedKeys(keys []string) (KeyProvider, error) { 47 | if len(keys) < 2 { 48 | return nil, errors.Errorf("at least 2 keys required (got %d)", len(keys)) 49 | } 50 | 51 | return &RotatedKeyProvider{keys: keys, index: 0}, nil 52 | } 53 | 54 | func (p *RotatedKeyProvider) Get() string { 55 | p.lock.Lock() 56 | defer p.lock.Unlock() 57 | 58 | current := p.index % len(p.keys) 59 | p.index++ 60 | 61 | return p.keys[current] 62 | } 63 | -------------------------------------------------------------------------------- /pkg/feed/key_test.go: -------------------------------------------------------------------------------- 1 | package feed 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestNewFixedKey(t *testing.T) { 10 | key, err := NewFixedKey("123") 11 | assert.NoError(t, err) 12 | 13 | assert.EqualValues(t, "123", key.Get()) 14 | assert.EqualValues(t, "123", key.Get()) 15 | } 16 | 17 | func TestNewRotatedKeys(t *testing.T) { 18 | key, err := NewRotatedKeys([]string{"123", "456"}) 19 | assert.NoError(t, err) 20 | 21 | assert.EqualValues(t, "123", key.Get()) 22 | assert.EqualValues(t, "456", key.Get()) 23 | 24 | assert.EqualValues(t, "123", key.Get()) 25 | assert.EqualValues(t, "456", key.Get()) 26 | } 27 | -------------------------------------------------------------------------------- /pkg/feed/opml.go: -------------------------------------------------------------------------------- 1 | package feed 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "strings" 7 | 8 | "github.com/gilliek/go-opml/opml" 9 | "github.com/pkg/errors" 10 | log "github.com/sirupsen/logrus" 11 | 12 | "github.com/mxpv/podsync/pkg/model" 13 | ) 14 | 15 | func BuildOPML(ctx context.Context, feeds map[string]*Config, db feedProvider, hostname string) (string, error) { 16 | doc := opml.OPML{Version: "1.0"} 17 | doc.Head = opml.Head{Title: "Podsync feeds"} 18 | doc.Body = opml.Body{} 19 | 20 | for _, feed := range feeds { 21 | f, err := db.GetFeed(ctx, feed.ID) 22 | if err == model.ErrNotFound { 23 | // As we update OPML on per-feed basis, some feeds may not yet be populated in database. 24 | log.Debugf("can't find configuration for feed %q, ignoring opml", feed.ID) 25 | continue 26 | } else if err != nil { 27 | return "", errors.Wrapf(err, "failed to query feed %q", feed.ID) 28 | } 29 | 30 | if !feed.OPML { 31 | continue 32 | } 33 | 34 | outline := opml.Outline{ 35 | Title: f.Title, 36 | Text: f.Description, 37 | Type: "rss", 38 | XMLURL: fmt.Sprintf("%s/%s.xml", strings.TrimRight(hostname, "/"), feed.ID), 39 | } 40 | 41 | doc.Body.Outlines = append(doc.Body.Outlines, outline) 42 | } 43 | 44 | out, err := doc.XML() 45 | if err != nil { 46 | return "", errors.Wrap(err, "failed to marshal OPML") 47 | } 48 | 49 | return out, nil 50 | } 51 | -------------------------------------------------------------------------------- /pkg/feed/opml_test.go: -------------------------------------------------------------------------------- 1 | package feed 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/golang/mock/gomock" 8 | "github.com/stretchr/testify/assert" 9 | 10 | "github.com/mxpv/podsync/pkg/model" 11 | ) 12 | 13 | func TestBuildOPML(t *testing.T) { 14 | expected := ` 15 | 16 | 17 | Podsync feeds 18 | 19 | 20 | 21 | 22 | ` 23 | 24 | ctrl := gomock.NewController(t) 25 | defer ctrl.Finish() 26 | 27 | dbMock := NewMockfeedProvider(ctrl) 28 | dbMock.EXPECT().GetFeed(gomock.Any(), "1").Return(&model.Feed{Title: "1", Description: "desc"}, nil) 29 | 30 | feeds := map[string]*Config{"any": {ID: "1", OPML: true}} 31 | out, err := BuildOPML(context.Background(), feeds, dbMock, "https://url/") 32 | assert.NoError(t, err) 33 | assert.Equal(t, expected, out) 34 | } 35 | -------------------------------------------------------------------------------- /pkg/feed/xml.go: -------------------------------------------------------------------------------- 1 | package feed 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sort" 7 | "strconv" 8 | "strings" 9 | "time" 10 | 11 | itunes "github.com/eduncan911/podcast" 12 | "github.com/pkg/errors" 13 | 14 | "github.com/mxpv/podsync/pkg/model" 15 | ) 16 | 17 | // sort.Interface implementation 18 | type timeSlice []*model.Episode 19 | 20 | func (p timeSlice) Len() int { 21 | return len(p) 22 | } 23 | 24 | // In descending order 25 | func (p timeSlice) Less(i, j int) bool { 26 | return p[i].PubDate.After(p[j].PubDate) 27 | } 28 | 29 | func (p timeSlice) Swap(i, j int) { 30 | p[i], p[j] = p[j], p[i] 31 | } 32 | 33 | func Build(_ctx context.Context, feed *model.Feed, cfg *Config, hostname string) (*itunes.Podcast, error) { 34 | const ( 35 | podsyncGenerator = "Podsync generator (support us at https://github.com/mxpv/podsync)" 36 | defaultCategory = "TV & Film" 37 | ) 38 | 39 | var ( 40 | now = time.Now().UTC() 41 | author = feed.Title 42 | title = feed.Title 43 | description = feed.Description 44 | feedLink = feed.ItemURL 45 | ) 46 | 47 | if cfg.Custom.Author != "" { 48 | author = cfg.Custom.Author 49 | } 50 | 51 | if cfg.Custom.Title != "" { 52 | title = cfg.Custom.Title 53 | } 54 | 55 | if cfg.Custom.Description != "" { 56 | description = cfg.Custom.Description 57 | } 58 | 59 | if cfg.Custom.Link != "" { 60 | feedLink = cfg.Custom.Link 61 | } 62 | 63 | p := itunes.New(title, feedLink, description, &feed.PubDate, &now) 64 | p.Generator = podsyncGenerator 65 | p.AddSubTitle(title) 66 | p.IAuthor = author 67 | p.AddSummary(description) 68 | 69 | if feed.PrivateFeed { 70 | p.IBlock = "yes" 71 | } 72 | 73 | if cfg.Custom.OwnerName != "" && cfg.Custom.OwnerEmail != "" { 74 | p.IOwner = &itunes.Author{ 75 | Name: cfg.Custom.OwnerName, 76 | Email: cfg.Custom.OwnerEmail, 77 | } 78 | } 79 | 80 | if cfg.Custom.CoverArt != "" { 81 | p.AddImage(cfg.Custom.CoverArt) 82 | } else { 83 | p.AddImage(feed.CoverArt) 84 | } 85 | 86 | if cfg.Custom.Category != "" { 87 | p.AddCategory(cfg.Custom.Category, cfg.Custom.Subcategories) 88 | } else { 89 | p.AddCategory(defaultCategory, cfg.Custom.Subcategories) 90 | } 91 | 92 | if cfg.Custom.Explicit { 93 | p.IExplicit = "yes" 94 | } else { 95 | p.IExplicit = "no" 96 | } 97 | 98 | if cfg.Custom.Language != "" { 99 | p.Language = cfg.Custom.Language 100 | } 101 | 102 | for _, episode := range feed.Episodes { 103 | if episode.PubDate.IsZero() { 104 | episode.PubDate = now 105 | } 106 | } 107 | 108 | // Sort all episodes in descending order 109 | sort.Sort(timeSlice(feed.Episodes)) 110 | 111 | for i, episode := range feed.Episodes { 112 | if episode.Status != model.EpisodeDownloaded { 113 | // Skip episodes that are not yet downloaded or have been removed 114 | continue 115 | } 116 | 117 | item := itunes.Item{ 118 | GUID: episode.ID, 119 | Link: episode.VideoURL, 120 | Title: episode.Title, 121 | Description: episode.Description, 122 | ISubtitle: episode.Title, 123 | // Some app prefer 1-based order 124 | IOrder: strconv.Itoa(i + 1), 125 | } 126 | 127 | item.AddPubDate(&episode.PubDate) 128 | item.AddSummary(episode.Description) 129 | item.AddImage(episode.Thumbnail) 130 | item.AddDuration(episode.Duration) 131 | 132 | enclosureType := itunes.MP4 133 | if feed.Format == model.FormatAudio { 134 | enclosureType = itunes.MP3 135 | } 136 | if feed.Format == model.FormatCustom { 137 | enclosureType = EnclosureFromExtension(cfg) 138 | } 139 | 140 | var ( 141 | episodeName = EpisodeName(cfg, episode) 142 | downloadURL = fmt.Sprintf("%s/%s/%s", strings.TrimRight(hostname, "/"), cfg.ID, episodeName) 143 | ) 144 | 145 | item.AddEnclosure(downloadURL, enclosureType, episode.Size) 146 | 147 | // p.AddItem requires description to be not empty, use workaround 148 | if item.Description == "" { 149 | item.Description = " " 150 | } 151 | 152 | if cfg.Custom.Explicit { 153 | item.IExplicit = "yes" 154 | } else { 155 | item.IExplicit = "no" 156 | } 157 | 158 | if _, err := p.AddItem(item); err != nil { 159 | return nil, errors.Wrapf(err, "failed to add item to podcast (id %q)", episode.ID) 160 | } 161 | } 162 | 163 | return &p, nil 164 | } 165 | 166 | func EpisodeName(feedConfig *Config, episode *model.Episode) string { 167 | ext := "mp4" 168 | if feedConfig.Format == model.FormatAudio { 169 | ext = "mp3" 170 | } 171 | if feedConfig.Format == model.FormatCustom { 172 | ext = feedConfig.CustomFormat.Extension 173 | } 174 | 175 | return fmt.Sprintf("%s.%s", episode.ID, ext) 176 | } 177 | 178 | func EnclosureFromExtension(feedConfig *Config) itunes.EnclosureType { 179 | ext := feedConfig.CustomFormat.Extension 180 | 181 | switch { 182 | case ext == "m4a": 183 | return itunes.M4A 184 | case ext == "m4v": 185 | return itunes.M4V 186 | case ext == "mp4": 187 | return itunes.MP4 188 | case ext == "mp3": 189 | return itunes.MP3 190 | case ext == "mov": 191 | return itunes.MOV 192 | case ext == "pdf": 193 | return itunes.PDF 194 | case ext == "epub": 195 | return itunes.EPUB 196 | } 197 | return -1 198 | } 199 | -------------------------------------------------------------------------------- /pkg/feed/xml_test.go: -------------------------------------------------------------------------------- 1 | package feed 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | itunes "github.com/eduncan911/podcast" 8 | "github.com/mxpv/podsync/pkg/model" 9 | "github.com/stretchr/testify/assert" 10 | "github.com/stretchr/testify/require" 11 | ) 12 | 13 | func TestBuildXML(t *testing.T) { 14 | feed := model.Feed{ 15 | Episodes: []*model.Episode{ 16 | { 17 | ID: "1", 18 | Status: model.EpisodeDownloaded, 19 | Title: "title", 20 | Description: "description", 21 | }, 22 | }, 23 | } 24 | 25 | cfg := Config{ 26 | ID: "test", 27 | Custom: Custom{Description: "description", Category: "Technology", Subcategories: []string{"Gadgets", "Podcasting"}}, 28 | } 29 | 30 | out, err := Build(context.Background(), &feed, &cfg, "http://localhost/") 31 | assert.NoError(t, err) 32 | 33 | assert.EqualValues(t, "description", out.Description) 34 | assert.EqualValues(t, "Technology", out.Category) 35 | 36 | require.Len(t, out.ICategories, 1) 37 | category := out.ICategories[0] 38 | assert.EqualValues(t, "Technology", category.Text) 39 | 40 | require.Len(t, category.ICategories, 2) 41 | assert.EqualValues(t, "Gadgets", category.ICategories[0].Text) 42 | assert.EqualValues(t, "Podcasting", category.ICategories[1].Text) 43 | 44 | require.Len(t, out.Items, 1) 45 | require.NotNil(t, out.Items[0].Enclosure) 46 | assert.EqualValues(t, out.Items[0].Enclosure.URL, "http://localhost/test/1.mp4") 47 | assert.EqualValues(t, out.Items[0].Enclosure.Type, itunes.MP4) 48 | } 49 | -------------------------------------------------------------------------------- /pkg/fs/local.go: -------------------------------------------------------------------------------- 1 | package fs 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "net/http" 7 | "os" 8 | "path/filepath" 9 | 10 | "github.com/pkg/errors" 11 | log "github.com/sirupsen/logrus" 12 | ) 13 | 14 | // LocalConfig is the storage configuration for local file system 15 | type LocalConfig struct { 16 | DataDir string `toml:"data_dir"` 17 | } 18 | 19 | // Local implements local file storage 20 | type Local struct { 21 | rootDir string 22 | } 23 | 24 | func NewLocal(rootDir string) (*Local, error) { 25 | return &Local{rootDir: rootDir}, nil 26 | } 27 | 28 | func (l *Local) Open(name string) (http.File, error) { 29 | path := filepath.Join(l.rootDir, name) 30 | return os.Open(path) 31 | } 32 | 33 | func (l *Local) Delete(_ctx context.Context, name string) error { 34 | path := filepath.Join(l.rootDir, name) 35 | return os.Remove(path) 36 | } 37 | 38 | func (l *Local) Create(_ctx context.Context, name string, reader io.Reader) (int64, error) { 39 | var ( 40 | logger = log.WithField("name", name) 41 | path = filepath.Join(l.rootDir, name) 42 | ) 43 | 44 | if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { 45 | return 0, errors.Wrapf(err, "failed to mkdir: %s", path) 46 | } 47 | 48 | logger.Infof("creating file: %s", path) 49 | written, err := l.copyFile(reader, path) 50 | if err != nil { 51 | return 0, errors.Wrap(err, "failed to copy file") 52 | } 53 | 54 | logger.Debugf("written %d bytes", written) 55 | return written, nil 56 | } 57 | 58 | func (l *Local) copyFile(source io.Reader, destinationPath string) (int64, error) { 59 | dest, err := os.Create(destinationPath) 60 | if err != nil { 61 | return 0, errors.Wrap(err, "failed to create destination file") 62 | } 63 | 64 | defer dest.Close() 65 | 66 | written, err := io.Copy(dest, source) 67 | if err != nil { 68 | return 0, errors.Wrap(err, "failed to copy data") 69 | } 70 | 71 | return written, nil 72 | } 73 | 74 | func (l *Local) Size(_ctx context.Context, name string) (int64, error) { 75 | file, err := l.Open(name) 76 | if err != nil { 77 | return 0, err 78 | } 79 | defer file.Close() 80 | 81 | stat, err := file.Stat() 82 | if err != nil { 83 | return 0, err 84 | } 85 | 86 | return stat.Size(), nil 87 | } 88 | -------------------------------------------------------------------------------- /pkg/fs/local_test.go: -------------------------------------------------------------------------------- 1 | package fs 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "os" 7 | "path/filepath" 8 | "testing" 9 | 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | var ( 14 | testCtx = context.Background() 15 | ) 16 | 17 | func TestNewLocal(t *testing.T) { 18 | local, err := NewLocal("") 19 | assert.NoError(t, err) 20 | assert.NotNil(t, local) 21 | } 22 | 23 | func TestLocal_Create(t *testing.T) { 24 | tmpDir, err := os.MkdirTemp("", "") 25 | assert.NoError(t, err) 26 | 27 | stor, err := NewLocal(tmpDir) 28 | assert.NoError(t, err) 29 | 30 | written, err := stor.Create(testCtx, "1/test", bytes.NewBuffer([]byte{1, 5, 7, 8, 3})) 31 | assert.NoError(t, err) 32 | assert.EqualValues(t, 5, written) 33 | 34 | stat, err := os.Stat(filepath.Join(tmpDir, "1", "test")) 35 | assert.NoError(t, err) 36 | assert.EqualValues(t, 5, stat.Size()) 37 | } 38 | 39 | func TestLocal_Size(t *testing.T) { 40 | tmpDir, err := os.MkdirTemp("", "") 41 | assert.NoError(t, err) 42 | 43 | defer os.RemoveAll(tmpDir) 44 | 45 | stor, err := NewLocal(tmpDir) 46 | assert.NoError(t, err) 47 | 48 | _, err = stor.Create(testCtx, "1/test", bytes.NewBuffer([]byte{1, 5, 7, 8, 3})) 49 | assert.NoError(t, err) 50 | 51 | sz, err := stor.Size(testCtx, "1/test") 52 | assert.NoError(t, err) 53 | assert.EqualValues(t, 5, sz) 54 | } 55 | 56 | func TestLocal_NoSize(t *testing.T) { 57 | stor, err := NewLocal("") 58 | assert.NoError(t, err) 59 | 60 | _, err = stor.Size(testCtx, "1/test") 61 | assert.True(t, os.IsNotExist(err)) 62 | } 63 | 64 | func TestLocal_Delete(t *testing.T) { 65 | tmpDir, err := os.MkdirTemp("", "") 66 | assert.NoError(t, err) 67 | 68 | stor, err := NewLocal(tmpDir) 69 | assert.NoError(t, err) 70 | 71 | _, err = stor.Create(testCtx, "1/test", bytes.NewBuffer([]byte{1, 5, 7, 8, 3})) 72 | assert.NoError(t, err) 73 | 74 | err = stor.Delete(testCtx, "1/test") 75 | assert.NoError(t, err) 76 | 77 | _, err = stor.Size(testCtx, "1/test") 78 | assert.True(t, os.IsNotExist(err)) 79 | 80 | _, err = os.Stat(filepath.Join(tmpDir, "1", "test")) 81 | assert.True(t, os.IsNotExist(err)) 82 | } 83 | 84 | func TestLocal_copyFile(t *testing.T) { 85 | reader := bytes.NewReader([]byte{1, 2, 4}) 86 | tmpDir, err := os.MkdirTemp("", "") 87 | assert.NoError(t, err) 88 | 89 | file := filepath.Join(tmpDir, "1") 90 | 91 | l := &Local{} 92 | size, err := l.copyFile(reader, file) 93 | assert.NoError(t, err) 94 | assert.EqualValues(t, 3, size) 95 | 96 | stat, err := os.Stat(file) 97 | assert.NoError(t, err) 98 | assert.EqualValues(t, 3, stat.Size()) 99 | } 100 | -------------------------------------------------------------------------------- /pkg/fs/s3.go: -------------------------------------------------------------------------------- 1 | package fs 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "net/http" 7 | "os" 8 | "path" 9 | 10 | "github.com/aws/aws-sdk-go/aws" 11 | "github.com/aws/aws-sdk-go/aws/awserr" 12 | "github.com/aws/aws-sdk-go/aws/session" 13 | "github.com/aws/aws-sdk-go/service/s3" 14 | "github.com/aws/aws-sdk-go/service/s3/s3iface" 15 | "github.com/aws/aws-sdk-go/service/s3/s3manager" 16 | "github.com/pkg/errors" 17 | log "github.com/sirupsen/logrus" 18 | ) 19 | 20 | // S3Config is the configuration for a S3-compatible storage provider 21 | type S3Config struct { 22 | // S3 Bucket to store files 23 | Bucket string `toml:"bucket"` 24 | // Region of the S3 service 25 | Region string `toml:"region"` 26 | // EndpointURL is an HTTP endpoint of the S3 API 27 | EndpointURL string `toml:"endpoint_url"` 28 | // Prefix is a prefix (subfolder) to use to build key names 29 | Prefix string `toml:"prefix"` 30 | } 31 | 32 | // S3 implements file storage for S3-compatible providers. 33 | type S3 struct { 34 | api s3iface.S3API 35 | uploader *s3manager.Uploader 36 | bucket string 37 | prefix string 38 | } 39 | 40 | func NewS3(c S3Config) (*S3, error) { 41 | cfg := aws.NewConfig(). 42 | WithEndpoint(c.EndpointURL). 43 | WithRegion(c.Region). 44 | WithLogger(s3logger{}). 45 | WithLogLevel(aws.LogDebug) 46 | sess, err := session.NewSessionWithOptions(session.Options{Config: *cfg}) 47 | if err != nil { 48 | return nil, errors.Wrap(err, "failed to initialize S3 session") 49 | } 50 | return &S3{ 51 | api: s3.New(sess), 52 | uploader: s3manager.NewUploader(sess), 53 | bucket: c.Bucket, 54 | prefix: c.Prefix, 55 | }, nil 56 | } 57 | 58 | func (s *S3) Open(_name string) (http.File, error) { 59 | return nil, errors.New("serving files from S3 is not supported") 60 | } 61 | 62 | func (s *S3) Delete(ctx context.Context, name string) error { 63 | key := s.buildKey(name) 64 | _, err := s.api.DeleteObjectWithContext(ctx, &s3.DeleteObjectInput{ 65 | Bucket: &s.bucket, 66 | Key: &key, 67 | }) 68 | return err 69 | } 70 | 71 | func (s *S3) Create(ctx context.Context, name string, reader io.Reader) (int64, error) { 72 | key := s.buildKey(name) 73 | logger := log.WithField("key", key) 74 | 75 | logger.Infof("uploading file to %s", s.bucket) 76 | r := &readerWithN{Reader: reader} 77 | _, err := s.uploader.UploadWithContext(ctx, &s3manager.UploadInput{ 78 | Bucket: &s.bucket, 79 | Key: &key, 80 | Body: r, 81 | }) 82 | if err != nil { 83 | return 0, errors.Wrap(err, "failed to upload file") 84 | } 85 | 86 | logger.Debugf("written %d bytes", r.n) 87 | return int64(r.n), nil 88 | } 89 | 90 | func (s *S3) Size(ctx context.Context, name string) (int64, error) { 91 | key := s.buildKey(name) 92 | logger := log.WithField("key", key) 93 | 94 | logger.Debugf("getting file size from %s", s.bucket) 95 | resp, err := s.api.HeadObjectWithContext(ctx, &s3.HeadObjectInput{ 96 | Bucket: &s.bucket, 97 | Key: &key, 98 | }) 99 | if err != nil { 100 | if awsErr, ok := err.(awserr.Error); ok { 101 | if awsErr.Code() == "NotFound" { 102 | return 0, os.ErrNotExist 103 | } 104 | } 105 | return 0, errors.Wrap(err, "failed to get file size") 106 | } 107 | 108 | return *resp.ContentLength, nil 109 | } 110 | 111 | func (s *S3) buildKey(name string) string { 112 | return path.Join(s.prefix, name) 113 | } 114 | 115 | type readerWithN struct { 116 | io.Reader 117 | n int 118 | } 119 | 120 | func (r *readerWithN) Read(p []byte) (n int, err error) { 121 | n, err = r.Reader.Read(p) 122 | r.n += n 123 | return 124 | } 125 | 126 | type s3logger struct{} 127 | 128 | func (s s3logger) Log(args ...interface{}) { 129 | log.Debug(args...) 130 | } 131 | -------------------------------------------------------------------------------- /pkg/fs/s3_test.go: -------------------------------------------------------------------------------- 1 | package fs 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "os" 7 | "testing" 8 | 9 | "github.com/aws/aws-sdk-go/aws" 10 | "github.com/aws/aws-sdk-go/aws/awserr" 11 | "github.com/aws/aws-sdk-go/aws/client/metadata" 12 | "github.com/aws/aws-sdk-go/aws/request" 13 | "github.com/aws/aws-sdk-go/service/s3" 14 | "github.com/aws/aws-sdk-go/service/s3/s3iface" 15 | "github.com/aws/aws-sdk-go/service/s3/s3manager" 16 | "github.com/stretchr/testify/assert" 17 | ) 18 | 19 | func TestS3_Create(t *testing.T) { 20 | files := make(map[string][]byte) 21 | stor, err := newMockS3(files, "") 22 | assert.NoError(t, err) 23 | 24 | written, err := stor.Create(testCtx, "1/test", bytes.NewBuffer([]byte{1, 5, 7, 8, 3})) 25 | assert.NoError(t, err) 26 | assert.EqualValues(t, 5, written) 27 | 28 | d, ok := files["1/test"] 29 | assert.True(t, ok) 30 | assert.EqualValues(t, 5, len(d)) 31 | } 32 | 33 | func TestS3_Size(t *testing.T) { 34 | files := make(map[string][]byte) 35 | stor, err := newMockS3(files, "") 36 | assert.NoError(t, err) 37 | 38 | _, err = stor.Create(testCtx, "1/test", bytes.NewBuffer([]byte{1, 5, 7, 8, 3})) 39 | assert.NoError(t, err) 40 | 41 | sz, err := stor.Size(testCtx, "1/test") 42 | assert.NoError(t, err) 43 | assert.EqualValues(t, 5, sz) 44 | } 45 | 46 | func TestS3_NoSize(t *testing.T) { 47 | files := make(map[string][]byte) 48 | stor, err := newMockS3(files, "") 49 | assert.NoError(t, err) 50 | 51 | _, err = stor.Size(testCtx, "1/test") 52 | assert.True(t, os.IsNotExist(err)) 53 | } 54 | 55 | func TestS3_Delete(t *testing.T) { 56 | files := make(map[string][]byte) 57 | stor, err := newMockS3(files, "") 58 | assert.NoError(t, err) 59 | 60 | _, err = stor.Create(testCtx, "1/test", bytes.NewBuffer([]byte{1, 5, 7, 8, 3})) 61 | assert.NoError(t, err) 62 | 63 | err = stor.Delete(testCtx, "1/test") 64 | assert.NoError(t, err) 65 | 66 | _, err = stor.Size(testCtx, "1/test") 67 | assert.True(t, os.IsNotExist(err)) 68 | 69 | _, ok := files["1/test"] 70 | assert.False(t, ok) 71 | } 72 | 73 | func TestS3_BuildKey(t *testing.T) { 74 | files := make(map[string][]byte) 75 | 76 | stor, _ := newMockS3(files, "") 77 | key := stor.buildKey("test-fn") 78 | assert.EqualValues(t, "test-fn", key) 79 | 80 | stor, _ = newMockS3(files, "mock-prefix") 81 | key = stor.buildKey("test-fn") 82 | assert.EqualValues(t, "mock-prefix/test-fn", key) 83 | } 84 | 85 | type mockS3API struct { 86 | s3iface.S3API 87 | files map[string][]byte 88 | } 89 | 90 | func newMockS3(files map[string][]byte, prefix string) (*S3, error) { 91 | api := &mockS3API{files: files} 92 | return &S3{ 93 | api: api, 94 | uploader: s3manager.NewUploaderWithClient(api), 95 | bucket: "mock-bucket", 96 | prefix: prefix, 97 | }, nil 98 | } 99 | 100 | func (m *mockS3API) PutObjectRequest(input *s3.PutObjectInput) (*request.Request, *s3.PutObjectOutput) { 101 | content, _ := io.ReadAll(input.Body) 102 | req := request.New(aws.Config{}, metadata.ClientInfo{}, request.Handlers{}, nil, &request.Operation{}, nil, nil) 103 | m.files[*input.Key] = content 104 | return req, &s3.PutObjectOutput{} 105 | } 106 | 107 | func (m *mockS3API) HeadObjectWithContext(ctx aws.Context, input *s3.HeadObjectInput, opts ...request.Option) (*s3.HeadObjectOutput, error) { 108 | if _, ok := m.files[*input.Key]; ok { 109 | return &s3.HeadObjectOutput{ContentLength: aws.Int64(int64(len(m.files[*input.Key])))}, nil 110 | } 111 | return nil, awserr.New("NotFound", "", nil) 112 | } 113 | 114 | func (m *mockS3API) DeleteObjectWithContext(ctx aws.Context, input *s3.DeleteObjectInput, opts ...request.Option) (*s3.DeleteObjectOutput, error) { 115 | if _, ok := m.files[*input.Key]; ok { 116 | delete(m.files, *input.Key) 117 | return &s3.DeleteObjectOutput{}, nil 118 | } 119 | return nil, awserr.New("NotFound", "", nil) 120 | } 121 | -------------------------------------------------------------------------------- /pkg/fs/storage.go: -------------------------------------------------------------------------------- 1 | package fs 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "net/http" 7 | ) 8 | 9 | // Storage is a file system interface to host downloaded episodes and feeds. 10 | type Storage interface { 11 | // FileSystem must be implemented to in order to pass Storage interface to HTTP file server. 12 | http.FileSystem 13 | 14 | // Create will create a new file from reader 15 | Create(ctx context.Context, name string, reader io.Reader) (int64, error) 16 | 17 | // Delete deletes the file 18 | Delete(ctx context.Context, name string) error 19 | 20 | // Size returns a storage object's size in bytes 21 | Size(ctx context.Context, name string) (int64, error) 22 | } 23 | 24 | // Config is a configuration for the file storage backend 25 | type Config struct { 26 | // Type is the type of file system to use 27 | Type string `toml:"type"` 28 | Local LocalConfig `toml:"local"` 29 | S3 S3Config `toml:"s3"` 30 | } 31 | -------------------------------------------------------------------------------- /pkg/model/defaults.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | const ( 8 | DefaultFormat = FormatVideo 9 | DefaultQuality = QualityHigh 10 | DefaultPageSize = 50 11 | DefaultUpdatePeriod = 6 * time.Hour 12 | DefaultLogMaxSize = 50 // megabytes 13 | DefaultLogMaxAge = 30 // days 14 | DefaultLogMaxBackups = 7 15 | PathRegex = `^[A-Za-z0-9]+$` 16 | ) 17 | -------------------------------------------------------------------------------- /pkg/model/errors.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "errors" 5 | ) 6 | 7 | var ( 8 | ErrAlreadyExists = errors.New("object already exists") 9 | ErrNotFound = errors.New("not found") 10 | ErrQuotaExceeded = errors.New("query limit is exceeded") 11 | ) 12 | -------------------------------------------------------------------------------- /pkg/model/feed.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | // Quality to use when downloading episodes 8 | type Quality string 9 | 10 | const ( 11 | QualityHigh = Quality("high") 12 | QualityLow = Quality("low") 13 | ) 14 | 15 | // Format to convert episode when downloading episodes 16 | type Format string 17 | 18 | const ( 19 | FormatAudio = Format("audio") 20 | FormatVideo = Format("video") 21 | FormatCustom = Format("custom") 22 | ) 23 | 24 | // Playlist sorting style 25 | type Sorting string 26 | 27 | const ( 28 | SortingDesc = Sorting("desc") 29 | SortingAsc = Sorting("asc") 30 | ) 31 | 32 | type Episode struct { 33 | // ID of episode 34 | ID string `json:"id"` 35 | Title string `json:"title"` 36 | Description string `json:"description"` 37 | Thumbnail string `json:"thumbnail"` 38 | Duration int64 `json:"duration"` 39 | VideoURL string `json:"video_url"` 40 | PubDate time.Time `json:"pub_date"` 41 | Size int64 `json:"size"` 42 | Order string `json:"order"` 43 | Status EpisodeStatus `json:"status"` // Disk status 44 | } 45 | 46 | type Feed struct { 47 | ID string `json:"feed_id"` 48 | ItemID string `json:"item_id"` 49 | LinkType Type `json:"link_type"` // Either group, channel or user 50 | Provider Provider `json:"provider"` // Youtube or Vimeo 51 | CreatedAt time.Time `json:"created_at"` 52 | LastAccess time.Time `json:"last_access"` 53 | ExpirationTime time.Time `json:"expiration_time"` 54 | Format Format `json:"format"` 55 | Quality Quality `json:"quality"` 56 | CoverArtQuality Quality `json:"cover_art_quality"` 57 | PageSize int `json:"page_size"` 58 | CoverArt string `json:"cover_art"` 59 | Title string `json:"title"` 60 | Description string `json:"description"` 61 | PubDate time.Time `json:"pub_date"` 62 | Author string `json:"author"` 63 | ItemURL string `json:"item_url"` // Platform specific URL 64 | Episodes []*Episode `json:"-"` // Array of episodes 65 | UpdatedAt time.Time `json:"updated_at"` 66 | PlaylistSort Sorting `json:"playlist_sort"` 67 | PrivateFeed bool `json:"private_feed"` 68 | } 69 | 70 | type EpisodeStatus string 71 | 72 | const ( 73 | EpisodeNew = EpisodeStatus("new") // New episode received via API 74 | EpisodeDownloaded = EpisodeStatus("downloaded") // Downloaded, encoded and available for download 75 | EpisodeError = EpisodeStatus("error") // Could not download, will retry 76 | EpisodeCleaned = EpisodeStatus("cleaned") // Downloaded and later removed from disk due to update strategy 77 | ) 78 | -------------------------------------------------------------------------------- /pkg/model/link.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | type Type string 4 | 5 | const ( 6 | TypeChannel = Type("channel") 7 | TypePlaylist = Type("playlist") 8 | TypeUser = Type("user") 9 | TypeGroup = Type("group") 10 | ) 11 | 12 | type Provider string 13 | 14 | const ( 15 | ProviderYoutube = Provider("youtube") 16 | ProviderVimeo = Provider("vimeo") 17 | ProviderSoundcloud = Provider("soundcloud") 18 | ) 19 | 20 | // Info represents data extracted from URL 21 | type Info struct { 22 | LinkType Type // Either group, channel or user 23 | Provider Provider // Youtube, Vimeo, or SoundCloud 24 | ItemID string 25 | } 26 | -------------------------------------------------------------------------------- /pkg/ytdl/temp_file.go: -------------------------------------------------------------------------------- 1 | package ytdl 2 | 3 | import ( 4 | "os" 5 | 6 | log "github.com/sirupsen/logrus" 7 | ) 8 | 9 | type tempFile struct { 10 | *os.File 11 | dir string 12 | } 13 | 14 | func (f *tempFile) Close() error { 15 | err := f.File.Close() 16 | err1 := os.RemoveAll(f.dir) 17 | if err1 != nil { 18 | log.Errorf("could not remove temp dir: %v", err1) 19 | } 20 | return err 21 | } 22 | -------------------------------------------------------------------------------- /pkg/ytdl/ytdl.go: -------------------------------------------------------------------------------- 1 | package ytdl 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "net/http" 8 | "os" 9 | "os/exec" 10 | "path/filepath" 11 | "strings" 12 | "sync" 13 | "time" 14 | 15 | "github.com/mxpv/podsync/pkg/feed" 16 | "github.com/pkg/errors" 17 | log "github.com/sirupsen/logrus" 18 | 19 | "github.com/mxpv/podsync/pkg/model" 20 | ) 21 | 22 | const ( 23 | DefaultDownloadTimeout = 10 * time.Minute 24 | UpdatePeriod = 24 * time.Hour 25 | ) 26 | 27 | var ( 28 | ErrTooManyRequests = errors.New(http.StatusText(http.StatusTooManyRequests)) 29 | ) 30 | 31 | // Config is a youtube-dl related configuration 32 | type Config struct { 33 | // SelfUpdate toggles self update every 24 hour 34 | SelfUpdate bool `toml:"self_update"` 35 | // Timeout in minutes for youtube-dl process to finish download 36 | Timeout int `toml:"timeout"` 37 | // CustomBinary is a custom path to youtube-dl, this allows using various youtube-dl forks. 38 | CustomBinary string `toml:"custom_binary"` 39 | } 40 | 41 | type YoutubeDl struct { 42 | path string 43 | timeout time.Duration 44 | updateLock sync.Mutex // Don't call youtube-dl while self updating 45 | } 46 | 47 | func New(ctx context.Context, cfg Config) (*YoutubeDl, error) { 48 | var ( 49 | path string 50 | err error 51 | ) 52 | 53 | if cfg.CustomBinary != "" { 54 | path = cfg.CustomBinary 55 | 56 | // Don't update custom youtube-dl binaries. 57 | log.Warnf("using custom youtube-dl binary, turning self updates off") 58 | cfg.SelfUpdate = false 59 | } else { 60 | path, err = exec.LookPath("youtube-dl") 61 | if err != nil { 62 | return nil, errors.Wrap(err, "youtube-dl binary not found") 63 | } 64 | 65 | log.Debugf("found youtube-dl binary at %q", path) 66 | } 67 | 68 | timeout := DefaultDownloadTimeout 69 | if cfg.Timeout > 0 { 70 | timeout = time.Duration(cfg.Timeout) * time.Minute 71 | } 72 | 73 | log.Debugf("download timeout: %d min(s)", int(timeout.Minutes())) 74 | 75 | ytdl := &YoutubeDl{ 76 | path: path, 77 | timeout: timeout, 78 | } 79 | 80 | // Make sure youtube-dl exists 81 | version, err := ytdl.exec(ctx, "--version") 82 | if err != nil { 83 | return nil, errors.Wrap(err, "could not find youtube-dl") 84 | } 85 | 86 | log.Infof("using youtube-dl %s", version) 87 | 88 | if err := ytdl.ensureDependencies(ctx); err != nil { 89 | return nil, err 90 | } 91 | 92 | if cfg.SelfUpdate { 93 | // Do initial blocking update at launch 94 | if err := ytdl.Update(ctx); err != nil { 95 | log.WithError(err).Error("failed to update youtube-dl") 96 | } 97 | 98 | go func() { 99 | for { 100 | time.Sleep(UpdatePeriod) 101 | 102 | if err := ytdl.Update(context.Background()); err != nil { 103 | log.WithError(err).Error("update failed") 104 | } 105 | } 106 | }() 107 | } 108 | 109 | return ytdl, nil 110 | } 111 | 112 | func (dl *YoutubeDl) ensureDependencies(ctx context.Context) error { 113 | found := false 114 | 115 | if path, err := exec.LookPath("ffmpeg"); err == nil { 116 | found = true 117 | 118 | output, err := exec.CommandContext(ctx, path, "-version").CombinedOutput() 119 | if err != nil { 120 | return errors.Wrap(err, "could not get ffmpeg version") 121 | } 122 | 123 | log.Infof("found ffmpeg: %s", output) 124 | } 125 | 126 | if path, err := exec.LookPath("avconv"); err == nil { 127 | found = true 128 | 129 | output, err := exec.CommandContext(ctx, path, "-version").CombinedOutput() 130 | if err != nil { 131 | return errors.Wrap(err, "could not get avconv version") 132 | } 133 | 134 | log.Infof("found avconv: %s", output) 135 | } 136 | 137 | if !found { 138 | return errors.New("either ffmpeg or avconv required to run Podsync") 139 | } 140 | 141 | return nil 142 | } 143 | 144 | func (dl *YoutubeDl) Update(ctx context.Context) error { 145 | dl.updateLock.Lock() 146 | defer dl.updateLock.Unlock() 147 | 148 | log.Info("updating youtube-dl") 149 | output, err := dl.exec(ctx, "--update", "--verbose") 150 | if err != nil { 151 | log.WithError(err).Error(output) 152 | return errors.Wrap(err, "failed to self update youtube-dl") 153 | } 154 | 155 | log.Info(output) 156 | return nil 157 | } 158 | 159 | func (dl *YoutubeDl) Download(ctx context.Context, feedConfig *feed.Config, episode *model.Episode) (r io.ReadCloser, err error) { 160 | tmpDir, err := os.MkdirTemp("", "podsync-") 161 | if err != nil { 162 | return nil, errors.Wrap(err, "failed to get temp dir for download") 163 | } 164 | 165 | defer func() { 166 | if err != nil { 167 | err1 := os.RemoveAll(tmpDir) 168 | if err1 != nil { 169 | log.Errorf("could not remove temp dir: %v", err1) 170 | } 171 | } 172 | }() 173 | 174 | // filePath with YoutubeDl template format 175 | filePath := filepath.Join(tmpDir, fmt.Sprintf("%s.%s", episode.ID, "%(ext)s")) 176 | 177 | args := buildArgs(feedConfig, episode, filePath) 178 | 179 | dl.updateLock.Lock() 180 | defer dl.updateLock.Unlock() 181 | 182 | output, err := dl.exec(ctx, args...) 183 | if err != nil { 184 | log.WithError(err).Errorf("youtube-dl error: %s", filePath) 185 | 186 | // YouTube might block host with HTTP Error 429: Too Many Requests 187 | if strings.Contains(output, "HTTP Error 429") { 188 | return nil, ErrTooManyRequests 189 | } 190 | 191 | log.Error(output) 192 | 193 | return nil, errors.New(output) 194 | } 195 | 196 | ext := "mp4" 197 | if feedConfig.Format == model.FormatAudio { 198 | ext = "mp3" 199 | } 200 | if feedConfig.Format == model.FormatCustom { 201 | ext = feedConfig.CustomFormat.Extension 202 | } 203 | 204 | // filePath now with the final extension 205 | filePath = filepath.Join(tmpDir, fmt.Sprintf("%s.%s", episode.ID, ext)) 206 | f, err := os.Open(filePath) 207 | if err != nil { 208 | return nil, errors.Wrap(err, "failed to open downloaded file") 209 | } 210 | 211 | return &tempFile{File: f, dir: tmpDir}, nil 212 | } 213 | 214 | func (dl *YoutubeDl) exec(ctx context.Context, args ...string) (string, error) { 215 | ctx, cancel := context.WithTimeout(ctx, dl.timeout) 216 | defer cancel() 217 | 218 | cmd := exec.CommandContext(ctx, dl.path, args...) 219 | output, err := cmd.CombinedOutput() 220 | if err != nil { 221 | return string(output), errors.Wrap(err, "failed to execute youtube-dl") 222 | } 223 | 224 | return string(output), nil 225 | } 226 | 227 | func buildArgs(feedConfig *feed.Config, episode *model.Episode, outputFilePath string) []string { 228 | var args []string 229 | 230 | if feedConfig.Format == model.FormatVideo { 231 | // Video, mp4, high by default 232 | 233 | format := "bestvideo[ext=mp4][vcodec^=avc1]+bestaudio[ext=m4a]/best[ext=mp4][vcodec^=avc1]/best[ext=mp4]/best" 234 | 235 | if feedConfig.Quality == model.QualityLow { 236 | format = "worstvideo[ext=mp4][vcodec^=avc1]+worstaudio[ext=m4a]/worst[ext=mp4][vcodec^=avc1]/worst[ext=mp4]/worst" 237 | } else if feedConfig.Quality == model.QualityHigh && feedConfig.MaxHeight > 0 { 238 | format = fmt.Sprintf("bestvideo[height<=%d][ext=mp4][vcodec^=avc1]+bestaudio[ext=m4a]/best[height<=%d][ext=mp4][vcodec^=avc1]/best[ext=mp4]/best", feedConfig.MaxHeight, feedConfig.MaxHeight) 239 | } 240 | 241 | args = append(args, "--format", format) 242 | } else if feedConfig.Format == model.FormatAudio { 243 | // Audio, mp3, high by default 244 | format := "bestaudio" 245 | if feedConfig.Quality == model.QualityLow { 246 | format = "worstaudio" 247 | } 248 | 249 | args = append(args, "--extract-audio", "--audio-format", "mp3", "--format", format) 250 | } else { 251 | args = append(args, "--audio-format", feedConfig.CustomFormat.Extension, "--format", feedConfig.CustomFormat.YouTubeDLFormat) 252 | } 253 | 254 | // Insert additional per-feed youtube-dl arguments 255 | args = append(args, feedConfig.YouTubeDLArgs...) 256 | 257 | args = append(args, "--output", outputFilePath, episode.VideoURL) 258 | return args 259 | } 260 | -------------------------------------------------------------------------------- /pkg/ytdl/ytdl_test.go: -------------------------------------------------------------------------------- 1 | package ytdl 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/mxpv/podsync/pkg/feed" 7 | "github.com/mxpv/podsync/pkg/model" 8 | 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestBuildArgs(t *testing.T) { 13 | tests := []struct { 14 | name string 15 | format model.Format 16 | customFormat feed.CustomFormat 17 | quality model.Quality 18 | maxHeight int 19 | output string 20 | videoURL string 21 | ytdlArgs []string 22 | expect []string 23 | }{ 24 | { 25 | name: "Audio unknown quality", 26 | format: model.FormatAudio, 27 | output: "/tmp/1", 28 | videoURL: "http://url", 29 | expect: []string{"--extract-audio", "--audio-format", "mp3", "--format", "bestaudio", "--output", "/tmp/1", "http://url"}, 30 | }, 31 | { 32 | name: "Audio low quality", 33 | format: model.FormatAudio, 34 | quality: model.QualityLow, 35 | output: "/tmp/1", 36 | videoURL: "http://url", 37 | expect: []string{"--extract-audio", "--audio-format", "mp3", "--format", "worstaudio", "--output", "/tmp/1", "http://url"}, 38 | }, 39 | { 40 | name: "Audio best quality", 41 | format: model.FormatAudio, 42 | quality: model.QualityHigh, 43 | output: "/tmp/1", 44 | videoURL: "http://url", 45 | expect: []string{"--extract-audio", "--audio-format", "mp3", "--format", "bestaudio", "--output", "/tmp/1", "http://url"}, 46 | }, 47 | { 48 | name: "Video unknown quality", 49 | format: model.FormatVideo, 50 | output: "/tmp/1", 51 | videoURL: "http://url", 52 | expect: []string{"--format", "bestvideo[ext=mp4][vcodec^=avc1]+bestaudio[ext=m4a]/best[ext=mp4][vcodec^=avc1]/best[ext=mp4]/best", "--output", "/tmp/1", "http://url"}, 53 | }, 54 | { 55 | name: "Video unknown quality with maxheight", 56 | format: model.FormatVideo, 57 | maxHeight: 720, 58 | output: "/tmp/1", 59 | videoURL: "http://url", 60 | expect: []string{"--format", "bestvideo[ext=mp4][vcodec^=avc1]+bestaudio[ext=m4a]/best[ext=mp4][vcodec^=avc1]/best[ext=mp4]/best", "--output", "/tmp/1", "http://url"}, 61 | }, 62 | { 63 | name: "Video low quality", 64 | format: model.FormatVideo, 65 | quality: model.QualityLow, 66 | output: "/tmp/2", 67 | videoURL: "http://url", 68 | expect: []string{"--format", "worstvideo[ext=mp4][vcodec^=avc1]+worstaudio[ext=m4a]/worst[ext=mp4][vcodec^=avc1]/worst[ext=mp4]/worst", "--output", "/tmp/2", "http://url"}, 69 | }, 70 | { 71 | name: "Video low quality with maxheight", 72 | format: model.FormatVideo, 73 | quality: model.QualityLow, 74 | maxHeight: 720, 75 | output: "/tmp/2", 76 | videoURL: "http://url", 77 | expect: []string{"--format", "worstvideo[ext=mp4][vcodec^=avc1]+worstaudio[ext=m4a]/worst[ext=mp4][vcodec^=avc1]/worst[ext=mp4]/worst", "--output", "/tmp/2", "http://url"}, 78 | }, 79 | { 80 | name: "Video high quality", 81 | format: model.FormatVideo, 82 | quality: model.QualityHigh, 83 | output: "/tmp/2", 84 | videoURL: "http://url1", 85 | expect: []string{"--format", "bestvideo[ext=mp4][vcodec^=avc1]+bestaudio[ext=m4a]/best[ext=mp4][vcodec^=avc1]/best[ext=mp4]/best", "--output", "/tmp/2", "http://url1"}, 86 | }, 87 | { 88 | name: "Video high quality with maxheight", 89 | format: model.FormatVideo, 90 | quality: model.QualityHigh, 91 | maxHeight: 1024, 92 | output: "/tmp/2", 93 | videoURL: "http://url1", 94 | expect: []string{"--format", "bestvideo[height<=1024][ext=mp4][vcodec^=avc1]+bestaudio[ext=m4a]/best[height<=1024][ext=mp4][vcodec^=avc1]/best[ext=mp4]/best", "--output", "/tmp/2", "http://url1"}, 95 | }, 96 | { 97 | name: "Video high quality with custom youtube-dl arguments", 98 | format: model.FormatVideo, 99 | quality: model.QualityHigh, 100 | output: "/tmp/2", 101 | videoURL: "http://url1", 102 | ytdlArgs: []string{"--write-sub", "--embed-subs", "--sub-lang", "en,en-US,en-GB"}, 103 | expect: []string{"--format", "bestvideo[ext=mp4][vcodec^=avc1]+bestaudio[ext=m4a]/best[ext=mp4][vcodec^=avc1]/best[ext=mp4]/best", "--write-sub", "--embed-subs", "--sub-lang", "en,en-US,en-GB", "--output", "/tmp/2", "http://url1"}, 104 | }, 105 | { 106 | name: "Custom format", 107 | format: model.FormatCustom, 108 | customFormat: feed.CustomFormat{YouTubeDLFormat: "bestaudio[ext=m4a]", Extension: "m4a"}, 109 | quality: model.QualityHigh, 110 | output: "/tmp/2", 111 | videoURL: "http://url1", 112 | expect: []string{"--audio-format", "m4a", "--format", "bestaudio[ext=m4a]", "--output", "/tmp/2", "http://url1"}, 113 | }, 114 | } 115 | 116 | for _, tst := range tests { 117 | t.Run(tst.name, func(t *testing.T) { 118 | result := buildArgs(&feed.Config{ 119 | Format: tst.format, 120 | Quality: tst.quality, 121 | CustomFormat: tst.customFormat, 122 | MaxHeight: tst.maxHeight, 123 | YouTubeDLArgs: tst.ytdlArgs, 124 | }, &model.Episode{ 125 | VideoURL: tst.videoURL, 126 | }, tst.output) 127 | 128 | assert.EqualValues(t, tst.expect, result) 129 | }) 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /services/update/matcher.go: -------------------------------------------------------------------------------- 1 | package update 2 | 3 | import ( 4 | "regexp" 5 | "time" 6 | 7 | "github.com/mxpv/podsync/pkg/feed" 8 | "github.com/mxpv/podsync/pkg/model" 9 | log "github.com/sirupsen/logrus" 10 | ) 11 | 12 | func matchRegexpFilter(pattern, str string, negative bool, logger log.FieldLogger) bool { 13 | if pattern != "" { 14 | matched, err := regexp.MatchString(pattern, str) 15 | if err != nil { 16 | logger.Warnf("pattern %q is not a valid") 17 | } else { 18 | if matched == negative { 19 | logger.Infof("skipping due to regexp mismatch") 20 | return false 21 | } 22 | } 23 | } 24 | return true 25 | } 26 | 27 | func matchFilters(episode *model.Episode, filters *feed.Filters) bool { 28 | logger := log.WithFields(log.Fields{"episode_id": episode.ID}) 29 | if !matchRegexpFilter(filters.Title, episode.Title, false, logger.WithField("filter", "title")) { 30 | return false 31 | } 32 | 33 | if !matchRegexpFilter(filters.NotTitle, episode.Title, true, logger.WithField("filter", "not_title")) { 34 | return false 35 | } 36 | 37 | if !matchRegexpFilter(filters.Description, episode.Description, false, logger.WithField("filter", "description")) { 38 | return false 39 | } 40 | 41 | if !matchRegexpFilter(filters.NotDescription, episode.Description, true, logger.WithField("filter", "not_description")) { 42 | return false 43 | } 44 | 45 | if filters.MaxDuration > 0 && episode.Duration > filters.MaxDuration { 46 | logger.WithField("filter", "max_duration").Infof("skipping due to duration filter (%ds)", episode.Duration) 47 | return false 48 | } 49 | 50 | if filters.MinDuration > 0 && episode.Duration < filters.MinDuration { 51 | logger.WithField("filter", "min_duration").Infof("skipping due to duration filter (%ds)", episode.Duration) 52 | return false 53 | } 54 | 55 | if filters.MaxAge > 0 { 56 | dateDiff := int(time.Since(episode.PubDate).Hours()) / 24 57 | if dateDiff > filters.MaxAge { 58 | logger.WithField("filter", "max_age").Infof("skipping due to max_age filter (%dd > %dd)", dateDiff, filters.MaxAge) 59 | return false 60 | } 61 | } 62 | 63 | return true 64 | } 65 | -------------------------------------------------------------------------------- /services/update/updater.go: -------------------------------------------------------------------------------- 1 | package update 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "io" 8 | "os" 9 | "sort" 10 | "time" 11 | 12 | "github.com/hashicorp/go-multierror" 13 | "github.com/pkg/errors" 14 | log "github.com/sirupsen/logrus" 15 | 16 | "github.com/mxpv/podsync/pkg/builder" 17 | "github.com/mxpv/podsync/pkg/db" 18 | "github.com/mxpv/podsync/pkg/feed" 19 | "github.com/mxpv/podsync/pkg/fs" 20 | "github.com/mxpv/podsync/pkg/model" 21 | "github.com/mxpv/podsync/pkg/ytdl" 22 | ) 23 | 24 | type Downloader interface { 25 | Download(ctx context.Context, feedConfig *feed.Config, episode *model.Episode) (io.ReadCloser, error) 26 | } 27 | 28 | type TokenList []string 29 | 30 | type Manager struct { 31 | hostname string 32 | downloader Downloader 33 | db db.Storage 34 | fs fs.Storage 35 | feeds map[string]*feed.Config 36 | keys map[model.Provider]feed.KeyProvider 37 | } 38 | 39 | func NewUpdater( 40 | feeds map[string]*feed.Config, 41 | keys map[model.Provider]feed.KeyProvider, 42 | hostname string, 43 | downloader Downloader, 44 | db db.Storage, 45 | fs fs.Storage, 46 | ) (*Manager, error) { 47 | return &Manager{ 48 | hostname: hostname, 49 | downloader: downloader, 50 | db: db, 51 | fs: fs, 52 | feeds: feeds, 53 | keys: keys, 54 | }, nil 55 | } 56 | 57 | func (u *Manager) Update(ctx context.Context, feedConfig *feed.Config) error { 58 | log.WithFields(log.Fields{ 59 | "feed_id": feedConfig.ID, 60 | "format": feedConfig.Format, 61 | "quality": feedConfig.Quality, 62 | }).Infof("-> updating %s", feedConfig.URL) 63 | 64 | started := time.Now() 65 | 66 | if err := u.updateFeed(ctx, feedConfig); err != nil { 67 | return errors.Wrap(err, "update failed") 68 | } 69 | 70 | if err := u.downloadEpisodes(ctx, feedConfig); err != nil { 71 | return errors.Wrap(err, "download failed") 72 | } 73 | 74 | if err := u.cleanup(ctx, feedConfig); err != nil { 75 | log.WithError(err).Error("cleanup failed") 76 | } 77 | 78 | if err := u.buildXML(ctx, feedConfig); err != nil { 79 | return errors.Wrap(err, "xml build failed") 80 | } 81 | 82 | if err := u.buildOPML(ctx); err != nil { 83 | return errors.Wrap(err, "opml build failed") 84 | } 85 | 86 | elapsed := time.Since(started) 87 | log.Infof("successfully updated feed in %s", elapsed) 88 | return nil 89 | } 90 | 91 | // updateFeed pulls API for new episodes and saves them to database 92 | func (u *Manager) updateFeed(ctx context.Context, feedConfig *feed.Config) error { 93 | info, err := builder.ParseURL(feedConfig.URL) 94 | if err != nil { 95 | return errors.Wrapf(err, "failed to parse URL: %s", feedConfig.URL) 96 | } 97 | 98 | keyProvider, ok := u.keys[info.Provider] 99 | if !ok { 100 | return errors.Errorf("key provider %q not loaded", info.Provider) 101 | } 102 | 103 | // Create an updater for this feed type 104 | provider, err := builder.New(ctx, info.Provider, keyProvider.Get()) 105 | if err != nil { 106 | return err 107 | } 108 | 109 | // Query API to get episodes 110 | log.Debug("building feed") 111 | result, err := provider.Build(ctx, feedConfig) 112 | if err != nil { 113 | return err 114 | } 115 | 116 | log.Debugf("received %d episode(s) for %q", len(result.Episodes), result.Title) 117 | 118 | episodeSet := make(map[string]struct{}) 119 | if err := u.db.WalkEpisodes(ctx, feedConfig.ID, func(episode *model.Episode) error { 120 | if episode.Status != model.EpisodeDownloaded && episode.Status != model.EpisodeCleaned { 121 | episodeSet[episode.ID] = struct{}{} 122 | } 123 | return nil 124 | }); err != nil { 125 | return err 126 | } 127 | 128 | if err := u.db.AddFeed(ctx, feedConfig.ID, result); err != nil { 129 | return err 130 | } 131 | 132 | for _, episode := range result.Episodes { 133 | delete(episodeSet, episode.ID) 134 | } 135 | 136 | // removing episodes that are no longer available in the feed and not downloaded or cleaned 137 | for id := range episodeSet { 138 | log.Infof("removing episode %q", id) 139 | err := u.db.DeleteEpisode(feedConfig.ID, id) 140 | if err != nil { 141 | return err 142 | } 143 | } 144 | 145 | log.Debug("successfully saved updates to storage") 146 | return nil 147 | } 148 | 149 | func (u *Manager) downloadEpisodes(ctx context.Context, feedConfig *feed.Config) error { 150 | var ( 151 | feedID = feedConfig.ID 152 | downloadList []*model.Episode 153 | pageSize = feedConfig.PageSize 154 | ) 155 | 156 | log.WithField("page_size", pageSize).Info("downloading episodes") 157 | 158 | // Build the list of files to download 159 | if err := u.db.WalkEpisodes(ctx, feedID, func(episode *model.Episode) error { 160 | var ( 161 | logger = log.WithFields(log.Fields{"episode_id": episode.ID}) 162 | ) 163 | if episode.Status != model.EpisodeNew && episode.Status != model.EpisodeError { 164 | // File already downloaded 165 | logger.Infof("skipping due to already downloaded") 166 | return nil 167 | } 168 | 169 | if !matchFilters(episode, &feedConfig.Filters) { 170 | return nil 171 | } 172 | 173 | // Limit the number of episodes downloaded at once 174 | pageSize-- 175 | if pageSize < 0 { 176 | return nil 177 | } 178 | 179 | log.Debugf("adding %s (%q) to queue", episode.ID, episode.Title) 180 | downloadList = append(downloadList, episode) 181 | return nil 182 | }); err != nil { 183 | return errors.Wrapf(err, "failed to build update list") 184 | } 185 | 186 | var ( 187 | downloadCount = len(downloadList) 188 | downloaded = 0 189 | ) 190 | 191 | if downloadCount > 0 { 192 | log.Infof("download count: %d", downloadCount) 193 | } else { 194 | log.Info("no episodes to download") 195 | return nil 196 | } 197 | 198 | // Download pending episodes 199 | 200 | for idx, episode := range downloadList { 201 | var ( 202 | logger = log.WithFields(log.Fields{"index": idx, "episode_id": episode.ID}) 203 | episodeName = feed.EpisodeName(feedConfig, episode) 204 | ) 205 | 206 | // Check whether episode already exists 207 | size, err := u.fs.Size(ctx, fmt.Sprintf("%s/%s", feedID, episodeName)) 208 | if err == nil { 209 | logger.Infof("episode %q already exists on disk", episode.ID) 210 | 211 | // File already exists, update file status and disk size 212 | if err := u.db.UpdateEpisode(feedID, episode.ID, func(episode *model.Episode) error { 213 | episode.Size = size 214 | episode.Status = model.EpisodeDownloaded 215 | return nil 216 | }); err != nil { 217 | logger.WithError(err).Error("failed to update file info") 218 | return err 219 | } 220 | 221 | continue 222 | } else if os.IsNotExist(err) { 223 | // Will download, do nothing here 224 | } else { 225 | logger.WithError(err).Error("failed to stat file") 226 | return err 227 | } 228 | 229 | // Download episode to disk 230 | // We download the episode to a temp directory first to avoid downloading this file by clients 231 | // while still being processed by youtube-dl (e.g. a file is being downloaded from YT or encoding in progress) 232 | 233 | logger.Infof("! downloading episode %s", episode.VideoURL) 234 | tempFile, err := u.downloader.Download(ctx, feedConfig, episode) 235 | if err != nil { 236 | // YouTube might block host with HTTP Error 429: Too Many Requests 237 | // We still need to generate XML, so just stop sending download requests and 238 | // retry next time 239 | if err == ytdl.ErrTooManyRequests { 240 | logger.Warn("server responded with a 'Too Many Requests' error") 241 | break 242 | } 243 | 244 | if err := u.db.UpdateEpisode(feedID, episode.ID, func(episode *model.Episode) error { 245 | episode.Status = model.EpisodeError 246 | return nil 247 | }); err != nil { 248 | return err 249 | } 250 | 251 | continue 252 | } 253 | 254 | logger.Debug("copying file") 255 | fileSize, err := u.fs.Create(ctx, fmt.Sprintf("%s/%s", feedID, episodeName), tempFile) 256 | tempFile.Close() 257 | if err != nil { 258 | logger.WithError(err).Error("failed to copy file") 259 | return err 260 | } 261 | 262 | // Update file status in database 263 | 264 | logger.Infof("successfully downloaded file %q", episode.ID) 265 | if err := u.db.UpdateEpisode(feedID, episode.ID, func(episode *model.Episode) error { 266 | episode.Size = fileSize 267 | episode.Status = model.EpisodeDownloaded 268 | return nil 269 | }); err != nil { 270 | return err 271 | } 272 | 273 | downloaded++ 274 | } 275 | 276 | log.Infof("downloaded %d episode(s)", downloaded) 277 | return nil 278 | } 279 | 280 | func (u *Manager) buildXML(ctx context.Context, feedConfig *feed.Config) error { 281 | f, err := u.db.GetFeed(ctx, feedConfig.ID) 282 | if err != nil { 283 | return err 284 | } 285 | 286 | // Build iTunes XML feed with data received from builder 287 | log.Debug("building iTunes podcast feed") 288 | podcast, err := feed.Build(ctx, f, feedConfig, u.hostname) 289 | if err != nil { 290 | return err 291 | } 292 | 293 | var ( 294 | reader = bytes.NewReader([]byte(podcast.String())) 295 | xmlName = fmt.Sprintf("%s.xml", feedConfig.ID) 296 | ) 297 | 298 | if _, err := u.fs.Create(ctx, xmlName, reader); err != nil { 299 | return errors.Wrap(err, "failed to upload new XML feed") 300 | } 301 | 302 | return nil 303 | } 304 | 305 | func (u *Manager) buildOPML(ctx context.Context) error { 306 | // Build OPML with data received from builder 307 | log.Debug("building podcast OPML") 308 | opml, err := feed.BuildOPML(ctx, u.feeds, u.db, u.hostname) 309 | if err != nil { 310 | return err 311 | } 312 | 313 | var ( 314 | reader = bytes.NewReader([]byte(opml)) 315 | xmlName = fmt.Sprintf("%s.opml", "podsync") 316 | ) 317 | 318 | if _, err := u.fs.Create(ctx, xmlName, reader); err != nil { 319 | return errors.Wrap(err, "failed to upload OPML") 320 | } 321 | 322 | return nil 323 | } 324 | 325 | func (u *Manager) cleanup(ctx context.Context, feedConfig *feed.Config) error { 326 | var ( 327 | feedID = feedConfig.ID 328 | logger = log.WithField("feed_id", feedID) 329 | count = feedConfig.Clean.KeepLast 330 | list []*model.Episode 331 | result *multierror.Error 332 | ) 333 | 334 | if count < 1 { 335 | logger.Info("nothing to clean") 336 | return nil 337 | } 338 | 339 | logger.WithField("count", count).Info("running cleaner") 340 | if err := u.db.WalkEpisodes(ctx, feedConfig.ID, func(episode *model.Episode) error { 341 | if episode.Status == model.EpisodeDownloaded { 342 | list = append(list, episode) 343 | } 344 | return nil 345 | }); err != nil { 346 | return err 347 | } 348 | 349 | if count > len(list) { 350 | return nil 351 | } 352 | 353 | sort.Slice(list, func(i, j int) bool { 354 | return list[i].PubDate.After(list[j].PubDate) 355 | }) 356 | 357 | for _, episode := range list[count:] { 358 | logger.WithField("episode_id", episode.ID).Infof("deleting %q", episode.Title) 359 | 360 | var ( 361 | episodeName = feed.EpisodeName(feedConfig, episode) 362 | path = fmt.Sprintf("%s/%s", feedConfig.ID, episodeName) 363 | ) 364 | 365 | if err := u.fs.Delete(ctx, path); err != nil { 366 | result = multierror.Append(result, errors.Wrapf(err, "failed to delete episode: %s", episode.ID)) 367 | continue 368 | } 369 | 370 | if err := u.db.UpdateEpisode(feedID, episode.ID, func(episode *model.Episode) error { 371 | episode.Status = model.EpisodeCleaned 372 | episode.Title = "" 373 | episode.Description = "" 374 | return nil 375 | }); err != nil { 376 | result = multierror.Append(result, errors.Wrapf(err, "failed to set state for cleaned episode: %s", episode.ID)) 377 | continue 378 | } 379 | } 380 | 381 | return result.ErrorOrNil() 382 | } 383 | -------------------------------------------------------------------------------- /services/web/server.go: -------------------------------------------------------------------------------- 1 | package web 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | 7 | log "github.com/sirupsen/logrus" 8 | ) 9 | 10 | type Server struct { 11 | http.Server 12 | } 13 | 14 | type Config struct { 15 | // Hostname to use for download links 16 | Hostname string `toml:"hostname"` 17 | // Port is a server port to listen to 18 | Port int `toml:"port"` 19 | // Bind a specific IP addresses for server 20 | // "*": bind all IP addresses which is default option 21 | // localhost or 127.0.0.1 bind a single IPv4 address 22 | BindAddress string `toml:"bind_address"` 23 | // Flag indicating if the server will use TLS 24 | TLS bool `toml:"tls"` 25 | // Path to a certificate file for TLS connections 26 | CertificatePath string `toml:"certificate_path"` 27 | // Path to a private key file for TLS connections 28 | KeyFilePath string `toml:"key_file_path"` 29 | // Specify path for reverse proxy and only [A-Za-z0-9] 30 | Path string `toml:"path"` 31 | // DataDir is a path to a directory to keep XML feeds and downloaded episodes, 32 | // that will be available to user via web server for download. 33 | DataDir string `toml:"data_dir"` 34 | } 35 | 36 | func New(cfg Config, storage http.FileSystem) *Server { 37 | port := cfg.Port 38 | if port == 0 { 39 | port = 8080 40 | } 41 | 42 | bindAddress := cfg.BindAddress 43 | if bindAddress == "*" { 44 | bindAddress = "" 45 | } 46 | 47 | srv := Server{} 48 | 49 | srv.Addr = fmt.Sprintf("%s:%d", bindAddress, port) 50 | log.Debugf("using address: %s:%s", bindAddress, srv.Addr) 51 | 52 | fileServer := http.FileServer(storage) 53 | 54 | log.Debugf("handle path: /%s", cfg.Path) 55 | http.Handle(fmt.Sprintf("/%s", cfg.Path), fileServer) 56 | 57 | return &srv 58 | } 59 | --------------------------------------------------------------------------------