├── .github └── FUNDING.yml ├── .gitignore ├── .travis.yml ├── LICENSE.GPLv3 ├── LICENSE.ISC ├── MANIFEST.in ├── README ├── bin ├── gpo └── gpodder-migrate2cuatro ├── examples └── after_download.py ├── makefile ├── po ├── cz.po ├── de.po └── messages.pot ├── requirements-test.txt ├── requirements.txt ├── setup.cfg ├── setup.py ├── share └── man │ └── man1 │ └── gpo.1 ├── src └── gpodder │ ├── __init__.py │ ├── api.py │ ├── common.py │ ├── config.py │ ├── core.py │ ├── coverart.py │ ├── directory.py │ ├── download.py │ ├── jsonconfig.py │ ├── log.py │ ├── model.py │ ├── opml.py │ ├── plugins │ ├── __init__.py │ ├── gpoddernet.py │ ├── itunes.py │ ├── podcast.py │ ├── podverse.py │ ├── soundcloud.py │ ├── vimeo.py │ └── youtube.py │ ├── query.py │ ├── registry.py │ ├── storage.py │ └── util.py └── test └── test_gpodder └── test_model.py /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: Keeper-of-the-Keys 2 | custom: ["https://paypal.me/esrosenberg", Paypal - Keeper-of-the-Keys] 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - "3.7" 4 | install: 5 | - "pip install -r requirements.txt" 6 | - "pip install -r requirements-test.txt" 7 | script: make test 8 | -------------------------------------------------------------------------------- /LICENSE.ISC: -------------------------------------------------------------------------------- 1 | gPodder: Media and podcast aggregator 2 | Copyright (c) 2005-2020 Thomas Perl and the gPodder Team 3 | 4 | Permission to use, copy, modify, and/or distribute this software for any 5 | purpose with or without fee is hereby granted, provided that the above 6 | copyright notice and this permission notice appear in all copies. 7 | 8 | THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH 9 | REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY 10 | AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, 11 | INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM 12 | LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR 13 | OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 14 | PERFORMANCE OF THIS SOFTWARE. 15 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README LICENSE.* MANIFEST.in ChangeLog makefile setup.py 2 | recursive-include share * 3 | recursive-include po * 4 | recursive-include test * 5 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | 2 | ___ _ _ 3 | __ _| _ \___ __| |__| |___ _ _ 4 | / _` | _/ _ \/ _` / _` / -_) '_| 5 | \__, |_| \___/\__,_\__,_\___|_| 6 | |___/ 7 | Media and podcast aggregator 8 | 9 | Copyright 2005-2024 Thomas Perl and the gPodder Team 10 | 11 | 12 | [ LICENSE ] 13 | 14 | Historically, gPodder was licensed under the terms of the "GNU GPLv2 or 15 | later", and has been upgraded to "GNU GPLv3 or later" in August 2007. 16 | 17 | Code that has been solely written by thp was re-licensed to a more 18 | permissive license (ISC license) in August 2013. The new license is 19 | DFSG-compatible, FSF-approved, OSI-approved and GPL-compatible (see 20 | http://en.wikipedia.org/wiki/ISC_license for more information). 21 | 22 | For the license that applies to a file, see the copyright header in it. 23 | 24 | [ WHAT IS THIS? ] 25 | 26 | This is the gPodder Core, including the core Python modules as well as 27 | the command-line interface "gpo". Since version 4, the user interfaces 28 | have been split out into different packages that can be found elsewhere: 29 | 30 | - QML UI: http://github.com/gpodder/gpodder-ui-qml 31 | 32 | [ DEPENDENCIES ] 33 | 34 | - Python 3.2 or newer http://python.org/ 35 | - podcastparser 0.4.0 http://gpodder.org/podcastparser/ 36 | - minidb 2 http://github.com/thp/minidb 37 | 38 | Use "pip install -r requirements.txt" to install Python dependencies. 39 | 40 | [ BUILD DEPENDENCIES ] 41 | 42 | - gettext 43 | 44 | [ TEST DEPENDENCIES ] 45 | 46 | - python3-nose 47 | - python3-minimock 48 | - python3-coverage 49 | 50 | [ TESTING ] 51 | 52 | To run automated tests, use... 53 | 54 | make test 55 | 56 | Tests in gPodder are written in two different ways: 57 | 58 | - doctests http://docs.python.org/3/library/doctest 59 | - unittests http://docs.python.org/3/library/unittest 60 | 61 | If you want to add unit tests for a specific module (ex: gpodder.model), 62 | you should add the tests as test_gpodder.test_model, or in other words: 63 | 64 | The file src/gpodder/model.py 65 | is tested by tests/test_gpodder/test_model.py 66 | 67 | 68 | [ RUNNING AND INSTALLATION ] 69 | 70 | To run gPodder from source, use.. 71 | 72 | bin/gpo for the command-line interface 73 | 74 | To install gPodder system-wide, use "make install". By default, this 75 | will install all translations. The following environment variables 76 | are processed by setup.py: 77 | 78 | LINGUAS space-separated list of languages to install 79 | GPODDER_MANPATH_NO_SHARE if set, install manpages to $PREFIX/man/man1 80 | 81 | See setup.py for a list of recognized UIs. 82 | 83 | Example: Install with German and Dutch translations: 84 | 85 | export LINGUAS="de nl" 86 | make install 87 | 88 | The "make install" target also supports DESTDIR and PREFIX for installing 89 | into an alternative root (default /) and prefix (default /usr): 90 | 91 | make install DESTDIR=tmp/ PREFIX=/usr/local/ 92 | 93 | 94 | [ PORTABLE MODE / ROAMING PROFILES ] 95 | 96 | The run-time environment variable GPODDER_HOME is used to set 97 | the location for storing the database and downloaded files. 98 | 99 | This can be used for multiple configurations or to store the 100 | download directory directly on a MP3 player or USB disk: 101 | 102 | export GPODDER_HOME=/media/usbdisk/gpodder-data/ 103 | 104 | By default, gPodder 4 uses the XDG Base Directory Specification 105 | for determining the location of data files ($XDG_DATA_HOME) and 106 | configuration files ($XDG_CONFIG_HOME): 107 | 108 | http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html 109 | 110 | 111 | [ CHANGING THE DOWNLOAD DIRECTORY ] 112 | 113 | The run-time environment variable GPODDER_DOWNLOAD_DIR is used to 114 | set the location for storing the downloads only (independent of the 115 | data directory GPODDER_HOME): 116 | 117 | export GPODDER_DOWNLOAD_DIR=/media/BigDisk/Podcasts/ 118 | 119 | In this case, the database and settings will be stored in the default 120 | location, with the downloads stored in /media/BigDisk/Podcasts/. 121 | 122 | Another example would be to set both environment variables: 123 | 124 | export GPODDER_HOME=~/bla/gpodder/ 125 | export GPODDER_DOWNLOAD_DIR=~/Music/Podcasts/ 126 | 127 | This will store the database and settings files in ~/bla/gpodder/ 128 | and the downloads in ~/Music/Podcasts/. If GPODDER_DOWNLOAD_DIR is 129 | not set, $GPODDER_HOME will be used if $GPODDER_HOME is set, or 130 | the XDG Base Directory (inside $XDG_DATA_HOME) otherwise. 131 | 132 | 133 | [ LOGGING ] 134 | 135 | By default, gPodder writes log files to a folder in $XDG_CACHE_HOME (or 136 | $GPODDER_HOME if it is set) and removes 137 | them after a certain amount of times. To avoid this behavior, you can set 138 | the environment variable GPODDER_WRITE_LOGS to "no", e.g: 139 | 140 | export GPODDER_WRITE_LOGS=no 141 | 142 | [ AUTOMATIC DOWNLOADING ] 143 | 144 | If you are using gPodder on a server, you can add the following command 145 | to your crontab(5) to automatically check for episodes and download them: 146 | 147 | gpo run --batch 148 | 149 | It will only show output if new episodes are found (or if there was an 150 | error updating feeds) and/or downloaded, which is useful for cronjobs. 151 | 152 | [ MORE INFORMATION ] 153 | 154 | - Homepage http://gpodder.org/ 155 | - Bug tracker http://bugs.gpodder.org/ 156 | - Mailing list http://freelists.org/list/gpodder 157 | - IRC channel #gpodder on irc.freenode.net 158 | 159 | ............................................................................ 160 | Last updated: 2014-12-23 by Thomas Perl 161 | 162 | -------------------------------------------------------------------------------- /bin/gpodder-migrate2cuatro: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # convert_sqlite_to_jsondb: Convert gPodder 3 SQLite DB to gPodder 4 JSON DB 4 | # Copyright (c) 2013, Thomas Perl 5 | # 6 | # Permission to use, copy, modify, and/or distribute this software for any 7 | # purpose with or without fee is hereby granted, provided that the above 8 | # copyright notice and this permission notice appear in all copies. 9 | # 10 | # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH 11 | # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY 12 | # AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, 13 | # INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM 14 | # LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR 15 | # OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 16 | # PERFORMANCE OF THIS SOFTWARE. 17 | # 18 | 19 | import sys 20 | import os 21 | 22 | sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) 23 | 24 | from gpodder import model 25 | from gpodder import storage 26 | 27 | import sqlite3.dbapi2 as sqlite3 28 | 29 | 30 | class Database(object): 31 | def __init__(self, filename): 32 | self.db = sqlite3.connect(filename) 33 | # Make sure you install the latest gPodder 3 version first before upgrading 34 | assert self.db.execute('SELECT version FROM version').fetchone()[0] == 5 35 | 36 | def load_podcasts(self): 37 | cur = self.db.cursor() 38 | cur.execute('SELECT * FROM podcast') 39 | keys = [desc[0] for desc in cur.description] 40 | for row in cur: 41 | yield dict(zip(keys, row)) 42 | 43 | def load_episodes(self, podcast): 44 | cur = self.db.cursor() 45 | cur.execute('SELECT * FROM episode WHERE podcast_id = ?', (podcast.id,)) 46 | keys = [desc[0] for desc in cur.description] 47 | for row in cur: 48 | yield dict(zip(keys, row)) 49 | 50 | 51 | if len(sys.argv) != 2: 52 | print(""" 53 | Usage: {progname} /path/to/Database 54 | """.format(progname=sys.argv[0]), file=sys.stderr) 55 | sys.exit(1) 56 | 57 | db_in = Database(sys.argv[1]) 58 | db_out = storage.Database(sys.argv[1]) 59 | 60 | for podcast_dict in db_in.load_podcasts(): 61 | podcast = model.PodcastChannel(None) 62 | for key, value in podcast_dict.items(): 63 | if hasattr(podcast, key): 64 | setattr(podcast, key, value) 65 | else: 66 | print("""Skipping key: {} = {}""".format(key, value)) 67 | db_out.save_podcast(podcast) 68 | 69 | for episode_dict in db_in.load_episodes(podcast): 70 | episode = model.PodcastEpisode(podcast) 71 | for key, value in episode_dict.items(): 72 | if key != 'sync_to_mp3_player': 73 | setattr(episode, key, value) 74 | db_out.save_episode(episode) 75 | 76 | # Update sequence numbers 77 | db_out.finish_migration() 78 | 79 | db_out.close() 80 | -------------------------------------------------------------------------------- /examples/after_download.py: -------------------------------------------------------------------------------- 1 | # Example post-download extension script for gPodder 4 2 | # To test this, you need to set the GPODDER_ADD_PLUGINS environment 3 | # variable and make sure this folder is in your PYTHONPATH: 4 | # env PYTHONPATH=examples GPODDER_ADD_PLUGINS=after_download bin/gpo 5 | 6 | from gpodder import registry 7 | 8 | @registry.after_download.register 9 | def on_episode_downloaded(episode): 10 | print('Downloaded episode: {}'.format(episode.title)) 11 | -------------------------------------------------------------------------------- /makefile: -------------------------------------------------------------------------------- 1 | # 2 | # gPodder: Media and podcast aggregator 3 | # Copyright (c) 2005-2020 Thomas Perl and the gPodder Team 4 | # 5 | # Permission to use, copy, modify, and/or distribute this software for any 6 | # purpose with or without fee is hereby granted, provided that the above 7 | # copyright notice and this permission notice appear in all copies. 8 | # 9 | # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH 10 | # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY 11 | # AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, 12 | # INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM 13 | # LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR 14 | # OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 15 | # PERFORMANCE OF THIS SOFTWARE. 16 | # 17 | 18 | ########################################################################## 19 | 20 | PYTHON ?= python3 21 | 22 | ########################################################################## 23 | 24 | help: 25 | @echo "" 26 | @echo " make messages Update translation files in po/ from source" 27 | @echo " make headlink Print commit URL for the current Git head" 28 | @echo "" 29 | @echo " make test Run automated tests" 30 | @echo " make pep8 Run pep8 utility to check code style" 31 | @echo " make clean Remove generated and compiled files" 32 | @echo " make distclean 'make clean' + remove dist/" 33 | @echo "" 34 | @echo " make release Create the source tarball in dist/" 35 | @echo "" 36 | @echo " make install Install gPodder into \$$DESTDIR/\$$PREFIX" 37 | @echo "" 38 | @echo " PREFIX ..... Installation prefix (default: /usr)" 39 | @echo " DESTDIR .... Installation destination (default: /)" 40 | @echo " LINGUAS .... Space-separated list of translations" 41 | @echo "" 42 | 43 | ########################################################################## 44 | 45 | test: 46 | LC_ALL=C $(PYTHON) -m nose 47 | 48 | releasetest: test $(POFILES) 49 | for lang in $(POFILES); do $(MSGFMT) --check $$lang; done 50 | 51 | pep8: 52 | $(PYTHON) -m pep8 53 | 54 | ########################################################################## 55 | 56 | release: releasetest distclean 57 | $(PYTHON) setup.py sdist 58 | 59 | DESTDIR ?= / 60 | PREFIX ?= /usr 61 | 62 | install: messages 63 | $(PYTHON) setup.py install --root=$(DESTDIR) --prefix=$(PREFIX) --optimize=1 64 | 65 | # This only works in a Git working commit, and assumes that the local Git 66 | # HEAD has already been pushed to the main repository. It's mainly useful 67 | # for the gPodder maintainer to quickly generate a commit link that can be 68 | # posted online in bug trackers and mailing lists. 69 | 70 | headlink: 71 | @echo http://gpodder.org/commit/$(shell git show-ref HEAD | head -c8) 72 | 73 | ########################################################################## 74 | 75 | XGETTEXT ?= xgettext 76 | MSGMERGE ?= msgmerge 77 | MSGFMT ?= msgfmt 78 | 79 | MESSAGES = po/messages.pot 80 | POFILES = $(wildcard po/*.po) 81 | LOCALEDIR = share/locale 82 | MOFILES = $(patsubst po/%.po,$(LOCALEDIR)/%/LC_MESSAGES/gpodder.mo,$(POFILES)) 83 | 84 | messages: $(MESSAGES) $(MOFILES) 85 | 86 | %.po: $(MESSAGES) 87 | $(MSGMERGE) --silent $@ $< --output-file=$@ 88 | 89 | $(LOCALEDIR)/%/LC_MESSAGES/gpodder.mo: po/%.po 90 | @mkdir -p $(@D) 91 | $(MSGFMT) $< -o $@ 92 | 93 | $(MESSAGES): bin/gpo 94 | $(XGETTEXT) --from-code=utf-8 --language=Python -k_:1 -kN_:1 -kN_:1,2 -o $(MESSAGES) $^ 95 | 96 | ########################################################################## 97 | 98 | clean: 99 | $(PYTHON) setup.py clean 100 | find src/ -type d -name '__pycache__' -exec rm -r '{}' + 101 | find test/ -type d -name '__pycache__' -exec rm -r '{}' + 102 | rm -f MANIFEST PKG-INFO .coverage messages.mo po/*.mo 103 | rm -rf build $(LOCALEDIR) 104 | 105 | distclean: clean 106 | rm -rf dist 107 | 108 | ########################################################################## 109 | 110 | .PHONY: help \ 111 | test releasetest \ 112 | release install headlink messages \ 113 | clean distclean \ 114 | -------------------------------------------------------------------------------- /po/cz.po: -------------------------------------------------------------------------------- 1 | # Czech gPodder translation. 2 | # Copyright (C) 2014 Jaroslav Lichtblau 3 | # This file is distributed under the same license as the gpodder package. 4 | # Jaroslav Lichtblau , 2014. 5 | # 6 | msgid "" 7 | msgstr "" 8 | "Project-Id-Version: gpodder-core\n" 9 | "Report-Msgid-Bugs-To: \n" 10 | "POT-Creation-Date: 2014-03-14 23:28+0100\n" 11 | "PO-Revision-Date: 2014-09-11 09:19+0100\n" 12 | "Last-Translator: Jaroslav Lichtblau \n" 13 | "Language-Team: \n" 14 | "Language: cs_CZ\n" 15 | "MIME-Version: 1.0\n" 16 | "Content-Type: text/plain; charset=UTF-8\n" 17 | "Content-Transfer-Encoding: 8bit\n" 18 | "Plural-Forms: nplurals=3; plural=(n==1) ? 0 : (n>=2 && n<=4) ? 1 : 2;\n" 19 | 20 | #: bin/gpo:172 21 | msgid "Install mygpoclient for gpodder.net features" 22 | msgstr "Nainstalujte mygpoclient pro správné fungování gpodder.net" 23 | 24 | #: bin/gpo:307 bin/gpo:946 25 | #, python-format 26 | msgid "Invalid URL: %(url)s" 27 | msgstr "Neplatná URL: %(url)s" 28 | 29 | #: bin/gpo:320 30 | #, python-format 31 | msgid "Not subscribed to %(url)s" 32 | msgstr "%(url)s není odebírána" 33 | 34 | #: bin/gpo:332 35 | #, python-format 36 | msgid "Subscribing to %(url)s" 37 | msgstr "Začínám odebírat %(url)s" 38 | 39 | #: bin/gpo:337 40 | #, python-format 41 | msgid "Already subscribed to %(url)s" 42 | msgstr "%(url)s je již odebírána" 43 | 44 | #: bin/gpo:343 45 | #, python-format 46 | msgid "Subscription to %(url)s failed" 47 | msgstr "Odběr %(url)s selhal" 48 | 49 | #: bin/gpo:352 50 | #, python-format 51 | msgid "Resolved feed URL: %(url)s" 52 | msgstr "Načtena adresa URL: %(url)s" 53 | 54 | #: bin/gpo:386 55 | #, python-format 56 | msgid "Configuration option %(key)s not found" 57 | msgstr "Konfigurační nastavení pro %(key)s nenalezeno" 58 | 59 | #: bin/gpo:390 60 | #, python-format 61 | msgid "Invalid configuration option: %(key)s" 62 | msgstr "Neplatné nastavení konfigurace: %(key)s" 63 | 64 | #: bin/gpo:425 65 | #, python-format 66 | msgid "Removed %(url)s" 67 | msgstr "%(url)s odstraněno" 68 | 69 | #: bin/gpo:516 70 | msgid "Usage: query EQL" 71 | msgstr "Použití: query EQL" 72 | 73 | #: bin/gpo:535 74 | #, python-format 75 | msgid "%(count)d episode matched" 76 | msgid_plural "%(count)d episodes matched" 77 | msgstr[0] "nalezena %(count)d epizoda" 78 | msgstr[1] "nalezeny %(count)d epizody" 79 | msgstr[2] "nalezeno %(count)d epizod" 80 | 81 | #: bin/gpo:537 82 | msgid "Use \"apply\" to apply a single-episode command" 83 | msgstr "Použít \"apply\" jako příkaz pro jednu epizodu" 84 | 85 | #: bin/gpo:545 86 | msgid "Subscription suspended" 87 | msgstr "Odběr ukončen" 88 | 89 | #: bin/gpo:569 90 | #, python-format 91 | msgid "%(count)d new episode" 92 | msgid_plural "%(count)d new episodes" 93 | msgstr[0] "%(count)d nová epizoda" 94 | msgstr[1] "%(count)d nové epizody" 95 | msgstr[2] "%(count)d nových epizod" 96 | 97 | #: bin/gpo:584 98 | msgid "Checking for new episodes" 99 | msgstr "Hledám nové epizody" 100 | 101 | #: bin/gpo:591 102 | #, python-format 103 | msgid "Skipping %(podcast)s" 104 | msgstr "Přeskakuji %(podcast)s" 105 | 106 | #: bin/gpo:636 107 | #, python-format 108 | msgid "Invalid episode ID: %(error)s" 109 | msgstr "Neplatné ID epizody: %(error)s" 110 | 111 | #: bin/gpo:644 112 | #, python-format 113 | msgid "Episode ID not found: %(id)x" 114 | msgstr "ID epizody nenalezeno: %(id)x" 115 | 116 | #: bin/gpo:663 117 | #, python-format 118 | msgid "Invalid action: %(action)s. Valid actions: %(valid_actions)s" 119 | msgstr "Neplatný příkaz: %(action)s. Dostupné příkazy: %(valid_actions)s" 120 | 121 | #: bin/gpo:672 122 | #, python-format 123 | msgid "Episode marked as old: %(title)s" 124 | msgstr "Epizoda označena jako stará: %(title)s" 125 | 126 | #: bin/gpo:675 127 | #, python-format 128 | msgid "Episode marked as new: %(title)s" 129 | msgstr "Epizoda označena jako nová: %(title)s" 130 | 131 | #: bin/gpo:715 132 | #, python-format 133 | msgid "Episode deleted: %(episode)s" 134 | msgstr "Epizoda smazána: %(episode)s" 135 | 136 | #: bin/gpo:768 137 | msgid "\"apply\" can only be used during an interactive session" 138 | msgstr "\"apply\" lze použít pouze během interaktivního sezení" 139 | 140 | #: bin/gpo:772 141 | msgid "Empty query result (use \"query\" first)" 142 | msgstr "Žádný výsledek hledání (nejdříve použít \"query\")" 143 | 144 | #: bin/gpo:776 145 | msgid "Cannot apply this command" 146 | msgstr "Tento příkaz nelze použít" 147 | 148 | #: bin/gpo:787 149 | msgid "Please provide a search query" 150 | msgstr "Zadejte hledaný výraz" 151 | 152 | #: bin/gpo:803 153 | #, python-format 154 | msgid "%(count)d episode downloaded" 155 | msgid_plural "%(count)d episodes downloaded" 156 | msgstr[0] "stažena %(count)d epizoda" 157 | msgstr[1] "staženy %(count)d epizody" 158 | msgstr[2] "staženo %(count)d epizod" 159 | 160 | #: bin/gpo:806 161 | #, python-format 162 | msgid "Downloading %(episode)s" 163 | msgstr "Stahuje se %(episode)s" 164 | 165 | #: bin/gpo:858 166 | #, python-format 167 | msgid "Subscription suspended: %(url)s" 168 | msgstr "Odběr ukončen: %(url)s" 169 | 170 | #: bin/gpo:879 171 | #, python-format 172 | msgid "Subscription resumed: %(url)s" 173 | msgstr "Odběr obnoven: %(url)s" 174 | 175 | #: bin/gpo:899 176 | msgid "No software updates available" 177 | msgstr "Není dostupná žádná novější verze" 178 | 179 | #: bin/gpo:901 180 | #, python-format 181 | msgid "New version %(latestversion)s available (released: %(latestdate)s)" 182 | msgstr "Nová verze %(latestversion)s je dostupná (sestavena: %(latestdate)s)" 183 | 184 | #: bin/gpo:902 185 | #, python-format 186 | msgid "You have version %(thisversion)s (released: %(thisdate)s)" 187 | msgstr "Aktuální verze %(thisversion)s (sestavena: %(thisdate)s)" 188 | 189 | #: bin/gpo:903 190 | #, python-format 191 | msgid "Download the new version from %(url)s" 192 | msgstr "Nová verze ke stažení na %(url)s" 193 | 194 | #: bin/gpo:965 195 | msgid "" 196 | msgstr "" 197 | 198 | #: bin/gpo:966 199 | #, python-format 200 | msgid "Use \"help %(command)s\" to get detailed help for a command" 201 | msgstr "Použij \"help %(command)s\" pro obdržení detailnějších informací k příkazu" 202 | 203 | #: bin/gpo:972 204 | msgid "No documentation available (command exists)" 205 | msgstr "Dokumentace není dostupná (příkaz existuje)" 206 | 207 | #: bin/gpo:978 208 | msgid "No such command" 209 | msgstr "Příkaz neexistuje" 210 | 211 | #: bin/gpo:1035 212 | #, python-format 213 | msgid "Syntax error: %(error)s" 214 | msgstr "Chyba syntaxe: %(error)s" 215 | 216 | #: bin/gpo:1127 217 | msgid "Incomplete command; matching commands:" 218 | msgstr "Nekompletní příkaz; odpovídající příkazy:" 219 | 220 | #: bin/gpo:1131 221 | msgid "Command not found" 222 | msgstr "Příkaz nenalezen" 223 | -------------------------------------------------------------------------------- /po/de.po: -------------------------------------------------------------------------------- 1 | # German gPodder translation. 2 | # Copyright (C) 2013 Thomas Perl 3 | # This file is distributed under the same license as the gpodder package. 4 | # Thomas Perl , 2013. 5 | # 6 | msgid "" 7 | msgstr "" 8 | "Project-Id-Version: gpodder 4.0.0\n" 9 | "Report-Msgid-Bugs-To: \n" 10 | "POT-Creation-Date: 2014-03-14 23:28+0100\n" 11 | "PO-Revision-Date: 2013-09-29 23:10+0200\n" 12 | "Last-Translator: Thomas Perl \n" 13 | "Language-Team: German \n" 14 | "Language: de\n" 15 | "MIME-Version: 1.0\n" 16 | "Content-Type: text/plain; charset=utf-8\n" 17 | "Content-Transfer-Encoding: 8bit\n" 18 | "Plural-Forms: nplurals=2; plural=n == 1 ? 0 : 1;\n" 19 | 20 | #: bin/gpo:172 21 | msgid "Install mygpoclient for gpodder.net features" 22 | msgstr "Für gpodder.net-Funktionen muss mygpoclient installiert sein" 23 | 24 | #: bin/gpo:307 bin/gpo:946 25 | #, python-format 26 | msgid "Invalid URL: %(url)s" 27 | msgstr "Ungültige URL: %(url)s" 28 | 29 | #: bin/gpo:320 30 | #, python-format 31 | msgid "Not subscribed to %(url)s" 32 | msgstr "Kein Abonnement für %(url)s" 33 | 34 | #: bin/gpo:332 35 | #, python-format 36 | msgid "Subscribing to %(url)s" 37 | msgstr "Abonniere %(url)s" 38 | 39 | #: bin/gpo:337 40 | #, python-format 41 | msgid "Already subscribed to %(url)s" 42 | msgstr "Abonement existiert bereits: %(url)s" 43 | 44 | #: bin/gpo:343 45 | #, python-format 46 | msgid "Subscription to %(url)s failed" 47 | msgstr "Abonnieren von %(url)s fehlgeschlagen" 48 | 49 | #: bin/gpo:352 50 | #, python-format 51 | msgid "Resolved feed URL: %(url)s" 52 | msgstr "Aufgelöste Feed-URL: %(url)s" 53 | 54 | #: bin/gpo:386 55 | #, python-format 56 | msgid "Configuration option %(key)s not found" 57 | msgstr "Konfigurations-Option %(key)s nicht gefunden" 58 | 59 | #: bin/gpo:390 60 | #, python-format 61 | msgid "Invalid configuration option: %(key)s" 62 | msgstr "Ungültige Konfigurations-Option: %(key)s" 63 | 64 | #: bin/gpo:425 65 | #, python-format 66 | msgid "Removed %(url)s" 67 | msgstr "%(url)s entfernt" 68 | 69 | #: bin/gpo:516 70 | msgid "Usage: query EQL" 71 | msgstr "Aufruf: query SQL" 72 | 73 | #: bin/gpo:535 74 | #, python-format 75 | msgid "%(count)d episode matched" 76 | msgid_plural "%(count)d episodes matched" 77 | msgstr[0] "%(count)d Episode gefunden" 78 | msgstr[1] "%(count)d Episoden gefunden" 79 | 80 | #: bin/gpo:537 81 | msgid "Use \"apply\" to apply a single-episode command" 82 | msgstr "Benutze \"apply\" um einen Episoden-Befehl anzuwenden" 83 | 84 | #: bin/gpo:545 85 | msgid "Subscription suspended" 86 | msgstr "Abonnement ausgesetzt" 87 | 88 | #: bin/gpo:569 89 | #, python-format 90 | msgid "%(count)d new episode" 91 | msgid_plural "%(count)d new episodes" 92 | msgstr[0] "%(count)d neue Episode" 93 | msgstr[1] "%(count)d neue Episoden" 94 | 95 | #: bin/gpo:584 96 | msgid "Checking for new episodes" 97 | msgstr "Neue Episoden werden gesucht" 98 | 99 | #: bin/gpo:591 100 | #, python-format 101 | msgid "Skipping %(podcast)s" 102 | msgstr "Überspringe %(podcast)s" 103 | 104 | #: bin/gpo:636 105 | #, python-format 106 | msgid "Invalid episode ID: %(error)s" 107 | msgstr "Fehlerhafte Episoden-ID: %(error)s" 108 | 109 | #: bin/gpo:644 110 | #, python-format 111 | msgid "Episode ID not found: %(id)x" 112 | msgstr "Episoden-ID nicht gefunden: %(id)x" 113 | 114 | #: bin/gpo:663 115 | #, python-format 116 | msgid "Invalid action: %(action)s. Valid actions: %(valid_actions)s" 117 | msgstr "Ungültiger Befehl: %(action)s. Verfügbare Befehle: %(valid_actions)s" 118 | 119 | #: bin/gpo:672 120 | #, python-format 121 | msgid "Episode marked as old: %(title)s" 122 | msgstr "Episode als alt markiert: %(title)s" 123 | 124 | #: bin/gpo:675 125 | #, python-format 126 | msgid "Episode marked as new: %(title)s" 127 | msgstr "Episode als neu markiert: %(title)s" 128 | 129 | #: bin/gpo:715 130 | #, python-format 131 | msgid "Episode deleted: %(episode)s" 132 | msgstr "Episode gelöscht: %(episode)s" 133 | 134 | #: bin/gpo:768 135 | msgid "\"apply\" can only be used during an interactive session" 136 | msgstr "\"apply\" kann nur während einer interaktiven Sitzung verwendet werden" 137 | 138 | #: bin/gpo:772 139 | msgid "Empty query result (use \"query\" first)" 140 | msgstr "Leeres Abfrage-Ergebnis (zuerst \"query\" benutzen)" 141 | 142 | #: bin/gpo:776 143 | msgid "Cannot apply this command" 144 | msgstr "Kann diesen Befehl nicht anwenden" 145 | 146 | #: bin/gpo:787 147 | msgid "Please provide a search query" 148 | msgstr "Bitte eine Suchanfrage eingeben" 149 | 150 | #: bin/gpo:803 151 | #, python-format 152 | msgid "%(count)d episode downloaded" 153 | msgid_plural "%(count)d episodes downloaded" 154 | msgstr[0] "%(count)d Episode heruntergeladen" 155 | msgstr[1] "%(count)d Episoden heruntergeladen" 156 | 157 | #: bin/gpo:806 158 | #, python-format 159 | msgid "Downloading %(episode)s" 160 | msgstr "Lade %(episode)s herunter" 161 | 162 | #: bin/gpo:858 163 | #, python-format 164 | msgid "Subscription suspended: %(url)s" 165 | msgstr "Abonnement ausgesetzt: %(url)s" 166 | 167 | #: bin/gpo:879 168 | #, python-format 169 | msgid "Subscription resumed: %(url)s" 170 | msgstr "Abonnement fortgesetzt: %(url)s" 171 | 172 | #: bin/gpo:899 173 | msgid "No software updates available" 174 | msgstr "Keine Software-Aktualisierungen verfügbar" 175 | 176 | #: bin/gpo:901 177 | #, python-format 178 | msgid "New version %(latestversion)s available (released: %(latestdate)s)" 179 | msgstr "Neue Version %(latestversion)s verfügbar (veröffentlicht: %(latestdate)s)" 180 | 181 | #: bin/gpo:902 182 | #, python-format 183 | msgid "You have version %(thisversion)s (released: %(thisdate)s)" 184 | msgstr "Sie haben Version %(thisversion)s (veröffentlicht: %(thisdate)s)" 185 | 186 | #: bin/gpo:903 187 | #, python-format 188 | msgid "Download the new version from %(url)s" 189 | msgstr "Die neue Version kann von %(url)s heruntergeladen werden" 190 | 191 | #: bin/gpo:965 192 | msgid "" 193 | msgstr "" 194 | 195 | #: bin/gpo:966 196 | #, python-format 197 | msgid "Use \"help %(command)s\" to get detailed help for a command" 198 | msgstr "Benutze \"help %(command)s\", um detaillierte Hilfe für ein Kommando zu bekommen" 199 | 200 | #: bin/gpo:972 201 | msgid "No documentation available (command exists)" 202 | msgstr "Keine Dokumentation verfügbar (Befehl existiert)" 203 | 204 | #: bin/gpo:978 205 | msgid "No such command" 206 | msgstr "Befehl existiert nicht" 207 | 208 | #: bin/gpo:1035 209 | #, python-format 210 | msgid "Syntax error: %(error)s" 211 | msgstr "Syntax-Fehler: %(error)s" 212 | 213 | #: bin/gpo:1127 214 | msgid "Incomplete command; matching commands:" 215 | msgstr "Unvollständiger Befehl; passende Befehle:" 216 | 217 | #: bin/gpo:1131 218 | msgid "Command not found" 219 | msgstr "Befehl nicht gefunden" 220 | -------------------------------------------------------------------------------- /po/messages.pot: -------------------------------------------------------------------------------- 1 | # SOME DESCRIPTIVE TITLE. 2 | # Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER 3 | # This file is distributed under the same license as the PACKAGE package. 4 | # FIRST AUTHOR , YEAR. 5 | # 6 | #, fuzzy 7 | msgid "" 8 | msgstr "" 9 | "Project-Id-Version: PACKAGE VERSION\n" 10 | "Report-Msgid-Bugs-To: \n" 11 | "POT-Creation-Date: 2014-03-14 23:28+0100\n" 12 | "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" 13 | "Last-Translator: FULL NAME \n" 14 | "Language-Team: LANGUAGE \n" 15 | "Language: \n" 16 | "MIME-Version: 1.0\n" 17 | "Content-Type: text/plain; charset=CHARSET\n" 18 | "Content-Transfer-Encoding: 8bit\n" 19 | "Plural-Forms: nplurals=INTEGER; plural=EXPRESSION;\n" 20 | 21 | #: bin/gpo:172 22 | msgid "Install mygpoclient for gpodder.net features" 23 | msgstr "" 24 | 25 | #: bin/gpo:307 bin/gpo:946 26 | #, python-format 27 | msgid "Invalid URL: %(url)s" 28 | msgstr "" 29 | 30 | #: bin/gpo:320 31 | #, python-format 32 | msgid "Not subscribed to %(url)s" 33 | msgstr "" 34 | 35 | #: bin/gpo:332 36 | #, python-format 37 | msgid "Subscribing to %(url)s" 38 | msgstr "" 39 | 40 | #: bin/gpo:337 41 | #, python-format 42 | msgid "Already subscribed to %(url)s" 43 | msgstr "" 44 | 45 | #: bin/gpo:343 46 | #, python-format 47 | msgid "Subscription to %(url)s failed" 48 | msgstr "" 49 | 50 | #: bin/gpo:352 51 | #, python-format 52 | msgid "Resolved feed URL: %(url)s" 53 | msgstr "" 54 | 55 | #: bin/gpo:386 56 | #, python-format 57 | msgid "Configuration option %(key)s not found" 58 | msgstr "" 59 | 60 | #: bin/gpo:390 61 | #, python-format 62 | msgid "Invalid configuration option: %(key)s" 63 | msgstr "" 64 | 65 | #: bin/gpo:425 66 | #, python-format 67 | msgid "Removed %(url)s" 68 | msgstr "" 69 | 70 | #: bin/gpo:516 71 | msgid "Usage: query EQL" 72 | msgstr "" 73 | 74 | #: bin/gpo:535 75 | #, python-format 76 | msgid "%(count)d episode matched" 77 | msgid_plural "%(count)d episodes matched" 78 | msgstr[0] "" 79 | msgstr[1] "" 80 | 81 | #: bin/gpo:537 82 | msgid "Use \"apply\" to apply a single-episode command" 83 | msgstr "" 84 | 85 | #: bin/gpo:545 86 | msgid "Subscription suspended" 87 | msgstr "" 88 | 89 | #: bin/gpo:569 90 | #, python-format 91 | msgid "%(count)d new episode" 92 | msgid_plural "%(count)d new episodes" 93 | msgstr[0] "" 94 | msgstr[1] "" 95 | 96 | #: bin/gpo:584 97 | msgid "Checking for new episodes" 98 | msgstr "" 99 | 100 | #: bin/gpo:591 101 | #, python-format 102 | msgid "Skipping %(podcast)s" 103 | msgstr "" 104 | 105 | #: bin/gpo:636 106 | #, python-format 107 | msgid "Invalid episode ID: %(error)s" 108 | msgstr "" 109 | 110 | #: bin/gpo:644 111 | #, python-format 112 | msgid "Episode ID not found: %(id)x" 113 | msgstr "" 114 | 115 | #: bin/gpo:663 116 | #, python-format 117 | msgid "Invalid action: %(action)s. Valid actions: %(valid_actions)s" 118 | msgstr "" 119 | 120 | #: bin/gpo:672 121 | #, python-format 122 | msgid "Episode marked as old: %(title)s" 123 | msgstr "" 124 | 125 | #: bin/gpo:675 126 | #, python-format 127 | msgid "Episode marked as new: %(title)s" 128 | msgstr "" 129 | 130 | #: bin/gpo:715 131 | #, python-format 132 | msgid "Episode deleted: %(episode)s" 133 | msgstr "" 134 | 135 | #: bin/gpo:768 136 | msgid "\"apply\" can only be used during an interactive session" 137 | msgstr "" 138 | 139 | #: bin/gpo:772 140 | msgid "Empty query result (use \"query\" first)" 141 | msgstr "" 142 | 143 | #: bin/gpo:776 144 | msgid "Cannot apply this command" 145 | msgstr "" 146 | 147 | #: bin/gpo:787 148 | msgid "Please provide a search query" 149 | msgstr "" 150 | 151 | #: bin/gpo:803 152 | #, python-format 153 | msgid "%(count)d episode downloaded" 154 | msgid_plural "%(count)d episodes downloaded" 155 | msgstr[0] "" 156 | msgstr[1] "" 157 | 158 | #: bin/gpo:806 159 | #, python-format 160 | msgid "Downloading %(episode)s" 161 | msgstr "" 162 | 163 | #: bin/gpo:858 164 | #, python-format 165 | msgid "Subscription suspended: %(url)s" 166 | msgstr "" 167 | 168 | #: bin/gpo:879 169 | #, python-format 170 | msgid "Subscription resumed: %(url)s" 171 | msgstr "" 172 | 173 | #: bin/gpo:899 174 | msgid "No software updates available" 175 | msgstr "" 176 | 177 | #: bin/gpo:901 178 | #, python-format 179 | msgid "New version %(latestversion)s available (released: %(latestdate)s)" 180 | msgstr "" 181 | 182 | #: bin/gpo:902 183 | #, python-format 184 | msgid "You have version %(thisversion)s (released: %(thisdate)s)" 185 | msgstr "" 186 | 187 | #: bin/gpo:903 188 | #, python-format 189 | msgid "Download the new version from %(url)s" 190 | msgstr "" 191 | 192 | #: bin/gpo:965 193 | msgid "" 194 | msgstr "" 195 | 196 | #: bin/gpo:966 197 | #, python-format 198 | msgid "Use \"help %(command)s\" to get detailed help for a command" 199 | msgstr "" 200 | 201 | #: bin/gpo:972 202 | msgid "No documentation available (command exists)" 203 | msgstr "" 204 | 205 | #: bin/gpo:978 206 | msgid "No such command" 207 | msgstr "" 208 | 209 | #: bin/gpo:1035 210 | #, python-format 211 | msgid "Syntax error: %(error)s" 212 | msgstr "" 213 | 214 | #: bin/gpo:1127 215 | msgid "Incomplete command; matching commands:" 216 | msgstr "" 217 | 218 | #: bin/gpo:1131 219 | msgid "Command not found" 220 | msgstr "" 221 | -------------------------------------------------------------------------------- /requirements-test.txt: -------------------------------------------------------------------------------- 1 | nose 2 | minimock 3 | coverage 4 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | podcastparser 2 | minidb 3 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [nosetests] 2 | with-doctest = true 3 | verbose = true 4 | all-modules = true 5 | with-coverage = true 6 | cover-erase = true 7 | cover-package = gpodder 8 | 9 | [pep8] 10 | max-line-length = 120 11 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # setup.py: gPodder Setup Script 4 | # Copyright (c) 2005-2020, Thomas Perl 5 | # 6 | # Permission to use, copy, modify, and/or distribute this software for any 7 | # purpose with or without fee is hereby granted, provided that the above 8 | # copyright notice and this permission notice appear in all copies. 9 | # 10 | # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH 11 | # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY 12 | # AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, 13 | # INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM 14 | # LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR 15 | # OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 16 | # PERFORMANCE OF THIS SOFTWARE. 17 | # 18 | 19 | 20 | import glob 21 | import os 22 | import re 23 | import sys 24 | 25 | from distutils.core import setup 26 | 27 | installing = ('install' in sys.argv and '--help' not in sys.argv) 28 | 29 | # Read the metadata from gPodder's __init__ module (doesn't need importing) 30 | main_module = open('src/gpodder/__init__.py', 'rb').read().decode('utf-8') 31 | metadata = dict(re.findall("__([a-z_]+)__\s*=\s*'([^']+)'", main_module)) 32 | 33 | author, email = re.match(r'^(.*) <(.*)>$', metadata['author']).groups() 34 | 35 | 36 | class MissingFile(Exception): 37 | pass 38 | 39 | 40 | def info(message, item=None): 41 | print('=>', message, item if item is not None else '') 42 | 43 | 44 | def find_data_files(scripts): 45 | # Support for installing only a subset of translations 46 | linguas = os.environ.get('LINGUAS', None) 47 | if linguas is not None: 48 | linguas = linguas.split() 49 | info('Selected languages (from $LINGUAS):', linguas) 50 | 51 | for dirpath, dirnames, filenames in os.walk('share'): 52 | if not filenames: 53 | continue 54 | 55 | # Skip translations if $LINGUAS is set 56 | share_locale = os.path.join('share', 'locale') 57 | if linguas is not None and dirpath.startswith(share_locale): 58 | _, _, language, _ = dirpath.split(os.sep, 3) 59 | if language not in linguas: 60 | info('Skipping translation:', language) 61 | continue 62 | 63 | # Skip manpages if their scripts are not going to be installed 64 | share_man = os.path.join('share', 'man') 65 | if dirpath.startswith(share_man): 66 | def have_script(filename): 67 | if not filename.endswith('.1'): 68 | return True 69 | 70 | basename, _ = os.path.splitext(filename) 71 | result = any(os.path.basename(s) == basename for s in scripts) 72 | if not result: 73 | info('Skipping manpage without script:', filename) 74 | return result 75 | filenames = list(filter(have_script, filenames)) 76 | 77 | def convert_filename(filename): 78 | filename = os.path.join(dirpath, filename) 79 | 80 | # Skip .in files, but check if their target exist 81 | if filename.endswith('.in'): 82 | filename = filename[:-3] 83 | if installing and not os.path.exists(filename): 84 | raise MissingFile(filename) 85 | return None 86 | 87 | return filename 88 | 89 | filenames = [_f for _f in map(convert_filename, filenames) if _f] 90 | if filenames: 91 | # Some distros/ports install manpages into $PREFIX/man instead 92 | # of $PREFIX/share/man (e.g. FreeBSD). To allow this, we strip 93 | # the "share/" part if the variable GPODDER_MANPATH_NO_SHARE is 94 | # set to any value in the environment. 95 | if dirpath.startswith(share_man): 96 | if 'GPODDER_MANPATH_NO_SHARE' in os.environ: 97 | dirpath = dirpath.replace(share_man, 'man') 98 | 99 | yield (dirpath, filenames) 100 | 101 | 102 | def find_packages(): 103 | src_gpodder = os.path.join('src', 'gpodder') 104 | for dirpath, dirnames, filenames in os.walk(src_gpodder): 105 | if '__init__.py' not in filenames: 106 | continue 107 | 108 | dirparts = dirpath.split(os.sep) 109 | dirparts.pop(0) 110 | package = '.'.join(dirparts) 111 | 112 | yield package 113 | 114 | 115 | def find_scripts(): 116 | for dirpath, dirnames, filenames in os.walk('bin'): 117 | for filename in filenames: 118 | yield os.path.join(dirpath, filename) 119 | 120 | 121 | try: 122 | packages = list(sorted(find_packages())) 123 | scripts = list(sorted(find_scripts())) 124 | data_files = list(sorted(find_data_files(scripts))) 125 | except MissingFile as mf: 126 | print(""" 127 | Missing file: %s 128 | 129 | If you want to install, use "make install" instead of using 130 | setup.py directly. See the README file for more information. 131 | """ % mf.message, file=sys.stderr) 132 | sys.exit(1) 133 | 134 | 135 | setup( 136 | name='gpodder-core', 137 | version=metadata['version'], 138 | description=metadata['tagline'], 139 | license=metadata['license'], 140 | url=metadata['url'], 141 | 142 | author=author, 143 | author_email=email, 144 | 145 | package_dir={'': 'src'}, 146 | packages=packages, 147 | scripts=scripts, 148 | data_files=data_files, 149 | ) 150 | -------------------------------------------------------------------------------- /share/man/man1/gpo.1: -------------------------------------------------------------------------------- 1 | .TH GPO "1" "December 2024" "gpodder 4.17.2" "User Commands" 2 | .SH NAME 3 | gpo \- gPodder command-line interface 4 | .SH SYNOPSIS 5 | .B gpo 6 | [\fI--verbose|-v\fR] 7 | [\fICOMMAND\fR] [\fIparams...\fR] 8 | 9 | .SH DESCRIPTION 10 | .PP 11 | gpo is the text mode interface to gPodder. gPodder downloads and manages free 12 | audio and video content ("podcasts") for you. Run it without any arguments to 13 | start the interactive shell, then type "help" for an overview of commands. 14 | 15 | .SH INTERACTIVE SHELL MODE 16 | .PP 17 | If you run "gpo" without \fICOMMAND\fR it will enter its interactive shell 18 | mode. From there, you can type commands directly. When readline is available, 19 | this shell supports completion using . Podcast feed URLs are also 20 | completed for commands that take the URL of a podcast as argument. 21 | 22 | .SH COMMAND PREFIXES 23 | .PP 24 | For all commands, you can use only the first few characters instead of the 25 | full command, as long as the command is unique. If not, gpo will show you all 26 | matching commands and the shortest prefix of each. 27 | .PP 28 | Please note that future additions to the command set could change the shortest 29 | prefix of any given command, so usage of the full command in scripts is 30 | recommended (e.g. use "gpo update" and not "gpo up" in scripts and cronjobs). 31 | The short command prefixes are mostly useful for interactive usage. 32 | 33 | .SH QUERYING USING EQL 34 | .PP 35 | Using the 36 | .I query 37 | command allows you to use the full range of Episode Query Language expressions 38 | that gPodder supports. Combined with the 39 | .I apply 40 | command, this can be very powerful. 41 | .PP 42 | For example, you can mark all episodes that 43 | are videos, not yet downloaded, smaller than 10 MB and released in the last 7 44 | days as new like this (in the interactive shell mode): 45 | .PP 46 | .RS 4 47 | .B query video and not downloaded and mb < 10 and since < 7 48 | .PP 49 | .B apply mark new 50 | .RE 51 | .PP 52 | Similarly, you can mark all new episodes as old with a combination of: 53 | .PP 54 | .RS 4 55 | .B query new 56 | .PP 57 | .B apply mark old 58 | .RE 59 | .PP 60 | Delete all downloaded episodes that have been downloaded more than 15 days ago: 61 | .PP 62 | .RS 4 63 | .B query downloaded and age > 15 64 | .PP 65 | .B apply rm 66 | .RE 67 | .PP 68 | And finally, download all audio files that are shorter than 30 minutes and that 69 | are not yet downloaded: 70 | .PP 71 | .RS 4 72 | .B query audio and minutes < 30 not downloaded 73 | .PP 74 | .B apply fetch 75 | .RE 76 | 77 | .SH EXAMPLES 78 | 79 | .PP 80 | .B gpo 81 | .RS 4 82 | Enter interactive shell mode 83 | .RE 84 | .PP 85 | .B gpo update && gpo download 86 | .RS 4 87 | Check for new episodes, then download all new episodes 88 | .RE 89 | 90 | .SH BUGS 91 | .PP 92 | Report bugs at \fIhttps://bugs.gpodder.org/\fR 93 | -------------------------------------------------------------------------------- /src/gpodder/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # gpodder: Main module with release metadata 3 | # 4 | 5 | """ 6 | gPodder: Media and podcast aggregator 7 | Copyright (c) 2005-2024 Thomas Perl and the gPodder Team 8 | 9 | Historically, gPodder was licensed under the terms of the "GNU GPLv2 or 10 | later", and has been upgraded to "GNU GPLv3 or later" in August 2007. 11 | 12 | Code that has been solely written by thp was re-licensed to a more 13 | permissive license (ISC license) in August 2013. The new license is 14 | DFSG-compatible, FSF-approved, OSI-approved and GPL-compatible (see 15 | http://en.wikipedia.org/wiki/ISC_license for more information). 16 | 17 | For the license that applies to a file, see the copyright header in it. 18 | 19 | 20 | ==== ISC License Text ==== 21 | 22 | Permission to use, copy, modify, and/or distribute this software for any 23 | purpose with or without fee is hereby granted, provided that the above 24 | copyright notice and this permission notice appear in all copies. 25 | 26 | THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH 27 | REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY 28 | AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, 29 | INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM 30 | LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR 31 | OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 32 | PERFORMANCE OF THIS SOFTWARE. 33 | 34 | ==== GPLv3 License Text ==== 35 | 36 | gPodder is free software; you can redistribute it and/or modify 37 | it under the terms of the GNU General Public License as published by 38 | the Free Software Foundation; either version 3 of the License, or 39 | (at your option) any later version. 40 | 41 | gPodder is distributed in the hope that it will be useful, 42 | but WITHOUT ANY WARRANTY; without even the implied warranty of 43 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 44 | GNU General Public License for more details. 45 | 46 | You should have received a copy of the GNU General Public License 47 | along with this program. If not, see . 48 | """ 49 | 50 | # This metadata block gets parsed by setup.py - use single quotes only 51 | __tagline__ = 'Media and podcast aggregator' 52 | __author__ = 'Thomas Perl ' 53 | __version__ = '4.17.2' 54 | __date__ = '2024-12-10' 55 | __relname__ = 'Achva' 56 | __copyright__ = '© 2005-2024 Thomas Perl and the gPodder Team' 57 | __license__ = 'ISC / GPLv3 or later' 58 | __url__ = 'http://gpodder.org/' 59 | 60 | __version_info__ = tuple(int(x) for x in __version__.split('.')) 61 | 62 | # The User-Agent string for downloads 63 | user_agent = 'gPodder/%s (+%s)' % (__version__, __url__) 64 | 65 | # Episode states used in the database 66 | STATE_NORMAL, STATE_DOWNLOADED, STATE_DELETED = list(range(3)) 67 | -------------------------------------------------------------------------------- /src/gpodder/api.py: -------------------------------------------------------------------------------- 1 | # 2 | # gpodder.api - Functions exposed to external applications (2013-05-23) 3 | # Copyright (c) 2013 Thomas Perl 4 | # 5 | # Permission to use, copy, modify, and/or distribute this software for any 6 | # purpose with or without fee is hereby granted, provided that the above 7 | # copyright notice and this permission notice appear in all copies. 8 | # 9 | # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH 10 | # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY 11 | # AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, 12 | # INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM 13 | # LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR 14 | # OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 15 | # PERFORMANCE OF THIS SOFTWARE. 16 | # 17 | 18 | # This module exposes top-level API functionality that can be imported 19 | # by applications not living in the gPodder source tree. It is used to 20 | # determine which functions must be provided by the gPodder tree. 21 | 22 | import gpodder.core 23 | import gpodder.util 24 | import gpodder.query 25 | import gpodder.registry 26 | 27 | 28 | class core: 29 | Core = gpodder.core.Core 30 | 31 | 32 | class util: 33 | run_in_background = gpodder.util.run_in_background 34 | normalize_feed_url = gpodder.util.normalize_feed_url 35 | remove_html_tags = gpodder.util.remove_html_tags 36 | format_date = gpodder.util.format_date 37 | 38 | 39 | class query: 40 | EQL = gpodder.query.EQL 41 | 42 | 43 | class registry: 44 | directory = gpodder.registry.directory 45 | -------------------------------------------------------------------------------- /src/gpodder/common.py: -------------------------------------------------------------------------------- 1 | # 2 | # gpodder.common - Common helper functions for all UIs (2012-08-16) 3 | # Copyright (c) 2012, 2013, Thomas Perl 4 | # 5 | # Permission to use, copy, modify, and/or distribute this software for any 6 | # purpose with or without fee is hereby granted, provided that the above 7 | # copyright notice and this permission notice appear in all copies. 8 | # 9 | # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH 10 | # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY 11 | # AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, 12 | # INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM 13 | # LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR 14 | # OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 15 | # PERFORMANCE OF THIS SOFTWARE. 16 | # 17 | 18 | 19 | import gpodder 20 | 21 | from gpodder import util 22 | 23 | import glob 24 | import os 25 | 26 | import logging 27 | logger = logging.getLogger(__name__) 28 | 29 | 30 | def clean_up_downloads(directory, delete_partial=False): 31 | """Clean up temporary files left behind by old gPodder versions 32 | 33 | delete_partial - If True, also delete in-progress downloads 34 | """ 35 | temporary_files = glob.glob('%s/*/.tmp-*' % directory) 36 | 37 | if delete_partial: 38 | temporary_files += glob.glob('%s/*/*.partial' % directory) 39 | 40 | for tempfile in temporary_files: 41 | util.delete_file(tempfile) 42 | 43 | 44 | def find_partial_downloads(directory, channels, start_progress_callback, progress_callback, 45 | finish_progress_callback): 46 | """Find partial downloads and match them with episodes 47 | 48 | directory - Download directory 49 | channels - A list of all model.PodcastChannel objects 50 | start_progress_callback - A callback(count) when partial files are searched 51 | progress_callback - A callback(title, progress) when an episode was found 52 | finish_progress_callback - A callback(resumable_episodes) when finished 53 | """ 54 | # Look for partial file downloads 55 | partial_files = glob.glob(os.path.join(directory, '*', '*.partial')) 56 | count = len(partial_files) 57 | resumable_episodes = [] 58 | if count: 59 | start_progress_callback(count) 60 | candidates = [f[:-len('.partial')] for f in partial_files] 61 | found = 0 62 | 63 | for channel in channels: 64 | for episode in channel.episodes: 65 | filename = episode.local_filename(create=False, check_only=True) 66 | if filename in candidates: 67 | found += 1 68 | progress_callback(episode.title, float(found)/count) 69 | candidates.remove(filename) 70 | partial_files.remove(filename+'.partial') 71 | 72 | if os.path.exists(filename): 73 | # The file has already been downloaded; 74 | # remove the leftover partial file 75 | util.delete_file(filename+'.partial') 76 | else: 77 | resumable_episodes.append(episode) 78 | 79 | if not candidates: 80 | break 81 | 82 | if not candidates: 83 | break 84 | 85 | for f in partial_files: 86 | logger.warn('Partial file without episode: %s', f) 87 | util.delete_file(f) 88 | 89 | finish_progress_callback(resumable_episodes) 90 | else: 91 | clean_up_downloads(directory, True) 92 | 93 | 94 | def get_expired_episodes(channels, config): 95 | for channel in channels: 96 | for index, episode in enumerate(channel.get_episodes(gpodder.STATE_DOWNLOADED)): 97 | # Never consider archived episodes as old 98 | if episode.archive: 99 | continue 100 | 101 | # Download strategy "Only keep latest" 102 | if (channel.download_strategy == channel.STRATEGY_LATEST and 103 | index > 0): 104 | logger.info('Removing episode (only keep latest strategy): %s', episode.title) 105 | yield episode 106 | continue 107 | 108 | # Only expire episodes if the age in days is positive 109 | if config.episode_old_age < 1: 110 | continue 111 | 112 | # Never consider fresh episodes as old 113 | if episode.age_in_days() < config.episode_old_age: 114 | continue 115 | 116 | # Do not delete played episodes (except if configured) 117 | if not episode.is_new: 118 | if not config.auto_remove_played_episodes: 119 | continue 120 | 121 | # Do not delete unfinished episodes (except if configured) 122 | if not episode.is_finished(): 123 | if not config.auto_remove_unfinished_episodes: 124 | continue 125 | 126 | # Do not delete unplayed episodes (except if configured) 127 | if episode.is_new: 128 | if not config.auto_remove_unplayed_episodes: 129 | continue 130 | 131 | yield episode 132 | -------------------------------------------------------------------------------- /src/gpodder/config.py: -------------------------------------------------------------------------------- 1 | # 2 | # gPodder: Media and podcast aggregator 3 | # Copyright (c) 2005-2020 Thomas Perl and the gPodder Team 4 | # 5 | # gPodder is free software; you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation; either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # gPodder is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | # 18 | 19 | 20 | # 21 | # config.py -- gPodder Configuration Manager 22 | # Thomas Perl 2007-11-02 23 | # 24 | 25 | 26 | import gpodder 27 | from gpodder import util 28 | 29 | from gpodder import jsonconfig 30 | 31 | import os 32 | import shutil 33 | import time 34 | import logging 35 | 36 | defaults = { 37 | # Various limits (downloading, updating, etc..) 38 | 'limit': { 39 | 'bandwidth': { 40 | 'enabled': False, 41 | 'kbps': 500.0, # maximum kB/s per download 42 | }, 43 | 'downloads': { 44 | 'enabled': True, 45 | 'concurrent': 1, 46 | }, 47 | 'episodes': 200, # max episodes per feed 48 | }, 49 | 50 | # Automatic feed updates, download removal and retry on download timeout 51 | 'auto': { 52 | 'update': { 53 | 'enabled': False, 54 | 'frequency': 20, # minutes 55 | }, 56 | 57 | 'cleanup': { 58 | 'days': 7, 59 | 'played': False, 60 | 'unplayed': False, 61 | 'unfinished': True, 62 | }, 63 | 64 | 'retries': 3, # number of retries when downloads time out 65 | }, 66 | 67 | 'fs': { 68 | 'downloads': '' 69 | }, 70 | 71 | 'ui': { 72 | # Settings for the Command-Line Interface 73 | 'cli': { 74 | 'colors': True, 75 | }, 76 | 77 | # Settings for the QML UI 78 | 'qml': { 79 | 'episode_list': { 80 | 'filter_eql': '', 81 | }, 82 | 'playback_speed': { 83 | 'stepSize': 0.1, 84 | 'minimumValue': 0.5, 85 | 'maximumValue': 3.0, 86 | }, 87 | }, 88 | }, 89 | 90 | 'plugins': { 91 | 'youtube': { 92 | 'preferred_fmt_id': 18, # default fmt_id (see fallbacks in youtube.py) 93 | 'preferred_fmt_ids': [], # for advanced uses (custom fallback sequence) 94 | 'api_key_v3': '', # API key, register for one at https://developers.google.com/youtube/v3/ 95 | }, 96 | 'vimeo': { 97 | 'fileformat': 'hd', # preferred format (hd, sd, mobile) 98 | }, 99 | }, 100 | } 101 | 102 | logger = logging.getLogger(__name__) 103 | 104 | 105 | def config_value_to_string(config_value): 106 | config_type = type(config_value) 107 | 108 | if config_type == list: 109 | return ','.join(map(config_value_to_string, config_value)) 110 | elif config_type in (str, str): 111 | return config_value 112 | else: 113 | return str(config_value) 114 | 115 | 116 | def string_to_config_value(new_value, old_value): 117 | config_type = type(old_value) 118 | 119 | if config_type == list: 120 | return [_f for _f in [x.strip() for x in new_value.split(',')] if _f] 121 | elif config_type == bool: 122 | return (new_value.strip().lower() in ('1', 'true')) 123 | else: 124 | return config_type(new_value) 125 | 126 | 127 | class Config(object): 128 | # Number of seconds after which settings are auto-saved 129 | WRITE_TO_DISK_TIMEOUT = 60 130 | 131 | def __init__(self, filename='gpodder.json'): 132 | self.__json_config = jsonconfig.JsonConfig(default=defaults, 133 | on_key_changed=self._on_key_changed) 134 | self.__save_thread = None 135 | self.__filename = filename 136 | self.__observers = [] 137 | 138 | self.load() 139 | 140 | # If there is no configuration file, we create one here (bug 1511) 141 | if not os.path.exists(self.__filename): 142 | self.save() 143 | 144 | def add_observer(self, callback): 145 | """ 146 | Add a callback function as observer. This callback 147 | will be called when a setting changes. It should 148 | have this signature: 149 | 150 | observer(name, old_value, new_value) 151 | 152 | The "name" is the setting name, the "old_value" is 153 | the value that has been overwritten with "new_value". 154 | """ 155 | if callback not in self.__observers: 156 | self.__observers.append(callback) 157 | else: 158 | logger.warn('Observer already added: %s', repr(callback)) 159 | 160 | def remove_observer(self, callback): 161 | """ 162 | Remove an observer previously added to this object. 163 | """ 164 | if callback in self.__observers: 165 | self.__observers.remove(callback) 166 | else: 167 | logger.warn('Observer not added: %s', repr(callback)) 168 | 169 | def all_keys(self): 170 | return self.__json_config._keys_iter() 171 | 172 | def schedule_save(self): 173 | if self.__save_thread is None: 174 | self.__save_thread = util.run_in_background(self.save_thread_proc, True) 175 | 176 | def save_thread_proc(self): 177 | time.sleep(self.WRITE_TO_DISK_TIMEOUT) 178 | if self.__save_thread is not None: 179 | self.save() 180 | 181 | def close(self): 182 | # If we have outstanding changes to the config, save them 183 | if self.__save_thread is not None: 184 | self.save() 185 | 186 | def save(self, filename=None): 187 | if filename is None: 188 | filename = self.__filename 189 | 190 | logger.info('Flushing settings to disk') 191 | 192 | try: 193 | with util.update_file_safely(filename) as temp_filename: 194 | with open(temp_filename, 'wt') as fp: 195 | fp.write(repr(self.__json_config)) 196 | except Exception as e: 197 | logger.error('Cannot write settings to %s: %s', filename, e) 198 | raise 199 | 200 | self.__save_thread = None 201 | 202 | def load(self, filename=None): 203 | if filename is not None: 204 | self.__filename = filename 205 | 206 | if os.path.exists(self.__filename): 207 | try: 208 | data = open(self.__filename, 'rt').read() 209 | new_keys_added = self.__json_config._restore(data) 210 | except: 211 | logger.warn('Cannot parse config file: %s', self.__filename, exc_info=True) 212 | new_keys_added = False 213 | 214 | if new_keys_added: 215 | logger.info('New default keys added - saving config.') 216 | self.save() 217 | 218 | def toggle_flag(self, name): 219 | setattr(self, name, not getattr(self, name)) 220 | 221 | def get_field(self, name): 222 | """Get the current value of a field""" 223 | return self._lookup(name) 224 | 225 | def update_field(self, name, new_value): 226 | """Update a config field, converting strings to the right types""" 227 | old_value = self._lookup(name) 228 | new_value = string_to_config_value(new_value, old_value) 229 | setattr(self, name, new_value) 230 | return True 231 | 232 | def _on_key_changed(self, name, old_value, value): 233 | logger.debug('%s: %s -> %s', name, old_value, value) 234 | for observer in self.__observers: 235 | try: 236 | observer(name, old_value, value) 237 | except Exception as exception: 238 | logger.error('Error while calling observer %r: %s', observer, exception, 239 | exc_info=True) 240 | 241 | self.schedule_save() 242 | 243 | def __getattr__(self, name): 244 | return getattr(self.__json_config, name) 245 | 246 | def __setattr__(self, name, value): 247 | if name.startswith('_'): 248 | object.__setattr__(self, name, value) 249 | return 250 | 251 | setattr(self.__json_config, name, value) 252 | -------------------------------------------------------------------------------- /src/gpodder/core.py: -------------------------------------------------------------------------------- 1 | # 2 | # gpodder.core - Common functionality used by all UIs (2011-02-06) 3 | # Copyright (c) 2011-2013, Thomas Perl 4 | # Copyright (c) 2011, Neal H. Walfield 5 | # Copyright (c) 2012, Bernd Schlapsi 6 | # 7 | # Permission to use, copy, modify, and/or distribute this software for any 8 | # purpose with or without fee is hereby granted, provided that the above 9 | # copyright notice and this permission notice appear in all copies. 10 | # 11 | # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH 12 | # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY 13 | # AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, 14 | # INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM 15 | # LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR 16 | # OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 17 | # PERFORMANCE OF THIS SOFTWARE. 18 | # 19 | 20 | 21 | import gpodder 22 | 23 | from gpodder import util 24 | from gpodder import config 25 | from gpodder import storage 26 | from gpodder import coverart 27 | from gpodder import model 28 | from gpodder import log 29 | 30 | import os 31 | import logging 32 | import socket 33 | 34 | 35 | class Core(object): 36 | def __init__(self, 37 | config_class=config.Config, 38 | database_class=storage.Database, 39 | model_class=model.Model, 40 | verbose=True, 41 | progname='gpodder', 42 | stdout=False): 43 | self._set_socket_timeout() 44 | 45 | home = os.path.expanduser('~') 46 | 47 | xdg_data_home = os.environ.get('XDG_DATA_HOME', os.path.join(home, '.local', 'share')) 48 | xdg_config_home = os.environ.get('XDG_CONFIG_HOME', os.path.join(home, '.config')) 49 | xdg_cache_home = os.environ.get('XDG_CACHE_HOME', os.path.join(home, '.cache')) 50 | 51 | self.data_home = os.path.join(xdg_data_home, progname) 52 | self.config_home = os.path.join(xdg_config_home, progname) 53 | self.cache_home = os.path.join(xdg_cache_home, progname) 54 | 55 | # Use $GPODDER_HOME to set a fixed config and data folder 56 | if 'GPODDER_HOME' in os.environ: 57 | home = os.environ['GPODDER_HOME'] 58 | self.data_home = self.config_home = self.cache_home = home 59 | 60 | # Setup logging 61 | log.setup(self.cache_home, verbose, stdout) 62 | self.logger = logging.getLogger(__name__) 63 | 64 | config_file = os.path.join(self.config_home, 'Settings.json') 65 | database_file = os.path.join(self.data_home, 'Database') 66 | # Downloads go to or $GPODDER_DOWNLOAD_DIR 67 | self.downloads = os.environ.get('GPODDER_DOWNLOAD_DIR', os.path.join(self.data_home)) 68 | 69 | # Read config and change default directories where needed 70 | self.config = config_class(config_file) 71 | 72 | if self.config.fs.downloads != '': 73 | self.downloads = self.config.fs.downloads 74 | 75 | # Initialize the gPodder home directories 76 | util.make_directory(self.data_home) 77 | util.make_directory(self.config_home) 78 | 79 | if self.data_home != self.downloads: 80 | if not util.make_directory(self.downloads): 81 | self.logger.warn('Custom downloads path [%s] not writable reverting to default', self.downloads) 82 | self.downloads = os.environ.get('GPODDER_DOWNLOAD_DIR', os.path.join(self.data_home)) 83 | 84 | # Open the database and configuration file 85 | self.db = database_class(database_file, verbose) 86 | self.model = model_class(self) 87 | 88 | # Load installed/configured plugins 89 | self._load_plugins() 90 | 91 | self.cover_downloader = coverart.CoverDownloader(self) 92 | 93 | def _set_socket_timeout(self): 94 | # Set up socket timeouts to fix bug 174 95 | SOCKET_TIMEOUT = 60 96 | socket.setdefaulttimeout(SOCKET_TIMEOUT) 97 | 98 | def _load_plugins(self): 99 | # Plugins to load by default 100 | DEFAULT_PLUGINS = [ 101 | # Custom handlers (tried in order, put most specific first) 102 | #'gpodder.plugins.soundcloud', 103 | 'gpodder.plugins.itunes', 104 | 'gpodder.plugins.youtube', 105 | 'gpodder.plugins.vimeo', 106 | 'gpodder.plugins.podverse', 107 | 108 | # Directory plugins 109 | #'gpodder.plugins.gpoddernet', 110 | 111 | # Fallback handlers (catch-all) 112 | 'gpodder.plugins.podcast', 113 | ] 114 | 115 | PLUGINS = os.environ.get('GPODDER_PLUGINS', None) 116 | if PLUGINS is None: 117 | PLUGINS = DEFAULT_PLUGINS 118 | else: 119 | PLUGINS = PLUGINS.split() 120 | ADD_PLUGINS = os.environ.get('GPODDER_ADD_PLUGINS', None) 121 | if ADD_PLUGINS is not None: 122 | PLUGINS += ADD_PLUGINS.split() 123 | 124 | for plugin in PLUGINS: 125 | try: 126 | __import__(plugin) 127 | except Exception as e: 128 | self.logger.warn('Cannot load plugin "%s": %s', plugin, e, exc_info=True) 129 | 130 | def save(self): 131 | # XXX: Although the function is called close(), this actually doesn't 132 | # close the DB, just saves the current state to disk 133 | self.db.commit() 134 | 135 | def shutdown(self): 136 | self.logger.info('Shutting down core') 137 | 138 | # Close the configuration and store outstanding changes 139 | self.config.close() 140 | 141 | # Close the database and store outstanding changes 142 | self.db.close() 143 | -------------------------------------------------------------------------------- /src/gpodder/coverart.py: -------------------------------------------------------------------------------- 1 | # 2 | # gpodder.coverart - Unified cover art downloading module (2012-03-04) 3 | # Copyright (c) 2012, 2013, Thomas Perl 4 | # 5 | # Permission to use, copy, modify, and/or distribute this software for any 6 | # purpose with or without fee is hereby granted, provided that the above 7 | # copyright notice and this permission notice appear in all copies. 8 | # 9 | # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH 10 | # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY 11 | # AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, 12 | # INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM 13 | # LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR 14 | # OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 15 | # PERFORMANCE OF THIS SOFTWARE. 16 | # 17 | 18 | 19 | import gpodder 20 | 21 | import logging 22 | logger = logging.getLogger(__name__) 23 | 24 | from gpodder import util 25 | from gpodder import registry 26 | 27 | import os 28 | 29 | 30 | class CoverDownloader(object): 31 | # File name extension dict, lists supported cover art extensions 32 | # Values: functions that check if some data is of that file type 33 | SUPPORTED_EXTENSIONS = { 34 | '.png': lambda d: d.startswith(b'\x89PNG\r\n\x1a\n\x00'), 35 | '.jpg': lambda d: d.startswith(b'\xff\xd8'), 36 | '.gif': lambda d: d.startswith(b'GIF89a') or d.startswith(b'GIF87a'), 37 | } 38 | 39 | EXTENSIONS = list(SUPPORTED_EXTENSIONS.keys()) 40 | 41 | # Low timeout to avoid unnecessary hangs of GUIs 42 | TIMEOUT = 5 43 | 44 | def __init__(self, core): 45 | self.core = core 46 | 47 | def get_cover(self, podcast, download=False, episode=None): 48 | if episode: 49 | # Get episode art. 50 | filename = episode.art_file 51 | cover_url = episode.episode_art_url 52 | 53 | else: 54 | # Get podcast cover. 55 | filename = podcast.cover_file 56 | cover_url = podcast.cover_url 57 | 58 | if not cover_url: 59 | return None 60 | 61 | username = podcast.auth_username 62 | password = podcast.auth_password 63 | 64 | # Return already existing files 65 | for extension in self.EXTENSIONS: 66 | if os.path.exists(filename + extension): 67 | return 'file://' + filename + extension 68 | 69 | # If allowed to download files, do so here 70 | if download: 71 | cover_url = registry.cover_art.resolve(podcast, cover_url) 72 | 73 | if not cover_url: 74 | return None 75 | 76 | # We have to add username/password, because password-protected 77 | # feeds might keep their cover art also protected (bug 1521) 78 | cover_url = util.url_add_authentication(cover_url, username, password) 79 | 80 | try: 81 | logger.info('Downloading cover art: %s', cover_url) 82 | data = util.urlopen(cover_url, timeout=self.TIMEOUT).read() 83 | except Exception as e: 84 | logger.warn('Cover art download failed: %s', e) 85 | return None 86 | 87 | try: 88 | extension = None 89 | 90 | for filetype, check in list(self.SUPPORTED_EXTENSIONS.items()): 91 | if check(data): 92 | extension = filetype 93 | break 94 | 95 | if not extension: 96 | msg = 'Unknown file type: %s (%r)' % (cover_url, data[:6]) 97 | raise ValueError(msg) 98 | 99 | # Successfully downloaded the cover art - save it! 100 | with util.update_file_safely(filename + extension) as temp_filename: 101 | with open(temp_filename, 'wb') as fp: 102 | fp.write(data) 103 | 104 | return 'file://' + filename + extension 105 | except Exception as e: 106 | logger.warn('Cannot save cover art', exc_info=True) 107 | else: 108 | return cover_url 109 | 110 | return None 111 | -------------------------------------------------------------------------------- /src/gpodder/directory.py: -------------------------------------------------------------------------------- 1 | # 2 | # gpodder.directory - Podcast directory and search providers (2014-10-26) 3 | # Copyright (c) 2014, Thomas Perl 4 | # 5 | # Permission to use, copy, modify, and/or distribute this software for any 6 | # purpose with or without fee is hereby granted, provided that the above 7 | # copyright notice and this permission notice appear in all copies. 8 | # 9 | # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH 10 | # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY 11 | # AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, 12 | # INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM 13 | # LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR 14 | # OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 15 | # PERFORMANCE OF THIS SOFTWARE. 16 | # 17 | 18 | import logging 19 | 20 | logger = logging.getLogger(__name__) 21 | 22 | from gpodder import opml 23 | from gpodder import util 24 | 25 | 26 | class DirectoryEntry(object): 27 | def __init__(self, title, url, image=None, subscribers=-1, description=None): 28 | self.title = title 29 | self.url = url 30 | self.image = image 31 | self.subscribers = subscribers 32 | self.description = description 33 | 34 | 35 | class DirectoryTag(object): 36 | def __init__(self, tag, weight): 37 | self.tag = tag 38 | self.weight = weight 39 | 40 | 41 | class Provider(object): 42 | PROVIDER_SEARCH, PROVIDER_URL, PROVIDER_FILE, PROVIDER_TAGCLOUD, PROVIDER_STATIC = range(5) 43 | 44 | PRIORITY_GETTING_STARTED = 900 45 | PRIORITY_PRIMARY_SEARCH = 800 46 | PRIORITY_PRIMARY_TOPLIST = 700 47 | PRIORITY_PRIMARY_TAGS = 600 48 | PRIORITY_SECONDARY_SEARCH = 200 49 | 50 | def __init__(self): 51 | self.name = '' 52 | self.kind = self.PROVIDER_SEARCH 53 | self.priority = 0 54 | 55 | def on_string(self, query): 56 | if self.kind == self.PROVIDER_SEARCH: 57 | return self.on_search(query) 58 | elif self.kind == self.PROVIDER_URL: 59 | return self.on_url(query) 60 | elif self.kind == self.PROVIDER_FILE: 61 | return self.on_file(query) 62 | elif self.kind == self.PROVIDER_TAGCLOUD: 63 | return self.on_tag(query) 64 | elif self.kind == self.PROVIDER_STATIC: 65 | return self.on_static() 66 | 67 | def on_search(self, query): 68 | # Should return a list of DirectoryEntry objects 69 | raise NotImplemented() 70 | 71 | def on_url(self, url): 72 | # Should return a list of DirectoryEntry objects 73 | raise NotImplemented() 74 | 75 | def on_file(self, filename): 76 | # Should return a list of DirectoryEntry objects 77 | raise NotImplemented() 78 | 79 | def on_tag(self, tag): 80 | # Should return a list of DirectoryEntry objects 81 | raise NotImplemented() 82 | 83 | def on_static(self): 84 | # Should return a list of DirectoryEntry objects 85 | raise NotImplemented() 86 | 87 | def get_tags(self): 88 | # Should return a list of DirectoryTag objects 89 | raise NotImplemented() 90 | 91 | 92 | def directory_entry_from_opml(url): 93 | return [DirectoryEntry(d['title'], d['url'], description=d['description']) for d in opml.Importer(url).items] 94 | 95 | 96 | def directory_entry_from_mygpo_json(url): 97 | return [DirectoryEntry(d['title'], d['url'], d['logo_url'], d['subscribers'], d['description']) 98 | for d in util.read_json(url)] 99 | -------------------------------------------------------------------------------- /src/gpodder/download.py: -------------------------------------------------------------------------------- 1 | # 2 | # gPodder: Media and podcast aggregator 3 | # Copyright (c) 2005-2020 Thomas Perl and the gPodder Team 4 | # 5 | # gPodder is free software; you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation; either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # gPodder is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | # 18 | 19 | 20 | # 21 | # download.py -- Download queue management 22 | # Thomas Perl 2007-09-15 23 | # 24 | # Based on libwget.py (2005-10-29) 25 | # 26 | 27 | import logging 28 | logger = logging.getLogger(__name__) 29 | 30 | from gpodder import util 31 | from gpodder import registry 32 | 33 | import gpodder 34 | 35 | import socket 36 | import threading 37 | import urllib.request 38 | import urllib.parse 39 | import urllib.error 40 | import urllib.parse 41 | import shutil 42 | import os.path 43 | import os 44 | import time 45 | import collections 46 | 47 | import mimetypes 48 | import email 49 | 50 | from email.header import decode_header 51 | 52 | 53 | def get_header_param(headers, param, header_name): 54 | """Extract a HTTP header parameter from a dict 55 | 56 | Uses the "email" module to retrieve parameters 57 | from HTTP headers. This can be used to get the 58 | "filename" parameter of the "content-disposition" 59 | header for downloads to pick a good filename. 60 | 61 | Returns None if the filename cannot be retrieved. 62 | """ 63 | value = None 64 | try: 65 | headers_string = ['%s:%s' % (k, v) for k, v in list(headers.items())] 66 | msg = email.message_from_string('\n'.join(headers_string)) 67 | if header_name in msg: 68 | raw_value = msg.get_param(param, header=header_name) 69 | if raw_value is not None: 70 | value = email.utils.collapse_rfc2231_value(raw_value) 71 | except Exception as e: 72 | logger.error('Cannot get %s from %s', param, header_name, exc_info=True) 73 | 74 | return value 75 | 76 | 77 | class ContentRange(object): 78 | # Based on: 79 | # http://svn.pythonpaste.org/Paste/WebOb/trunk/webob/byterange.py 80 | # 81 | # Copyright (c) 2007 Ian Bicking and Contributors 82 | # 83 | # Permission is hereby granted, free of charge, to any person obtaining 84 | # a copy of this software and associated documentation files (the 85 | # "Software"), to deal in the Software without restriction, including 86 | # without limitation the rights to use, copy, modify, merge, publish, 87 | # distribute, sublicense, and/or sell copies of the Software, and to 88 | # permit persons to whom the Software is furnished to do so, subject to 89 | # the following conditions: 90 | # 91 | # The above copyright notice and this permission notice shall be 92 | # included in all copies or substantial portions of the Software. 93 | # 94 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 95 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 96 | # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 97 | # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 98 | # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 99 | # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 100 | # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 101 | """ 102 | Represents the Content-Range header 103 | 104 | This header is ``start-stop/length``, where stop and length can be 105 | ``*`` (represented as None in the attributes). 106 | """ 107 | 108 | def __init__(self, start, stop, length): 109 | if start < 0: 110 | raise Exception("Bad start: {}, stop: {}, length: {}".format(str(start), str(stop), str(length))) 111 | if stop is None or (stop >= 0 and stop <= start): 112 | raise Exception("Bad stop: {}, start: {}, length: {}".format(str(stop), str(start), str(length))) 113 | 114 | self.start = start 115 | self.stop = stop 116 | self.length = length 117 | 118 | def __repr__(self): 119 | return '<%s %s>' % ( 120 | self.__class__.__name__, 121 | self) 122 | 123 | def __str__(self): 124 | if self.stop is None: 125 | stop = '*' 126 | else: 127 | stop = self.stop + 1 128 | if self.length is None: 129 | length = '*' 130 | else: 131 | length = self.length 132 | return 'bytes %s-%s/%s' % (self.start, stop, length) 133 | 134 | def __iter__(self): 135 | """ 136 | Mostly so you can unpack this, like: 137 | 138 | start, stop, length = res.content_range 139 | """ 140 | return iter([self.start, self.stop, self.length]) 141 | 142 | @classmethod 143 | def parse(cls, value): 144 | """ 145 | Parse the header. May return None if it cannot parse. 146 | """ 147 | if value is None: 148 | return None 149 | value = value.strip() 150 | if not value.startswith('bytes '): 151 | # Unparseable 152 | return None 153 | value = value[len('bytes '):].strip() 154 | if '/' not in value: 155 | # Invalid, no length given 156 | return None 157 | range, length = value.split('/', 1) 158 | if '-' not in range: 159 | # Invalid, no range 160 | return None 161 | start, end = range.split('-', 1) 162 | try: 163 | start = int(start) 164 | if end == '*': 165 | end = None 166 | else: 167 | end = int(end) 168 | if length == '*': 169 | length = None 170 | else: 171 | length = int(length) 172 | except ValueError: 173 | # Parse problem 174 | return None 175 | if end is None: 176 | return cls(start, None, length) 177 | else: 178 | return cls(start, end-1, length) 179 | 180 | 181 | class DownloadCancelledException(Exception): 182 | pass 183 | 184 | 185 | class AuthenticationError(Exception): 186 | pass 187 | 188 | 189 | class gPodderDownloadHTTPError(Exception): 190 | def __init__(self, url, error_code, error_message): 191 | self.url = url 192 | self.error_code = error_code 193 | self.error_message = error_message 194 | 195 | 196 | class DownloadURLOpener(urllib.request.FancyURLopener): 197 | version = gpodder.user_agent 198 | 199 | # Sometimes URLs are not escaped correctly - try to fix them 200 | # (see RFC2396; Section 2.4.3. Excluded US-ASCII Characters) 201 | # FYI: The omission of "%" in the list is to avoid double escaping! 202 | ESCAPE_CHARS = dict((ord(c), '%%%x' % ord(c)) for c in ' <>#"{}|\\^[]`') 203 | 204 | def __init__(self, podcast): 205 | self.podcast = podcast 206 | self._auth_retry_counter = 0 207 | urllib.request.FancyURLopener.__init__(self, None) 208 | 209 | def http_error_default(self, url, fp, errcode, errmsg, headers): 210 | """ 211 | FancyURLopener by default does not raise an exception when 212 | there is some unknown HTTP error code. We want to override 213 | this and provide a function to log the error and raise an 214 | exception, so we don't download the HTTP error page here. 215 | """ 216 | # The following two lines are copied from urllib.URLopener's 217 | # implementation of http_error_default 218 | void = fp.read() 219 | fp.close() 220 | raise gPodderDownloadHTTPError(url, errcode, errmsg) 221 | 222 | def redirect_internal(self, url, fp, errcode, errmsg, headers, data): 223 | """ This is the exact same function that's included with urllib 224 | except with "void = fp.read()" commented out. """ 225 | 226 | if 'location' in headers: 227 | newurl = headers['location'] 228 | elif 'uri' in headers: 229 | newurl = headers['uri'] 230 | else: 231 | return 232 | 233 | # This blocks forever(?) with certain servers (see bug #465) 234 | # void = fp.read() 235 | fp.close() 236 | 237 | # In case the server sent a relative URL, join with original: 238 | newurl = urllib.parse.urljoin(self.type + ":" + url, newurl) 239 | return self.open(newurl) 240 | 241 | # The following is based on Python's urllib.py "URLopener.retrieve" 242 | # Also based on http://mail.python.org/pipermail/python-list/2001-October/110069.html 243 | 244 | def http_error_206(self, url, fp, errcode, errmsg, headers, data=None): 245 | # The next line is taken from urllib's URLopener.open_http 246 | # method, at the end after the line "if errcode == 200:" 247 | return urllib.addinfourl(fp, headers, 'http:' + url) 248 | 249 | def retrieve_resume(self, url, filename, reporthook=None, data=None): 250 | """Download files from an URL; return (headers, real_url) 251 | 252 | Resumes a download if the local filename exists and 253 | the server supports download resuming. 254 | """ 255 | 256 | current_size = 0 257 | tfp = None 258 | if os.path.exists(filename): 259 | try: 260 | current_size = os.path.getsize(filename) 261 | tfp = open(filename, 'ab') 262 | # If the file exists, then only download the remainder 263 | if current_size > 0: 264 | self.addheader('Range', 'bytes=%s-' % (current_size)) 265 | except: 266 | logger.warn('Cannot resume download: %s', filename, exc_info=True) 267 | tfp = None 268 | current_size = 0 269 | 270 | if tfp is None: 271 | tfp = open(filename, 'wb') 272 | 273 | # XXX Fix a problem with bad URLs that are not encoded correctly (bug 549) 274 | 275 | fp = self.open(url, data) 276 | headers = fp.info() 277 | 278 | if current_size > 0: 279 | # We told the server to resume - see if she agrees 280 | # See RFC2616 (206 Partial Content + Section 14.16) 281 | # XXX check status code here, too... 282 | range = ContentRange.parse(headers.get('content-range', '')) 283 | if range is None or range.start != current_size: 284 | # Ok, that did not work. Reset the download 285 | # TODO: seek and truncate if content-range differs from request 286 | tfp.close() 287 | tfp = open(filename, 'wb') 288 | current_size = 0 289 | logger.warn('Cannot resume: Invalid Content-Range (RFC2616).') 290 | 291 | result = headers, fp.geturl() 292 | bs = 1024*8 293 | size = -1 294 | read = current_size 295 | blocknum = int(current_size/bs) 296 | if reporthook: 297 | if 'content-length' in headers: 298 | size = int(headers['content-length']) + current_size 299 | reporthook(blocknum, bs, size) 300 | while read < size or size == -1: 301 | if size == -1: 302 | block = fp.read(bs) 303 | else: 304 | block = fp.read(min(size-read, bs)) 305 | if block == "": 306 | break 307 | read += len(block) 308 | tfp.write(block) 309 | blocknum += 1 310 | if reporthook: 311 | reporthook(blocknum, bs, size) 312 | fp.close() 313 | tfp.close() 314 | del fp 315 | del tfp 316 | 317 | # raise exception if actual size does not match content-length header 318 | if size >= 0 and read < size: 319 | raise urllib.error.ContentTooShortError("retrieval incomplete: got only %i out " 320 | "of %i bytes" % (read, size), result) 321 | 322 | return result 323 | 324 | # end code based on urllib.py 325 | 326 | def prompt_user_passwd(self, host, realm): 327 | # Keep track of authentication attempts, fail after the third one 328 | self._auth_retry_counter += 1 329 | if self._auth_retry_counter > 3: 330 | raise AuthenticationError('Wrong username/password') 331 | 332 | if self.podcast.auth_username or self.podcast.auth_password: 333 | logger.debug('Authenticating as "%s" to "%s" for realm "%s".', 334 | self.podcast.auth_username, host, realm) 335 | return (self.podcast.auth_username, self.podcast.auth_password) 336 | 337 | return (None, None) 338 | 339 | 340 | class DownloadQueueWorker(object): 341 | def __init__(self, queue, exit_callback, continue_check_callback, minimum_tasks): 342 | self.queue = queue 343 | self.exit_callback = exit_callback 344 | self.continue_check_callback = continue_check_callback 345 | 346 | # The minimum amount of tasks that should be downloaded by this worker 347 | # before using the continue_check_callback to determine if it might 348 | # continue accepting tasks. This can be used to forcefully start a 349 | # download, even if a download limit is in effect. 350 | self.minimum_tasks = minimum_tasks 351 | 352 | def __repr__(self): 353 | return threading.current_thread().getName() 354 | 355 | def run(self): 356 | logger.info('Starting new thread: %s', self) 357 | while True: 358 | # Check if this thread is allowed to continue accepting tasks 359 | # (But only after reducing minimum_tasks to zero - see above) 360 | if self.minimum_tasks > 0: 361 | self.minimum_tasks -= 1 362 | elif not self.continue_check_callback(self): 363 | return 364 | 365 | try: 366 | task = self.queue.pop() 367 | logger.info('%s is processing: %s', self, task) 368 | task.run() 369 | task.recycle() 370 | except IndexError as e: 371 | logger.info('No more tasks for %s to carry out.', self) 372 | break 373 | self.exit_callback(self) 374 | 375 | 376 | class DownloadQueueManager(object): 377 | def __init__(self, config): 378 | self._config = config 379 | self.tasks = collections.deque() 380 | 381 | self.worker_threads_access = threading.RLock() 382 | self.worker_threads = [] 383 | 384 | def __exit_callback(self, worker_thread): 385 | with self.worker_threads_access: 386 | self.worker_threads.remove(worker_thread) 387 | 388 | def __continue_check_callback(self, worker_thread): 389 | with self.worker_threads_access: 390 | if len(self.worker_threads) > self._config.limit.downloads.concurrent and \ 391 | self._config.limit.downloads.enabled: 392 | self.worker_threads.remove(worker_thread) 393 | return False 394 | else: 395 | return True 396 | 397 | def spawn_threads(self, force_start=False): 398 | """Spawn new worker threads if necessary 399 | 400 | If force_start is True, forcefully spawn a thread and 401 | let it process at least one episodes, even if a download 402 | limit is in effect at the moment. 403 | """ 404 | with self.worker_threads_access: 405 | if not len(self.tasks): 406 | return 407 | 408 | if force_start or len(self.worker_threads) == 0 or \ 409 | len(self.worker_threads) < self._config.limit.downloads.concurrent or \ 410 | not self._config.limit.downloads.enabled: 411 | # We have to create a new thread here, there's work to do 412 | logger.info('Starting new worker thread.') 413 | 414 | # The new worker should process at least one task (the one 415 | # that we want to forcefully start) if force_start is True. 416 | if force_start: 417 | minimum_tasks = 1 418 | else: 419 | minimum_tasks = 0 420 | 421 | worker = DownloadQueueWorker(self.tasks, self.__exit_callback, 422 | self.__continue_check_callback, minimum_tasks) 423 | self.worker_threads.append(worker) 424 | util.run_in_background(worker.run) 425 | 426 | def add_task(self, task, force_start=False): 427 | """Add a new task to the download queue 428 | 429 | If force_start is True, ignore the download limit 430 | and forcefully start the download right away. 431 | """ 432 | if task.status != DownloadTask.INIT: 433 | # Remove the task from its current position in the 434 | # download queue (if any) to avoid race conditions 435 | # where two worker threads download the same file 436 | try: 437 | self.tasks.remove(task) 438 | except ValueError as e: 439 | pass 440 | task.status = DownloadTask.QUEUED 441 | if force_start: 442 | # Add the task to be taken on next pop 443 | self.tasks.append(task) 444 | else: 445 | # Add the task to the end of the queue 446 | self.tasks.appendleft(task) 447 | self.spawn_threads(force_start) 448 | 449 | 450 | class DownloadTask(object): 451 | """An object representing the download task of an episode 452 | 453 | You can create a new download task like this: 454 | 455 | task = DownloadTask(episode, gpodder.config.Config(CONFIGFILE)) 456 | task.status = DownloadTask.QUEUED 457 | task.run() 458 | 459 | While the download is in progress, you can access its properties: 460 | 461 | task.total_size # in bytes 462 | task.progress # from 0.0 to 1.0 463 | task.speed # in bytes per second 464 | str(task) # name of the episode 465 | task.status # current status 466 | task.status_changed # True if the status has been changed (see below) 467 | task.url # URL of the episode being downloaded 468 | task.podcast_url # URL of the podcast this download belongs to 469 | 470 | You can cancel a running download task by setting its status: 471 | 472 | task.status = DownloadTask.CANCELLED 473 | 474 | The task will then abort as soon as possible (due to the nature 475 | of downloading data, this can take a while when the Internet is 476 | busy). 477 | 478 | The "status_changed" attribute gets set to True everytime the 479 | "status" attribute changes its value. After you get the value of 480 | the "status_changed" attribute, it is always reset to False: 481 | 482 | if task.status_changed: 483 | new_status = task.status 484 | # .. update the UI accordingly .. 485 | 486 | Obviously, this also means that you must have at most *one* 487 | place in your UI code where you check for status changes and 488 | broadcast the status updates from there. 489 | 490 | While the download is taking place and after the .run() method 491 | has finished, you can get the final status to check if the download 492 | was successful: 493 | 494 | if task.status == DownloadTask.DONE: 495 | # .. everything ok .. 496 | elif task.status == DownloadTask.FAILED: 497 | # .. an error happened, and the 498 | # error_message attribute is set .. 499 | print task.error_message 500 | elif task.status == DownloadTask.PAUSED: 501 | # .. user paused the download .. 502 | elif task.status == DownloadTask.CANCELLED: 503 | # .. user cancelled the download .. 504 | 505 | The difference between cancelling and pausing a DownloadTask is 506 | that the temporary file gets deleted when cancelling, but does 507 | not get deleted when pausing. 508 | 509 | Be sure to call .removed_from_list() on this task when removing 510 | it from the UI, so that it can carry out any pending clean-up 511 | actions (e.g. removing the temporary file when the task has not 512 | finished successfully; i.e. task.status != DownloadTask.DONE). 513 | 514 | The UI can call the method "notify_as_finished()" to determine if 515 | this episode still has still to be shown as "finished" download 516 | in a notification window. This will return True only the first time 517 | it is called when the status is DONE. After returning True once, 518 | it will always return False afterwards. 519 | 520 | The same thing works for failed downloads ("notify_as_failed()"). 521 | """ 522 | (INIT, QUEUED, DOWNLOADING, DONE, FAILED, CANCELLED, PAUSED) = list(range(7)) 523 | 524 | # Minimum time between progress updates (in seconds) 525 | MIN_TIME_BETWEEN_UPDATES = 1. 526 | 527 | def __str__(self): 528 | return self.__episode.title 529 | 530 | def __get_status(self): 531 | return self.__status 532 | 533 | def __set_status(self, status): 534 | if status != self.__status: 535 | self.__status_changed = True 536 | self.__status = status 537 | 538 | status = property(fget=__get_status, fset=__set_status) 539 | 540 | def __get_status_changed(self): 541 | if self.__status_changed: 542 | self.__status_changed = False 543 | return True 544 | else: 545 | return False 546 | 547 | status_changed = property(fget=__get_status_changed) 548 | 549 | def __get_url(self): 550 | return self.__episode.url 551 | 552 | url = property(fget=__get_url) 553 | 554 | def __get_podcast_url(self): 555 | return self.__episode.podcast.url 556 | 557 | podcast_url = property(fget=__get_podcast_url) 558 | 559 | def cancel(self): 560 | if self.status in (self.DOWNLOADING, self.QUEUED): 561 | self.status = self.CANCELLED 562 | 563 | def removed_from_list(self): 564 | if self.status != self.DONE: 565 | util.delete_file(self.tempname) 566 | 567 | def __init__(self, episode): 568 | if episode.download_task is not None: 569 | raise Exception('Download already in progress.') 570 | 571 | self.__status = DownloadTask.INIT 572 | self.__status_changed = True 573 | self.__episode = episode 574 | self._config = episode.podcast.model.core.config 575 | 576 | # Create the target filename and save it in the database 577 | self.filename = self.__episode.local_filename(create=True) 578 | self.tempname = self.filename + '.partial' 579 | 580 | self.total_size = self.__episode.file_size 581 | self.speed = 0.0 582 | self.progress = 0.0 583 | self.error_message = None 584 | 585 | # Have we already shown this task in a notification? 586 | self._notification_shown = False 587 | 588 | # Variables for speed limit and speed calculation 589 | self.__start_time = 0 590 | self.__start_blocks = 0 591 | self.__limit_rate_value = self._config.limit.bandwidth.kbps 592 | self.__limit_rate = self._config.limit.bandwidth.enabled 593 | 594 | # Progress update functions 595 | self._progress_updated = None 596 | self._last_progress_updated = 0. 597 | 598 | # If the tempname already exists, set progress accordingly 599 | if os.path.exists(self.tempname): 600 | try: 601 | already_downloaded = os.path.getsize(self.tempname) 602 | if self.total_size > 0: 603 | self.progress = max(0.0, min(1.0, float(already_downloaded)/self.total_size)) 604 | except OSError as os_error: 605 | logger.error('Cannot get size for %s', os_error) 606 | else: 607 | # "touch self.tempname", so we also get partial 608 | # files for resuming when the file is queued 609 | open(self.tempname, 'w').close() 610 | 611 | # Store a reference to this task in the episode 612 | episode.download_task = self 613 | 614 | def notify_as_finished(self): 615 | if self.status == DownloadTask.DONE: 616 | if self._notification_shown: 617 | return False 618 | else: 619 | self._notification_shown = True 620 | return True 621 | 622 | return False 623 | 624 | def notify_as_failed(self): 625 | if self.status == DownloadTask.FAILED: 626 | if self._notification_shown: 627 | return False 628 | else: 629 | self._notification_shown = True 630 | return True 631 | 632 | return False 633 | 634 | def add_progress_callback(self, callback): 635 | self._progress_updated = callback 636 | # Send immediate feedback about the started download 637 | self._progress_updated(self.progress) 638 | 639 | def status_updated(self, count, blockSize, totalSize): 640 | # We see a different "total size" while downloading, 641 | # so correct the total size variable in the thread 642 | if totalSize != self.total_size and totalSize > 0: 643 | self.total_size = float(totalSize) 644 | if self.__episode.file_size != self.total_size: 645 | logger.debug('Updating file size of %s to %s', self.filename, self.total_size) 646 | self.__episode.file_size = self.total_size 647 | self.__episode.save() 648 | 649 | if self.total_size > 0: 650 | self.progress = max(0.0, min(1.0, float(count*blockSize)/self.total_size)) 651 | if self._progress_updated is not None: 652 | diff = time.time() - self._last_progress_updated 653 | if diff > self.MIN_TIME_BETWEEN_UPDATES or self.progress == 1.: 654 | self._progress_updated(self.progress) 655 | self._last_progress_updated = time.time() 656 | 657 | self.calculate_speed(count, blockSize) 658 | 659 | if self.status == DownloadTask.CANCELLED: 660 | raise DownloadCancelledException() 661 | 662 | if self.status == DownloadTask.PAUSED: 663 | raise DownloadCancelledException() 664 | 665 | def calculate_speed(self, count, blockSize): 666 | if count % 5 == 0: 667 | now = time.time() 668 | if self.__start_time > 0: 669 | # Has rate limiting been enabled or disabled? 670 | if self.__limit_rate != self._config.limit.bandwidth.enabled: 671 | # If it has been enabled then reset base time and block count 672 | if self._config.limit.bandwidth.enabled: 673 | self.__start_time = now 674 | self.__start_blocks = count 675 | self.__limit_rate = self._config.limit.bandwidth.enabled 676 | 677 | # Has the rate been changed and are we currently limiting? 678 | if self.__limit_rate_value != self._config.limit.bandwith.kbps and \ 679 | self.__limit_rate: 680 | self.__start_time = now 681 | self.__start_blocks = count 682 | self.__limit_rate_value = self._config.limit.bandwidth.kbps 683 | 684 | passed = now - self.__start_time 685 | if passed > 0: 686 | speed = ((count-self.__start_blocks)*blockSize)/passed 687 | else: 688 | speed = 0 689 | else: 690 | self.__start_time = now 691 | self.__start_blocks = count 692 | passed = now - self.__start_time 693 | speed = count*blockSize 694 | 695 | self.speed = float(speed) 696 | 697 | if self._config.limit.bandwidth.enabled and speed > self._config.limit.bandwidth.kbps: 698 | # calculate the time that should have passed to reach 699 | # the desired download rate and wait if necessary 700 | should_have_passed = (float((count-self.__start_blocks)*blockSize) / 701 | (self._config.limit.bandwidth.kbps*1024.0)) 702 | if should_have_passed > passed: 703 | # sleep a maximum of 10 seconds to not cause time-outs 704 | delay = min(10.0, float(should_have_passed-passed)) 705 | time.sleep(delay) 706 | 707 | def recycle(self): 708 | self.__episode.download_task = None 709 | 710 | def run(self): 711 | # Speed calculation (re-)starts here 712 | self.__start_time = 0 713 | self.__start_blocks = 0 714 | 715 | # If the download has already been cancelled, skip it 716 | if self.status == DownloadTask.CANCELLED: 717 | util.delete_file(self.tempname) 718 | self.progress = 0.0 719 | self.speed = 0.0 720 | return False 721 | 722 | # We only start this download if its status is "queued" 723 | if self.status != DownloadTask.QUEUED: 724 | return False 725 | 726 | # We are downloading this file right now 727 | self.status = DownloadTask.DOWNLOADING 728 | self._notification_shown = False 729 | 730 | try: 731 | # Resolve URL and start downloading the episode 732 | url = registry.download_url.resolve(self.__episode, self.url, self._config) 733 | 734 | downloader = DownloadURLOpener(self.__episode.podcast) 735 | 736 | # HTTP Status codes for which we retry the download 737 | retry_codes = (408, 418, 504, 598, 599) 738 | max_retries = max(0, self._config.auto.retries) 739 | 740 | # Retry the download on timeout (bug 1013) 741 | for retry in range(max_retries + 1): 742 | if retry > 0: 743 | logger.info('Retrying download of %s (%d)', url, retry) 744 | time.sleep(1) 745 | 746 | try: 747 | headers, real_url = downloader.retrieve_resume(url, self.tempname, 748 | reporthook=self.status_updated) 749 | # If we arrive here, the download was successful 750 | break 751 | except urllib.error.ContentTooShortError as ctse: 752 | if retry < max_retries: 753 | logger.info('Content too short: %s - will retry.', url) 754 | continue 755 | raise 756 | except socket.timeout as tmout: 757 | if retry < max_retries: 758 | logger.info('Socket timeout: %s - will retry.', url) 759 | continue 760 | raise 761 | except gPodderDownloadHTTPError as http: 762 | if retry < max_retries and http.error_code in retry_codes: 763 | logger.info('HTTP error %d: %s - will retry.', http.error_code, url) 764 | continue 765 | raise 766 | 767 | new_mimetype = headers.get('content-type', self.__episode.mime_type) 768 | old_mimetype = self.__episode.mime_type 769 | _basename, ext = os.path.splitext(self.filename) 770 | if new_mimetype != old_mimetype or util.wrong_extension(ext): 771 | logger.info('Updating mime type: %s => %s', old_mimetype, new_mimetype) 772 | old_extension = self.__episode.extension() 773 | self.__episode.mime_type = new_mimetype 774 | new_extension = self.__episode.extension() 775 | 776 | # If the desired filename extension changed due to the new 777 | # mimetype, we force an update of the local filename to fix the 778 | # extension. 779 | if old_extension != new_extension or util.wrong_extension(ext): 780 | self.filename = self.__episode.local_filename(create=True, force_update=True) 781 | 782 | # In some cases, the redirect of a URL causes the real filename to 783 | # be revealed in the final URL (e.g. http://gpodder.org/bug/1423) 784 | if real_url != url: 785 | realname, realext = util.filename_from_url(real_url) 786 | 787 | # Only update from redirect if the redirected-to filename has 788 | # a proper extension (this is needed for e.g. YouTube) 789 | if not util.wrong_extension(realext): 790 | real_filename = ''.join((realname, realext)) 791 | self.filename = self.__episode.local_filename(create=True, force_update=True, 792 | template=real_filename) 793 | logger.info('Download was redirected (%s). New filename: %s', real_url, 794 | os.path.basename(self.filename)) 795 | 796 | # Look at the Content-disposition header; use if if available 797 | disposition_filename = get_header_param(headers, 'filename', 'content-disposition') 798 | 799 | # Some servers do send the content-disposition header, but provide 800 | # an empty filename, resulting in an empty string here (bug 1440) 801 | if disposition_filename is not None and disposition_filename != '': 802 | # The server specifies a download filename - try to use it 803 | disposition_filename = os.path.basename(disposition_filename) 804 | self.filename = self.__episode.local_filename(create=True, force_update=True, 805 | template=disposition_filename) 806 | new_mimetype, encoding = mimetypes.guess_type(self.filename) 807 | if new_mimetype is not None: 808 | logger.info('Using content-disposition mimetype: %s', new_mimetype) 809 | self.__episode.mime_type = new_mimetype 810 | 811 | # Re-evaluate filename and tempname to take care of podcast renames 812 | # while downloads are running (which will change both file names) 813 | self.filename = self.__episode.local_filename(create=False) 814 | self.tempname = os.path.join(os.path.dirname(self.filename), 815 | os.path.basename(self.tempname)) 816 | shutil.move(self.tempname, self.filename) 817 | 818 | # Model- and database-related updates after a download has finished 819 | self.__episode.on_downloaded(self.filename) 820 | except DownloadCancelledException: 821 | logger.info('Download has been cancelled/paused: %s', self) 822 | if self.status == DownloadTask.CANCELLED: 823 | util.delete_file(self.tempname) 824 | self.progress = 0.0 825 | self.speed = 0.0 826 | except urllib.error.ContentTooShortError as ctse: 827 | self.status = DownloadTask.FAILED 828 | self.error_message = 'Missing content from server' 829 | except IOError as ioe: 830 | logger.error('%s while downloading "%s": %s', ioe.strerror, 831 | self.__episode.title, ioe.filename, exc_info=True) 832 | self.status = DownloadTask.FAILED 833 | d = {'error': ioe.strerror, 'filename': ioe.filename} 834 | self.error_message = 'I/O Error: %(error)s: %(filename)s' % d 835 | except gPodderDownloadHTTPError as gdhe: 836 | logger.error('HTTP %s while downloading "%s": %s', 837 | gdhe.error_code, self.__episode.title, gdhe.error_message, exc_info=True) 838 | self.status = DownloadTask.FAILED 839 | d = {'code': gdhe.error_code, 'message': gdhe.error_message} 840 | self.error_message = 'HTTP Error %(code)s: %(message)s' % d 841 | except Exception as e: 842 | self.status = DownloadTask.FAILED 843 | logger.error('Download failed: %s', str(e), exc_info=True) 844 | self.error_message = 'Error: %s' % (str(e),) 845 | 846 | if self.status == DownloadTask.DOWNLOADING: 847 | # Everything went well - we're done 848 | self.status = DownloadTask.DONE 849 | if self.total_size <= 0: 850 | self.total_size = util.calculate_size(self.filename) 851 | logger.info('Total size updated to %d', self.total_size) 852 | self.progress = 1.0 853 | registry.after_download.call_each(self.__episode) 854 | return True 855 | 856 | self.speed = 0.0 857 | 858 | # We finished, but not successfully (at least not really) 859 | return False 860 | -------------------------------------------------------------------------------- /src/gpodder/jsonconfig.py: -------------------------------------------------------------------------------- 1 | # 2 | # jsonconfig - JSON-based configuration backend (2012-01-18) 3 | # Copyright (c) 2012, 2013, Thomas Perl 4 | # 5 | # Permission to use, copy, modify, and/or distribute this software for any 6 | # purpose with or without fee is hereby granted, provided that the above 7 | # copyright notice and this permission notice appear in all copies. 8 | # 9 | # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH 10 | # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY 11 | # AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, 12 | # INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM 13 | # LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR 14 | # OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 15 | # PERFORMANCE OF THIS SOFTWARE. 16 | # 17 | 18 | import copy 19 | from functools import reduce 20 | 21 | import json 22 | 23 | 24 | class JsonConfigSubtree(object): 25 | def __init__(self, parent, name): 26 | self._parent = parent 27 | self._name = name 28 | 29 | def __repr__(self): 30 | return '' % (self._name,) 31 | 32 | def _attr(self, name): 33 | return '.'.join((self._name, name)) 34 | 35 | def __getitem__(self, name): 36 | return self._parent._lookup(self._name).__getitem__(name) 37 | 38 | def __delitem__(self, name): 39 | self._parent._lookup(self._name).__delitem__(name) 40 | 41 | def __setitem__(self, name, value): 42 | self._parent._lookup(self._name).__setitem__(name, value) 43 | 44 | def __getattr__(self, name): 45 | if name == 'keys': 46 | # Kludge for using dict() on a JsonConfigSubtree 47 | return getattr(self._parent._lookup(self._name), name) 48 | 49 | return getattr(self._parent, self._attr(name)) 50 | 51 | def __setattr__(self, name, value): 52 | if name.startswith('_'): 53 | object.__setattr__(self, name, value) 54 | else: 55 | self._parent.__setattr__(self._attr(name), value) 56 | 57 | 58 | class JsonConfig(object): 59 | _INDENT = 2 60 | 61 | def __init__(self, data=None, default=None, on_key_changed=None): 62 | """ 63 | Create a new JsonConfig object 64 | 65 | data: A JSON string that contains the data to load (optional) 66 | default: A dict that contains default config values (optional) 67 | on_key_changed: Callback when a value changes (optional) 68 | 69 | The signature of on_key_changed looks like this: 70 | 71 | func(name, old_value, new_value) 72 | 73 | name: The key name, e.g. "ui.gtk.show_toolbar" 74 | old_value: The old value, e.g. False 75 | new_value: The new value, e.g. True 76 | 77 | For newly-set keys, on_key_changed is also called. In this case, 78 | None will be the old_value: 79 | 80 | >>> def callback(*args): print('callback:', args) 81 | >>> c = JsonConfig(on_key_changed=callback) 82 | >>> c.a.b = 10 83 | callback: ('a.b', None, 10) 84 | >>> c.a.b = 11 85 | callback: ('a.b', 10, 11) 86 | >>> c.x.y.z = [1,2,3] 87 | callback: ('x.y.z', None, [1, 2, 3]) 88 | >>> c.x.y.z = 42 89 | callback: ('x.y.z', [1, 2, 3], 42) 90 | 91 | Please note that dict-style access will not call on_key_changed: 92 | 93 | >>> def callback(*args): print('callback:', args) 94 | >>> c = JsonConfig(on_key_changed=callback) 95 | >>> c.a.b = 1 # This works as expected 96 | callback: ('a.b', None, 1) 97 | >>> c.a['c'] = 10 # This doesn't call on_key_changed! 98 | >>> del c.a['c'] # This also doesn't call on_key_changed! 99 | """ 100 | self._default = default 101 | self._data = copy.deepcopy(self._default) or {} 102 | self._on_key_changed = on_key_changed 103 | if data is not None: 104 | self._restore(data) 105 | 106 | def _restore(self, backup): 107 | """ 108 | Restore a previous state saved with repr() 109 | 110 | This function allows you to "snapshot" the current values of 111 | the configuration and reload them later on. Any missing 112 | default values will be added on top of the restored config. 113 | 114 | Returns True if new keys from the default config have been added, 115 | False if no keys have been added (backup contains all default keys) 116 | 117 | >>> c = JsonConfig() 118 | >>> c.a.b = 10 119 | >>> backup = repr(c) 120 | >>> print(c.a.b) 121 | 10 122 | >>> c.a.b = 11 123 | >>> print(c.a.b) 124 | 11 125 | >>> c._restore(backup) 126 | False 127 | >>> print(c.a.b) 128 | 10 129 | """ 130 | self._data = json.loads(backup) 131 | # Add newly-added default configuration options 132 | if self._default is not None: 133 | return self._merge_keys(self._default) 134 | 135 | return False 136 | 137 | def _merge_keys(self, merge_source): 138 | """Merge keys from merge_source into this config object 139 | 140 | Return True if new keys were merged, False otherwise 141 | """ 142 | added_new_key = False 143 | # Recurse into the data and add missing items 144 | work_queue = [(self._data, merge_source)] 145 | while work_queue: 146 | data, default = work_queue.pop() 147 | for key, value in default.items(): 148 | if key not in data: 149 | # Copy defaults for missing key 150 | data[key] = copy.deepcopy(value) 151 | added_new_key = True 152 | elif isinstance(value, dict): 153 | # Recurse into sub-dictionaries 154 | work_queue.append((data[key], value)) 155 | elif isinstance(value, type(data[key])): 156 | # Type mismatch of current value and default 157 | if type(value) == int and type(data[key]) == float: 158 | # Convert float to int if default value is int 159 | data[key] = int(data[key]) 160 | 161 | return added_new_key 162 | 163 | def __repr__(self): 164 | """ 165 | >>> c = JsonConfig('{"a": 1}') 166 | >>> print(c) 167 | { 168 | "a": 1 169 | } 170 | """ 171 | return json.dumps(self._data, indent=self._INDENT, sort_keys=True) 172 | 173 | def _lookup(self, name): 174 | return reduce(lambda d, k: d[k], name.split('.'), self._data) 175 | 176 | def _keys_iter(self): 177 | work_queue = [] 178 | work_queue.append(([], self._data)) 179 | while work_queue: 180 | path, data = work_queue.pop(0) 181 | 182 | if isinstance(data, dict): 183 | for key in sorted(data.keys()): 184 | work_queue.append((path + [key], data[key])) 185 | else: 186 | yield '.'.join(path) 187 | 188 | def __getattr__(self, name): 189 | try: 190 | value = self._lookup(name) 191 | if not isinstance(value, dict): 192 | return value 193 | except KeyError: 194 | pass 195 | 196 | return JsonConfigSubtree(self, name) 197 | 198 | def __setattr__(self, name, value): 199 | if name.startswith('_'): 200 | object.__setattr__(self, name, value) 201 | return 202 | 203 | attrs = name.split('.') 204 | target_dict = self._data 205 | 206 | while attrs: 207 | attr = attrs.pop(0) 208 | if not attrs: 209 | old_value = target_dict.get(attr, None) 210 | if old_value != value or attr not in target_dict: 211 | target_dict[attr] = value 212 | if self._on_key_changed is not None: 213 | self._on_key_changed(name, old_value, value) 214 | break 215 | 216 | target = target_dict.get(attr, None) 217 | if target is None or not isinstance(target, dict): 218 | target_dict[attr] = target = {} 219 | target_dict = target 220 | -------------------------------------------------------------------------------- /src/gpodder/log.py: -------------------------------------------------------------------------------- 1 | # 2 | # gpodder.log - Logging setup (2013-03-02) 3 | # Copyright (c) 2012, 2013, Thomas Perl 4 | # 5 | # Permission to use, copy, modify, and/or distribute this software for any 6 | # purpose with or without fee is hereby granted, provided that the above 7 | # copyright notice and this permission notice appear in all copies. 8 | # 9 | # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH 10 | # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY 11 | # AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, 12 | # INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM 13 | # LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR 14 | # OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 15 | # PERFORMANCE OF THIS SOFTWARE. 16 | # 17 | 18 | 19 | import gpodder 20 | 21 | import glob 22 | import logging 23 | import os 24 | import sys 25 | import time 26 | import traceback 27 | 28 | logger = logging.getLogger(__name__) 29 | 30 | 31 | def setup(home=None, verbose=True, stdout=False): 32 | # Configure basic stdout logging 33 | STDOUT_FMT = '%(asctime)s [%(name)s] %(levelname)s: %(message)s' 34 | logging.basicConfig(format=STDOUT_FMT, level=logging.DEBUG if verbose else logging.WARNING, 35 | stream=sys.stdout if stdout else sys.stderr) 36 | 37 | # Replace except hook with a custom one that logs it as an error 38 | original_excepthook = sys.excepthook 39 | 40 | def on_uncaught_exception(exctype, value, tb): 41 | message = ''.join(traceback.format_exception(exctype, value, tb)) 42 | logger.error('Uncaught exception: %s', message) 43 | original_excepthook(exctype, value, tb) 44 | sys.excepthook = on_uncaught_exception 45 | 46 | if home and os.environ.get('GPODDER_WRITE_LOGS', 'yes') != 'no': 47 | # Configure file based logging 48 | logging_basename = time.strftime('%Y-%m-%d.log') 49 | logging_directory = os.path.join(home, 'Logs') 50 | if not os.path.isdir(logging_directory): 51 | try: 52 | os.makedirs(logging_directory) 53 | except: 54 | logger.warn('Cannot create output directory: %s', logging_directory) 55 | return False 56 | 57 | # Keep logs around for 5 days 58 | LOG_KEEP_DAYS = 5 59 | 60 | # Purge old logfiles if they are older than LOG_KEEP_DAYS days 61 | old_logfiles = glob.glob(os.path.join(logging_directory, '*-*-*.log')) 62 | for old_logfile in old_logfiles: 63 | st = os.stat(old_logfile) 64 | if time.time() - st.st_mtime > 60*60*24*LOG_KEEP_DAYS: 65 | logger.info('Purging old logfile: %s', old_logfile) 66 | try: 67 | os.remove(old_logfile) 68 | except: 69 | logger.warn('Cannot purge logfile: %s', exc_info=True) 70 | 71 | root = logging.getLogger() 72 | logfile = os.path.join(logging_directory, logging_basename) 73 | file_handler = logging.FileHandler(logfile, 'a', 'utf-8') 74 | FILE_FMT = '%(asctime)s [%(name)s] %(levelname)s: %(message)s' 75 | file_handler.setFormatter(logging.Formatter(FILE_FMT)) 76 | root.addHandler(file_handler) 77 | 78 | logger.debug('==== gPodder starts up ====') 79 | 80 | return True 81 | -------------------------------------------------------------------------------- /src/gpodder/opml.py: -------------------------------------------------------------------------------- 1 | # 2 | # gpodder.opml: OPML import and export functionality (2007-08-19) 3 | # Copyright (c) 2007-2013, Thomas Perl 4 | # 5 | # Based on: 6 | # libopmlreader.py (2006-06-13) 7 | # libopmlwriter.py (2005-12-08) 8 | # Copyright (c) 2005-2007 Thomas Perl 9 | # 10 | # Permission to use, copy, modify, and/or distribute this software for any 11 | # purpose with or without fee is hereby granted, provided that the above 12 | # copyright notice and this permission notice appear in all copies. 13 | # 14 | # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH 15 | # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY 16 | # AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, 17 | # INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM 18 | # LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR 19 | # OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 20 | # PERFORMANCE OF THIS SOFTWARE. 21 | # 22 | 23 | 24 | """OPML import and export functionality 25 | 26 | This module contains helper classes to import subscriptions 27 | from OPML files on the web and to export a list of channel 28 | objects to valid OPML 1.1 files that can be used to backup 29 | or distribute gPodder's channel subscriptions. 30 | """ 31 | 32 | import logging 33 | logger = logging.getLogger(__name__) 34 | 35 | from gpodder import util 36 | 37 | import xml.dom.minidom 38 | 39 | import os.path 40 | import os 41 | import shutil 42 | 43 | from email.utils import formatdate 44 | import gpodder 45 | 46 | 47 | class Importer(object): 48 | """ 49 | Helper class to import an OPML feed from protocols 50 | supported by urllib2 (e.g. HTTP) and return a GTK 51 | ListStore that can be displayed in the GUI. 52 | 53 | This class should support standard OPML feeds and 54 | contains workarounds to support odeo.com feeds. 55 | """ 56 | 57 | VALID_TYPES = ('rss', 'link') 58 | 59 | def __init__(self, url, opml_str=None): 60 | """ 61 | Parses the OPML feed from the given URL into 62 | a local data structure containing channel metadata. 63 | """ 64 | self.items = [] 65 | try: 66 | if os.path.exists(url): 67 | doc = xml.dom.minidom.parse(url) 68 | elif opml_str is not None: 69 | doc = xml.dom.minidom.parseString(opml_str) 70 | else: 71 | doc = xml.dom.minidom.parseString(util.urlopen(url).read()) 72 | 73 | section = None 74 | for outline in doc.getElementsByTagName('outline'): 75 | # Make sure we are dealing with a valid link type (ignore case) 76 | otl_type = outline.getAttribute('type') 77 | if otl_type is None or otl_type.lower() not in self.VALID_TYPES: 78 | otl_title = outline.getAttribute('title') 79 | otl_text = outline.getAttribute('text') 80 | #gPodder sections will have name == text, if OPML accepts it type=section 81 | if otl_title is not None and otl_title == otl_text: 82 | section = otl_title 83 | continue 84 | 85 | if outline.getAttribute('xmlUrl') or outline.getAttribute('url'): 86 | channel = {'url': (outline.getAttribute('xmlUrl') or 87 | outline.getAttribute('url')), 88 | 'title': (outline.getAttribute('title') or 89 | outline.getAttribute('text') or 90 | outline.getAttribute('xmlUrl') or 91 | outline.getAttribute('url')), 92 | 'description': (outline.getAttribute('text') or 93 | outline.getAttribute('xmlUrl') or 94 | outline.getAttribute('url')), 95 | 'section': section or 'audio' 96 | } 97 | 98 | if channel['description'] == channel['title']: 99 | channel['description'] = channel['url'] 100 | 101 | for attr in ('url', 'title', 'description'): 102 | channel[attr] = channel[attr].strip() 103 | 104 | self.items.append(channel) 105 | if not len(self.items): 106 | logger.info('OPML import finished, but no items found: %s', url) 107 | except: 108 | logger.error('Cannot import OPML from URL: %s', url, exc_info=True) 109 | 110 | 111 | class Exporter(object): 112 | """ 113 | Helper class to export a list of channel objects 114 | to a local file in OPML 1.1 format. 115 | 116 | See www.opml.org for the OPML specification. 117 | """ 118 | 119 | FEED_TYPE = 'rss' 120 | 121 | def __init__(self, filename): 122 | if filename is None: 123 | self.filename = None 124 | elif filename.endswith('.opml') or filename.endswith('.xml'): 125 | self.filename = filename 126 | else: 127 | self.filename = '%s.opml' % (filename,) 128 | 129 | def create_node(self, doc, name, content): 130 | """ 131 | Creates a simple XML Element node in a document 132 | with tag name "name" and text content "content", 133 | as in content and returns the element. 134 | """ 135 | node = doc.createElement(name) 136 | node.appendChild(doc.createTextNode(content)) 137 | return node 138 | 139 | def create_outline(self, doc, channel): 140 | """ 141 | Creates a OPML outline as XML Element node in a 142 | document for the supplied channel. 143 | """ 144 | outline = doc.createElement('outline') 145 | outline.setAttribute('title', channel.title) 146 | outline.setAttribute('text', channel.description) 147 | outline.setAttribute('xmlUrl', channel.url) 148 | outline.setAttribute('type', self.FEED_TYPE) 149 | return outline 150 | 151 | def create_section(self, doc, name): 152 | """ 153 | Creates an empty OPML ouline element used to divide sections. 154 | """ 155 | section = doc.createElement('outline') 156 | section.setAttribute('title', name) 157 | section.setAttribute('text', name) 158 | return section 159 | 160 | def write(self, channels): 161 | """ 162 | Creates a XML document containing metadata for each 163 | channel object in the "channels" parameter, which 164 | should be a list of channel objects. 165 | 166 | OPML 2.0 specification: http://www.opml.org/spec2 167 | 168 | Returns True on success or False when there was an 169 | error writing the file. 170 | """ 171 | 172 | doc = xml.dom.minidom.Document() 173 | 174 | opml = doc.createElement('opml') 175 | opml.setAttribute('version', '2.0') 176 | doc.appendChild(opml) 177 | 178 | head = doc.createElement('head') 179 | head.appendChild(self.create_node(doc, 'title', 'gPodder subscriptions')) 180 | head.appendChild(self.create_node(doc, 'dateCreated', formatdate(localtime=True))) 181 | opml.appendChild(head) 182 | 183 | body = doc.createElement('body') 184 | sections = {} 185 | for channel in channels: 186 | if channel.section not in sections.keys(): 187 | sections[channel.section] = self.create_section(doc, channel.section) 188 | 189 | sections[channel.section].appendChild(self.create_outline(doc, channel)) 190 | 191 | for section in sections.values(): 192 | body.appendChild(section) 193 | 194 | opml.appendChild(body) 195 | 196 | if self.filename is None: 197 | return doc.toprettyxml(indent=' ', newl=os.linesep) 198 | else: 199 | try: 200 | with util.update_file_safely(self.filename) as temp_filename: 201 | with open(temp_filename, 'w', encoding='utf-8') as fp: 202 | fp.write(doc.toprettyxml(indent=' ', newl=os.linesep)) 203 | except: 204 | logger.error('Could not open file for writing: %s', self.filename, exc_info=True) 205 | return False 206 | 207 | return True 208 | -------------------------------------------------------------------------------- /src/gpodder/plugins/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # gpodder.plugins - Plugins for parsing different types of media feeds 3 | # Copyright (c) 2013, Thomas Perl 4 | # 5 | # Permission to use, copy, modify, and/or distribute this software for any 6 | # purpose with or without fee is hereby granted, provided that the above 7 | # copyright notice and this permission notice appear in all copies. 8 | # 9 | # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH 10 | # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY 11 | # AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, 12 | # INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM 13 | # LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR 14 | # OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 15 | # PERFORMANCE OF THIS SOFTWARE. 16 | # 17 | -------------------------------------------------------------------------------- /src/gpodder/plugins/gpoddernet.py: -------------------------------------------------------------------------------- 1 | # 2 | # gpodder.plugins.gpoddernet: gpodder.net directory integration (2014-10-26) 3 | # Copyright (c) 2014, Thomas Perl 4 | # 5 | # Permission to use, copy, modify, and/or distribute this software for any 6 | # purpose with or without fee is hereby granted, provided that the above 7 | # copyright notice and this permission notice appear in all copies. 8 | # 9 | # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH 10 | # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY 11 | # AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, 12 | # INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM 13 | # LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR 14 | # OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 15 | # PERFORMANCE OF THIS SOFTWARE. 16 | # 17 | 18 | 19 | import gpodder 20 | 21 | from gpodder import util 22 | from gpodder import registry 23 | from gpodder import directory 24 | 25 | import logging 26 | import urllib.parse 27 | 28 | logger = logging.getLogger(__name__) 29 | 30 | 31 | @registry.directory.register_instance 32 | class GPodderNetSearchProvider(directory.Provider): 33 | def __init__(self): 34 | self.name = 'gpodder.net search' 35 | self.kind = directory.Provider.PROVIDER_SEARCH 36 | self.priority = directory.Provider.PRIORITY_PRIMARY_SEARCH 37 | 38 | def on_search(self, query): 39 | return directory.directory_entry_from_mygpo_json('http://gpodder.net/search.json?q=' + 40 | urllib.parse.quote(query)) 41 | 42 | 43 | @registry.directory.register_instance 44 | class GPodderRecommendationsProvider(directory.Provider): 45 | def __init__(self): 46 | self.name = 'Getting started' 47 | self.kind = directory.Provider.PROVIDER_STATIC 48 | self.priority = directory.Provider.PRIORITY_GETTING_STARTED 49 | 50 | def on_static(self): 51 | return directory.directory_entry_from_opml('http://gpodder.org/directory.opml') 52 | 53 | 54 | @registry.directory.register_instance 55 | class GPodderNetToplistProvider(directory.Provider): 56 | def __init__(self): 57 | self.name = 'gpodder.net Top 50' 58 | self.kind = directory.Provider.PROVIDER_STATIC 59 | self.priority = directory.Provider.PRIORITY_PRIMARY_TOPLIST 60 | 61 | def on_static(self): 62 | return directory.directory_entry_from_mygpo_json('http://gpodder.net/toplist/50.json') 63 | 64 | 65 | @registry.directory.register_instance 66 | class GPodderNetTagsProvider(directory.Provider): 67 | def __init__(self): 68 | self.name = 'gpodder.net Tags' 69 | self.kind = directory.Provider.PROVIDER_TAGCLOUD 70 | self.priority = directory.Provider.PRIORITY_PRIMARY_TAGS 71 | 72 | def on_tag(self, tag): 73 | return directory.directory_entry_from_mygpo_json('http://gpodder.net/api/2/tag/%s/50.json' % 74 | urllib.parse.quote(tag)) 75 | 76 | def get_tags(self): 77 | return [DirectoryTag(d['tag'], d['usage']) 78 | for d in json.load(util.urlopen('http://gpodder.net/api/2/tags/40.json'))] 79 | -------------------------------------------------------------------------------- /src/gpodder/plugins/itunes.py: -------------------------------------------------------------------------------- 1 | 2 | # 3 | # gpodder.plugins.itunes: Resolve iTunes feed URLs (based on a gist by Yepoleb, 2014-03-09) 4 | # Copyright (c) 2014, Thomas Perl 5 | # 6 | # Permission to use, copy, modify, and/or distribute this software for any 7 | # purpose with or without fee is hereby granted, provided that the above 8 | # copyright notice and this permission notice appear in all copies. 9 | # 10 | # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH 11 | # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY 12 | # AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, 13 | # INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM 14 | # LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR 15 | # OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 16 | # PERFORMANCE OF THIS SOFTWARE. 17 | # 18 | 19 | 20 | import gpodder 21 | 22 | from gpodder import util, registry, directory 23 | 24 | import re 25 | import logging 26 | import urllib.parse 27 | 28 | logger = logging.getLogger(__name__) 29 | 30 | class ITunesFeedException(Exception): 31 | pass 32 | 33 | 34 | @registry.feed_handler.register 35 | def itunes_feed_handler(channel, max_episodes, config): 36 | m = re.match(r'https?://(podcasts|itunes)\.apple\.com/(?:[^/]*/)?podcast/.*id(?P[0-9]+).*$', channel.url, re.I) 37 | if m is None: 38 | return None 39 | 40 | logger.debug('Detected iTunes feed.') 41 | 42 | itunes_lookup_url = 'https://itunes.apple.com/lookup?entity=podcast&id=' + m.group('podcast_id') 43 | try: 44 | json_data = util.read_json(itunes_lookup_url) 45 | 46 | if len(json_data['results']) != 1: 47 | raise ITunesFeedException('Unsupported number of results: ' + str(len(json_data['results']))) 48 | 49 | feed_url = util.normalize_feed_url(json_data['results'][0]['feedUrl']) 50 | 51 | if not feed_url: 52 | raise ITunesFeedException('Could not resolve real feed URL from iTunes feed.\nDetected URL: ' + json_data['results'][0]['feedUrl']) 53 | 54 | logger.info('Resolved iTunes feed URL: {} -> {}'.format(channel.url, feed_url)) 55 | channel.url = feed_url 56 | 57 | # Delegate further processing of the feed to the normal podcast parser 58 | # by returning None (will try the next handler in the resolver chain) 59 | return None 60 | except Exception as ex: 61 | logger.warn('Cannot resolve iTunes feed: {}'.format(str(ex))) 62 | raise 63 | 64 | @registry.directory.register_instance 65 | class ApplePodcastsSearchProvider(directory.Provider): 66 | def __init__(self): 67 | self.name = 'Apple Podcasts' 68 | self.kind = directory.Provider.PROVIDER_SEARCH 69 | self.priority = directory.Provider.PRIORITY_SECONDARY_SEARCH 70 | 71 | def on_search(self, query): 72 | json_url = 'https://itunes.apple.com/search?media=podcast&term={}'.format(urllib.parse.quote(query)) 73 | 74 | return [directory.DirectoryEntry(entry['collectionName'], entry['feedUrl'], entry['artworkUrl100']) for entry in util.read_json(json_url)['results']] 75 | -------------------------------------------------------------------------------- /src/gpodder/plugins/podcast.py: -------------------------------------------------------------------------------- 1 | # 2 | # gpodder.plugins.podcast: Faster Podcast Parser module for gPodder (2012-12-29) 3 | # Copyright (c) 2012, 2013, Thomas Perl 4 | # 5 | # Permission to use, copy, modify, and/or distribute this software for any 6 | # purpose with or without fee is hereby granted, provided that the above 7 | # copyright notice and this permission notice appear in all copies. 8 | # 9 | # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH 10 | # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY 11 | # AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, 12 | # INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM 13 | # LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR 14 | # OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 15 | # PERFORMANCE OF THIS SOFTWARE. 16 | # 17 | 18 | 19 | import gpodder 20 | 21 | from gpodder import registry 22 | from gpodder import util 23 | 24 | import podcastparser 25 | 26 | import urllib.request 27 | import urllib.error 28 | import urllib.parse 29 | import re 30 | 31 | import logging 32 | 33 | logger = logging.getLogger(__name__) 34 | 35 | 36 | class PodcastParserFeed(object): 37 | # Maximum number of pages for paged feeds 38 | PAGED_FEED_MAX_PAGES = 50 39 | 40 | def __init__(self, channel, max_episodes): 41 | url = channel.authenticate_url(channel.url) 42 | 43 | logger.info('Parsing via podcastparser: %s', url) 44 | 45 | headers = {} 46 | if channel.http_etag: 47 | headers['If-None-Match'] = channel.http_etag 48 | if channel.http_last_modified: 49 | headers['If-Modified-Since'] = channel.http_last_modified 50 | 51 | try: 52 | stream = util.urlopen(url, headers) 53 | self.status = 200 54 | info = stream.info() 55 | self.etag = info.get('etag') 56 | self.modified = info.get('last-modified') 57 | self.parsed = podcastparser.parse(url, stream, max_episodes) 58 | self._handle_paged_feed(max_episodes) 59 | except urllib.error.HTTPError as error: 60 | self.status = error.code 61 | if error.code == 304: 62 | logger.info('Not modified') 63 | else: 64 | logger.warn('Feed update failed: %s', error) 65 | raise error 66 | 67 | self.etag = None 68 | self.modified = None 69 | self.parsed = None 70 | 71 | def was_updated(self): 72 | return (self.status == 200) 73 | 74 | def get_etag(self, default): 75 | return self.etag or default 76 | 77 | def get_modified(self, default): 78 | return self.modified or default 79 | 80 | def get_title(self): 81 | return self.parsed['title'] 82 | 83 | def get_image(self): 84 | return self.parsed.get('cover_url') 85 | 86 | def get_link(self): 87 | return self.parsed.get('link', '') 88 | 89 | def get_description(self): 90 | return self.parsed.get('description', '') 91 | 92 | def get_payment_url(self): 93 | return self.parsed.get('payment_url') 94 | 95 | def _handle_paged_feed(self, max_episodes): 96 | page = 2 97 | remaining_episodes = max_episodes - len(self.parsed['episodes']) 98 | while ('paged_feed_next' in self.parsed and 99 | page < self.PAGED_FEED_MAX_PAGES and 100 | remaining_episodes > 0): 101 | # Take the next page from the paged feed 102 | url = self.parsed['paged_feed_next'] 103 | del self.parsed['paged_feed_next'] 104 | 105 | if not url: 106 | break 107 | 108 | try: 109 | logger.debug('Downloading page %d from %s', page, url) 110 | stream = util.urlopen(url) 111 | parsed = podcastparser.parse(url, stream, remaining_episodes) 112 | added_episodes = len(parsed['episodes']) 113 | remaining_episodes -= added_episodes 114 | logger.debug('Page %d contains %d additional episodes', page, 115 | added_episodes) 116 | self.parsed['episodes'].extend(parsed['episodes']) 117 | 118 | # Next iteration if we still have a next page 119 | if 'paged_feed_next' in parsed: 120 | self.parsed['paged_feed_next'] = parsed['paged_feed_next'] 121 | except Exception as e: 122 | logger.warn('Error while fetching feed page %d from %s: %s', page, url, e) 123 | # Give up, don't try to download additional pages here 124 | break 125 | 126 | page += 1 127 | 128 | def _pick_enclosure(self, episode_dict): 129 | if not episode_dict['enclosures']: 130 | del episode_dict['enclosures'] 131 | return False 132 | 133 | # FIXME: Pick the right enclosure from multiple ones 134 | episode_dict.update(episode_dict['enclosures'][0]) 135 | del episode_dict['enclosures'] 136 | 137 | return True 138 | 139 | def get_new_episodes(self, channel): 140 | existing_guids = dict((episode.guid, episode) for episode in channel.episodes) 141 | seen_guids = [entry['guid'] for entry in self.parsed['episodes']] 142 | new_episodes = [] 143 | 144 | for episode_dict in self.parsed['episodes']: 145 | if not self._pick_enclosure(episode_dict): 146 | continue 147 | 148 | episode = existing_guids.get(episode_dict['guid']) 149 | if episode is None: 150 | episode = channel.episode_factory(episode_dict.items()) 151 | new_episodes.append(episode) 152 | logger.info('Found new episode: %s', episode.guid) 153 | else: 154 | episode.update_from_dict(episode_dict) 155 | logger.info('Updating existing episode: %s', episode.guid) 156 | episode.save() 157 | 158 | return new_episodes, seen_guids 159 | 160 | 161 | class PodcastParserEnclosureFallbackFeed(PodcastParserFeed): 162 | # Implement this in a subclass to determine a fallback enclosure 163 | # for feeds that don't list their media files as enclosures 164 | def _get_enclosure_url(self, episode_dict): 165 | return None 166 | 167 | def _pick_enclosure(self, episode_dict): 168 | if not episode_dict['enclosures']: 169 | url = self._get_enclosure_url(episode_dict) 170 | if url is not None: 171 | del episode_dict['enclosures'] 172 | episode_dict['url'] = url 173 | return True 174 | 175 | return super(PodcastParserEnclosureFallbackFeed, self)._pick_enclosure(episode_dict) 176 | 177 | 178 | class PodcastParserLinkFallbackFeed(PodcastParserEnclosureFallbackFeed): 179 | # Tries to use the episode link if the URL looks like an audio/video file 180 | # link for episodes that do not have enclosures 181 | 182 | def _get_enclosure_url(self, episode_dict): 183 | url = episode_dict.get('link') 184 | if url is not None: 185 | base, extension = util.filename_from_url(url) 186 | if util.file_type_by_extension(extension) in ('audio', 'video'): 187 | logger.debug('Using link for enclosure URL: %s', url) 188 | return url 189 | 190 | return None 191 | 192 | 193 | @registry.fallback_feed_handler.register 194 | def podcast_parser_handler(channel, max_episodes, config): 195 | return PodcastParserLinkFallbackFeed(channel, max_episodes) 196 | 197 | 198 | @registry.url_shortcut.register 199 | def podcast_resolve_url_shortcut(): 200 | return {'fb': 'http://feeds.feedburner.com/%s'} 201 | -------------------------------------------------------------------------------- /src/gpodder/plugins/podverse.py: -------------------------------------------------------------------------------- 1 | # 2 | # gpodder.plugins.podverse: podverse.fm directory integration (2024-03-12) 3 | # Copyright (c) 2024, kirbylife 4 | # 5 | # Permission to use, copy, modify, and/or distribute this software for any 6 | # purpose with or without fee is hereby granted, provided that the above 7 | # copyright notice and this permission notice appear in all copies. 8 | # 9 | # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH 10 | # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY 11 | # AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, 12 | # INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM 13 | # LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR 14 | # OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 15 | # PERFORMANCE OF THIS SOFTWARE. 16 | # 17 | 18 | 19 | import gpodder 20 | 21 | from gpodder import util 22 | from gpodder import registry 23 | from gpodder import directory 24 | 25 | import logging 26 | import urllib.parse 27 | 28 | logger = logging.getLogger(__name__) 29 | PAGE_SIZE = 20 30 | 31 | @registry.directory.register_instance 32 | class PodverseSearchProvider(directory.Provider): 33 | def __init__(self): 34 | self.name = 'Podverse search' 35 | self.kind = directory.Provider.PROVIDER_SEARCH 36 | self.priority = directory.Provider.PRIORITY_SECONDARY_SEARCH 37 | 38 | def on_search(self, query): 39 | page = 1 40 | 41 | while True: 42 | json_url = "https://api.podverse.fm/api/v1/podcast?page={}&searchTitle={}&sort=top-past-week".format(page, urllib.parse.quote(query)) 43 | 44 | json_data, entry_count = util.read_json(json_url) 45 | 46 | if entry_count > 0: 47 | for entry in json_data: 48 | if entry["credentialsRequired"]: 49 | continue 50 | 51 | title = entry["title"] 52 | url = entry["feedUrls"][0]["url"] 53 | image = entry["imageUrl"] 54 | description = entry["description"] 55 | 56 | yield(directory.DirectoryEntry(title, url, image, -1, description)) 57 | 58 | if entry_count < PAGE_SIZE: 59 | break 60 | 61 | page += 1 62 | 63 | -------------------------------------------------------------------------------- /src/gpodder/plugins/soundcloud.py: -------------------------------------------------------------------------------- 1 | # 2 | # gPodder - A media aggregator and podcast client 3 | # Copyright (c) 2005-2020 The gPodder Team 4 | # 5 | # gPodder is free software; you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation; either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # gPodder is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | # 18 | 19 | # Soundcloud.com API client module for gPodder 20 | # Thomas Perl ; 2009-11-03 21 | 22 | import email 23 | import json 24 | import logging 25 | logger = logging.getLogger(__name__) 26 | import os 27 | import re 28 | import time 29 | import urllib.error 30 | import urllib.parse 31 | import urllib.request 32 | 33 | import gpodder 34 | from gpodder import model, util, registry, directory 35 | 36 | # _ = qsTr 37 | 38 | # gPodder's consumer key for the Soundcloud API 39 | CONSUMER_KEY = 'zrweghtEtnZLpXf3mlm8mQ' 40 | 41 | 42 | def soundcloud_parsedate(s): 43 | """Parse a string into a unix timestamp 44 | 45 | Only strings provided by Soundcloud's API are 46 | parsed with this function (2009/11/03 13:37:00). 47 | """ 48 | m = re.match(r'(\d{4})/(\d{2})/(\d{2}) (\d{2}):(\d{2}):(\d{2})', s) 49 | return time.mktime(tuple([int(x) for x in m.groups()] + [0, 0, -1])) 50 | 51 | 52 | def get_param(s, param='filename', header='content-disposition'): 53 | """Get a parameter from a string of headers 54 | 55 | By default, this gets the "filename" parameter of 56 | the content-disposition header. This works fine 57 | for downloads from Soundcloud. 58 | """ 59 | msg = email.message_from_string(s) 60 | if header in msg: 61 | value = msg.get_param(param, header=header) 62 | decoded_list = email.header.decode_header(value) 63 | value = [] 64 | for part, encoding in decoded_list: 65 | if encoding: 66 | value.append(part.decode(encoding)) 67 | else: 68 | value.append(str(part)) 69 | return ''.join(value) 70 | 71 | return None 72 | 73 | 74 | def get_metadata(url): 75 | """Get file download metadata 76 | 77 | Returns a (size, type, name) from the given download 78 | URL. Will use the network connection to determine the 79 | metadata via the HTTP header fields. 80 | """ 81 | track_fp = util.urlopen(url) 82 | headers = track_fp.info() 83 | filesize = headers['content-length'] or '0' 84 | filetype = headers['content-type'] or 'application/octet-stream' 85 | headers_s = '\n'.join('%s:%s' % (k, v) for k, v in list(headers.items())) 86 | filename = get_param(headers_s) or os.path.basename(os.path.dirname(url)) 87 | track_fp.close() 88 | return filesize, filetype, filename 89 | 90 | 91 | class SoundcloudUser(object): 92 | def __init__(self, username): 93 | self.username = username 94 | self.cache = {} 95 | 96 | def get_user_info(self): 97 | global CONSUMER_KEY 98 | key = ':'.join((self.username, 'user_info')) 99 | 100 | if key not in self.cache: 101 | json_url = 'https://api.soundcloud.com/users/%s.json?consumer_key=%s' % (self.username, CONSUMER_KEY) 102 | logger.debug('get_user_info url: %s', json_url) 103 | user_info = json.loads(util.urlopen(json_url).read().decode('utf-8')) 104 | self.cache[key] = user_info 105 | 106 | return self.cache[key] 107 | 108 | def get_coverart(self): 109 | user_info = self.get_user_info() 110 | avatar_url = user_info.get('avatar_url', None) 111 | if avatar_url: 112 | # Soundcloud API by default returns the URL to "large" artwork - 100x100 113 | # by replacing "-large" with "-original" in the URL we get unresized files. 114 | return avatar_url.replace("-large", "-original") 115 | return avatar_url 116 | 117 | def get_user_id(self): 118 | user_info = self.get_user_info() 119 | return user_info.get('id', None) 120 | 121 | def get_username(self): 122 | user_info = self.get_user_info() 123 | return user_info.get('username', None) 124 | 125 | def get_tracks(self, feed, channel): 126 | """Get a generator of tracks from a SC user 127 | 128 | The generator will give you a dictionary for every 129 | track it can find for its user.""" 130 | global CONSUMER_KEY 131 | 132 | json_url = ('https://api.soundcloud.com/users/%(user)s/%(feed)s.' 133 | 'json?consumer_key=%' 134 | '(consumer_key)s&limit=200&linked_partitioning=1' 135 | % {"user": self.get_user_id(), 136 | "feed": feed, 137 | "consumer_key": CONSUMER_KEY}) 138 | 139 | logger.debug('get_tracks url: %s', json_url) 140 | 141 | json_tracks = json.loads(util.urlopen(json_url).read().decode('utf-8')) 142 | tracks = [track for track in json_tracks['collection'] if track['streamable'] or track['downloadable']] 143 | 144 | try: 145 | while(json_tracks['next_href'] != ''): 146 | next_url = json_tracks['next_href'] 147 | logger.debug('get page: %s', next_url) 148 | json_tracks = json.loads(util.urlopen(next_url).read().decode('utf-8')) 149 | tracks += [track for track in json_tracks['collection'] if track['streamable'] or track['downloadable']] 150 | except: 151 | logger.debug('No pagination/end of pagination for this feed.') 152 | 153 | logger.debug('Starting to add %d tracks, this may take a while.', len(tracks)) 154 | 155 | self.cache['episodes'] = { episode.guid: 156 | { "filesize": episode.file_size, 157 | "filetype": episode.mime_type, 158 | } for episode in channel.episodes 159 | } 160 | 161 | read_from_cache = 0 162 | logger.debug('%d Episodes in database for Soundcloud:%s', len(self.cache['episodes']), self.username) 163 | 164 | for track in tracks: 165 | # Prefer stream URL (MP3), fallback to download URL 166 | base_url = track.get('stream_url') if track['streamable'] else track.get('download_url') 167 | if base_url: 168 | url = base_url + '?consumer_key=%(consumer_key)s' % {'consumer_key': CONSUMER_KEY} 169 | else: 170 | logger.debug('Skipping track with no base_url') 171 | continue 172 | 173 | track_guid = track.get('permalink', track.get('id')) 174 | 175 | if track_guid not in self.cache['episodes']: 176 | filesize, filetype, filename = get_metadata(url) 177 | else: 178 | filesize = self.cache['episodes'][track_guid]['filesize'] 179 | filetype = self.cache['episodes'][track_guid]['filetype'] 180 | read_from_cache += 1 181 | 182 | artwork_url = track.get('artwork_url') 183 | if artwork_url: 184 | artwork_url = artwork_url.replace("-large", "-original") 185 | 186 | yield { 187 | 'title': track.get('title', track.get('permalink')) or ('Unknown track'), 188 | 'link': track.get('permalink_url') or 'https://soundcloud.com/' + self.username, 189 | 'description': track.get('description') or ('No description available'), 190 | 'url': url, 191 | 'file_size': int(filesize), 192 | 'mime_type': filetype, 193 | 'guid': track_guid, 194 | 'published': soundcloud_parsedate(track.get('created_at', None)), 195 | 'total_time': int(track.get('duration') / 1000), 196 | 'episode_art_url' : artwork_url, 197 | } 198 | 199 | logger.debug('Read %d episodes from %d cached episodes', read_from_cache, len(self.cache['episodes'])) 200 | 201 | 202 | class SoundcloudFeed(object): 203 | def __init__(self, username): 204 | self.username = username 205 | self.sc_user = SoundcloudUser(username) 206 | 207 | def was_updated(self): 208 | return True 209 | 210 | def get_etag(self, default): 211 | return default 212 | 213 | def get_modified(self, default): 214 | return default 215 | 216 | def get_title(self): 217 | return '%s on Soundcloud' % self.sc_user.get_username() 218 | 219 | def get_image(self): 220 | return self.sc_user.get_coverart() 221 | 222 | def get_link(self): 223 | return 'https://soundcloud.com/%s' % self.username 224 | 225 | def get_description(self): 226 | return 'Tracks published by %s on Soundcloud.' % self.username 227 | 228 | def get_payment_url(self): 229 | return None 230 | 231 | def get_new_episodes(self, channel): 232 | return self._get_new_episodes(channel, 'tracks') 233 | 234 | def _get_new_episodes(self, channel, track_type): 235 | tracks = [t for t in self.sc_user.get_tracks(track_type, channel)] 236 | existing_guids = dict((episode.guid, episode) for episode in channel.episodes) 237 | seen_guids = [track['guid'] for track in tracks] 238 | new_episodes = [] 239 | 240 | for track in tracks: 241 | episode = existing_guids.get(track['guid']) 242 | 243 | if not episode: 244 | episode = channel.episode_factory(track.items()) 245 | new_episodes.append(episode) 246 | logger.info('Found new episode: %s', episode.guid) 247 | else: 248 | episode.update_from_dict(track) 249 | logger.info('Updating existing episode: %s', episode.guid) 250 | episode.save() 251 | 252 | return new_episodes, seen_guids 253 | 254 | 255 | class SoundcloudFavFeed(SoundcloudFeed): 256 | def __init__(self, username): 257 | super(SoundcloudFavFeed, self).__init__(username) 258 | 259 | def get_title(self): 260 | return '%s\'s favorites on Soundcloud' % self.username 261 | 262 | def get_link(self): 263 | return 'https://soundcloud.com/%s/favorites' % self.username 264 | 265 | def get_description(self): 266 | return 'Tracks favorited by %s on Soundcloud.' % self.username 267 | 268 | def get_new_episodes(self, channel): 269 | return self._get_new_episodes(channel, 'favorites') 270 | 271 | 272 | @registry.feed_handler.register 273 | def soundcloud_feed_handler(channel, max_episodes, config): 274 | m = re.match(r'https?://([a-z]+\.)?soundcloud\.com/([^/]+)$', channel.url, re.I) 275 | 276 | if m is not None: 277 | subdomain, username = m.groups() 278 | return SoundcloudFeed(username) 279 | 280 | 281 | @registry.feed_handler.register 282 | def soundcloud_fav_feed_handler(channel, max_episodes, config): 283 | m = re.match(r'https?://([a-z]+\.)?soundcloud\.com/([^/]+)/favorites', channel.url, re.I) 284 | 285 | if m is not None: 286 | subdomain, username = m.groups() 287 | return SoundcloudFavFeed(username) 288 | 289 | 290 | @registry.url_shortcut.register 291 | def soundcloud_resolve_url_shortcut(): 292 | return {'sc': 'https://soundcloud.com/%s', 293 | 'scfav': 'https://soundcloud.com/%s/favorites'} 294 | 295 | 296 | @registry.directory.register_instance 297 | class SoundcloudSearchProvider(directory.Provider): 298 | def __init__(self): 299 | self.name = 'Soundcloud search' 300 | self.kind = directory.Provider.PROVIDER_SEARCH 301 | self.priority = directory.Provider.PRIORITY_SECONDARY_SEARCH 302 | 303 | def on_search(self, query): 304 | json_url = 'https://api.soundcloud.com/users.json?q=%s&consumer_key=%s' % (urllib.parse.quote(query), 305 | CONSUMER_KEY) 306 | return [directory.DirectoryEntry(entry['username'], entry['permalink_url']) 307 | for entry in util.read_json(json_url)] 308 | -------------------------------------------------------------------------------- /src/gpodder/plugins/vimeo.py: -------------------------------------------------------------------------------- 1 | # 2 | # gpodder.plugins.vimeo: Vimeo download magic (2012-01-03) 3 | # Copyright (c) 2012, 2013, Thomas Perl 4 | # 5 | # Permission to use, copy, modify, and/or distribute this software for any 6 | # purpose with or without fee is hereby granted, provided that the above 7 | # copyright notice and this permission notice appear in all copies. 8 | # 9 | # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH 10 | # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY 11 | # AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, 12 | # INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM 13 | # LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR 14 | # OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 15 | # PERFORMANCE OF THIS SOFTWARE. 16 | # 17 | 18 | 19 | import gpodder 20 | 21 | from gpodder import util 22 | from gpodder import registry 23 | 24 | from gpodder.plugins import podcast 25 | 26 | import logging 27 | logger = logging.getLogger(__name__) 28 | 29 | import re 30 | 31 | VIMEOCOM_RE = re.compile(r'http[s]?://vimeo\.com/(\d+)$', re.IGNORECASE) 32 | VIMEOCHANNEL_RE = re.compile(r'http[s]?://vimeo\.com/(channels/[^/]+|\d+)$', re.IGNORECASE) 33 | MOOGALOOP_RE = re.compile(r'http[s]?://vimeo\.com/moogaloop\.swf\?clip_id=(\d+)$', re.IGNORECASE) 34 | VIMEO_VIDEO_RE = re.compile(r'http[s]?://vimeo.com/channels/(?:[^/])+/(\d+)$', re.IGNORECASE) 35 | SIGNATURE_RE = re.compile(r'"timestamp":(\d+),"signature":"([^"]+)"') 36 | 37 | # List of qualities, from lowest to highest 38 | FILEFORMAT_RANKING = ['mobile', 'sd', 'hd'] 39 | 40 | 41 | class VimeoError(Exception): 42 | pass 43 | 44 | 45 | @registry.download_url.register 46 | def vimeo_resolve_download_url(episode, config): 47 | url = episode.url 48 | 49 | video_id = get_vimeo_id(url) 50 | 51 | if video_id is None: 52 | return None 53 | 54 | data_config_url = 'https://player.vimeo.com/video/%s/config' % (video_id,) 55 | 56 | def get_urls(data_config_url): 57 | data_config = util.read_json(data_config_url) 58 | for fileinfo in data_config['request']['files'].values(): 59 | if not isinstance(fileinfo, list): 60 | continue 61 | 62 | for item in fileinfo: 63 | yield (item['quality'], item['url']) 64 | 65 | fileformat_to_url = dict(get_urls(data_config_url)) 66 | 67 | preferred_fileformat = config.plugins.vimeo.fileformat 68 | if preferred_fileformat is not None and preferred_fileformat in fileformat_to_url: 69 | logger.debug('Picking preferredformat: %s', preferred_fileformat) 70 | return fileformat_to_url[preferred_fileformat] 71 | 72 | def fileformat_sort_key_func(fileformat): 73 | if fileformat in FILEFORMAT_RANKING: 74 | return FILEFORMAT_RANKING.index(fileformat) 75 | 76 | return 0 77 | 78 | for fileformat in sorted(fileformat_to_url, key=fileformat_sort_key_func, reverse=True): 79 | logger.debug('Picking best format: %s', fileformat) 80 | return fileformat_to_url[fileformat] 81 | 82 | 83 | def get_vimeo_id(url): 84 | result = MOOGALOOP_RE.match(url) 85 | if result is not None: 86 | return result.group(1) 87 | 88 | result = VIMEOCOM_RE.match(url) 89 | if result is not None: 90 | return result.group(1) 91 | 92 | result = VIMEO_VIDEO_RE.match(url) 93 | if result is not None: 94 | return result.group(1) 95 | 96 | return None 97 | 98 | 99 | def is_video_link(url): 100 | return (get_vimeo_id(url) is not None) 101 | 102 | 103 | def get_real_channel_url(url): 104 | result = VIMEOCHANNEL_RE.match(url) 105 | if result is not None: 106 | return 'http://vimeo.com/%s/videos/rss' % result.group(1) 107 | 108 | return None 109 | 110 | 111 | def get_real_cover(url): 112 | return None 113 | 114 | 115 | class PodcastParserVimeoFeed(podcast.PodcastParserEnclosureFallbackFeed): 116 | def _get_enclosure_url(self, episode_dict): 117 | if is_video_link(episode_dict['link']): 118 | return episode_dict['link'] 119 | 120 | return None 121 | 122 | 123 | @registry.feed_handler.register 124 | def vimeo_feed_handler(channel, max_episodes, config): 125 | url = get_real_channel_url(channel.url) 126 | if url is None: 127 | return None 128 | 129 | logger.info('Vimeo feed resolved: {} -> {}'.format(channel.url, url)) 130 | channel.url = url 131 | 132 | return PodcastParserVimeoFeed(channel, max_episodes) 133 | 134 | 135 | @registry.episode_basename.register 136 | def vimeo_resolve_episode_basename(episode, sanitized): 137 | if sanitized and is_video_link(episode.url): 138 | return sanitized 139 | 140 | 141 | @registry.podcast_title.register 142 | def vimeo_resolve_podcast_title(podcast, new_title): 143 | VIMEO_PREFIX = 'Vimeo / ' 144 | if new_title.startswith(VIMEO_PREFIX): 145 | return new_title[len(VIMEO_PREFIX):] + ' on Vimeo' 146 | 147 | 148 | @registry.content_type.register 149 | def vimeo_resolve_content_type(episode): 150 | if is_video_link(episode.url): 151 | return 'video' 152 | -------------------------------------------------------------------------------- /src/gpodder/plugins/youtube.py: -------------------------------------------------------------------------------- 1 | # 2 | # gPodder: Media and podcast aggregator 3 | # Copyright (c) 2005-2020 Thomas Perl and the gPodder Team 4 | # 5 | # gPodder is free software; you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation; either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # gPodder is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with this program. If not, see . 17 | # 18 | # gpodder.youtube - YouTube and related magic 19 | # Justin Forest 2008-10-13 20 | # 21 | 22 | import gpodder 23 | 24 | from gpodder import util 25 | from gpodder import registry 26 | from gpodder import directory 27 | from gpodder.plugins import podcast 28 | 29 | import os.path 30 | 31 | import logging 32 | logger = logging.getLogger(__name__) 33 | 34 | import re 35 | import json 36 | import urllib.request 37 | import urllib.parse 38 | import urllib.error 39 | 40 | 41 | # http://en.wikipedia.org/wiki/YouTube#Quality_and_codecs 42 | # format id, (preferred ids, path(?), description) # video bitrate, audio bitrate 43 | formats = [ 44 | # WebM VP8 video, Vorbis audio 45 | # Fallback to an MP4 version of same quality. 46 | # Try 34 (FLV 360p H.264 AAC) if 18 (MP4 360p) fails. 47 | # Fallback to 6 or 5 (FLV Sorenson H.263 MP3) if all fails. 48 | (46, ([46, 37, 45, 22, 44, 35, 43, 18, 6, 34, 5], '45/1280x720/99/0/0', 49 | 'WebM 1080p (1920x1080)')), 50 | (45, ([45, 22, 44, 35, 43, 18, 6, 34, 5], '45/1280x720/99/0/0', 51 | 'WebM 720p (1280x720)')), 52 | (44, ([44, 35, 43, 18, 6, 34, 5], '44/854x480/99/0/0', 53 | 'WebM 480p (854x480)')), 54 | (43, ([43, 18, 6, 34, 5], '43/640x360/99/0/0', 55 | 'WebM 360p (640x360)')), 56 | 57 | # MP4 H.264 video, AAC audio 58 | # Try 35 (FLV 480p H.264 AAC) between 720p and 360p because there's no MP4 480p. 59 | # Try 34 (FLV 360p H.264 AAC) if 18 (MP4 360p) fails. 60 | # Fallback to 6 or 5 (FLV Sorenson H.263 MP3) if all fails. 61 | (38, ([38, 37, 22, 35, 18, 34, 6, 5], '38/1920x1080/9/0/115', 'MP4 4K 3072p (4096x3072)')), 62 | (37, ([37, 22, 35, 18, 34, 6, 5], '37/1920x1080/9/0/115', 'MP4 HD 1080p (1920x1080)')), 63 | (22, ([22, 35, 18, 34, 6, 5], '22/1280x720/9/0/115', 'MP4 HD 720p (1280x720)')), 64 | (18, ([18, 34, 6, 5], '18/640x360/9/0/115', 'MP4 360p (640x360)')), 65 | 66 | # FLV H.264 video, AAC audio 67 | # Does not check for 360p MP4. 68 | # Fallback to 6 or 5 (FLV Sorenson H.263 MP3) if all fails. 69 | (35, ([35, 34, 6, 5], '35/854x480/9/0/115', 'FLV 480p (854x480)')), # 1 - 0.80 Mbps, 128 kbps 70 | (34, ([34, 6, 5], '34/640x360/9/0/115', 'FLV 360p (640x360)')), # 0.50 Mbps, 128 kbps 71 | 72 | # FLV Sorenson H.263 video, MP3 audio 73 | (6, ([6, 5], '5/480x270/7/0/0', 'FLV 270p (480x270)')), # 0.80 Mbps, 64 kbps 74 | (5, ([5], '5/320x240/7/0/0', 'FLV 240p (320x240)')), # 0.25 Mbps, 64 kbps 75 | ] 76 | formats_dict = dict(formats) 77 | 78 | V3_API_ENDPOINT = 'https://www.googleapis.com/youtube/v3' 79 | CHANNEL_VIDEOS_XML = 'https://www.youtube.com/feeds/videos.xml' 80 | 81 | 82 | class YouTubeError(Exception): 83 | pass 84 | 85 | 86 | def get_fmt_ids(youtube_config): 87 | fmt_ids = youtube_config.preferred_fmt_ids 88 | if not fmt_ids: 89 | format = formats_dict.get(youtube_config.preferred_fmt_id) 90 | if format is None: 91 | fmt_ids = [] 92 | else: 93 | fmt_ids, path, description = format 94 | 95 | return fmt_ids 96 | 97 | 98 | @registry.download_url.register 99 | def youtube_resolve_download_url(episode, config): 100 | url = episode.url 101 | preferred_fmt_ids = get_fmt_ids(config.plugins.youtube) 102 | 103 | if not preferred_fmt_ids: 104 | preferred_fmt_ids, _, _ = formats_dict[22] # MP4 720p 105 | 106 | vid = get_youtube_id(url) 107 | if vid is None: 108 | return None 109 | 110 | page = None 111 | url = 'https://www.youtube.com/get_video_info?&el=detailpage&video_id=' + vid 112 | 113 | while page is None: 114 | req = util.http_request(url, method='GET') 115 | if 'location' in req.msg: 116 | url = req.msg['location'] 117 | else: 118 | page = req.read().decode('utf-8') 119 | 120 | # Try to find the best video format available for this video 121 | # (http://forum.videohelp.com/topic336882-1800.html#1912972) 122 | def find_urls(page): 123 | r4 = re.search('.*&url_encoded_fmt_stream_map=([^&]+)&.*', page) 124 | if r4 is not None: 125 | fmt_url_map = urllib.parse.unquote(r4.group(1)) 126 | for fmt_url_encoded in fmt_url_map.split(','): 127 | video_info = urllib.parse.parse_qs(fmt_url_encoded) 128 | yield (int(video_info['itag'][0]), video_info['url'][0]) 129 | else: 130 | error_info = urllib.parse.parse_qs(page) 131 | error_message = util.remove_html_tags(error_info['reason'][0]) 132 | raise YouTubeError('Cannot download video: %s' % error_message) 133 | 134 | fmt_id_url_map = sorted(find_urls(page), reverse=True) 135 | 136 | if not fmt_id_url_map: 137 | raise YouTubeError('fmt_url_map not found for video ID "%s"' % vid) 138 | 139 | # Default to the highest fmt_id if we don't find a match below 140 | _, url = fmt_id_url_map[0] 141 | 142 | formats_available = set(fmt_id for fmt_id, url in fmt_id_url_map) 143 | fmt_id_url_map = dict(fmt_id_url_map) 144 | 145 | for id in preferred_fmt_ids: 146 | id = int(id) 147 | if id in formats_available: 148 | format = formats_dict.get(id) 149 | if format is not None: 150 | _, _, description = format 151 | else: 152 | description = 'Unknown' 153 | 154 | logger.info('Found YouTube format: %s (fmt_id=%d)', description, id) 155 | return fmt_id_url_map[id] 156 | 157 | 158 | def get_youtube_id(url): 159 | r = re.compile('http[s]?://(?:[a-z]+\.)?youtube\.com/v/(.*)\.swf', re.IGNORECASE).match(url) 160 | if r is not None: 161 | return r.group(1) 162 | 163 | r = re.compile('http[s]?://(?:[a-z]+\.)?youtube\.com/watch\?v=([^&]*)', 164 | re.IGNORECASE).match(url) 165 | if r is not None: 166 | return r.group(1) 167 | 168 | r = re.compile('http[s]?://(?:[a-z]+\.)?youtube\.com/v/(.*)[?]', re.IGNORECASE).match(url) 169 | if r is not None: 170 | return r.group(1) 171 | 172 | return None 173 | 174 | 175 | def is_video_link(url): 176 | return (get_youtube_id(url) is not None) 177 | 178 | 179 | def is_youtube_guid(guid): 180 | return guid.startswith('tag:youtube.com,2008:video:') 181 | 182 | 183 | def for_each_feed_pattern(func, url, fallback_result): 184 | """ 185 | Try to find the username for all possible YouTube feed/webpage URLs 186 | Will call func(url, channel) for each match, and if func() returns 187 | a result other than None, returns this. If no match is found or 188 | func() returns None, return fallback_result. 189 | """ 190 | CHANNEL_MATCH_PATTERNS = [ 191 | 'http[s]?://(?:[a-z]+\.)?youtube\.com/user/([a-z0-9]+)', 192 | 'http[s]?://(?:[a-z]+\.)?youtube\.com/profile?user=([a-z0-9]+)', 193 | 'http[s]?://(?:[a-z]+\.)?youtube\.com/channel/([-_a-zA-Z0-9]+)', 194 | 'http[s]?://(?:[a-z]+\.)?youtube\.com/rss/user/([a-z0-9]+)/videos\.rss', 195 | 'http[s]?://gdata.youtube.com/feeds/users/([^/]+)/uploads', 196 | 'http[s]?://(?:[a-z]+\.)?youtube\.com/feeds/videos.xml\?channel_id=([-_a-zA-Z0-9]+)', 197 | ] 198 | 199 | for pattern in CHANNEL_MATCH_PATTERNS: 200 | m = re.match(pattern, url, re.IGNORECASE) 201 | if m is not None: 202 | result = func(url, m.group(1)) 203 | if result is not None: 204 | return result 205 | 206 | return fallback_result 207 | 208 | 209 | def get_channels_for_user(username, api_key_v3): 210 | stream = util.urlopen('{0}/channels?forUsername={1}&part=id&key={2}'.format(V3_API_ENDPOINT, username, api_key_v3)) 211 | data = json.loads(stream.read().decode('utf-8')) 212 | return ['{0}?channel_id={1}'.format(CHANNEL_VIDEOS_XML, item['id']) for item in data['items']] 213 | 214 | 215 | def get_real_channel_url(url, api_key_v3): 216 | # Check if it's a YouTube feed, and if we have an API key, auto-resolve the channel 217 | if url and api_key_v3: 218 | _, user = for_each_feed_pattern(lambda url, channel: (url, channel), url, (None, None)) 219 | if user is not None: 220 | logger.info('Getting channels for YouTube user %s', user) 221 | new_urls = get_channels_for_user(user, api_key_v3) 222 | logger.debug('YouTube channels retrieved: %r', new_urls) 223 | if len(new_urls) == 1: 224 | return new_urls[0] 225 | 226 | return None 227 | 228 | 229 | @registry.cover_art.register 230 | def youtube_resolve_cover_art(podcast): 231 | url = podcast.url 232 | r = re.compile('http://www\.youtube\.com/rss/user/([^/]+)/videos\.rss', re.IGNORECASE) 233 | m = r.match(url) 234 | 235 | if m is not None: 236 | username = m.group(1) 237 | api_url = 'http://gdata.youtube.com/feeds/api/users/%s?v=2' % username 238 | data = util.urlopen(api_url).read().decode('utf-8', 'ignore') 239 | match = re.search('', data) 240 | if match is not None: 241 | return match.group(1) 242 | 243 | return None 244 | 245 | 246 | class PodcastParserYouTubeFeed(podcast.PodcastParserEnclosureFallbackFeed): 247 | def _get_enclosure_url(self, episode_dict): 248 | if is_video_link(episode_dict['link']): 249 | return episode_dict['link'] 250 | 251 | return None 252 | 253 | 254 | @registry.feed_handler.register 255 | def youtube_feed_handler(channel, max_episodes, config): 256 | url = get_real_channel_url(channel.url, config.plugins.youtube.api_key_v3) 257 | if url is None: 258 | return None 259 | 260 | channel.url = url 261 | 262 | return PodcastParserYouTubeFeed(channel, max_episodes) 263 | 264 | 265 | @registry.episode_basename.register 266 | def youtube_resolve_episode_basename(episode, sanitized): 267 | if sanitized and is_video_link(episode.url): 268 | return sanitized 269 | 270 | 271 | @registry.podcast_title.register 272 | def youtube_resolve_podcast_title(podcast, new_title): 273 | YOUTUBE_PREFIX = 'Uploads by ' 274 | if new_title.startswith(YOUTUBE_PREFIX): 275 | return new_title[len(YOUTUBE_PREFIX):] + ' on YouTube' 276 | 277 | 278 | @registry.content_type.register 279 | def youtube_resolve_content_type(episode): 280 | if is_video_link(episode.url): 281 | return 'video' 282 | 283 | 284 | @registry.url_shortcut.register 285 | def youtube_resolve_url_shortcut(): 286 | return {'yt': 'http://www.youtube.com/rss/user/%s/videos.rss', 287 | # YouTube playlists. To get a list of playlists per-user, use: 288 | # https://gdata.youtube.com/feeds/api/users//playlists 289 | 'ytpl': 'http://gdata.youtube.com/feeds/api/playlists/%s'} 290 | 291 | 292 | @registry.directory.register_instance 293 | class YouTubeSearchProvider(directory.Provider): 294 | def __init__(self): 295 | self.name = 'YouTube search' 296 | self.kind = directory.Provider.PROVIDER_SEARCH 297 | self.priority = directory.Provider.PRIORITY_SECONDARY_SEARCH 298 | 299 | def on_search(self, query): 300 | url = 'http://gdata.youtube.com/feeds/api/videos?alt=json&q=%s' % urllib.parse.quote(query) 301 | data = util.read_json(url) 302 | 303 | result = [] 304 | 305 | seen_users = set() 306 | for entry in data['feed']['entry']: 307 | user = os.path.basename(entry['author'][0]['uri']['$t']) 308 | title = entry['title']['$t'] 309 | url = 'http://www.youtube.com/rss/user/%s/videos.rss' % user 310 | if user not in seen_users: 311 | result.append(directory.DirectoryEntry(user, url)) 312 | seen_users.add(user) 313 | 314 | return result 315 | -------------------------------------------------------------------------------- /src/gpodder/query.py: -------------------------------------------------------------------------------- 1 | # 2 | # gpodder.query - Episode Query Language (EQL) implementation (2010-11-29) 3 | # Copyright (c) 2010-2013, Thomas Perl 4 | # 5 | # Permission to use, copy, modify, and/or distribute this software for any 6 | # purpose with or without fee is hereby granted, provided that the above 7 | # copyright notice and this permission notice appear in all copies. 8 | # 9 | # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH 10 | # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY 11 | # AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, 12 | # INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM 13 | # LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR 14 | # OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 15 | # PERFORMANCE OF THIS SOFTWARE. 16 | # 17 | 18 | 19 | import gpodder 20 | 21 | import re 22 | import datetime 23 | 24 | 25 | class Matcher(object): 26 | """Match implementation for EQL 27 | 28 | This class implements the low-level matching of 29 | EQL statements against episode objects. 30 | """ 31 | 32 | def __init__(self, episode): 33 | self._episode = episode 34 | 35 | def match(self, term): 36 | try: 37 | return bool(eval(term, {'__builtins__': None}, self)) 38 | except Exception as e: 39 | print(e) 40 | return False 41 | 42 | def __getitem__(self, k): 43 | episode = self._episode 44 | 45 | # Adjectives (for direct usage) 46 | if k == 'new': 47 | return (episode.state == gpodder.STATE_NORMAL and episode.is_new) 48 | elif k == 'old': 49 | return not self['new'] 50 | elif k in ('downloaded', 'dl'): 51 | return episode.state == gpodder.STATE_DOWNLOADED 52 | elif k in ('deleted', 'rm'): 53 | return episode.state == gpodder.STATE_DELETED 54 | elif k == 'played': 55 | return not episode.is_new 56 | elif k == 'downloading': 57 | return episode.downloading 58 | elif k == 'archive': 59 | return episode.archive 60 | elif k in ('finished', 'fin'): 61 | return episode.is_finished() 62 | elif k in ('video', 'audio'): 63 | return episode.file_type() == k 64 | elif k == 'torrent': 65 | return episode.url.endswith('.torrent') or 'torrent' in episode.mime_type 66 | 67 | # Nouns (for comparisons) 68 | if k in ('megabytes', 'mb'): 69 | return float(episode.file_size) / (1024*1024) 70 | elif k == 'title': 71 | return episode.title 72 | elif k == 'description': 73 | return episode.description 74 | elif k == 'since': 75 | return (datetime.datetime.now() - 76 | datetime.datetime.fromtimestamp(episode.published)).days 77 | elif k == 'age': 78 | return episode.age_in_days() 79 | elif k in ('minutes', 'min'): 80 | return float(episode.total_time) / 60 81 | elif k in ('remaining', 'rem'): 82 | return float(episode.total_time - episode.current_position) / 60 83 | 84 | raise KeyError(k) 85 | 86 | 87 | class EQL(object): 88 | """A Query in EQL 89 | 90 | Objects of this class represent a query on episodes 91 | using EQL. Example usage: 92 | 93 | >>> q = EQL('downloaded and megabytes > 10') 94 | 95 | Regular expression queries are also supported: 96 | 97 | >>> q = EQL('/^The.*/') 98 | 99 | >>> q = EQL('/community/i') 100 | 101 | Normal string matches are also supported: 102 | 103 | >>> q = EQL('"S04"') 104 | 105 | >>> q = EQL("'linux'") 106 | """ 107 | 108 | def __init__(self, query): 109 | self._query = query 110 | self._flags = 0 111 | self._regex = False 112 | self._string = False 113 | 114 | # Regular expression based query 115 | match = re.match(r'^/(.*)/(i?)$', query) 116 | if match is not None: 117 | self._regex = True 118 | self._query, flags = match.groups() 119 | if flags == 'i': 120 | self._flags |= re.I 121 | 122 | # String based query 123 | match = re.match("^([\"'])(.*)(\\1)$", query) 124 | if match is not None: 125 | self._string = True 126 | a, query, b = match.groups() 127 | self._query = query.lower() 128 | 129 | # For everything else, compile the expression 130 | if not self._regex and not self._string: 131 | try: 132 | self._query = compile(query, '', 'eval') 133 | except Exception as e: 134 | print(e) 135 | self._query = None 136 | 137 | def match(self, episode): 138 | if self._query is None: 139 | return False 140 | 141 | if self._regex: 142 | return re.search(self._query, episode.title, self._flags) is not None 143 | elif self._string: 144 | return self._query in episode.title.lower() or self._query in episode.description.lower() 145 | 146 | return Matcher(episode).match(self._query) 147 | 148 | def filter(self, episodes): 149 | return list(filter(self.match, episodes)) 150 | 151 | 152 | def UserEQL(query): 153 | """EQL wrapper for user input 154 | 155 | Automatically adds missing quotes around a 156 | non-EQL string for user-based input. In this 157 | case, EQL queries need to be enclosed in (). 158 | """ 159 | 160 | if query is None: 161 | return None 162 | 163 | if query == '' or (query and query[0] not in "(/'\""): 164 | return EQL("'%s'" % query) 165 | else: 166 | return EQL(query) 167 | -------------------------------------------------------------------------------- /src/gpodder/registry.py: -------------------------------------------------------------------------------- 1 | # 2 | # gpodder.registry - Central hub for exchanging plugin resolvers (2014-03-09) 3 | # Copyright (c) 2014, Thomas Perl 4 | # 5 | # Permission to use, copy, modify, and/or distribute this software for any 6 | # purpose with or without fee is hereby granted, provided that the above 7 | # copyright notice and this permission notice appear in all copies. 8 | # 9 | # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH 10 | # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY 11 | # AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, 12 | # INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM 13 | # LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR 14 | # OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 15 | # PERFORMANCE OF THIS SOFTWARE. 16 | # 17 | 18 | import logging 19 | 20 | logger = logging.getLogger(__name__) 21 | 22 | 23 | class Resolver(object): 24 | def __init__(self, name, description): 25 | self._name = name 26 | self._description = description 27 | self._resolvers = [] 28 | 29 | def resolve(self, item, default, *args): 30 | for resolver in self._resolvers: 31 | result = resolver(item, *args) 32 | if result is not None: 33 | logger.info('{} resolved by {}: {} -> {}'.format(self._name, self._info(resolver), 34 | default, result)) 35 | return result 36 | 37 | return default 38 | 39 | def each(self, *args): 40 | for resolver in self._resolvers: 41 | result = resolver(*args) 42 | if result is not None: 43 | yield result 44 | 45 | def call_each(self, *args): 46 | list(self.each(*args)) 47 | 48 | def select(self, selector=None): 49 | for resolver in self._resolvers: 50 | if selector is None or selector(resolver): 51 | yield resolver 52 | 53 | def register(self, func): 54 | logger.debug('Registering {} resolver: {}'.format(self._name, func)) 55 | self._resolvers.append(func) 56 | return func 57 | 58 | def register_instance(self, klass): 59 | self._resolvers.append(klass()) 60 | return klass 61 | 62 | def _info(self, resolver): 63 | return '%s from %s' % (resolver.__name__ if hasattr(resolver, '__name__') 64 | else resolver.__class__.__name__, resolver.__module__) 65 | 66 | def _dump(self, indent=''): 67 | print('== {} ({}) =='.format(self._name, self._description)) 68 | print('\n'.join('%s- %s' % (indent, self._info(resolver)) for resolver in self._resolvers)) 69 | print() 70 | 71 | RESOLVER_NAMES = {'cover_art': 'Resolve the real cover art URL of an episode', 72 | 'download_url': 'Resolve the real download URL of an episode', 73 | 'episode_basename': 'Resolve a good, unique download filename for an episode', 74 | 'podcast_title': 'Resolve a good title for a podcast', 75 | 'content_type': 'Resolve the content type (audio, video) of an episode', 76 | 'feed_handler': 'Handle parsing of a feed', 77 | 'fallback_feed_handler': 'Handle parsing of a feed (catch-all)', 78 | 'url_shortcut': 'Expand shortcuts when adding a new URL', 79 | 'after_download': 'Function to call with episodes after download finishes', 80 | 'directory': 'Podcast directory and search provider'} 81 | 82 | LOCALS = locals() 83 | 84 | for name, description in RESOLVER_NAMES.items(): 85 | LOCALS[name] = Resolver(name, description) 86 | 87 | 88 | def dump(module_dict=LOCALS): 89 | for name in sorted(RESOLVER_NAMES): 90 | module_dict[name]._dump(' ') 91 | -------------------------------------------------------------------------------- /src/gpodder/storage.py: -------------------------------------------------------------------------------- 1 | # 2 | # gpodder.storage - JSON-based Database Backend (2013-05-20) 3 | # Copyright (c) 2013, Thomas Perl 4 | # 5 | # Permission to use, copy, modify, and/or distribute this software for any 6 | # purpose with or without fee is hereby granted, provided that the above 7 | # copyright notice and this permission notice appear in all copies. 8 | # 9 | # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH 10 | # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY 11 | # AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, 12 | # INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM 13 | # LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR 14 | # OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 15 | # PERFORMANCE OF THIS SOFTWARE. 16 | # 17 | 18 | 19 | import minidb 20 | 21 | from gpodder import model 22 | from gpodder import util 23 | 24 | import json 25 | import os 26 | import gzip 27 | import re 28 | import sys 29 | 30 | import logging 31 | logger = logging.getLogger(__name__) 32 | 33 | 34 | class MigrateJSONDBToMiniDB: 35 | def __init__(self, db): 36 | self.db = db 37 | self.jsondb_filename = re.sub(r'\.minidb$', '.jsondb', self.db.filename) 38 | 39 | def _append_podcast(self, podcast): 40 | # Dummy function to work as drop-in model.Model replacement 41 | ... 42 | 43 | def migrate(self): 44 | podcasts = {} 45 | classes = { 46 | 'podcast': model.PodcastChannel, 47 | 'episode': model.PodcastEpisode, 48 | } 49 | 50 | if os.path.exists(self.jsondb_filename): 51 | logger.info('Migrating from jsondb to minidb') 52 | data = json.loads(str(gzip.open(self.jsondb_filename, 'rb').read(), 'utf-8')) 53 | for table in ('podcast', 'episode'): 54 | cls = classes[table] 55 | for key, item in data[table].items(): 56 | if table == 'podcast': 57 | o = cls(self) 58 | podcasts[int(key)] = o 59 | elif table == 'episode': 60 | if item['podcast_id'] not in podcasts: 61 | logger.warn('Skipping orphaned episode: %s (podcast_id=%r)', 62 | item['title'], item['podcast_id']) 63 | continue 64 | o = cls(podcasts[item['podcast_id']]) 65 | 66 | for k, v in item.items(): 67 | if k == 'podcast_id': 68 | # Don't set the podcast id (will be set automatically) 69 | continue 70 | 71 | if hasattr(o, k): 72 | setattr(o, k, v) 73 | else: 74 | logger.warn('Skipping %s attribute: %s', table, k) 75 | 76 | o.save() 77 | 78 | os.rename(self.jsondb_filename, self.jsondb_filename + '.migrated') 79 | 80 | 81 | class Database: 82 | def __init__(self, filename, debug=False): 83 | self.filename = filename + '.minidb' 84 | 85 | need_migration = not os.path.exists(self.filename) 86 | 87 | self.db = minidb.Store(self.filename, debug=debug, smartupdate=True) 88 | self.db.register(model.PodcastEpisode) 89 | self.db.register(model.PodcastChannel) 90 | 91 | if need_migration: 92 | try: 93 | MigrateJSONDBToMiniDB(self).migrate() 94 | except Exception as e: 95 | logger.fatal('Could not migrate database: %s', e, exc_info=True) 96 | self.db.close() 97 | self.db = None 98 | util.delete_file(self.filename) 99 | sys.exit(1) 100 | 101 | def load_podcasts(self, *args): 102 | return model.PodcastChannel.load(self.db)(*args) 103 | 104 | def load_episodes(self, podcast, *args): 105 | return model.PodcastEpisode.load(self.db, podcast_id=podcast.id)(*args) 106 | 107 | def commit(self): 108 | self.db.commit() 109 | 110 | def close(self): 111 | self.db.commit() 112 | self.db.close() 113 | -------------------------------------------------------------------------------- /src/gpodder/util.py: -------------------------------------------------------------------------------- 1 | # 2 | # gPodder: Media and podcast aggregator 3 | # Copyright (c) 2005-2020 Thomas Perl and the gPodder Team 4 | # Copyright (c) 2011 Neal H. Walfield 5 | # 6 | # gPodder is free software; you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation; either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # gPodder is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | # 19 | 20 | # 21 | # util.py -- Misc utility functions 22 | # Thomas Perl 2007-08-04 23 | # 24 | 25 | """Miscellaneous helper functions for gPodder 26 | 27 | This module provides helper and utility functions for gPodder that 28 | are not tied to any specific part of gPodder. 29 | 30 | """ 31 | 32 | import gpodder 33 | 34 | import logging 35 | logger = logging.getLogger(__name__) 36 | 37 | import os 38 | import os.path 39 | import stat 40 | import sys 41 | import string 42 | 43 | import re 44 | from html.entities import entitydefs 45 | import time 46 | import datetime 47 | import threading 48 | import tempfile 49 | 50 | import json 51 | import urllib.parse 52 | import urllib.request 53 | import http.client 54 | import mimetypes 55 | import itertools 56 | import contextlib 57 | 58 | 59 | import locale 60 | try: 61 | locale.setlocale(locale.LC_ALL, '') 62 | except Exception as e: 63 | logger.warn('Cannot set locale (%s)', e, exc_info=True) 64 | 65 | try: 66 | import platform 67 | win32 = (platform.system() == 'Windows') 68 | except Exception as e: 69 | logger.warn('Cannot determine platform (%s)', e, exc_info=True) 70 | win32 = False 71 | 72 | # Native filesystem encoding detection 73 | encoding = sys.getfilesystemencoding() 74 | 75 | if encoding is None: 76 | if 'LANG' in os.environ and '.' in os.environ['LANG']: 77 | lang = os.environ['LANG'] 78 | (language, encoding) = lang.rsplit('.', 1) 79 | logger.info('Detected encoding: %s', encoding) 80 | else: 81 | encoding = 'utf-8' 82 | 83 | 84 | # Filename / folder name sanitization 85 | def _sanitize_char(c): 86 | if c in string.whitespace: 87 | return ' ' 88 | elif c in ',-.()': 89 | return c 90 | elif c in string.punctuation or ord(c) <= 31: 91 | return '_' 92 | 93 | return c 94 | 95 | SANITIZATION_TABLE = ''.join(map(_sanitize_char, list(map(chr, list(range(256)))))) 96 | del _sanitize_char 97 | 98 | _MIME_TYPE_LIST = [ 99 | ('.aac', 'audio/aac'), 100 | ('.axa', 'audio/annodex'), 101 | ('.flac', 'audio/flac'), 102 | ('.m4b', 'audio/m4b'), 103 | ('.m4a', 'audio/mp4'), 104 | ('.mp3', 'audio/mpeg'), 105 | ('.spx', 'audio/ogg'), 106 | ('.oga', 'audio/ogg'), 107 | ('.ogg', 'audio/ogg'), 108 | ('.wma', 'audio/x-ms-wma'), 109 | ('.3gp', 'video/3gpp'), 110 | ('.axv', 'video/annodex'), 111 | ('.divx', 'video/divx'), 112 | ('.m4v', 'video/m4v'), 113 | ('.mp4', 'video/mp4'), 114 | ('.ogv', 'video/ogg'), 115 | ('.mov', 'video/quicktime'), 116 | ('.flv', 'video/x-flv'), 117 | ('.mkv', 'video/x-matroska'), 118 | ('.wmv', 'video/x-ms-wmv'), 119 | ('.opus', 'audio/opus'), 120 | ] 121 | 122 | _MIME_TYPES = dict((k, v) for v, k in _MIME_TYPE_LIST) 123 | _MIME_TYPES_EXT = dict(_MIME_TYPE_LIST) 124 | 125 | 126 | def make_directory(path): 127 | """ 128 | Tries to create a directory if it does not exist already. 129 | Returns True if the directory exists after the function 130 | call, False otherwise. 131 | If the directory already exists it returns True if it is 132 | writable. 133 | """ 134 | if os.path.isdir(path): 135 | return os.access(path, os.W_OK) 136 | 137 | try: 138 | os.makedirs(path) 139 | except: 140 | logger.warn('Could not create directory: %s', path) 141 | return False 142 | 143 | return True 144 | 145 | 146 | def normalize_feed_url(url): 147 | """ 148 | Converts any URL to http:// or ftp:// so that it can be 149 | used with "wget". If the URL cannot be converted (invalid 150 | or unknown scheme), "None" is returned. 151 | 152 | This will also normalize feed:// and itpc:// to http://. 153 | 154 | >>> normalize_feed_url('itpc://example.org/podcast.rss') 155 | 'http://example.org/podcast.rss' 156 | 157 | If no URL scheme is defined (e.g. "curry.com"), we will 158 | simply assume the user intends to add a http:// feed. 159 | 160 | >>> normalize_feed_url('curry.com') 161 | 'http://curry.com/' 162 | 163 | It will also take care of converting the domain name to 164 | all-lowercase (because domains are not case sensitive): 165 | 166 | >>> normalize_feed_url('http://Example.COM/') 167 | 'http://example.com/' 168 | 169 | Some other minimalistic changes are also taken care of, 170 | e.g. a ? with an empty query is removed: 171 | 172 | >>> normalize_feed_url('http://example.org/test?') 173 | 'http://example.org/test' 174 | """ 175 | if not url or len(url) < 8: 176 | return None 177 | 178 | # Assume HTTP for URLs without scheme 179 | if '://' not in url: 180 | url = 'http://' + url 181 | 182 | scheme, netloc, path, query, fragment = urllib.parse.urlsplit(url) 183 | 184 | # Schemes and domain names are case insensitive 185 | scheme, netloc = scheme.lower(), netloc.lower() 186 | 187 | # Normalize empty paths to "/" 188 | if path == '': 189 | path = '/' 190 | 191 | # feed://, itpc://, itms:// and itmss:// are really http:// 192 | if scheme in ('feed', 'itpc', 'itms', 'itmss'): 193 | scheme = 'http' 194 | 195 | if scheme not in ('http', 'https', 'ftp', 'file'): 196 | return None 197 | 198 | # urlunsplit might return "a slighty different, but equivalent URL" 199 | return urllib.parse.urlunsplit((scheme, netloc, path, query, fragment)) 200 | 201 | 202 | def username_password_from_url(url): 203 | r""" 204 | Returns a tuple (username,password) containing authentication 205 | data from the specified URL or (None,None) if no authentication 206 | data can be found in the URL. 207 | 208 | See Section 3.1 of RFC 1738 (http://www.ietf.org/rfc/rfc1738.txt) 209 | 210 | >>> username_password_from_url('https://@host.com/') 211 | ('', None) 212 | >>> username_password_from_url('telnet://host.com/') 213 | (None, None) 214 | >>> username_password_from_url('ftp://foo:@host.com/') 215 | ('foo', '') 216 | >>> username_password_from_url('http://a:b@host.com/') 217 | ('a', 'b') 218 | >>> username_password_from_url(1) 219 | Traceback (most recent call last): 220 | ... 221 | ValueError: URL has to be a string or unicode object. 222 | >>> username_password_from_url(None) 223 | Traceback (most recent call last): 224 | ... 225 | ValueError: URL has to be a string or unicode object. 226 | >>> username_password_from_url('http://a@b:c@host.com/') 227 | ('a@b', 'c') 228 | >>> username_password_from_url('ftp://a:b:c@host.com/') 229 | ('a', 'b:c') 230 | >>> username_password_from_url('http://i%2Fo:P%40ss%3A@host.com/') 231 | ('i/o', 'P@ss:') 232 | >>> username_password_from_url('ftp://%C3%B6sterreich@host.com/') 233 | ('\xf6sterreich', None) 234 | >>> username_password_from_url('http://w%20x:y%20z@example.org/') 235 | ('w x', 'y z') 236 | >>> username_password_from_url('http://example.com/x@y:z@test.com/') 237 | (None, None) 238 | """ 239 | if type(url) not in (str, str): 240 | raise ValueError('URL has to be a string or unicode object.') 241 | 242 | (username, password) = (None, None) 243 | 244 | (scheme, netloc, path, params, query, fragment) = urllib.parse.urlparse(url) 245 | 246 | if '@' in netloc: 247 | (authentication, netloc) = netloc.rsplit('@', 1) 248 | if ':' in authentication: 249 | (username, password) = authentication.split(':', 1) 250 | 251 | # RFC1738 dictates that we should not allow ['/', '@', ':'] 252 | # characters in the username and password field (Section 3.1): 253 | # 254 | # 1. The "/" can't be in there at this point because of the way 255 | # urlparse (which we use above) works. 256 | # 2. Due to gPodder bug 1521, we allow "@" in the username and 257 | # password field. We use netloc.rsplit('@', 1), which will 258 | # make sure that we split it at the last '@' in netloc. 259 | # 3. The colon must be excluded (RFC2617, Section 2) in the 260 | # username, but is apparently allowed in the password. This 261 | # is handled by the authentication.split(':', 1) above, and 262 | # will cause any extraneous ':'s to be part of the password. 263 | 264 | username = urllib.parse.unquote(username) 265 | password = urllib.parse.unquote(password) 266 | else: 267 | username = urllib.parse.unquote(authentication) 268 | 269 | return (username, password) 270 | 271 | 272 | def calculate_size(path): 273 | """ 274 | Tries to calculate the size of a directory, including any 275 | subdirectories found. The returned value might not be 276 | correct if the user doesn't have appropriate permissions 277 | to list all subdirectories of the given path. 278 | """ 279 | if path is None: 280 | return 0 281 | 282 | if os.path.dirname(path) == '/': 283 | return 0 284 | 285 | if os.path.isfile(path): 286 | return os.path.getsize(path) 287 | 288 | if os.path.isdir(path) and not os.path.islink(path): 289 | sum = os.path.getsize(path) 290 | 291 | try: 292 | for item in os.listdir(path): 293 | try: 294 | sum += calculate_size(os.path.join(path, item)) 295 | except: 296 | logger.warn('Cannot get size for %s', path, exc_info=True) 297 | except: 298 | logger.warn('Cannot access %s', path, exc_info=True) 299 | 300 | return sum 301 | 302 | return 0 303 | 304 | 305 | def file_modification_datetime(filename): 306 | """ 307 | Returns the modification date of the specified file 308 | as a datetime.datetime object or None if the modification 309 | date cannot be determined. 310 | """ 311 | if filename is None: 312 | return None 313 | 314 | if not os.access(filename, os.R_OK): 315 | return None 316 | 317 | try: 318 | s = os.stat(filename) 319 | timestamp = s[stat.ST_MTIME] 320 | return datetime.datetime.fromtimestamp(timestamp) 321 | except: 322 | logger.warn('Cannot get mtime for %s', filename, exc_info=True) 323 | return None 324 | 325 | 326 | def file_age_in_days(filename): 327 | """ 328 | Returns the age of the specified filename in days or 329 | zero if the modification date cannot be determined. 330 | """ 331 | dt = file_modification_datetime(filename) 332 | if dt is None: 333 | return 0 334 | else: 335 | return (datetime.datetime.now()-dt).days 336 | 337 | 338 | def format_date(timestamp): 339 | """ 340 | Converts a UNIX timestamp to a date representation. 341 | 342 | Returns None if there has been an error converting the 343 | timestamp to a string representation. 344 | """ 345 | if timestamp is None: 346 | return None 347 | 348 | seconds_in_a_day = 60*60*24 349 | 350 | try: 351 | timestamp_date = time.localtime(timestamp)[:3] 352 | except ValueError as ve: 353 | logger.warn('Cannot convert timestamp', exc_info=True) 354 | return None 355 | 356 | try: 357 | diff = int((time.time() - timestamp)/seconds_in_a_day) 358 | except: 359 | logger.warn('Cannot convert "%s" to date.', timestamp, exc_info=True) 360 | return None 361 | 362 | try: 363 | timestamp = datetime.datetime.fromtimestamp(timestamp) 364 | except: 365 | return None 366 | 367 | if diff < 7: 368 | # Weekday name 369 | return timestamp.strftime('%A') 370 | else: 371 | # Locale's appropriate date representation 372 | return timestamp.strftime('%x') 373 | 374 | 375 | def delete_file(filename): 376 | """Delete a file from the filesystem 377 | 378 | Errors (permissions errors or file not found) 379 | are silently ignored. 380 | """ 381 | try: 382 | os.remove(filename) 383 | except Exception as e: 384 | logger.warn('Cannot delete file: %s', filename, exc_info=True) 385 | 386 | 387 | def remove_html_tags(html): 388 | """ 389 | Remove HTML tags from a string and replace numeric and 390 | named entities with the corresponding character, so the 391 | HTML text can be displayed in a simple text view. 392 | """ 393 | if html is None: 394 | return None 395 | 396 | # If we would want more speed, we could make these global 397 | re_strip_tags = re.compile('<[^>]*>') 398 | re_unicode_entities = re.compile('&#(\d{2,4});') 399 | re_html_entities = re.compile('&(.{2,8});') 400 | re_newline_tags = re.compile('(]*>|<[/]?ul[^>]*>|)', re.I) 401 | re_listing_tags = re.compile(']*>', re.I) 402 | 403 | result = html 404 | 405 | # Convert common HTML elements to their text equivalent 406 | result = re_newline_tags.sub('\n', result) 407 | result = re_listing_tags.sub('\n * ', result) 408 | result = re.sub('<[Pp]>', '\n\n', result) 409 | 410 | # Remove all HTML/XML tags from the string 411 | result = re_strip_tags.sub('', result) 412 | 413 | # Convert numeric XML entities to their unicode character 414 | result = re_unicode_entities.sub(lambda x: chr(int(x.group(1))), result) 415 | 416 | # Convert named HTML entities to their unicode character 417 | result = re_html_entities.sub(lambda x: entitydefs.get(x.group(1), ''), result) 418 | 419 | # Convert more than two newlines to two newlines 420 | result = re.sub('([\r\n]{2})([\r\n])+', '\\1', result) 421 | 422 | return result.strip() 423 | 424 | 425 | def wrong_extension(extension): 426 | """ 427 | Determine if a given extension looks like it's 428 | wrong (e.g. empty, extremely long or spaces) 429 | 430 | Returns True if the extension most likely is a 431 | wrong one and should be replaced. 432 | 433 | >>> wrong_extension('.mp3') 434 | False 435 | >>> wrong_extension('.divx') 436 | False 437 | >>> wrong_extension('mp3') 438 | True 439 | >>> wrong_extension('') 440 | True 441 | >>> wrong_extension('.12 - Everybody') 442 | True 443 | >>> wrong_extension('.mp3 ') 444 | True 445 | >>> wrong_extension('.') 446 | True 447 | >>> wrong_extension('.42') 448 | True 449 | """ 450 | if not extension: 451 | return True 452 | elif len(extension) > 5: 453 | return True 454 | elif ' ' in extension: 455 | return True 456 | elif extension == '.': 457 | return True 458 | elif not extension.startswith('.'): 459 | return True 460 | else: 461 | try: 462 | # "." is an invalid extension 463 | float(extension) 464 | return True 465 | except: 466 | pass 467 | 468 | return False 469 | 470 | 471 | def extension_from_mimetype(mimetype): 472 | """ 473 | Simply guesses what the file extension should be from the mimetype 474 | 475 | >>> extension_from_mimetype('audio/mp4') 476 | '.m4a' 477 | >>> extension_from_mimetype('audio/ogg') 478 | '.ogg' 479 | >>> extension_from_mimetype('audio/mpeg') 480 | '.mp3' 481 | >>> extension_from_mimetype('video/x-matroska') 482 | '.mkv' 483 | >>> extension_from_mimetype('wrong-mimetype') 484 | '' 485 | """ 486 | if mimetype in _MIME_TYPES: 487 | return _MIME_TYPES[mimetype] 488 | return mimetypes.guess_extension(mimetype) or '' 489 | 490 | 491 | def filename_from_url(url): 492 | """ 493 | Extracts the filename and (lowercase) extension (with dot) 494 | from a URL, e.g. http://server.com/file.MP3?download=yes 495 | will result in the string ("file", ".mp3") being returned. 496 | 497 | This function will also try to best-guess the "real" 498 | extension for a media file (audio, video) by 499 | trying to match an extension to these types and recurse 500 | into the query string to find better matches, if the 501 | original extension does not resolve to a known type. 502 | 503 | >>> filename_from_url('http://my.net/redirect.php?my.net/file.ogg') 504 | ('file', '.ogg') 505 | >>> filename_from_url('http://server/get.jsp?file=/episode0815.MOV') 506 | ('episode0815', '.mov') 507 | >>> filename_from_url('http://s/redirect.mp4?http://serv2/test.mp4') 508 | ('test', '.mp4') 509 | >>> filename_from_url('http://example.com/?download_media_file=25&ptm_context=x-mp3&ptm_file=DTM006_Zeppelin.mp3') 510 | ('DTM006_Zeppelin', '.mp3') 511 | """ 512 | (scheme, netloc, path, para, query, fragid) = urllib.parse.urlparse(url) 513 | (filename, extension) = os.path.splitext(os.path.basename(urllib.parse.unquote(path))) 514 | 515 | if file_type_by_extension(extension) is not None and not query.startswith(scheme+'://'): 516 | # We have found a valid extension (audio, video) 517 | # and the query string doesn't look like a URL 518 | return (filename, extension.lower()) 519 | 520 | # If the query string looks like a possible URL, try that first 521 | if len(query.strip()) > 0 and query.find('/') != -1: 522 | query_url = '://'.join((scheme, urllib.parse.unquote(query))) 523 | (query_filename, query_extension) = filename_from_url(query_url) 524 | 525 | if file_type_by_extension(query_extension) is not None: 526 | filename, extension = os.path.splitext(os.path.basename(query_url)) 527 | return (filename, extension.lower()) 528 | 529 | # Nothing found so far - try to see if any of the query parameters look like a filename 530 | # extracts the filename from e.g. 'http://example.com?foo=bar.mp3' to ('bar', '.mp3') 531 | parsed = urllib.parse.urlparse(url) 532 | for k, v in urllib.parse.parse_qsl(parsed.query): 533 | value_filename, value_extension = os.path.splitext(os.path.basename(v)) 534 | if file_type_by_extension(value_extension) is not None: 535 | return (value_filename, value_extension.lower()) 536 | 537 | # No exact match found, simply return the original filename & extension 538 | return (filename, extension.lower()) 539 | 540 | 541 | def file_type_by_extension(extension): 542 | """ 543 | Tries to guess the file type by looking up the filename 544 | extension from a table of known file types. Will return 545 | "audio", "video" or None. 546 | 547 | >>> file_type_by_extension('.aif') 548 | 'audio' 549 | >>> file_type_by_extension('.3GP') 550 | 'video' 551 | >>> file_type_by_extension('.m4a') 552 | 'audio' 553 | >>> file_type_by_extension('.txt') is None 554 | True 555 | >>> file_type_by_extension(None) is None 556 | True 557 | >>> file_type_by_extension('ogg') 558 | Traceback (most recent call last): 559 | ... 560 | ValueError: Extension does not start with a dot: ogg 561 | """ 562 | if not extension: 563 | return None 564 | 565 | if not extension.startswith('.'): 566 | raise ValueError('Extension does not start with a dot: %s' % extension) 567 | 568 | extension = extension.lower() 569 | 570 | if extension in _MIME_TYPES_EXT: 571 | return _MIME_TYPES_EXT[extension].split('/')[0] 572 | 573 | # Need to prepend something to the extension, so guess_type works 574 | type, encoding = mimetypes.guess_type('file'+extension) 575 | 576 | if type is not None and '/' in type: 577 | filetype, rest = type.split('/', 1) 578 | if filetype in ('audio', 'video', 'image'): 579 | return filetype 580 | 581 | return None 582 | 583 | 584 | def url_strip_authentication(url): 585 | """ 586 | Strips authentication data from an URL. Returns the URL with 587 | the authentication data removed from it. 588 | 589 | >>> url_strip_authentication('https://host.com/') 590 | 'https://host.com/' 591 | >>> url_strip_authentication('telnet://foo:bar@host.com/') 592 | 'telnet://host.com/' 593 | >>> url_strip_authentication('ftp://billy@example.org') 594 | 'ftp://example.org' 595 | >>> url_strip_authentication('ftp://billy:@example.org') 596 | 'ftp://example.org' 597 | >>> url_strip_authentication('http://aa:bc@localhost/x') 598 | 'http://localhost/x' 599 | >>> url_strip_authentication('http://i%2Fo:P%40ss%3A@blubb.lan/u.html') 600 | 'http://blubb.lan/u.html' 601 | >>> url_strip_authentication('http://c:d@x.org/') 602 | 'http://x.org/' 603 | >>> url_strip_authentication('http://P%40%3A:i%2F@cx.lan') 604 | 'http://cx.lan' 605 | >>> url_strip_authentication('http://x@x.com:s3cret@example.com/') 606 | 'http://example.com/' 607 | """ 608 | url_parts = list(urllib.parse.urlsplit(url)) 609 | # url_parts[1] is the HOST part of the URL 610 | 611 | # Remove existing authentication data 612 | if '@' in url_parts[1]: 613 | url_parts[1] = url_parts[1].rsplit('@', 1)[1] 614 | 615 | return urllib.parse.urlunsplit(url_parts) 616 | 617 | 618 | def url_add_authentication(url, username, password): 619 | """ 620 | Adds authentication data (username, password) to a given 621 | URL in order to construct an authenticated URL. 622 | 623 | >>> url_add_authentication('https://host.com/', '', None) 624 | 'https://host.com/' 625 | >>> url_add_authentication('http://example.org/', None, None) 626 | 'http://example.org/' 627 | >>> url_add_authentication('telnet://host.com/', 'foo', 'bar') 628 | 'telnet://foo:bar@host.com/' 629 | >>> url_add_authentication('ftp://example.org', 'billy', None) 630 | 'ftp://billy@example.org' 631 | >>> url_add_authentication('ftp://example.org', 'billy', '') 632 | 'ftp://billy:@example.org' 633 | >>> url_add_authentication('http://localhost/x', 'aa', 'bc') 634 | 'http://aa:bc@localhost/x' 635 | >>> url_add_authentication('http://blubb.lan/u.html', 'i/o', 'P@ss:') 636 | 'http://i%2Fo:P@ss:@blubb.lan/u.html' 637 | >>> url_add_authentication('http://a:b@x.org/', 'c', 'd') 638 | 'http://c:d@x.org/' 639 | >>> url_add_authentication('http://i%2F:P%40%3A@cx.lan', 'P@x', 'i/') 640 | 'http://P@x:i%2F@cx.lan' 641 | >>> url_add_authentication('http://x.org/', 'a b', 'c d') 642 | 'http://a%20b:c%20d@x.org/' 643 | """ 644 | if username is None or username == '': 645 | return url 646 | 647 | # Relaxations of the strict quoting rules (bug 1521): 648 | # 1. Accept '@' in username and password 649 | # 2. Acecpt ':' in password only 650 | username = urllib.parse.quote(username, safe='@') 651 | 652 | if password is not None: 653 | password = urllib.parse.quote(password, safe='@:') 654 | auth_string = ':'.join((username, password)) 655 | else: 656 | auth_string = username 657 | 658 | url = url_strip_authentication(url) 659 | 660 | url_parts = list(urllib.parse.urlsplit(url)) 661 | # url_parts[1] is the HOST part of the URL 662 | url_parts[1] = '@'.join((auth_string, url_parts[1])) 663 | 664 | return urllib.parse.urlunsplit(url_parts) 665 | 666 | 667 | def urlopen(url, headers=None, data=None, timeout=None): 668 | """ 669 | An URL opener with the User-agent set to gPodder (with version) 670 | """ 671 | username, password = username_password_from_url(url) 672 | if username is not None or password is not None: 673 | url = url_strip_authentication(url) 674 | password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm() 675 | password_mgr.add_password(None, url, username, password) 676 | handler = urllib.request.HTTPBasicAuthHandler(password_mgr) 677 | opener = urllib.request.build_opener(handler) 678 | else: 679 | opener = urllib.request.build_opener() 680 | 681 | if headers is None: 682 | headers = {} 683 | else: 684 | headers = dict(headers) 685 | 686 | # Allow the calling code to supply a custom user-agent 687 | if 'User-agent' not in headers: 688 | headers['User-agent'] = gpodder.user_agent 689 | 690 | request = urllib.request.Request(url, data=data, headers=headers) 691 | if timeout is None: 692 | return opener.open(request) 693 | else: 694 | return opener.open(request, timeout=timeout) 695 | 696 | 697 | def http_request(url, method='HEAD'): 698 | (scheme, netloc, path, parms, qry, fragid) = urllib.parse.urlparse(url) 699 | if scheme == 'https': 700 | conn = http.client.HTTPSConnection(netloc) 701 | else: 702 | conn = http.client.HTTPConnection(netloc) 703 | start = len(scheme) + len('://') + len(netloc) 704 | conn.request(method, url[start:]) 705 | return conn.getresponse() 706 | 707 | 708 | def sanitize_filename(filename, max_length=0, use_ascii=False): 709 | """ 710 | Generate a sanitized version of a filename that can 711 | be written on disk (i.e. remove/replace invalid 712 | characters and encode in the native language) and 713 | trim filename if greater than max_length (0 = no limit). 714 | 715 | If use_ascii is True, don't encode in the native language, 716 | but use only characters from the ASCII character set. 717 | """ 718 | if not isinstance(filename, str): 719 | raise Exception('filename is not a string') 720 | 721 | if max_length > 0 and len(filename) > max_length: 722 | logger.info('Limiting file/folder name "%s" to %d characters.', filename, max_length) 723 | filename = filename[:max_length] 724 | 725 | if use_ascii: 726 | filename = filename.encode('ascii', 'ignore').decode('ascii') 727 | 728 | filename = filename.translate(SANITIZATION_TABLE) 729 | filename = filename.strip('.' + string.whitespace) 730 | 731 | return filename 732 | 733 | 734 | def generate_names(filename): 735 | basename, ext = os.path.splitext(filename) 736 | for i in itertools.count(): 737 | if i: 738 | yield '%s (%d)%s' % (basename, i+1, ext) 739 | else: 740 | yield filename 741 | 742 | 743 | def get_update_info(url='http://gpodder.org/downloads'): 744 | """ 745 | Get up to date release information from gpodder.org. 746 | 747 | Returns a tuple: (up_to_date, latest_version, release_date, days_since) 748 | 749 | Example result (up to date version, 20 days after release): 750 | (True, '3.0.4', '2012-01-24', 20) 751 | 752 | Example result (outdated version, 10 days after release): 753 | (False, '3.0.5', '2012-02-29', 10) 754 | """ 755 | data = urlopen(url).read().decode('utf-8') 756 | id_field_re = re.compile(r'<([a-z]*)[^>]*id="([^"]*)"[^>]*>([^<]*)') 757 | info = dict((m.group(2), m.group(3)) for m in id_field_re.finditer(data)) 758 | 759 | latest_version = info['latest-version'] 760 | release_date = info['release-date'] 761 | 762 | release_parsed = datetime.datetime.strptime(release_date, '%Y-%m-%d') 763 | days_since_release = (datetime.datetime.today() - release_parsed).days 764 | 765 | convert = lambda s: tuple(int(x) for x in s.split('.')) 766 | up_to_date = (convert(gpodder.__version__) >= convert(latest_version)) 767 | 768 | return up_to_date, latest_version, release_date, days_since_release 769 | 770 | 771 | def run_in_background(function, daemon=False): 772 | logger.debug('run_in_background: %s (%s)', function, str(daemon)) 773 | thread = threading.Thread(target=function) 774 | thread.setDaemon(daemon) 775 | thread.start() 776 | return thread 777 | 778 | 779 | @contextlib.contextmanager 780 | def update_file_safely(target_filename): 781 | """Update file in a safe way using atomic renames 782 | 783 | Example usage: 784 | 785 | >>> filename = tempfile.NamedTemporaryFile(delete=False).name 786 | >>> with update_file_safely(filename) as temp_filename: 787 | ... with open(temp_filename, 'w') as fp: 788 | ... fp.write('Try to write this safely') 789 | 24 790 | >>> open(filename).read() 791 | 'Try to write this safely' 792 | >>> with update_file_safely(filename) as temp_filename: 793 | ... with open(temp_filename, 'w') as fp: 794 | ... fp.write('Updated!') 795 | ... raise ValueError('something bad happened') 796 | Traceback (most recent call last): 797 | ... 798 | ValueError: something bad happened 799 | >>> open(filename).read() 800 | 'Try to write this safely' 801 | >>> os.remove(filename) 802 | 803 | Note that the temporary file will be deleted and the atomic 804 | rename will not take place if something in the "with"-block 805 | raises an exception. 806 | 807 | Does not take care of race conditions, as the name of the 808 | temporary file is predictable and not unique between different 809 | (possibly simultaneous) invocations of this function. 810 | """ 811 | dirname = os.path.dirname(target_filename) 812 | basename = os.path.basename(target_filename) 813 | 814 | tmp_filename = os.path.join(dirname, '.tmp-' + basename) 815 | bak_filename = os.path.join(dirname, basename + '.bak') 816 | try: 817 | yield tmp_filename 818 | except Exception as e: 819 | logger.warn('Exception while atomic-saving file: %s', e, exc_info=True) 820 | delete_file(tmp_filename) 821 | raise 822 | 823 | # No atomic rename on Windows (http://bugs.python.org/issue1704547) 824 | if win32: 825 | if os.path.exists(target_filename): 826 | if os.path.exists(bak_filename): 827 | os.unlink(bak_filename) 828 | os.rename(target_filename, bak_filename) 829 | os.rename(tmp_filename, target_filename) 830 | if os.path.exists(bak_filename): 831 | os.unlink(bak_filename) 832 | return 833 | 834 | os.rename(tmp_filename, target_filename) 835 | 836 | 837 | def format_time(seconds): 838 | dt = datetime.datetime.utcfromtimestamp(seconds) 839 | return dt.strftime('%H:%M:%S') 840 | 841 | 842 | def find_command(command): 843 | """ 844 | Searches the system's PATH for a specific command that is 845 | executable by the user. Returns the first occurence of an 846 | executable binary in the PATH, or None if the command is 847 | not available. 848 | """ 849 | 850 | if 'PATH' not in os.environ: 851 | return None 852 | 853 | for path in os.environ['PATH'].split(os.pathsep): 854 | command_file = os.path.join(path, command) 855 | if os.path.isfile(command_file) and os.access(command_file, os.X_OK): 856 | return command_file 857 | 858 | return None 859 | 860 | 861 | def read_json(url): 862 | return json.loads(urlopen(url).read().decode('utf-8')) 863 | -------------------------------------------------------------------------------- /test/test_gpodder/test_model.py: -------------------------------------------------------------------------------- 1 | # 2 | # test_gpodder.model - Unit tests for gpodder.model (2013-02-12) 3 | # Copyright (c) 2013, Thomas Perl 4 | # 5 | # Permission to use, copy, modify, and/or distribute this software for any 6 | # purpose with or without fee is hereby granted, provided that the above 7 | # copyright notice and this permission notice appear in all copies. 8 | # 9 | # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH 10 | # REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY 11 | # AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, 12 | # INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM 13 | # LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR 14 | # OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 15 | # PERFORMANCE OF THIS SOFTWARE. 16 | # 17 | 18 | 19 | import unittest 20 | 21 | import gpodder 22 | 23 | from gpodder import model 24 | 25 | 26 | class TestEpisodePublishedProperties(unittest.TestCase): 27 | PUBLISHED_UNIXTIME = 1360666744 28 | PUBLISHED_SORT = '2013-02-12' 29 | 30 | def setUp(self): 31 | self.podcast = model.PodcastChannel(None) 32 | self.episode = model.PodcastEpisode(self.podcast) 33 | self.episode.published = self.PUBLISHED_UNIXTIME 34 | 35 | def test_sortdate(self): 36 | self.assertEqual(self.episode.sortdate, self.PUBLISHED_SORT) 37 | 38 | 39 | class TestSectionFromContentType(unittest.TestCase): 40 | def setUp(self): 41 | self.podcast = model.PodcastChannel(None) 42 | self.podcast.url = 'http://example.com/feed.rss' 43 | self.audio_episode = model.PodcastEpisode(self.podcast) 44 | self.audio_episode.mime_type = 'audio/mpeg' 45 | self.video_episode = model.PodcastEpisode(self.podcast) 46 | self.video_episode.mime_type = 'video/mp4' 47 | 48 | def test_audio(self): 49 | self.podcast._children = [self.audio_episode] 50 | self.assertEqual(self.podcast._get_content_type(), 'audio') 51 | 52 | def test_video(self): 53 | self.podcast._children = [self.video_episode] 54 | self.assertEqual(self.podcast._get_content_type(), 'video') 55 | 56 | def test_more_video_than_audio(self): 57 | self.podcast._children = [self.audio_episode, self.video_episode, self.video_episode] 58 | self.assertEqual(self.podcast._get_content_type(), 'video') 59 | --------------------------------------------------------------------------------