├── pakrat ├── log.py ├── yumbase.py ├── repos.py ├── __init__.py ├── util.py ├── repo.py └── progress.py ├── COPYING ├── setup.py ├── packaging └── pakrat.spec ├── bin └── pakrat └── README.md /pakrat/log.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import syslog 3 | 4 | def write(pri, message): 5 | """ Record a log message. 6 | 7 | Currently just uses syslog, and if unattended, writes the log messages 8 | to stdout so they can be piped elsewhere. 9 | """ 10 | syslog.openlog('pakrat') # sets log ident 11 | syslog.syslog(pri, message) 12 | if not sys.stdout.isatty(): 13 | print message # print if running unattended 14 | 15 | def debug(message): 16 | """ Record a debugging message. """ 17 | write(syslog.LOG_DEBUG, 'debug: %s' % message) 18 | 19 | def trace(message): 20 | """ Record a trace message """ 21 | write(syslog.LOG_DEBUG, 'trace: %s' % message) 22 | 23 | def error(message): 24 | """ Record an error message. """ 25 | write(syslog.LOG_ERR, 'error: %s' % message) 26 | 27 | def info(message): 28 | """ Record an informational message. """ 29 | write(syslog.LOG_INFO, message) 30 | -------------------------------------------------------------------------------- /pakrat/yumbase.py: -------------------------------------------------------------------------------- 1 | import yum 2 | 3 | """ YumBase object specifically for pakrat. 4 | 5 | This class is a simple extension of the yum.YumBase class. It is required 6 | because pakrat uses YUM in an atypical way. When we load a YUM object, we want 7 | only the scaffolding, and none of the system repositories or other inherited 8 | configuration. This sounds easy but is not really built-in to YUM. This method 9 | also uses the YUM client library to create its own cachedir on instantiation. 10 | """ 11 | class YumBase(yum.YumBase): 12 | 13 | def __init__(self): 14 | """ Create a new YumBase object for use in pakrat. """ 15 | yum.YumBase.__init__(self) 16 | self.preconf = yum._YumPreBaseConf() 17 | self.preconf.debuglevel = 0 18 | self.prerepoconf = yum._YumPreRepoConf() 19 | self.setCacheDir(force=True, reuse=False, 20 | tmpdir=yum.misc.getCacheDir()) 21 | self.repos.repos = {} 22 | -------------------------------------------------------------------------------- /COPYING: -------------------------------------------------------------------------------- 1 | MIT LICENSE 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from setuptools import setup 3 | 4 | def required_module(module): 5 | """ Test for the presence of a given module. 6 | 7 | This function attempts to load a module, and if it fails to load, a message 8 | is displayed and installation is aborted. This is required because YUM and 9 | createrepo are not compatible with setuptools, and pakrat cannot function 10 | without either one of them. 11 | """ 12 | try: 13 | __import__(module) 14 | return True 15 | except: 16 | print '\n'.join([ 17 | 'The "%s" module is required, but was not found.' % module, 18 | 'Please install the module and try again.' 19 | ]) 20 | sys.exit(1) 21 | 22 | required_module('yum') 23 | required_module('createrepo') 24 | 25 | setup(name='pakrat', 26 | version='0.3.2', 27 | description='A tool for mirroring and versioning YUM repositories', 28 | author='Ryan Uber', 29 | author_email='ru@ryanuber.com', 30 | url='https://github.com/ryanuber/pakrat', 31 | packages=['pakrat'], 32 | scripts=['bin/pakrat'], 33 | package_data={'pakrat': ['LICENSE', 'README.md']} 34 | ) 35 | -------------------------------------------------------------------------------- /packaging/pakrat.spec: -------------------------------------------------------------------------------- 1 | %{!?python_sitelib: %define python_sitelib %(%{__python} -c "from distutils.sysconfig import get_python_lib; print get_python_lib()")} 2 | %define pakrat_dir %(tar -tzf %{SOURCE0} | egrep '^(\./)?pakrat(-[^/]*)?/$') 3 | 4 | name: pakrat 5 | summary: A Python library for mirroring and versioning YUM repositories 6 | version: 0.3.2 7 | release: 1%{?dist} 8 | buildarch: noarch 9 | license: MIT 10 | source0: %{name}.tar.gz 11 | buildrequires: yum, createrepo, python-setuptools 12 | requires: yum, createrepo 13 | 14 | %description 15 | Pakrat is a Pythonic library used to mirror YUM repositories using 16 | a snapshot-based approach with common package file storage to reduce 17 | the footprint of storing versioned repositories. Pakrat uses the 18 | standard YUM repository configuration format and supports baseurls 19 | as well as mirrorlists. Pakrat provides both a command-line 20 | interface as well as an easy-to-use Python api for integration with 21 | other projects. 22 | 23 | %prep 24 | %setup -n %{pakrat_dir} 25 | 26 | %build 27 | %{__python} setup.py build 28 | 29 | %install 30 | %{__python} setup.py install -O1 --skip-build --root %{buildroot} 31 | 32 | %clean 33 | rm -rf %{buildroot} 34 | 35 | %files 36 | %defattr(-,root,root,-) 37 | %{python_sitelib}/%{name}* 38 | %attr(0755, root, root) %{_bindir}/%{name} 39 | 40 | %changelog 41 | * %(date "+%a %b %d %Y") %{name} - %{version}-%{release} 42 | - Automatic build 43 | -------------------------------------------------------------------------------- /pakrat/repos.py: -------------------------------------------------------------------------------- 1 | import os 2 | import yum 3 | from pakrat import util, log 4 | 5 | def from_file(path): 6 | """ Read repository configuration from a YUM config file. 7 | 8 | Using the YUM client library, read in a configuration file and return 9 | a list of repository objects. 10 | """ 11 | if not os.path.exists(path): 12 | raise Exception('No such file or directory: %s' % path) 13 | yb = util.get_yum() 14 | yb.getReposFromConfigFile(path) 15 | for repo in yb.repos.findRepos('*'): 16 | yb.doSackSetup(thisrepo=repo.getAttribute('name')) 17 | repos = [] 18 | for repo in yb.repos.findRepos('*'): 19 | if repo.isEnabled(): 20 | log.info('Added repo %s from file %s' % (repo.id, path)) 21 | repos.append(repo) 22 | else: 23 | log.debug('Not adding repo %s because it is disabled' % repo.id) 24 | return repos 25 | 26 | def from_dir(path): 27 | """ Read repository configuration from a directory containing YUM configs. 28 | 29 | This method will look through a directory for YUM repository configuration 30 | files (*.repo) and read them in using the from_file method. 31 | """ 32 | repos = [] 33 | if os.path.isdir(path): 34 | for _file in sorted(os.listdir(path)): 35 | _file = os.path.join(path, _file) 36 | if not _file.endswith('.repo'): 37 | continue 38 | repos += from_file(_file) 39 | return repos 40 | -------------------------------------------------------------------------------- /bin/pakrat: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python -tt 2 | 3 | import os 4 | import sys 5 | import pakrat 6 | from optparse import OptionParser 7 | 8 | parser = OptionParser(version='pakrat %s' % pakrat.__version__, 9 | usage='\n'.join([ 10 | 'pakrat [options]\n', 11 | 'Repositories are declared using two options: name and', 12 | 'url. The "--name" and "--url" options may be repeated', 13 | 'to declare multiple repositories. Repository names and', 14 | 'URLs are matched up according to their position on the', 15 | 'command line.', 16 | '', 17 | 'The exit code will be one of:', 18 | ' 0: All repos successfully synced', 19 | ' 1: All repos failed to sync', 20 | ' 2: Some repos failed to sync, others succeeded.' 21 | ])) 22 | parser.add_option('--repoversion', default=None, 23 | help=('The version of the repository to create. By default, the ' 24 | 'repositories will not be versioned.')) 25 | parser.add_option('--name', action='append', 26 | help='The name of a YUM repository. (repeatable)') 27 | parser.add_option('--baseurl', action='append', 28 | help='The baseurl of a repository (repeatable)') 29 | parser.add_option('--outdir', type='string', default=os.getcwd(), 30 | help='The root output directory to store repository data. Default is the ' 31 | 'current working directory.') 32 | parser.add_option('--repofile', default=[], action='append', 33 | help=('The path to a YUM repository configuration file. (repeatable)')) 34 | parser.add_option('--repodir', default=[], action='append', 35 | help=('The path to a YUM repos.d-style directory of configs (repeatable)')) 36 | parser.add_option('--delete', action='store_true', default=False, 37 | help=('Delete packages that are no longer at the remote source. ' 38 | 'This option does nothing if --repoversion is passed.')) 39 | parser.add_option('--combined', action='store_true', default=False, 40 | help=('Create repository metadata for the entire packages directory. ' 41 | 'The resulting repository will be a combined package history of ' 42 | 'all packages ever synced for each repository. Only useful with ' 43 | 'the --repoversion option.')) 44 | 45 | options, args = parser.parse_args() 46 | 47 | repos = [] 48 | 49 | if options.name and options.baseurl: 50 | if len(options.name) != len(options.baseurl): 51 | print 'Each repository must have a name and URL.' 52 | sys.exit(1) 53 | for name, baseurl in zip(options.name, options.baseurl): 54 | repos.append(pakrat.repo.factory(name=name, baseurls=[baseurl])) 55 | 56 | if len(repos) < 1 and len(options.repofile) < 1 and len(options.repodir) < 1: 57 | parser.print_help() 58 | sys.exit(1) 59 | 60 | if options.combined and not options.repoversion: 61 | print '--combined is only usable with --repoversion.' 62 | sys.exit(1) 63 | 64 | try: 65 | numrepos, numfails, elapsed = pakrat.sync( 66 | basedir=options.outdir, 67 | repoversion=options.repoversion, 68 | objrepos=repos, 69 | repofiles=options.repofile, 70 | repodirs=options.repodir, 71 | delete=options.delete, 72 | combined=options.combined 73 | ) 74 | except: 75 | sys.exit(1) 76 | 77 | print 'Finished in %s' % elapsed 78 | 79 | if numfails == 0: 80 | sys.exit(0) 81 | elif numrepos == numfails: 82 | sys.exit(1) 83 | else: 84 | sys.exit(2) 85 | -------------------------------------------------------------------------------- /pakrat/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import multiprocessing 4 | import signal 5 | import urlparse 6 | from pakrat import util, log, repo, repos, progress 7 | 8 | __version__ = '0.3.2' 9 | 10 | def sync(basedir=None, objrepos=[], repodirs=[], repofiles=[], 11 | repoversion=None, delete=False, combined=False, callback=None): 12 | """ Mirror repositories with configuration data from multiple sources. 13 | 14 | Handles all input validation and higher-level logic before passing control 15 | on to threads for doing the actual syncing. One thread is created per 16 | repository to alleviate the impact of slow mirrors on faster ones. 17 | """ 18 | util.validate_repos(objrepos) 19 | util.validate_repofiles(repofiles) 20 | util.validate_repodirs(repodirs) 21 | 22 | if not basedir: 23 | basedir = os.getcwd() # default current working directory 24 | 25 | util.validate_basedir(basedir) 26 | 27 | if repoversion: 28 | delete = False # versioned repos have nothing to delete 29 | 30 | for _file in repofiles: 31 | objrepos += repos.from_file(_file) 32 | 33 | for _dir in repodirs: 34 | objrepos += repos.from_dir(_dir) 35 | 36 | prog = progress.Progress() # callbacks talk to this object 37 | manager = multiprocessing.Manager() 38 | queue = manager.Queue() 39 | processes = [] 40 | for objrepo in objrepos: 41 | prog.update(objrepo.id) # Add the repo to the progress object 42 | yumcallback = progress.YumProgress(objrepo.id, queue, callback) 43 | repocallback = progress.ProgressCallback(queue, callback) 44 | dest = util.get_repo_dir(basedir, objrepo.id) 45 | p = multiprocessing.Process(target=repo.sync, args=(objrepo, dest, 46 | repoversion, delete, combined, yumcallback, 47 | repocallback)) 48 | p.start() 49 | processes.append(p) 50 | 51 | def stop(*args): 52 | """ Inner method for terminating threads on signal events. 53 | 54 | This method uses os.kill() to send a SIGKILL directly to the process ID 55 | because the child processes are running blocking calls that will likely 56 | take a long time to complete. 57 | """ 58 | log.error('Caught exit signal - aborting') 59 | while len(processes) > 0: 60 | for p in processes: 61 | os.kill(p.pid, signal.SIGKILL) 62 | if not p.is_alive(): 63 | processes.remove(p) 64 | sys.exit(1) # safe to do exit() here because we are a worker 65 | 66 | # Catch user-cancelled or killed signals to terminate threads. 67 | signal.signal(signal.SIGINT, stop) 68 | signal.signal(signal.SIGTERM, stop) 69 | 70 | while len(processes) > 0: 71 | # If data is waiting in the queue from the workers, process it. This 72 | # needs to be done in the current scope so that one progress object may 73 | # hold all of the results. (This might be easier with Python 3's 74 | # nonlocal keyword). 75 | while not queue.empty(): 76 | e = queue.get() 77 | if not e.has_key('action'): 78 | continue 79 | if e['action'] == 'repo_init' and e.has_key('value'): 80 | prog.update(e['repo_id'], set_total=e['value']) 81 | elif e['action'] == 'download_end' and e.has_key('value'): 82 | prog.update(e['repo_id'], pkgs_downloaded=e['value']) 83 | elif e['action'] == 'repo_metadata': 84 | prog.update(e['repo_id'], repo_metadata=e['value']) 85 | elif e['action'] == 'repo_complete': 86 | pass # should already know this, but handle it anyways. 87 | elif e['action'] == 'repo_error': 88 | prog.update(e['repo_id'], repo_error=e['value']) 89 | elif e['action'] == 'local_pkg_exists': 90 | prog.update(e['repo_id'], pkgs_downloaded=1) 91 | for p in processes: 92 | if not p.is_alive(): 93 | processes.remove(p) 94 | 95 | # Return tuple (#repos, #fail, elapsed time) 96 | return (len(objrepos), prog.totals['errors'], prog.elapsed()) 97 | -------------------------------------------------------------------------------- /pakrat/util.py: -------------------------------------------------------------------------------- 1 | import os 2 | import yum 3 | from pakrat.yumbase import YumBase 4 | from pakrat import log 5 | 6 | PACKAGESDIR = 'Packages' 7 | METADATADIR = 'repodata' 8 | LATESTREPO = 'latest' 9 | 10 | def get_yum(): 11 | """ Retrieve a YumBase object, pre-configured. """ 12 | return YumBase() 13 | 14 | def get_repo_dir(basedir, name): 15 | """ Return the path to a repository directory. 16 | 17 | This is the directory in which all of the repository data will live. The 18 | path can be relative or fully qualified. 19 | """ 20 | return os.path.join(basedir, name) 21 | 22 | def get_packages_dir(repodir): 23 | """ Return the path to the packages directory of a repository. """ 24 | return os.path.join(repodir, PACKAGESDIR) 25 | 26 | def get_package_path(repodir, packagename): 27 | """ Return the path to an individual package file. """ 28 | return os.path.join(repodir, PACKAGESDIR, packagename) 29 | 30 | def get_relative_packages_dir(): 31 | """ Return the relative path to the packages directory. """ 32 | return os.path.join('..', PACKAGESDIR) 33 | 34 | def get_package_relativedir(packagename): 35 | """ Return the relative path to an individual package file. 36 | 37 | This is used during repository metadata creation so that fragments of the 38 | local filesystem layout are not found in the repository index. 39 | """ 40 | return os.path.join(PACKAGESDIR, packagename) 41 | 42 | def get_versioned_dir(repodir, version): 43 | """ Return the path to a specific version of a repository. """ 44 | return os.path.join(repodir, version) 45 | 46 | def get_latest_symlink_path(repodir): 47 | """ Return the path to the latest repository directory. 48 | 49 | The latest directory will be created as a symbolic link, pointing back to 50 | the newest versioned copy. 51 | """ 52 | return os.path.join(repodir, LATESTREPO) 53 | 54 | def get_package_filename(pkg): 55 | """ From a repository object, return the name of the RPM file. """ 56 | return '%s-%s-%s.%s.rpm' % (pkg.name, pkg.version, pkg.release, pkg.arch) 57 | 58 | def get_metadata_dir(repodir): 59 | """ Return the path to the repository metadata directory """ 60 | return os.path.join(repodir, METADATADIR) 61 | 62 | def validate_basedir(basedir): 63 | """ Validate the input of a basedir. 64 | 65 | Since a basedir can be either absolute or relative, the only thing we can 66 | really validate here is that the value is a regular string. 67 | """ 68 | if type(basedir) is not str: 69 | raise Exception('basedir must be a string, not "%s"' % type(basedir)) 70 | 71 | def validate_url(url): 72 | """ Validate a source URL. http(s) or file-based accepted. """ 73 | if not (url.startswith('http://') or url.startswith('https://') or 74 | url.startswith('file://')): 75 | raise Exception('Unsupported URL format "%s"' % url) 76 | 77 | def validate_baseurl(baseurl): 78 | """ Validate user input of a repository baseurl. """ 79 | if type(baseurl) is not str: 80 | raise Exception('baseurl must be a string') 81 | validate_url(baseurl) 82 | 83 | def validate_baseurls(baseurls): 84 | """ Validate multiple baseurls from a list. """ 85 | if type(baseurls) is not list: 86 | raise Exception('baseurls must be a list') 87 | for baseurl in baseurls: 88 | validate_baseurl(baseurl) 89 | 90 | def validate_mirrorlist(mirrorlist): 91 | """ Validate a repository mirrorlist source. """ 92 | if type(mirrorlist) is not str: 93 | raise Exception('mirrorlist must be a string, not "%s"' % 94 | type(mirrorlist)) 95 | if mirrorlist.startswith('file://'): 96 | raise Exception('mirrorlist cannot use a file:// source.') 97 | validate_url(mirrorlist) 98 | 99 | def validate_repo(repo): 100 | """ Validate a repository object. """ 101 | if type(repo) is not yum.yumRepo.YumRepository: 102 | raise Exception('repo must be a YumRepository, not "%s"' % type(repo)) 103 | 104 | def validate_repos(repos): 105 | """ Validate repository objects. """ 106 | if type(repos) is not list: 107 | raise Exception('repos must be a list, not "%s"' % type(repos)) 108 | for repo in repos: 109 | validate_repo(repo) 110 | 111 | def validate_repofile(repofile): 112 | """ Validate a repository file """ 113 | if type(repofile) is not str: 114 | raise Exception('repofile must be a string, not "%s"' % type(repofile)) 115 | if not os.path.exists(repofile): 116 | raise Exception('repofile does not exist: "%s"' % repofile) 117 | 118 | def validate_repofiles(repofiles): 119 | """ Validate paths to repofiles. """ 120 | if type(repofiles) is not list: 121 | raise Exception('repofiles must be a list, not "%s"' % type(repofiles)) 122 | for repofile in repofiles: 123 | validate_repofile(repofile) 124 | 125 | def validate_repodir(repodir): 126 | """ Validate a repository configuration directory path """ 127 | if type(repodir) is not str: 128 | raise Exception('repodir must be a string, not "%s"' % type(repodir)) 129 | if not os.path.isdir(repodir): 130 | raise Exception('Path does not exist or is not a directory: "%s"' % 131 | repodir) 132 | 133 | def validate_repodirs(repodirs): 134 | """ Validate directories of repository files. """ 135 | if type(repodirs) is not list: 136 | raise Exception('repodirs must be a list, not "%s"' % type(repodirs)) 137 | for repodir in repodirs: 138 | validate_repodir(repodir) 139 | 140 | def make_dir(dir): 141 | """ Create a directory recursively, if it does not exist. """ 142 | if not os.path.exists(dir): 143 | log.trace('Creating directory %s' % dir) 144 | os.makedirs(dir) 145 | 146 | def symlink(path, target): 147 | """ Create a symbolic link. 148 | 149 | Determines if a link in the destination already exists, and if it does, 150 | updates its target. If the destination exists but is not a link, throws an 151 | exception. If the link does not exist, it is created. 152 | """ 153 | if not os.path.islink(path): 154 | if os.path.exists(path): 155 | raise Exception('%s exists - Cannot create symlink' % path) 156 | dir = os.path.dirname(path) 157 | if not os.path.exists(dir): 158 | make_dir(dir) 159 | elif os.readlink(path) != target: 160 | log.trace('Unlinking %s because its target is changing' % path) 161 | os.unlink(path) 162 | if not os.path.lexists(path): 163 | log.trace('Linking %s to %s' % (path, target)) 164 | os.symlink(target, path) 165 | -------------------------------------------------------------------------------- /pakrat/repo.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import tempfile 4 | import shutil 5 | import yum 6 | import createrepo 7 | import copy 8 | from contextlib import contextmanager 9 | from pakrat import util, log 10 | 11 | def factory(name, baseurls=None, mirrorlist=None): 12 | """ Generate a pakrat.yumbase.YumBase object on-the-fly. 13 | 14 | This makes it possible to mirror YUM repositories without having any stored 15 | configuration anywhere. Simply pass in the name of the repository, and 16 | either one or more baseurl's or a mirrorlist URL, and you will get an 17 | object in return that you can pass to a mirroring function. 18 | """ 19 | yb = util.get_yum() 20 | if baseurls is not None: 21 | util.validate_baseurls(baseurls) 22 | repo = yb.add_enable_repo(name, baseurls=baseurls) 23 | elif mirrorlist is not None: 24 | util.validate_mirrorlist(mirrorlist) 25 | repo = yb.add_enable_repo(name, mirrorlist=mirrorlist) 26 | else: 27 | raise Exception('One or more baseurls or mirrorlist required') 28 | return repo 29 | 30 | def set_path(repo, path): 31 | """ Set the local filesystem path to use for a repository object. """ 32 | util.validate_repo(repo) 33 | result = copy.copy(repo) # make a copy so the original is untouched 34 | 35 | # The following is wrapped in a try-except to suppress an anticipated 36 | # exception from YUM's yumRepo.py, line 530 and 557. 37 | try: result.pkgdir = path 38 | except yum.Errors.RepoError: pass 39 | 40 | return result 41 | 42 | def create_metadata(repo, packages=None, comps=None): 43 | """ Generate YUM metadata for a repository. 44 | 45 | This method accepts a repository object and, based on its configuration, 46 | generates YUM metadata for it using the createrepo sister library. 47 | """ 48 | util.validate_repo(repo) 49 | conf = createrepo.MetaDataConfig() 50 | conf.directory = os.path.dirname(repo.pkgdir) 51 | conf.outputdir = os.path.dirname(repo.pkgdir) 52 | if packages: 53 | conf.pkglist = packages 54 | conf.quiet = True 55 | 56 | if comps: 57 | groupdir = tempfile.mkdtemp() 58 | conf.groupfile = os.path.join(groupdir, 'groups.xml') 59 | with open(conf.groupfile, 'w') as f: 60 | f.write(comps) 61 | 62 | generator = createrepo.SplitMetaDataGenerator(conf) 63 | generator.doPkgMetadata() 64 | generator.doRepoMetadata() 65 | generator.doFinalMove() 66 | 67 | if comps and os.path.exists(groupdir): 68 | shutil.rmtree(groupdir) 69 | 70 | def create_combined_metadata(repo, dest, comps=None): 71 | """ Creates YUM metadata for the entire Packages directory. 72 | 73 | When used with versioning, this creates a combined repository of all 74 | packages ever synced for the repository. 75 | """ 76 | combined_repo = set_path(repo, util.get_packages_dir(dest)) 77 | create_metadata(combined_repo, None, comps) 78 | 79 | def retrieve_group_comps(repo): 80 | """ Retrieve group comps XML data from a remote repository. 81 | 82 | This data can be used while running createrepo to provide package groups 83 | data that clients can use while installing software. 84 | """ 85 | if repo.enablegroups: 86 | try: 87 | yb = util.get_yum() 88 | yb.repos.add(repo) 89 | comps = yb._getGroups().xml() 90 | log.info('Group data retrieved for repository %s' % repo.id) 91 | return comps 92 | except yum.Errors.GroupsError: 93 | log.debug('No group data available for repository %s' % repo.id) 94 | return None 95 | 96 | def sync(repo, dest, version, delete=False, combined=False, yumcallback=None, 97 | repocallback=None): 98 | """ Sync repository contents from a remote source. 99 | 100 | Accepts a repository, destination path, and an optional version, and uses 101 | the YUM client library to download all available packages from the mirror. 102 | If the delete flag is passed, any packages found on the local filesystem 103 | which are not present in the remote repository will be deleted. 104 | """ 105 | util.make_dir(util.get_packages_dir(dest)) # Make package storage dir 106 | 107 | @contextmanager 108 | def suppress(): 109 | """ Suppress stdout within a context. 110 | 111 | This is necessary in this use case because, unfortunately, the YUM 112 | library will do direct printing to stdout in many error conditions. 113 | Since we are maintaining a real-time, in-place updating presentation 114 | of progress, we must suppress this, as we receive exceptions for our 115 | reporting purposes anyways. 116 | """ 117 | stdout = sys.stdout 118 | sys.stdout = open(os.devnull, 'w') 119 | yield 120 | sys.stdout = stdout 121 | 122 | if version: 123 | dest_dir = util.get_versioned_dir(dest, version) 124 | util.make_dir(dest_dir) 125 | packages_dir = util.get_packages_dir(dest_dir) 126 | util.symlink(packages_dir, util.get_relative_packages_dir()) 127 | else: 128 | dest_dir = dest 129 | packages_dir = util.get_packages_dir(dest_dir) 130 | try: 131 | yb = util.get_yum() 132 | repo = set_path(repo, packages_dir) 133 | if yumcallback: 134 | repo.setCallback(yumcallback) 135 | yb.repos.add(repo) 136 | yb.repos.enableRepo(repo.id) 137 | with suppress(): 138 | # showdups allows us to get multiple versions of the same package. 139 | ygh = yb.doPackageLists(showdups=True) 140 | 141 | # reinstall_available = Available packages which are installed. 142 | packages = ygh.available + ygh.reinstall_available 143 | 144 | # Inform about number of packages total in the repo. 145 | callback(repocallback, repo, 'repo_init', len(packages)) 146 | 147 | # Check if the packages are already downloaded. This is probably a bit 148 | # expensive, but the alternative is simply not knowing, which is 149 | # horrible for progress indication. 150 | for po in packages: 151 | local = po.localPkg() 152 | if os.path.exists(local): 153 | if yb.verifyPkg(local, po, False): 154 | callback(repocallback, repo, 'local_pkg_exists', 155 | util.get_package_filename(po)) 156 | 157 | with suppress(): 158 | yb.downloadPkgs(packages) 159 | 160 | except (KeyboardInterrupt, SystemExit): 161 | pass 162 | except Exception, e: 163 | callback(repocallback, repo, 'repo_error', str(e)) 164 | log.error(str(e)) 165 | return False 166 | callback(repocallback, repo, 'repo_complete') 167 | 168 | if delete: 169 | package_names = [] 170 | for package in packages: 171 | package_names.append(util.get_package_filename(package)) 172 | for _file in os.listdir(util.get_packages_dir(dest)): 173 | if not _file in package_names: 174 | package_path = util.get_package_path(dest, _file) 175 | log.debug('Deleting file %s' % package_path) 176 | os.remove(package_path) 177 | log.info('Finished downloading packages from repository %s' % repo.id) 178 | 179 | log.info('Creating metadata for repository %s' % repo.id) 180 | callback(repocallback, repo, 'repo_metadata', 'working') 181 | comps = retrieve_group_comps(repo) # try group data 182 | pkglist = [] 183 | for pkg in packages: 184 | pkglist.append( 185 | util.get_package_relativedir(util.get_package_filename(pkg)) 186 | ) 187 | 188 | create_metadata(repo, pkglist, comps) 189 | if combined and version: 190 | create_combined_metadata(repo, dest, comps) 191 | elif os.path.exists(util.get_metadata_dir(dest)): 192 | # At this point the combined metadata is stale, so remove it. 193 | log.debug('Removing combined metadata for repository %s' % repo.id) 194 | shutil.rmtree(util.get_metadata_dir(dest)) 195 | callback(repocallback, repo, 'repo_metadata', 'complete') 196 | log.info('Finished creating metadata for repository %s' % repo.id) 197 | 198 | if version: 199 | latest_symlink = util.get_latest_symlink_path(dest) 200 | util.symlink(latest_symlink, version) 201 | 202 | def callback(callback_obj, repo, event, data=None): 203 | """ Abstracts calling class callbacks. 204 | 205 | Since callbacks are optional, a function should check if the callback is 206 | set or not, and then call it, so we don't repeat this code many times. 207 | """ 208 | if callback_obj and hasattr(callback_obj, event): 209 | method = getattr(callback_obj, event) 210 | if data: 211 | method(repo.id, data) 212 | else: 213 | method(repo.id) 214 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Pakrat 2 | ------- 3 | 4 | A tool to mirror and version YUM repositories 5 | 6 | What does it do? 7 | ---------------- 8 | 9 | * You invoke pakrat and pass it some information about your repositories. 10 | * Pakrat mirrors the YUM repositories, and optionally arranges the data in a 11 | versioned manner. 12 | 13 | It is easiest to demonstrate what Pakrat does by shell example: 14 | ``` 15 | $ pakrat --repodir /etc/yum.repos.d 16 | 17 | repo done/total complete metadata 18 | ------------------------------------------------------- 19 | base 357/6381 5% - 20 | updates 112/1100 10% - 21 | extras 13/13 100% complete 22 | 23 | total: 482/7494 6% 24 | 25 | ``` 26 | 27 | Features 28 | -------- 29 | 30 | * Mirror repository packages from remote sources 31 | * Optional repository versioning with user-defined version schema 32 | * Mirror YUM group metadata 33 | * Supports standard YUM configuration files 34 | * Supports YUM configuration directories (repos.d style) 35 | * Supports command-line repos for zero-configuration (`--name` and `--baseurl`) 36 | * Command-line interface with real-time progress indicator 37 | * Parallel repository downloads for maximum effeciency 38 | * Syslog integration 39 | * Supports user-specified callbacks 40 | 41 | Installation 42 | ------------ 43 | 44 | Pakrat is available in PyPI as `pakrat`. That means you can install it with 45 | easy_install: 46 | 47 | ``` 48 | # easy_install pakrat 49 | ``` 50 | 51 | *NOTE* 52 | Installation from PyPI should work on any Linux. However, since Pakrat depends 53 | on YUM and Createrepo, which are not available in PyPI, these dependencies will 54 | not be detected as missing. The easiest install path is to install on some kind 55 | of RHEL like so: 56 | 57 | ``` 58 | # yum -y install createrepo 59 | # easy_install pakrat 60 | ``` 61 | 62 | How to use it 63 | ------------- 64 | 65 | The simplest possible example would involve mirroring a YUM repository in a 66 | very basic way, using the CLI: 67 | 68 | ``` 69 | $ pakrat --name centos --baseurl http://mirror.centos.org/centos/6/os/x86_64 70 | $ tree -d centos 71 | centos/ 72 | ├── Packages 73 | └── repodata 74 | ``` 75 | 76 | A slightly more complex example would be to version the same repository. To 77 | do this, you must pass in a version number. An easy example is to mirror a 78 | repository daily. 79 | ``` 80 | $ pakrat \ 81 | --repoversion $(date +%Y-%m-%d) \ 82 | --name centos \ 83 | --baseurl http://mirror.centos.org/centos/6/os/x86_64 84 | $ tree -d centos 85 | centos/ 86 | ├── 2013-07-29 87 | │   ├── Packages -> ../Packages 88 | │   └── repodata 89 | ├── latest -> 2013-07-29 90 | └── Packages 91 | ``` 92 | 93 | If you were to configure the above to command to run on a daily schedule, 94 | eventually you would see something like: 95 | ``` 96 | $ tree -d centos 97 | centos/ 98 | ├── 2013-07-29 99 | │   ├── Packages -> ../Packages 100 | │   └── repodata 101 | ├── 2013-07-30 102 | │   ├── Packages -> ../Packages 103 | │   └── repodata 104 | ├── 2013-07-31 105 | │   ├── Packages -> ../Packages 106 | │   └── repodata 107 | ├── latest -> 2013-07-31 108 | └── Packages 109 | ``` 110 | 111 | You can also opt to have a combined repository for each of your repos. This is 112 | useful because you could simply point your clients to the root of your 113 | repository, and they will have access to its complete history of RPMs. You can 114 | do this by passing in the `--combined` option when versioning repositories. 115 | 116 | Pakrat is also capable of handling multiple YUM repositories in the same mirror 117 | run. If multiple repositories are specified, each repository will get its own 118 | download thread. This is handy if you are syncing from a mirror that is not 119 | particularly quick. The other repositories do not need to wait on it to finish. 120 | ``` 121 | $ pakrat \ 122 | --repoversion $(date +%Y-%m-%d) \ 123 | --name centos --baseurl http://mirror.centos.org/centos/6/os/x86_64 \ 124 | --name epel --baseurl http://dl.fedoraproject.org/pub/epel/6/x86_64 125 | $ tree -d centos epel 126 | centos/ 127 | ├── 2013-07-29 128 | │   ├── Packages -> ../Packages 129 | │   └── repodata 130 | ├── latest -> 2013-07-29 131 | └── Packages 132 | epel/ 133 | ├── 2013-07-29 134 | │   ├── Packages -> ../Packages 135 | │   └── repodata 136 | ├── latest -> 2013-07-29 137 | └── Packages 138 | ``` 139 | 140 | Configuration can also be passed in from YUM configuration files. See the CLI 141 | `--help` for details. 142 | 143 | Pakrat also exposes its interfaces in plain python for integration with other 144 | projects and software. A good starting point for using Pakrat via the python 145 | API is to take a look at the `pakrat.sync` method. The CLI calls this method 146 | almost exclusively, so it should be fairly straightforward in its usage (all 147 | arguments are named and optional): 148 | ``` 149 | pakrat.sync(basedir, objrepos, repodirs, repofiles, repoversion, delete, callback) 150 | ``` 151 | 152 | Another handy python method is `pakrat.repo.factory`, which creates YUM 153 | repository objects so that no file-based configuration is needed. 154 | ``` 155 | pakrat.repo.factory(name, baseurls=None, mirrorlist=None) 156 | ``` 157 | 158 | User-defined callbacks 159 | ---------------------- 160 | 161 | Since the YUM team did a decent job at externalizing the progress data, 162 | pakrat will return the favor by exposing the same data, plus some extras 163 | via user callbacks. 164 | 165 | A user callback is a simple class that implements some methods for handling 166 | received data. It is not mandatory to implement any of the methods. 167 | 168 | A few of the available user callbacks in pakrat come directly from the 169 | `urlgrabber` interface (namely, any user callback beginning with `download_`. 170 | The other methods are called by pakrat, which explains why the interfaces 171 | are varied. 172 | 173 | The supported user callbacks are listed in the following method signatures: 174 | ```python 175 | """ Called when the number of packages a repository contains becomes known """ 176 | repo_init(repo_id, num_pkgs) 177 | 178 | """ Called when 'createrepo' begins running and when it completes """ 179 | repo_metadata(repo_id, status) 180 | 181 | """ Called when a repository finishes downloading all packages """ 182 | repo_complete(repo_id) 183 | 184 | """ Called whenever an exception is thrown from a repo thread """ 185 | repo_error(repo_id, error) 186 | 187 | """ Called when a package becomes known as 'already downloaded' """ 188 | local_pkg_exists(repo_id, pkgname) 189 | 190 | """ Called when a file begins downloading (non-exclusive) """ 191 | download_start(repo_id, fpath, url, fname, fsize, text) 192 | 193 | """ Called during downloads, 'size' is bytes downloaded """ 194 | download_update(repo_id, size) 195 | 196 | """ Called when a file download completes, 'size' is file size in bytes """ 197 | download_end(repo_id, size) 198 | ``` 199 | 200 | The following is a basic example of how to use user callbacks in pakrat. 201 | Note that an instance of the class is passed into the `pakrat.sync()` call 202 | as the named argument `callback`. 203 | 204 | ```python 205 | import pakrat 206 | 207 | class mycallback(object): 208 | def log(self, msg): 209 | with open('log.txt', 'a') as logfile: 210 | logfile.write('%s\n' % msg) 211 | 212 | def repo_init(self, repo_id, num_pkgs): 213 | self.log('Found %d packages in repo %s' % (num_pkgs, repo_id)) 214 | 215 | def download_start(self, repo_id, _file, url, basename, size, text): 216 | self.fname = basename 217 | 218 | def download_end(self, repo_id, size): 219 | if self.fname.endswith('.rpm'): 220 | self.log('%s, repo %s, size %d' % (self.fname, repo_id, size)) 221 | 222 | def repo_metadata(self, repo_id, status): 223 | self.log('Metadata for repo %s is now %s' % (repo_id, status)) 224 | 225 | myrepo = pakrat.repo.factory( 226 | 'extras', 227 | mirrorlist='http://mirrorlist.centos.org/?repo=extras&release=6&arch=x86_64' 228 | ) 229 | 230 | mycallback_instance = mycallback() 231 | pakrat.sync(objrepos=[myrepo], callback=mycallback_instance) 232 | ``` 233 | 234 | If you run the above example, and then take a look in the `log.txt` file (which 235 | the user callbacks should have created), you will see something like: 236 | 237 | ``` 238 | Found 13 packages in repo extras 239 | bakefile-0.2.8-3.el6.centos.x86_64.rpm, repo extras, size 256356 240 | centos-release-cr-6-0.el6.centos.x86_64.rpm, repo extras, size 3996 241 | centos-release-xen-6-2.el6.centos.x86_64.rpm, repo extras, size 4086 242 | freenx-0.7.3-9.4.el6.centos.x86_64.rpm, repo extras, size 99256 243 | jfsutils-1.1.13-9.el6.x86_64.rpm, repo extras, size 244104 244 | nx-3.5.0-2.1.el6.centos.x86_64.rpm, repo extras, size 2807864 245 | opennx-0.16-724.el6.centos.1.x86_64.rpm, repo extras, size 1244240 246 | python-empy-3.3-5.el6.centos.noarch.rpm, repo extras, size 104632 247 | wxBase-2.8.12-1.el6.centos.x86_64.rpm, repo extras, size 586068 248 | wxGTK-2.8.12-1.el6.centos.x86_64.rpm, repo extras, size 3081804 249 | wxGTK-devel-2.8.12-1.el6.centos.x86_64.rpm, repo extras, size 1005036 250 | wxGTK-gl-2.8.12-1.el6.centos.x86_64.rpm, repo extras, size 31824 251 | wxGTK-media-2.8.12-1.el6.centos.x86_64.rpm, repo extras, size 38644 252 | Metadata for repo extras is now working 253 | Metadata for repo extras is now complete 254 | ``` 255 | 256 | Building an RPM 257 | --------------- 258 | 259 | Pakrat can be easily packaged into an RPM. 260 | 261 | 1. Download a release and name the tarball `pakrat.tar.gz`: 262 | ``` 263 | curl -o pakrat.tar.gz -L https://github.com/ryanuber/pakrat/archive/master.tar.gz 264 | ``` 265 | 266 | 2. Build it into an RPM: 267 | ``` 268 | rpmbuild -tb pakrat.tar.gz 269 | ``` 270 | 271 | What's missing 272 | -------------- 273 | 274 | * Unit tests (preliminary work done in unit_test branch) 275 | 276 | Thanks 277 | ------ 278 | 279 | Thanks to [Keith Chambers](https://github.com/keithchambers) for help with the 280 | ideas and useful input on CLI design. 281 | -------------------------------------------------------------------------------- /pakrat/progress.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import datetime 3 | 4 | class Progress(object): 5 | """ Handle progress indication using callbacks. 6 | 7 | This class will create an object that stores information about a 8 | running pakrat process. It stores information about each repository 9 | being synced, including total packages, completed packages, and 10 | the status of the repository metadata. This makes it possible to 11 | display aggregated status of multiple repositories during a sync. 12 | """ 13 | repos = {} 14 | totals = {'numpkgs':0, 'dlpkgs':0, 'errors':0} 15 | errors = [] 16 | prevlines = 0 17 | 18 | def __init__(self): 19 | """ Simply records the time the sync started. """ 20 | self.start = datetime.datetime.now() 21 | 22 | def update(self, repo_id, set_total=None, pkgs_downloaded=None, 23 | local_pkg_exists=None, repo_metadata=None, repo_error=None): 24 | """ Handles updating the object itself. 25 | 26 | This method will be called any time the number of packages in 27 | a repository becomes known, when any package finishes downloading, 28 | when repository metadata begins indexing and when it completes. 29 | """ 30 | if not self.repos.has_key(repo_id): 31 | self.repos[repo_id] = {'numpkgs':0, 'dlpkgs':0, 'repomd':'-'} 32 | if set_total: 33 | self.repos[repo_id]['numpkgs'] = set_total 34 | self.totals['numpkgs'] += set_total 35 | if pkgs_downloaded: 36 | self.repos[repo_id]['dlpkgs'] += pkgs_downloaded 37 | self.totals['dlpkgs'] += pkgs_downloaded 38 | if repo_metadata: 39 | self.repos[repo_id]['repomd'] = repo_metadata 40 | if repo_error: 41 | self.totals['errors'] += 1 42 | self.errors.append((repo_id, repo_error)) 43 | self.formatted() 44 | 45 | @staticmethod 46 | def pct(current, total): 47 | """ Calculate a percentage. """ 48 | return int((current / float(total)) * 100) 49 | 50 | def elapsed(self): 51 | """ Calculate and return elapsed time. 52 | 53 | This function does dumb rounding by just plucking off anything past a 54 | dot "." in a time delta between two datetime.datetime()'s. 55 | """ 56 | return str(datetime.datetime.now() - self.start).split('.')[0] 57 | 58 | def format_line(self, reponame, package_counts, percent, repomd): 59 | """ Return a string formatted for output. 60 | 61 | Since there is a common column layout in the progress indicator, we can 62 | we can implement the printf-style formatter in a function. 63 | """ 64 | return '%-15s %-15s %-10s %s' % (reponame, package_counts, percent, 65 | repomd) 66 | 67 | def represent_repo_pkgs(self, repo_id): 68 | """ Format the ratio of packages in a repository. """ 69 | numpkgs = self.repos[repo_id]['numpkgs'] 70 | dlpkgs = self.repos[repo_id]['dlpkgs'] 71 | return self.represent_pkgs(dlpkgs, numpkgs) 72 | 73 | def represent_total_pkgs(self): 74 | """ Format the total number of packages in all repositories. """ 75 | numpkgs = self.totals['numpkgs'] 76 | dlpkgs = self.totals['dlpkgs'] 77 | return self.represent_pkgs(dlpkgs, numpkgs) 78 | 79 | def represent_pkgs(self, dlpkgs, numpkgs): 80 | """ Represent a package ratio. 81 | 82 | This will display nothing if the number of packages is 0 or unknown, or 83 | typical done/total if total is > 0. 84 | """ 85 | if numpkgs == 0: 86 | return '%6s%10s' % ('-', ' ') 87 | else: 88 | return '%5s/%-10s' % (dlpkgs, numpkgs) 89 | 90 | def represent_repo_percent(self, repo_id): 91 | """ Display the percentage of packages downloaded in a repository. """ 92 | numpkgs = self.repos[repo_id]['numpkgs'] 93 | dlpkgs = self.repos[repo_id]['dlpkgs'] 94 | return self.represent_percent(dlpkgs, numpkgs) 95 | 96 | def represent_total_percent(self): 97 | """ Display the overall percentage of downloaded packages. """ 98 | numpkgs = self.totals['numpkgs'] 99 | dlpkgs = self.totals['dlpkgs'] 100 | return self.represent_percent(dlpkgs, numpkgs) 101 | 102 | def represent_percent(self, dlpkgs, numpkgs): 103 | """ Display a percentage of completion. 104 | 105 | If the number of packages is unknown, nothing is displayed. Otherwise, 106 | a number followed by the percent sign is displayed. 107 | """ 108 | if numpkgs == 0: 109 | return '-' 110 | else: 111 | return '%s%%' % self.pct(dlpkgs, numpkgs) 112 | 113 | def represent_repomd(self, repo_id): 114 | """ Display the current status of repository metadata. """ 115 | return self.repos[repo_id]['repomd'] 116 | 117 | def represent_repo(self, repo_id): 118 | """ Represent an entire repository in one line. 119 | 120 | This makes calls to the other methods of this class to create a 121 | formatted string, which makes nice columns. 122 | """ 123 | if self.repos[repo_id].has_key('error'): 124 | packages = ' error' 125 | percent = '' 126 | metadata = '' 127 | else: 128 | packages = self.represent_repo_pkgs(repo_id) 129 | percent = self.represent_repo_percent(repo_id) 130 | metadata = self.represent_repomd(repo_id) 131 | return self.format_line(repo_id, packages, percent, metadata) 132 | 133 | def emit(self, line=''): 134 | self.prevlines += len(line.split('\n')) 135 | sys.stdout.write('%s\n' % line) 136 | 137 | def formatted(self): 138 | """ Print all known progress data in a nicely formatted table. 139 | 140 | This method keeps track of what it has printed before, so that it can 141 | backtrack over the console screen, clearing out the previous flush and 142 | printing out a new one. This method is called any time any value is 143 | updated, which is what gives us that real-time feeling. 144 | 145 | Unforutnately, the YUM library calls print directly rather than just 146 | throwing exceptions and handling them in the presentation layer, so 147 | this means that pakrat's output will be slightly flawed if YUM prints 148 | something directly to the screen from a worker process. 149 | """ 150 | if not sys.stdout.isatty(): 151 | return 152 | sys.stdout.write('\033[F\033[K' * self.prevlines) # clears lines 153 | self.prevlines = 0 # reset line counter 154 | header = self.format_line('repo', '%5s/%-10s' % ('done', 'total'), 155 | 'complete', 'metadata') 156 | self.emit('\n%s' % header) 157 | self.emit(('-' * len(header))) 158 | 159 | # Remove repos with errors from totals 160 | if self.totals['errors'] > 0: 161 | for repo_id, error in self.errors: 162 | if repo_id in self.repos.keys(): 163 | self.totals['dlpkgs'] -= self.repos[repo_id]['dlpkgs'] 164 | self.totals['numpkgs'] -= self.repos[repo_id]['numpkgs'] 165 | #del self.repos[repo_id] 166 | self.repos[repo_id]['error'] = True 167 | 168 | for repo_id in self.repos.keys(): 169 | self.emit(self.represent_repo(repo_id)) 170 | self.emit() 171 | self.emit(self.format_line('total:', self.represent_total_pkgs(), 172 | self.represent_total_percent(), '')) 173 | self.emit() 174 | 175 | # Append errors to output if any found. 176 | if self.totals['errors'] > 0: 177 | self.emit('errors(%d):' % self.totals['errors']) 178 | for repo_id, error in self.errors: 179 | self.emit(error) 180 | self.emit() 181 | 182 | sys.stdout.flush() 183 | 184 | class YumProgress(object): 185 | """ Creates an object for passing to YUM for status updates. 186 | 187 | YUM allows you to pass in your own callback object, which urlgrabber will 188 | use directly by calling some methods on it. Here we have an object that can 189 | be prepared with a repository ID, so that we can know which repository it 190 | is that is making the calls back. 191 | """ 192 | def __init__(self, repo_id, queue, usercallback): 193 | """ Create the instance and set prepared config """ 194 | self.repo_id = repo_id 195 | self.queue = queue 196 | self.usercallback = usercallback 197 | 198 | def callback(self, method, *args): 199 | """ Abstracted callback function to reduce boilerplate. 200 | 201 | This is actually quite useful, because it checks that the method exists 202 | on the callback object before trying to invoke it, making all methods 203 | optional. 204 | """ 205 | if self.usercallback and hasattr(self.usercallback, method): 206 | method = getattr(self.usercallback, method) 207 | try: method(self.repo_id, *args) 208 | except: pass 209 | 210 | def start(self, filename=None, url=None, basename=None, size=None, text=None): 211 | """ Called by urlgrabber when a file download starts. 212 | 213 | All we use this for is storing the name of the file being downloaded so 214 | we can check that it is an RPM later on. 215 | """ 216 | if basename: 217 | self.package = basename 218 | self.callback('download_start', filename, url, basename, size, text) 219 | 220 | def update(self, size): 221 | """ Called during the course of a download. 222 | 223 | Pakrat does not use this for anyting, but we'll be a good neighbor and 224 | pass the data on to the user callback. 225 | """ 226 | self.callback('download_update', size) 227 | 228 | def end(self, size): 229 | """ Called by urlgrabber when it completes a download. 230 | 231 | Here we have to check the file name saved earlier to make sure it is an 232 | RPM we are getting the event for. 233 | """ 234 | if self.package.endswith('.rpm'): 235 | self.queue.put({'repo_id':self.repo_id, 'action':'download_end', 236 | 'value':1}) 237 | self.callback('download_end', size) 238 | 239 | class ProgressCallback(object): 240 | """ Register our own callback for progress indication. 241 | 242 | This class allows pakrat to stuff a user callback into an object before 243 | forking a thread, so that we don't have to keep making calls to multiple 244 | callbacks everywhere. 245 | """ 246 | def __init__(self, queue, usercallback): 247 | """ Create a new progress object. 248 | 249 | This method allows the main process to pass its multiprocessing.Queue() 250 | object in so we can talk back to it. 251 | """ 252 | self.queue = queue 253 | self.usercallback = usercallback 254 | 255 | def callback(self, repo_id, event, value): 256 | """ Abstracts calling the user callback. """ 257 | if self.usercallback and hasattr(self.usercallback, event): 258 | method = getattr(self.usercallback, event) 259 | try: method(repo_id, value) 260 | except: pass 261 | 262 | def send(self, repo_id, action, value=None): 263 | """ Send an event to the main queue for processing. 264 | 265 | This gives us the ability to pass data back to the parent process, 266 | which is mandatory to do aggregated progress indication. This method 267 | also calls the user callback, if any is defined. 268 | """ 269 | self.queue.put({'repo_id':repo_id, 'action': action, 270 | 'value':value}) 271 | self.callback(repo_id, action, value) 272 | 273 | def repo_metadata(self, repo_id, value): 274 | """ Update the status of metadata creation. """ 275 | self.send(repo_id, 'repo_metadata', value) 276 | 277 | def repo_init(self, repo_id, numpkgs): 278 | """ Share the total packages in a repository, when known. """ 279 | self.send(repo_id, 'repo_init', numpkgs) 280 | 281 | def repo_complete(self, repo_id): 282 | """ Called when a repository completes downloading all packages. """ 283 | self.send(repo_id, 'repo_complete') 284 | 285 | def repo_error(self, repo_id, error): 286 | """ Called when a repository throws an exception. """ 287 | self.send(repo_id, 'repo_error', error) 288 | 289 | def local_pkg_exists(self, repo_id, pkgname): 290 | """ Called when a download will be skipped because it already exists """ 291 | self.send(repo_id, 'local_pkg_exists', pkgname) 292 | --------------------------------------------------------------------------------