├── g-sorcery.cfg ├── g_sorcery ├── __init__.py ├── file_bson │ ├── __init__.py │ └── file_bson.py ├── git_syncer │ ├── __init__.py │ └── git_syncer.py ├── compatibility.py ├── exceptions.py ├── mangler.py ├── data │ └── g-sorcery.eclass ├── eclass.py ├── g_sorcery.py ├── logger.py ├── syncer.py ├── serialization.py ├── g_collections.py ├── ebuild.py ├── db_layout.py ├── fileutils.py ├── metadata.py └── backend.py ├── gs_db_tool ├── __init__.py └── gs_db_tool.py ├── tests ├── __init__.py ├── dummy_backend │ ├── __init__.py │ └── backend.py ├── base.py ├── serializable.py ├── test_eclass.py ├── server.py ├── test_metadata.py ├── test_FileBSON.py ├── test_ebuild.py ├── test_FileJSON.py ├── test_PackageDB.py └── test_DBGenerator.py ├── scripts ├── all_pythons.sh └── run_tests.py ├── docs ├── Makefile ├── g-sorcery.cfg.8.rst ├── g-sorcery.cfg.8 ├── g-sorcery.8.rst ├── g-sorcery.8 └── developer_instructions.rst ├── bin ├── g-sorcery └── gs-db-tool ├── .gitignore ├── setup.py ├── README.md ├── pylint.rc └── LICENSE /g-sorcery.cfg: -------------------------------------------------------------------------------- 1 | [main] 2 | package_manager=portage 3 | -------------------------------------------------------------------------------- /g_sorcery/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | -------------------------------------------------------------------------------- /gs_db_tool/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | -------------------------------------------------------------------------------- /g_sorcery/file_bson/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | -------------------------------------------------------------------------------- /g_sorcery/git_syncer/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | -------------------------------------------------------------------------------- /tests/dummy_backend/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | __all__ = ['backend'] 5 | -------------------------------------------------------------------------------- /scripts/all_pythons.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 4 | 5 | for VER in 2.7 3.3 3.4 3.5 6 | do 7 | echo 8 | echo "testing python${VER}" 9 | python${VER} ${DIR}/run_tests.py 10 | done 11 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | HTML_SOURCES=developer_instructions 2 | HTML_DOCS=$(HTML_SOURCES:=.html) 3 | 4 | MAN_SOURCES=g-sorcery g-sorcery.cfg 5 | MANS=$(MAN_SOURCES:=.8) 6 | 7 | RST2HTML=rst2html.py 8 | RST2MAN=rst2man.py 9 | 10 | all: ${MANS} ${HTML_DOCS} 11 | 12 | %.8: %.8.rst 13 | $(RST2MAN) $< $@ 14 | 15 | %.html: %.rst 16 | $(RST2HTML) $< $@ 17 | -------------------------------------------------------------------------------- /tests/dummy_backend/backend.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | class Test(object): 5 | def __init__(self): 6 | self.tst = 'test backend' 7 | 8 | def __call__(self, args): 9 | print(args[0]) 10 | return 0 11 | 12 | def __eq__(self, other): 13 | return self.tst == other.tst 14 | 15 | dispatcher = Test() 16 | -------------------------------------------------------------------------------- /bin/g-sorcery: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | g-sorcery 6 | ~~~~~~~~~ 7 | 8 | g-sorcery executable 9 | 10 | :copyright: (c) 2013 by Jauhien Piatlicki 11 | :license: GPL-2, see LICENSE for more details. 12 | """ 13 | 14 | import sys 15 | from g_sorcery import g_sorcery 16 | 17 | if __name__ == "__main__": 18 | sys.exit(g_sorcery.main()) 19 | -------------------------------------------------------------------------------- /bin/gs-db-tool: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | gs-db-tool 6 | ~~~~~~~~~~ 7 | 8 | CLI to manipulate with package DB 9 | 10 | :copyright: (c) 2013 by Jauhien Piatlicki 11 | :license: GPL-2, see LICENSE for more details. 12 | """ 13 | 14 | import sys 15 | from gs_db_tool import gs_db_tool 16 | 17 | if __name__ == "__main__": 18 | sys.exit(gs_db_tool.main()) 19 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[co] 2 | 3 | # Packages 4 | *.egg 5 | *.egg-info 6 | dist 7 | build 8 | eggs 9 | parts 10 | var 11 | sdist 12 | develop-eggs 13 | .installed.cfg 14 | 15 | # Installer logs 16 | pip-log.txt 17 | 18 | # Unit test / coverage reports 19 | .coverage 20 | .tox 21 | 22 | #Translations 23 | *.mo 24 | 25 | #Mr Developer 26 | .mr.developer.cfg 27 | 28 | #tmp files 29 | *\~ 30 | 31 | tst 32 | 33 | #layman 34 | layman 35 | -------------------------------------------------------------------------------- /tests/base.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | base.py 6 | ~~~~~~~ 7 | 8 | base class for tests 9 | 10 | :copyright: (c) 2013 by Jauhien Piatlicki 11 | :license: GPL-2, see LICENSE for more details. 12 | """ 13 | 14 | import unittest 15 | 16 | from g_sorcery.compatibility import TemporaryDirectory 17 | 18 | class BaseTest(unittest.TestCase): 19 | 20 | def setUp(self): 21 | self.tempdir = TemporaryDirectory() 22 | 23 | def tearDown(self): 24 | del self.tempdir 25 | -------------------------------------------------------------------------------- /tests/serializable.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | serializable.py 6 | ~~~~~~~~~~~~~~~ 7 | 8 | test classes for serialization 9 | 10 | :copyright: (c) 2013-2015 by Jauhien Piatlicki 11 | :license: GPL-2, see LICENSE for more details. 12 | """ 13 | 14 | class NonSerializableClass(object): 15 | pass 16 | 17 | 18 | class SerializableClass(object): 19 | 20 | __slots__ = ("field1", "field2") 21 | 22 | def __init__(self, field1, field2): 23 | self.field1 = field1 24 | self.field2 = field2 25 | 26 | def __eq__(self, other): 27 | return self.field1 == other.field1 \ 28 | and self.field2 == other.field2 29 | 30 | def serialize(self): 31 | return {"field1": self.field1, "field2": self.field2} 32 | 33 | 34 | class DeserializableClass(SerializableClass): 35 | 36 | @classmethod 37 | def deserialize(cls, value): 38 | return DeserializableClass(value["field1"], value["field2"]) 39 | -------------------------------------------------------------------------------- /g_sorcery/compatibility.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | compatibility.py 6 | ~~~~~~~~~~~~~~~~ 7 | 8 | utilities for py2 compatibility 9 | 10 | :copyright: (c) 2013 by Jauhien Piatlicki 11 | :license: GPL-2, see LICENSE for more details. 12 | """ 13 | 14 | import shutil, sys 15 | 16 | py2k = sys.version_info < (3, 0) 17 | 18 | if py2k: 19 | 20 | from tempfile import mkdtemp 21 | import ConfigParser as configparser 22 | 23 | class TemporaryDirectory(object): 24 | def __init__(self): 25 | self.name = mkdtemp() 26 | 27 | def __del__(self): 28 | shutil.rmtree(self.name) 29 | else: 30 | from tempfile import TemporaryDirectory 31 | import configparser 32 | 33 | #basestring removed in py3k 34 | #fix for it from https://github.com/oxplot/fysom/issues/1 35 | 36 | if py2k: 37 | str = str 38 | unicode = unicode 39 | bytes = str 40 | basestring = basestring 41 | else: 42 | str = str 43 | unicode = str 44 | bytes = bytes 45 | basestring = (str, bytes) 46 | -------------------------------------------------------------------------------- /g_sorcery/exceptions.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | exceptions.py 6 | ~~~~~~~~~~~~~ 7 | 8 | Exceptions hierarchy 9 | 10 | :copyright: (c) 2013-2015 by Jauhien Piatlicki 11 | :license: GPL-2, see LICENSE for more details. 12 | """ 13 | 14 | class GSorceryError(Exception): 15 | pass 16 | 17 | class DBError(GSorceryError): 18 | pass 19 | 20 | class DBLayoutError(GSorceryError): 21 | pass 22 | 23 | class InvalidKeyError(DBError): 24 | pass 25 | 26 | class SyncError(DBError): 27 | pass 28 | 29 | class IntegrityError(DBError): 30 | pass 31 | 32 | class DBStructureError(DBError): 33 | pass 34 | 35 | class FileJSONError(GSorceryError): 36 | pass 37 | 38 | class XMLGeneratorError(GSorceryError): 39 | pass 40 | 41 | class DescriptionError(GSorceryError): 42 | pass 43 | 44 | class DependencyError(GSorceryError): 45 | pass 46 | 47 | class EclassError(GSorceryError): 48 | pass 49 | 50 | class DigestError(GSorceryError): 51 | pass 52 | 53 | class DownloadingError(GSorceryError): 54 | pass 55 | -------------------------------------------------------------------------------- /tests/test_eclass.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | test_eclass.py 6 | ~~~~~~~~~~~~~~ 7 | 8 | eclass test suite 9 | 10 | :copyright: (c) 2013 by Jauhien Piatlicki 11 | :license: GPL-2, see LICENSE for more details. 12 | """ 13 | 14 | import os 15 | import unittest 16 | 17 | from g_sorcery.eclass import EclassGenerator 18 | 19 | from tests.base import BaseTest 20 | 21 | 22 | class TestEclassGenerator(BaseTest): 23 | 24 | def test_eclass_generator(self): 25 | eclasses = ["test1", "test2"] 26 | for eclass in eclasses: 27 | os.system("echo 'eclass " + eclass + "' > " + os.path.join(self.tempdir.name, eclass + ".eclass")) 28 | 29 | eclass_g = EclassGenerator(self.tempdir.name) 30 | self.assertEqual(set(eclass_g.list()), set(eclasses) | set(["g-sorcery"])) 31 | 32 | for eclass in eclasses: 33 | self.assertEqual(eclass_g.generate(eclass), ["eclass " + eclass]) 34 | 35 | 36 | def suite(): 37 | suite = unittest.TestSuite() 38 | suite.addTest(TestEclassGenerator('test_eclass_generator')) 39 | return suite 40 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | setup.py 6 | ~~~~~~~~ 7 | 8 | installation script 9 | 10 | :copyright: (c) 2013-2015 by Jauhien Piatlicki 11 | :copyright: (c) 2014 by Brian Dolbec 12 | (code for conditional module installation 13 | is taken from the layman project) 14 | :license: GPL-2, see LICENSE for more details. 15 | """ 16 | 17 | import os 18 | 19 | from distutils.core import setup 20 | 21 | SELECTABLE = {'bson': 'file_bson', 'git': 'git_syncer'} 22 | 23 | use_defaults = ' '.join(list(SELECTABLE)) 24 | USE = os.environ.get("USE", use_defaults).split() 25 | 26 | optional_modules = [] 27 | for mod in SELECTABLE: 28 | if mod in USE: 29 | optional_modules.append('g_sorcery.%s' % SELECTABLE[mod]) 30 | 31 | setup(name = 'g-sorcery', 32 | version = '0.2.1', 33 | description = 'framework for automated ebuild generators', 34 | author = 'Jauhien Piatlicki', 35 | author_email = 'jauhien@gentoo.org', 36 | packages = ['g_sorcery', 'gs_db_tool'] + optional_modules, 37 | package_data = {'g_sorcery': ['data/*']}, 38 | scripts = ['bin/g-sorcery', 'bin/gs-db-tool'], 39 | data_files = [('/etc/g-sorcery/', ['g-sorcery.cfg'])], 40 | license = 'GPL-2', 41 | ) 42 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Usage 2 | ===== 3 | 4 | This project is a framework, you may be interested in it only if 5 | you want to develop your own ebuild generator. 6 | 7 | As user you may be interested in already implemented ones: 8 | [gs-elpa](https://github.com/jauhien/gs-elpa) and 9 | [gs-pypi](https://github.com/jauhien/gs-pypi). 10 | 11 | User instructions in gs-elpa are more complete, so consult them for how to use. 12 | 13 | Objective 14 | ========= 15 | 16 | There is a lot of 3rd party software providers that resemble overlays 17 | or repositories of Linux distributions in some way. For example: pypi, 18 | CRAN, CPAN, CTAN, octave-forge, ELPA. It's clear that all this 19 | software available through different mechanisms (such as package.el 20 | for Emacs or pkg command in Octave) will never have separately 21 | maintained ebuilds in Gentoo tree or even in overlays. Installing such 22 | a software with its own distribution system does not seem like a good 23 | idea, especially if one needs to install it system-wide. 24 | 25 | This project is aimed to create a framework for ebuild-generators for 26 | 3rd party software providers. 27 | 28 | If you want to develop a new backend see [developer's instructions](https://github.com/jauhien/g-sorcery/blob/master/docs/developer_instructions.rst). 29 | 30 | [TODO list](https://trello.com/b/8WdY2ZIs/framework-for-automated-ebuild-generators). 31 | -------------------------------------------------------------------------------- /g_sorcery/file_bson/file_bson.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | file_bson.py 6 | ~~~~~~~~~~~~ 7 | 8 | bson file format support 9 | 10 | :copyright: (c) 2015 by Jauhien Piatlicki 11 | :license: GPL-2, see LICENSE for more details. 12 | """ 13 | 14 | import bson 15 | 16 | from g_sorcery.exceptions import FileJSONError 17 | from g_sorcery.fileutils import FileJSONData 18 | from g_sorcery.serialization import from_raw_serializable, to_raw_serializable 19 | 20 | class FileBSON(FileJSONData): 21 | """ 22 | Class for BSON files. Supports custom JSON serialization 23 | provided by g_sorcery.serialization. 24 | """ 25 | def read_content(self): 26 | """ 27 | Read BSON file. 28 | """ 29 | content = {} 30 | bcnt = None 31 | with open(self.path, 'rb') as f: 32 | bcnt = bson.BSON(f.read()) 33 | if not bcnt: 34 | raise FileJSONError('failed to read: ', self.path) 35 | rawcnt = bcnt.decode() 36 | content = from_raw_serializable(rawcnt) 37 | return content 38 | 39 | 40 | def write_content(self, content): 41 | """ 42 | Write BSON file. 43 | """ 44 | rawcnt = to_raw_serializable(content) 45 | bcnt = bson.BSON.encode(rawcnt) 46 | with open(self.path, 'wb') as f: 47 | f.write(bcnt) 48 | -------------------------------------------------------------------------------- /g_sorcery/mangler.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | mangler.py 6 | ~~~~~~~~~~ 7 | 8 | package manager interaction 9 | 10 | :copyright: (c) 2013 by Jauhien Piatlicki 11 | :license: GPL-2, see LICENSE for more details. 12 | """ 13 | 14 | import os 15 | 16 | from .logger import Logger 17 | 18 | class PackageManager(object): 19 | """ 20 | Base class for package manager abstraction 21 | """ 22 | 23 | executable = "" 24 | logger = Logger() 25 | 26 | def __init__(self): 27 | pass 28 | 29 | def run_command(self, *args): 30 | command = self.executable + " " + " ".join(args) 31 | self.logger.info("running a package mangler: " + command) 32 | return os.system(self.executable + " " + " ".join(args)) 33 | 34 | def install(self, pkgname, *args): 35 | """ 36 | It supports intallation by package name currently, 37 | will add support of atoms with version specified later 38 | """ 39 | raise NotImplementedError 40 | 41 | 42 | class Portage(PackageManager): 43 | """ 44 | Portage package manager abstraction. 45 | """ 46 | def __init__(self): 47 | super(Portage, self).__init__() 48 | self.executable = "/usr/bin/emerge" 49 | 50 | def install(self, pkgname, *args): 51 | return self.run_command("-va", pkgname, *args) 52 | 53 | #list of supported package managers. 54 | package_managers = {'portage' : Portage} 55 | -------------------------------------------------------------------------------- /g_sorcery/data/g-sorcery.eclass: -------------------------------------------------------------------------------- 1 | # Copyright 1999-2013 Gentoo Foundation 2 | # Distributed under the terms of the GNU General Public License v2 3 | # $Header: $ 4 | # automatically generated by g-sorcery 5 | # please do not edit this file 6 | # 7 | # Original Author: Jauhien Piatlicki 8 | # Purpose: base routines for g-sorcery backends' eclasses 9 | # 10 | # Bugs to piatlicki@gmail.com 11 | # 12 | # @ECLASS: g-sorcery.eclass 13 | # 14 | # @ECLASS-VARIABLE: REPO_URI 15 | # @DESCRIPTION: address of a repository with sources 16 | # 17 | # @ECLASS-VARIABLE: DIGEST_SOURCES 18 | # @DESCRIPTION: whether manifest for sources exists 19 | # 20 | # @ECLASS-VARIABLE: SOURCEFILE 21 | # @DESCRIPTION: source file name 22 | # 23 | # @ECLASS-VARIABLE: GSORCERY_STORE_DIR 24 | # @DESCRIPTION: store location for downloaded sources 25 | GSORCERY_STORE_DIR="${PORTAGE_ACTUAL_DISTDIR:-${DISTDIR}}" 26 | # 27 | # @ECLASS-VARIABLE: GSORCERY_FETCH_CMD 28 | # @DESCRIPTION: fetch command 29 | GSORCERY_FETCH_CMD="wget" 30 | 31 | EXPORT_FUNCTIONS src_unpack 32 | 33 | g-sorcery_fetch() { 34 | addwrite "${GSORCERY_STORE_DIR}" 35 | pushd "${GSORCERY_STORE_DIR}" >/dev/null || die "can't chdir to ${GSORCERY_STORE_DIR}" 36 | if [[ ! -f "${SOURCEFILE}" ]]; then 37 | $GSORCERY_FETCH_CMD ${REPO_URI}${SOURCEFILE} || die 38 | fi 39 | popd >/dev/null || die 40 | } 41 | 42 | g-sorcery_src_unpack() { 43 | if [[ x${DIGEST_SOURCES} = x ]]; then 44 | g-sorcery_fetch 45 | fi 46 | 47 | cp ${GSORCERY_STORE_DIR}/${SOURCEFILE} . || die 48 | unpack ./${SOURCEFILE} 49 | } 50 | -------------------------------------------------------------------------------- /tests/server.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | server.py 6 | ~~~~~~~~~ 7 | 8 | test server 9 | 10 | :copyright: (c) 2013 by Jauhien Piatlicki 11 | :license: GPL-2, see LICENSE for more details. 12 | """ 13 | 14 | import os 15 | import threading 16 | 17 | from g_sorcery.compatibility import py2k 18 | 19 | if py2k: 20 | from SocketServer import TCPServer as HTTPServer 21 | from SimpleHTTPServer import SimpleHTTPRequestHandler 22 | else: 23 | from http.server import HTTPServer 24 | from http.server import SimpleHTTPRequestHandler 25 | 26 | def HTTPRequestHandlerGenerator(direct): 27 | 28 | class HTTPRequestHandler(SimpleHTTPRequestHandler, object): 29 | directory = direct 30 | 31 | def __init__(self, request, client_address, server): 32 | super(HTTPRequestHandler, self).__init__(request, client_address, server) 33 | 34 | def translate_path(self, path): 35 | return os.path.join(self.directory, path[1:]) 36 | 37 | return HTTPRequestHandler 38 | 39 | 40 | class Server(threading.Thread): 41 | def __init__(self, directory, port=8080): 42 | super(Server, self).__init__() 43 | HTTPServer.allow_reuse_address = True 44 | server_address = ('127.0.0.1', port) 45 | self.httpd = HTTPServer(server_address, HTTPRequestHandlerGenerator(directory)) 46 | 47 | def run(self): 48 | self.httpd.serve_forever() 49 | 50 | def shutdown(self): 51 | self.httpd.shutdown() 52 | -------------------------------------------------------------------------------- /docs/g-sorcery.cfg.8.rst: -------------------------------------------------------------------------------- 1 | ============= 2 | g-sorcery.cfg 3 | ============= 4 | 5 | ----------------------------- 6 | custom settings for g-sorcery 7 | ----------------------------- 8 | 9 | :Author: Written by Jauhien Piatlicki . GSoC idea 10 | and mentorship by Rafael Martins. Lots of help and improvements 11 | by Brian Dolbec. 12 | :Date: 2015-04-20 13 | :Copyright: Copyright (c) 2013-2015 Jauhien Piatlicki, License: GPL-2 14 | :Version: 0.2.1 15 | :Manual section: 8 16 | :Manual group: g-sorcery 17 | 18 | 19 | SYNOPSIS 20 | ======== 21 | 22 | **/etc/g-sorcery/g-sorcery.cfg** 23 | 24 | DESCRIPTION 25 | =========== 26 | 27 | **g-sorcery.cfg** various **g-sorcery** settings aimed to be changeable by user. 28 | 29 | SECTIONS AND VARIABLES 30 | ====================== 31 | 32 | \[main\] 33 | ~~~~~~~~ 34 | Section with common settings. Contains only one variable: *package_manager*. 35 | Currently it only can have value *portage*. 36 | 37 | \[BACKEND\] 38 | ~~~~~~~~~~~ 39 | Section with settings for a given backend. **BACKEND** should be a backend name. 40 | It can contain entries named **REPO_packages** with a list of space separated names 41 | of packages which ebuilds should be generated for. **REPO** is a name of a repository. 42 | 43 | 44 | EXAMPLE 45 | ======= 46 | 47 | .. code-block:: 48 | 49 | [main] 50 | package_manager=portage 51 | 52 | [gs-elpa] 53 | marmalade_packages = clojure-mode clojurescript-mode 54 | 55 | 56 | SEE ALSO 57 | ======== 58 | 59 | **g-sorcery**\(8), **gs-elpa**\(8), **gs-pypi**\(8), **portage**\(5), **emerge**\(1), **layman**\(8) 60 | -------------------------------------------------------------------------------- /scripts/run_tests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | run_tests.py 6 | ~~~~~~~~~~~~ 7 | 8 | a simple script that runs all the tests from the *tests* directory 9 | (.py files) 10 | 11 | :copyright: (c) 2010 by Rafael Goncalves Martins 12 | :copyright: (c) 2013 by Jauhien Piatlicki 13 | :license: GPL-2, see LICENSE for more details. 14 | """ 15 | 16 | import os 17 | import sys 18 | import unittest 19 | 20 | root_dir = os.path.realpath(os.path.join( 21 | os.path.dirname(os.path.abspath(__file__)), '..' 22 | )) 23 | 24 | tests_dir = os.path.join(root_dir, 'tests') 25 | 26 | # adding the tests directory to the top of the PYTHONPATH 27 | sys.path = [root_dir, tests_dir] + sys.path 28 | 29 | os.environ["PYTHONPATH"] = ':'.join([root_dir, tests_dir]) 30 | 31 | suites = [] 32 | 33 | # getting the test suites from the python modules (files that ends whit .py) 34 | for f in os.listdir(tests_dir): 35 | if not f.endswith('.py') or not f.startswith('test_'): 36 | continue 37 | try: 38 | my_test = __import__(f[:-len('.py')]) 39 | except ImportError: 40 | continue 41 | 42 | # all the python modules MUST have a 'suite' method, that returns the 43 | # test suite of the module 44 | suites.append(my_test.suite()) 45 | 46 | # unifying all the test suites in only one 47 | suites = unittest.TestSuite(suites) 48 | 49 | # creating the Test Runner object 50 | test_runner = unittest.TextTestRunner(descriptions=2, verbosity=2) 51 | 52 | # running the tests 53 | result = test_runner.run(suites) 54 | 55 | if result.failures or result.errors: 56 | sys.exit(1) 57 | -------------------------------------------------------------------------------- /g_sorcery/eclass.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | eclass.py 6 | ~~~~~~~~~ 7 | 8 | eclass generation 9 | 10 | :copyright: (c) 2013 by Jauhien Piatlicki 11 | :license: GPL-2, see LICENSE for more details. 12 | """ 13 | 14 | import glob 15 | import os 16 | 17 | from .exceptions import EclassError 18 | from .fileutils import get_pkgpath 19 | 20 | class EclassGenerator(object): 21 | """ 22 | Generates eclasses from files in a given dir. 23 | """ 24 | 25 | def __init__(self, eclass_dir): 26 | """ 27 | __init__ in a subclass should take no parameters. 28 | """ 29 | self.eclass_dir = eclass_dir 30 | 31 | def list(self): 32 | """ 33 | List all eclasses. 34 | 35 | Returns: 36 | List of all eclasses with string entries. 37 | """ 38 | result = [] 39 | 40 | for directory in [self.eclass_dir, os.path.join(get_pkgpath(), 'data')]: 41 | if directory: 42 | for f_name in glob.iglob(os.path.join(directory, '*.eclass')): 43 | result.append(os.path.basename(f_name)[:-7]) 44 | 45 | return list(set(result)) 46 | 47 | def generate(self, eclass): 48 | """ 49 | Generate a given eclass. 50 | 51 | Args: 52 | eclass: String containing eclass name. 53 | 54 | Returns: 55 | Eclass source as a list of strings. 56 | """ 57 | for directory in [self.eclass_dir, os.path.join(get_pkgpath(), 'data')]: 58 | f_name = os.path.join(directory, eclass + '.eclass') 59 | if os.path.isfile(f_name): 60 | with open(f_name, 'r') as f: 61 | eclass = f.read().split('\n') 62 | if eclass[-1] == '': 63 | eclass = eclass[:-1] 64 | return eclass 65 | 66 | raise EclassError('No eclass ' + eclass) 67 | -------------------------------------------------------------------------------- /tests/test_metadata.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | test_metadata.py 6 | ~~~~~~~~~~~~~~~~ 7 | 8 | metadata test suite 9 | 10 | :copyright: (c) 2013 by Jauhien Piatlicki 11 | :license: GPL-2, see LICENSE for more details. 12 | """ 13 | 14 | import os 15 | import unittest 16 | 17 | from g_sorcery.compatibility import TemporaryDirectory 18 | from g_sorcery.g_collections import Package 19 | from g_sorcery.metadata import MetadataGenerator 20 | from g_sorcery.package_db import PackageDB 21 | 22 | from tests.base import BaseTest 23 | 24 | 25 | class TestMetadataGenerator(BaseTest): 26 | 27 | def test_metadata(self): 28 | pkg_db = PackageDB(self.tempdir.name) 29 | pkg_db.add_category("app-test") 30 | ebuild_data = {"herd": ["testers", "crackers"], 31 | 'maintainer': [{'email': 'test@example.com', 32 | 'name': 'tux'}], 33 | "longdescription": "very long description here", 34 | "use": {"flag": {"use1": "testing use1", "use2": "testing use2"}}} 35 | package = Package("app-test", "metadata_tester", "0.1") 36 | pkg_db.add_package(package, ebuild_data) 37 | metadata_g = MetadataGenerator(pkg_db) 38 | metadata = metadata_g.generate(package) 39 | self.assertEqual(metadata, 40 | ['', 41 | '', 42 | '', 43 | '\ttesters', '\tcrackers', 44 | '\t', '\t\ttest@example.com', '\t\ttux', '\t', 45 | '\tvery long description here', 46 | '\t', '\t\ts', '\t\ts', '\t', 47 | '']) 48 | 49 | 50 | def suite(): 51 | suite = unittest.TestSuite() 52 | suite.addTest(TestMetadataGenerator('test_metadata')) 53 | return suite 54 | -------------------------------------------------------------------------------- /docs/g-sorcery.cfg.8: -------------------------------------------------------------------------------- 1 | .\" Man page generated from reStructuredText. 2 | . 3 | .TH G-SORCERY.CFG 8 "2015-04-20" "0.2.1" "g-sorcery" 4 | .SH NAME 5 | g-sorcery.cfg \- custom settings for g-sorcery 6 | . 7 | .nr rst2man-indent-level 0 8 | . 9 | .de1 rstReportMargin 10 | \\$1 \\n[an-margin] 11 | level \\n[rst2man-indent-level] 12 | level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] 13 | - 14 | \\n[rst2man-indent0] 15 | \\n[rst2man-indent1] 16 | \\n[rst2man-indent2] 17 | .. 18 | .de1 INDENT 19 | .\" .rstReportMargin pre: 20 | . RS \\$1 21 | . nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] 22 | . nr rst2man-indent-level +1 23 | .\" .rstReportMargin post: 24 | .. 25 | .de UNINDENT 26 | . RE 27 | .\" indent \\n[an-margin] 28 | .\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] 29 | .nr rst2man-indent-level -1 30 | .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] 31 | .in \\n[rst2man-indent\\n[rst2man-indent-level]]u 32 | .. 33 | .SH SYNOPSIS 34 | .sp 35 | \fB/etc/g\-sorcery/g\-sorcery.cfg\fP 36 | .SH DESCRIPTION 37 | .sp 38 | \fBg\-sorcery.cfg\fP various \fBg\-sorcery\fP settings aimed to be changeable by user. 39 | .SH SECTIONS AND VARIABLES 40 | .SS [main] 41 | .sp 42 | Section with common settings. Contains only one variable: \fIpackage_manager\fP\&. 43 | Currently it only can have value \fIportage\fP\&. 44 | .SS [BACKEND] 45 | .sp 46 | Section with settings for a given backend. \fBBACKEND\fP should be a backend name. 47 | It can contain entries named \fBREPO_packages\fP with a list of space separated names 48 | of packages which ebuilds should be generated for. \fBREPO\fP is a name of a repository. 49 | .SH EXAMPLE 50 | .INDENT 0.0 51 | .INDENT 3.5 52 | .sp 53 | .nf 54 | .ft C 55 | [main] 56 | package_manager=portage 57 | 58 | [gs\-elpa] 59 | marmalade_packages = clojure\-mode clojurescript\-mode 60 | .ft P 61 | .fi 62 | .UNINDENT 63 | .UNINDENT 64 | .SH SEE ALSO 65 | .sp 66 | \fBg\-sorcery\fP(8), \fBgs\-elpa\fP(8), \fBgs\-pypi\fP(8), \fBportage\fP(5), \fBemerge\fP(1), \fBlayman\fP(8) 67 | .SH AUTHOR 68 | Written by Jauhien Piatlicki . GSoC idea 69 | and mentorship by Rafael Martins. Lots of help and improvements 70 | by Brian Dolbec. 71 | .SH COPYRIGHT 72 | Copyright (c) 2013-2015 Jauhien Piatlicki, License: GPL-2 73 | .\" Generated by docutils manpage writer. 74 | . 75 | -------------------------------------------------------------------------------- /g_sorcery/g_sorcery.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | g_sorcery.py 6 | ~~~~~~~~~~~~ 7 | 8 | the main module 9 | 10 | :copyright: (c) 2013 by Jauhien Piatlicki 11 | :license: GPL-2, see LICENSE for more details. 12 | """ 13 | 14 | 15 | import importlib 16 | import os 17 | import sys 18 | 19 | from .compatibility import configparser 20 | from .fileutils import FileJSON 21 | from .exceptions import FileJSONError 22 | from .logger import Logger 23 | 24 | def main(): 25 | logger = Logger() 26 | if len(sys.argv) < 2: 27 | logger.error("no backend specified") 28 | return -1 29 | name = sys.argv[1] 30 | cfg = name + '.json' 31 | cfg_path = None 32 | for path in '.', '~', '/etc/g-sorcery': 33 | current = os.path.join(path, cfg) 34 | if (os.path.isfile(current)): 35 | cfg_path = path 36 | break 37 | if not cfg_path: 38 | logger.error('no config file for ' + name + ' backend\n') 39 | return -1 40 | cfg_f = FileJSON(cfg_path, cfg, ['package']) 41 | try: 42 | config = cfg_f.read() 43 | except FileJSONError as e: 44 | logger.error('error loading config file for ' \ 45 | + name + ': ' + str(e) + '\n') 46 | return -1 47 | backend = get_backend(config['package']) 48 | 49 | if not backend: 50 | logger.error("backend initialization failed, exiting") 51 | return -1 52 | 53 | config_file = None 54 | for path in '.', '~', '/etc/g-sorcery': 55 | config_file = os.path.join(path, "g-sorcery.cfg") 56 | if (os.path.isfile(config_file)): 57 | break 58 | else: 59 | config_file = None 60 | 61 | if not config_file: 62 | logger.error('no global config file\n') 63 | return -1 64 | 65 | global_config = configparser.ConfigParser() 66 | global_config.read(config_file) 67 | 68 | return backend.instance(sys.argv[2:], config, global_config) 69 | 70 | 71 | def get_backend(package): 72 | """ 73 | Load backend by package name. 74 | """ 75 | logger = Logger() 76 | try: 77 | module = importlib.import_module(package + '.backend') 78 | except ImportError as error: 79 | logger.error("error importing backend: " + str(error)) 80 | return None 81 | 82 | return module 83 | 84 | 85 | if __name__ == "__main__": 86 | sys.exit(main()) 87 | -------------------------------------------------------------------------------- /g_sorcery/logger.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | logger.py 6 | ~~~~~~~~~ 7 | 8 | logging classes 9 | 10 | :copyright: (c) 2013 by Jauhien Piatlicki 11 | :license: GPL-2, see LICENSE for more details. 12 | """ 13 | 14 | import sys 15 | 16 | import portage 17 | 18 | 19 | class Logger(object): 20 | """ 21 | A simple logger object. Uses portage out facilities. 22 | """ 23 | def __init__(self): 24 | self.out = portage.output.EOutput() 25 | 26 | def error(self, message): 27 | self.out.eerror(message) 28 | 29 | def info(self, message): 30 | self.out.einfo(message) 31 | 32 | def warn(self, message): 33 | self.out.ewarn(message) 34 | 35 | 36 | class ProgressBar(object): 37 | """ 38 | A progress bar for CLI 39 | """ 40 | 41 | __slots__ = ('length', 'total', 'processed', 'chars') 42 | 43 | def __init__(self, length, total, processed = 0): 44 | """ 45 | Args: 46 | length: Length of the progress bar. 47 | total: The overall number of items to process. 48 | processe: Number of processed items. 49 | """ 50 | self.length = length 51 | self.total = total 52 | self.chars = ['-', '\\', '|', '/'] 53 | self.processed = processed 54 | 55 | def begin(self): 56 | """ 57 | Start displaying the progress bar with 0% progress. 58 | """ 59 | self.processed = 0 60 | self.display() 61 | 62 | def display(self, processed = None): 63 | """ 64 | Show the progress bar with current progress. 65 | 66 | Args: 67 | processed: Number of processed items. 68 | """ 69 | if processed: 70 | self.processed = processed 71 | 72 | show = self.chars[self.processed % 4] 73 | percent = (self.processed * 100)//self.total 74 | progress = (percent * self.length)//100 75 | blank = self.length - progress 76 | sys.stderr.write("\r %s [%s%s] %s%%" % \ 77 | (show, "#" * progress, " " * blank, percent)) 78 | sys.stderr.flush() 79 | 80 | def increment(self, count = 1): 81 | """ 82 | Increment number of processed items. 83 | 84 | Args: 85 | count: Step of incrementation. 86 | """ 87 | self.processed += count 88 | self.display() 89 | 90 | def end(self): 91 | """ 92 | Show 100%. 93 | """ 94 | self.processed = self.total 95 | self.display() 96 | -------------------------------------------------------------------------------- /tests/test_FileBSON.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | test_FileBSON.py 6 | ~~~~~~~~~~~~~~~~ 7 | 8 | FileBSON test suite 9 | 10 | :copyright: (c) 2013-2015 by Jauhien Piatlicki 11 | :license: GPL-2, see LICENSE for more details. 12 | """ 13 | 14 | import os 15 | import unittest 16 | 17 | from g_sorcery.g_collections import serializable_elist 18 | 19 | from tests.base import BaseTest 20 | from tests.serializable import NonSerializableClass, SerializableClass, DeserializableClass 21 | 22 | BSON_INSTALLED = False 23 | 24 | try: 25 | from g_sorcery.file_bson.file_bson import FileBSON 26 | BSON_INSTALLED = True 27 | except ImportError as e: 28 | pass 29 | 30 | if BSON_INSTALLED: 31 | 32 | class TestFileJSON(BaseTest): 33 | def setUp(self): 34 | super(TestFileJSON, self).setUp() 35 | self.directory = os.path.join(self.tempdir.name, 'tst') 36 | self.name = 'tst.json' 37 | self.path = os.path.join(self.directory, self.name) 38 | 39 | def test_write_read(self): 40 | fj = FileBSON(self.directory, self.name, ["mandatory"]) 41 | content = {"mandatory":"1", "test":"2"} 42 | fj.write(content) 43 | content_r = fj.read() 44 | self.assertEqual(content, content_r) 45 | 46 | def test_serializable(self): 47 | fj = FileBSON(self.directory, self.name, []) 48 | content = SerializableClass("1", "2") 49 | fj.write(content) 50 | content_r = fj.read() 51 | self.assertEqual(content_r, {"field1":"1", "field2":"2"}) 52 | self.assertRaises(TypeError, fj.write, NonSerializableClass()) 53 | 54 | def test_deserializable(self): 55 | fj = FileBSON(self.directory, self.name, []) 56 | content = DeserializableClass("1", "2") 57 | fj.write(content) 58 | content_r = fj.read() 59 | self.assertEqual(content, content_r) 60 | 61 | def test_deserializable_collection(self): 62 | fj = FileBSON(self.directory, self.name, []) 63 | content1 = DeserializableClass("1", "2") 64 | content2 = DeserializableClass("3", "4") 65 | content = serializable_elist([content1, content2]) 66 | fj.write(content) 67 | content_r = fj.read() 68 | self.assertEqual(content, content_r) 69 | 70 | def suite(): 71 | suite = unittest.TestSuite() 72 | suite.addTest(TestFileJSON('test_write_read')) 73 | suite.addTest(TestFileJSON('test_serializable')) 74 | suite.addTest(TestFileJSON('test_deserializable')) 75 | suite.addTest(TestFileJSON('test_deserializable_collection')) 76 | return suite 77 | 78 | else: 79 | def suite(): 80 | suite = unittest.TestSuite() 81 | return suite 82 | -------------------------------------------------------------------------------- /g_sorcery/git_syncer/git_syncer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | git_syncer.py 6 | ~~~~~~~~~~~~~ 7 | 8 | git sync helper 9 | 10 | :copyright: (c) 2015 by Jauhien Piatlicki 11 | :license: GPL-2, see LICENSE for more details. 12 | """ 13 | 14 | import os 15 | import shutil 16 | import subprocess 17 | 18 | from g_sorcery.compatibility import TemporaryDirectory 19 | 20 | from g_sorcery.exceptions import SyncError 21 | from g_sorcery.syncer import Syncer, SyncedData, TmpSyncedData 22 | 23 | 24 | class GITSyncer(Syncer): 25 | """ 26 | Class used to sync with git repos. 27 | """ 28 | 29 | def sync(self, db_uri, repository_config): 30 | """ 31 | Synchronize local directory with remote source. 32 | 33 | Args: 34 | db_uri: URI for synchronization with remote source. 35 | repository_config: repository config. 36 | 37 | Returns: 38 | SyncedData object that gives access to the directory with data. 39 | """ 40 | if self.persistent_datadir is None: 41 | tmp_dir = TemporaryDirectory() 42 | path = os.path.join(tmp_dir.name, "remote") 43 | else: 44 | path = self.persistent_datadir 45 | try: 46 | branch = repository_config["branch"] 47 | except KeyError: 48 | branch = "master" 49 | 50 | if os.path.exists(path): 51 | if self.branch_not_changed(path, branch) and self.remote_url_not_changed(path, db_uri): 52 | self.pull(path) 53 | else: 54 | shutil.rmtree(path) 55 | self.clone(db_uri, branch, path) 56 | else: 57 | self.clone(db_uri, branch, path) 58 | 59 | if self.persistent_datadir is None: 60 | return TmpSyncedData(path, tmp_dir) 61 | else: 62 | return SyncedData(path) 63 | 64 | 65 | def clone(self, db_uri, branch, path): 66 | if os.system("git clone --depth 1 --branch " + branch + " " + db_uri + " " + path): 67 | raise SyncError("sync failed (clonning): " + db_uri) 68 | 69 | 70 | def pull(self, path): 71 | if os.system("cd " + path + " && git pull"): 72 | raise SyncError("sync failed (pulling): " + path) 73 | 74 | 75 | def branch_not_changed(self, path, branch): 76 | try: 77 | result = subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=path).rstrip().decode("utf-8") 78 | except Exception: 79 | return False 80 | return result == branch 81 | 82 | 83 | def remote_url_not_changed(self, path, url): 84 | try: 85 | result = subprocess.check_output(["git", "config", "--get", "remote.origin.url"], cwd=path).rstrip().decode("utf-8") 86 | except Exception: 87 | return False 88 | return result == url 89 | -------------------------------------------------------------------------------- /g_sorcery/syncer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | syncer.py 6 | ~~~~~~~~~ 7 | 8 | sync helper 9 | 10 | :copyright: (c) 2013-2015 by Jauhien Piatlicki 11 | :license: GPL-2, see LICENSE for more details. 12 | """ 13 | 14 | import glob 15 | import os 16 | 17 | from .compatibility import TemporaryDirectory 18 | 19 | from .exceptions import SyncError 20 | from .fileutils import wget 21 | 22 | 23 | class SyncedData(object): 24 | """ 25 | Synced data. 26 | 27 | Directory with sync data is guaranted to exist only as long as this 28 | object does. 29 | """ 30 | def __init__(self, directory): 31 | self.directory = os.path.abspath(directory) 32 | 33 | def get_path(self): 34 | return self.directory 35 | 36 | 37 | class TmpSyncedData(SyncedData): 38 | """ 39 | Synced data that lives in a temporary directory. 40 | """ 41 | 42 | def __init__(self, directory, tmpdirobj): 43 | super(TmpSyncedData, self).__init__(directory) 44 | self.tmpdirobj = tmpdirobj 45 | 46 | 47 | class Syncer(object): 48 | """ 49 | Class used to sync data with remote source. 50 | """ 51 | 52 | def __init__(self, persistent_datadir): 53 | self.persistent_datadir = persistent_datadir 54 | 55 | def sync(self, db_uri, repository_config): 56 | """ 57 | Synchronize local directory with remote source. 58 | 59 | Args: 60 | db_uri: URI for synchronization with remote source. 61 | repository_config: repository config. 62 | 63 | Returns: 64 | SyncedData object that gives access to the directory with data. 65 | """ 66 | raise NotImplementedError 67 | 68 | 69 | class TGZSyncer(Syncer): 70 | """ 71 | Class used to download and unpack tarballs. 72 | """ 73 | 74 | def sync(self, db_uri, repository_config): 75 | """ 76 | Synchronize local directory with remote source. 77 | 78 | Args: 79 | db_uri: URI for synchronization with remote source. 80 | repository_config: repository config. 81 | 82 | Returns: 83 | SyncedData object that gives access to the directory with data. 84 | """ 85 | download_dir = TemporaryDirectory() 86 | if wget(db_uri, download_dir.name): 87 | raise SyncError('sync failed: ' + db_uri) 88 | 89 | tmp_dir = TemporaryDirectory() 90 | for f_name in glob.iglob(os.path.join(download_dir.name, '*.tar.gz')): 91 | if os.system("tar -xvzf " + f_name + " -C " + tmp_dir.name): 92 | raise SyncError('sync failed (unpacking)') 93 | 94 | tmp_path = os.path.join(tmp_dir.name, os.listdir(tmp_dir.name)[0]) 95 | del download_dir 96 | return TmpSyncedData(tmp_path, tmp_dir) 97 | 98 | 99 | SUPPORTED_SYNCERS = {"tgz": TGZSyncer} 100 | 101 | # git_syncer module is optional, we should check if it is installed 102 | try: 103 | from .git_syncer.git_syncer import GITSyncer 104 | SUPPORTED_SYNCERS["git"] = GITSyncer 105 | 106 | except ImportError as e: 107 | pass 108 | -------------------------------------------------------------------------------- /tests/test_ebuild.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | test_ebuild.py 6 | ~~~~~~~~~~~~~~ 7 | 8 | ebuild test suite 9 | 10 | :copyright: (c) 2013 by Jauhien Piatlicki 11 | :license: GPL-2, see LICENSE for more details. 12 | """ 13 | 14 | import collections 15 | import os 16 | import unittest 17 | 18 | from g_sorcery.compatibility import TemporaryDirectory 19 | from g_sorcery.g_collections import Package 20 | from g_sorcery.ebuild import EbuildGeneratorFromFile, DefaultEbuildGenerator 21 | from g_sorcery.package_db import PackageDB 22 | 23 | from tests.base import BaseTest 24 | 25 | 26 | Layout = collections.namedtuple("Layout", 27 | ["vars_before_inherit", "inherit", 28 | "vars_after_description", "vars_after_keywords"]) 29 | 30 | class TestEbuildGenerator(BaseTest): 31 | 32 | ebuild_data = {"herd": ["testers", "crackers"], 33 | 'maintainer': [{'email': 'test@example.com', 34 | 'name': 'tux'}], 35 | "longdescription": "very long description here", 36 | "use": {"flag": {"use1": "testing use1", "use2": "testing use2"}}, 37 | "homepage": "example.com", 38 | "description": "testing ebuild", 39 | "array": "(a b c d)"} 40 | package = Package("app-test", "metadata_tester", "0.1") 41 | 42 | def setUp(self): 43 | super(TestEbuildGenerator, self).setUp() 44 | self.pkg_db = PackageDB(self.tempdir.name) 45 | self.pkg_db.add_category("app-test") 46 | self.pkg_db.add_package(self.package, self.ebuild_data) 47 | 48 | def test_ebuild_generator_from_file(self): 49 | template = os.path.join(self.tempdir.name, "test.tmpl") 50 | os.system("echo 'TEST_SUBST=%(array)s' > " + template) 51 | 52 | ebuild_g = EbuildGeneratorFromFile(self.pkg_db, template) 53 | ebuild = ebuild_g.generate(self.package) 54 | self.assertEqual(ebuild, ['TEST_SUBST=(a b c d)']) 55 | 56 | def test_default_ebuild_generator(self): 57 | vars_before_inherit = \ 58 | [{"name":"test_raw_value", "value":"raw_value", "raw":True}, 59 | {"name":"test_value", "value":"value"}] 60 | 61 | inherit = ["g-test"] 62 | 63 | vars_after_description = \ 64 | ["homepage"] 65 | 66 | vars_after_keywords = \ 67 | [{"name":"array"}, 68 | {"name":"array", "raw":True}] 69 | 70 | layout = Layout(vars_before_inherit, 71 | inherit, vars_after_description, vars_after_keywords) 72 | 73 | ebuild_g = DefaultEbuildGenerator(self.pkg_db, layout) 74 | ebuild = ebuild_g.generate(self.package) 75 | self.assertEqual(ebuild, ['# automatically generated by g-sorcery', 76 | '# please do not edit this file', '', 77 | 'EAPI=5', '', 78 | 'TEST_RAW_VALUE=raw_value', 'TEST_VALUE="value"', '', 79 | 'inherit g-test', '', 80 | 'DESCRIPTION="testing ebuild"', '', 81 | 'HOMEPAGE="example.com"', '', 82 | 'SLOT="0"', 'KEYWORDS="~amd64 ~x86"', '', 83 | 'ARRAY="(a b c d)"', 'ARRAY=(a b c d)', '']) 84 | 85 | 86 | def suite(): 87 | suite = unittest.TestSuite() 88 | suite.addTest(TestEbuildGenerator('test_ebuild_generator_from_file')) 89 | suite.addTest(TestEbuildGenerator('test_default_ebuild_generator')) 90 | return suite 91 | -------------------------------------------------------------------------------- /g_sorcery/serialization.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | serialization.py 6 | ~~~~~~~~~~~~~~~~ 7 | 8 | json serialization 9 | 10 | :copyright: (c) 2013-2015 by Jauhien Piatlicki 11 | :license: GPL-2, see LICENSE for more details. 12 | """ 13 | 14 | import json 15 | import importlib 16 | 17 | from .compatibility import basestring 18 | 19 | def step_to_raw_serializable(obj): 20 | """ 21 | Make one step of convertion of object 22 | to the type that is serializable 23 | by the json library. 24 | 25 | None return value signifies an error. 26 | """ 27 | if hasattr(obj, "serialize"): 28 | if hasattr(obj, "deserialize"): 29 | module = obj.__class__.__module__ 30 | name = obj.__class__.__name__ 31 | value = obj.serialize() 32 | return {"python_module" : module, 33 | "python_class" : name, 34 | "value" : value} 35 | else: 36 | return obj.serialize() 37 | return None 38 | 39 | 40 | def to_raw_serializable(obj): 41 | """ 42 | Convert object to the raw serializable type. 43 | Logic is the same as in the standard json encoder. 44 | """ 45 | if isinstance(obj, basestring) \ 46 | or obj is None \ 47 | or obj is True \ 48 | or obj is False \ 49 | or isinstance(obj, int) \ 50 | or isinstance(obj, float): 51 | return obj 52 | elif isinstance(obj, dict): 53 | return {k: to_raw_serializable(v) for k, v in obj.items()} 54 | elif isinstance(obj, (list, tuple)): 55 | return [to_raw_serializable(item) for item in obj] 56 | 57 | else: 58 | sobj = step_to_raw_serializable(obj) 59 | if not sobj: 60 | raise TypeError('Non serializable object: ', obj) 61 | return to_raw_serializable(sobj) 62 | 63 | 64 | def step_from_raw_serializable(sobj): 65 | """ 66 | Make one step of building of object from the 67 | raw json serializable type. 68 | """ 69 | if "python_class" in sobj: 70 | module = importlib.import_module(sobj["python_module"]) 71 | cls = getattr(module, sobj["python_class"]) 72 | return cls.deserialize(sobj["value"]) 73 | return sobj 74 | 75 | 76 | def from_raw_serializable(sobj): 77 | """ 78 | Build object from the raw serializable object. 79 | """ 80 | if isinstance(sobj, dict): 81 | res = {k: from_raw_serializable(v) for k, v in sobj.items()} 82 | return step_from_raw_serializable(res) 83 | elif isinstance(sobj, list): 84 | return [from_raw_serializable(item) for item in sobj] 85 | else: 86 | return sobj 87 | 88 | 89 | class JSONSerializer(json.JSONEncoder): 90 | """ 91 | Custom JSON encoder. 92 | 93 | Each serializable class should have a method serialize 94 | that returns JSON serializable value. If class additionally 95 | has a classmethod deserialize that it can be deserialized 96 | and additional metainformation is added to the resulting JSON. 97 | """ 98 | def default(self, obj): 99 | res = step_to_raw_serializable(obj) 100 | if res: 101 | return res 102 | else: 103 | return json.JSONEncoder.default(self, obj) 104 | 105 | 106 | def deserializeHook(json_object): 107 | """ 108 | Custom JSON decoder. 109 | 110 | Each class that can be deserialized should have classmethod deserialize 111 | that takes value (previously returned by serialize method) and transforms 112 | it into class instance. 113 | """ 114 | return step_from_raw_serializable(json_object) 115 | -------------------------------------------------------------------------------- /tests/test_FileJSON.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | test_FileJSON.py 6 | ~~~~~~~~~~~~~~~~ 7 | 8 | FileJSON test suite 9 | 10 | :copyright: (c) 2013-2015 by Jauhien Piatlicki 11 | :license: GPL-2, see LICENSE for more details. 12 | """ 13 | 14 | import json 15 | import os 16 | import unittest 17 | 18 | from g_sorcery.fileutils import FileJSON 19 | from g_sorcery.exceptions import FileJSONError 20 | from g_sorcery.g_collections import serializable_elist 21 | 22 | from tests.base import BaseTest 23 | from tests.serializable import NonSerializableClass, SerializableClass, DeserializableClass 24 | 25 | class TestFileJSON(BaseTest): 26 | def setUp(self): 27 | super(TestFileJSON, self).setUp() 28 | self.directory = os.path.join(self.tempdir.name, 'tst') 29 | self.name = 'tst.json' 30 | self.path = os.path.join(self.directory, self.name) 31 | 32 | def test_read_nonexistent(self): 33 | fj = FileJSON(self.directory, self.name, []) 34 | content = fj.read() 35 | self.assertEqual(content, {}) 36 | self.assertTrue(os.path.isfile(self.path)) 37 | 38 | def test_read_nonexistent_mandatory_key(self): 39 | fj = FileJSON(self.directory, self.name, ["mandatory1", "mandatory2"]) 40 | content = fj.read() 41 | self.assertEqual(content, {"mandatory1":"", "mandatory2":""}) 42 | self.assertTrue(os.path.isfile(self.path)) 43 | 44 | def test_read_luck_of_mandatory_key(self): 45 | fj = FileJSON(self.directory, self.name, ["mandatory"]) 46 | os.makedirs(self.directory) 47 | with open(self.path, 'w') as f: 48 | json.dump({"test":"test"}, f) 49 | self.assertRaises(FileJSONError, fj.read) 50 | 51 | def test_write_luck_of_mandatory_key(self): 52 | fj = FileJSON(self.directory, self.name, ["mandatory"]) 53 | self.assertRaises(FileJSONError, fj.write, {"test":"test"}) 54 | 55 | def test_write_read(self): 56 | fj = FileJSON(self.directory, self.name, ["mandatory"]) 57 | content = {"mandatory":"1", "test":"2"} 58 | fj.write(content) 59 | content_r = fj.read() 60 | self.assertEqual(content, content_r) 61 | 62 | def test_serializable(self): 63 | fj = FileJSON(self.directory, self.name, []) 64 | content = SerializableClass("1", "2") 65 | fj.write(content) 66 | content_r = fj.read() 67 | self.assertEqual(content_r, {"field1":"1", "field2":"2"}) 68 | self.assertRaises(TypeError, fj.write, NonSerializableClass()) 69 | 70 | def test_deserializable(self): 71 | fj = FileJSON(self.directory, self.name, []) 72 | content = DeserializableClass("1", "2") 73 | fj.write(content) 74 | content_r = fj.read() 75 | self.assertEqual(content, content_r) 76 | 77 | def test_deserializable_collection(self): 78 | fj = FileJSON(self.directory, self.name, []) 79 | content1 = DeserializableClass("1", "2") 80 | content2 = DeserializableClass("3", "4") 81 | content = serializable_elist([content1, content2]) 82 | fj.write(content) 83 | content_r = fj.read() 84 | self.assertEqual(content, content_r) 85 | 86 | def suite(): 87 | suite = unittest.TestSuite() 88 | suite.addTest(TestFileJSON('test_read_nonexistent')) 89 | suite.addTest(TestFileJSON('test_read_nonexistent_mandatory_key')) 90 | suite.addTest(TestFileJSON('test_read_luck_of_mandatory_key')) 91 | suite.addTest(TestFileJSON('test_write_luck_of_mandatory_key')) 92 | suite.addTest(TestFileJSON('test_write_read')) 93 | suite.addTest(TestFileJSON('test_serializable')) 94 | suite.addTest(TestFileJSON('test_deserializable')) 95 | suite.addTest(TestFileJSON('test_deserializable_collection')) 96 | return suite 97 | -------------------------------------------------------------------------------- /docs/g-sorcery.8.rst: -------------------------------------------------------------------------------- 1 | ========= 2 | g-sorcery 3 | ========= 4 | 5 | ------------------------------------------------ 6 | manage overlays for 3rd party software providers 7 | ------------------------------------------------ 8 | 9 | :Author: Written by Jauhien Piatlicki . GSoC idea 10 | and mentorship by Rafael Martins. Lots of help and improvements 11 | by Brian Dolbec. Integration with layman based on work of Auke Booij. 12 | :Date: 2015-04-20 13 | :Copyright: Copyright (c) 2013-2015 Jauhien Piatlicki, License: GPL-2 14 | :Version: 0.2.1 15 | :Manual section: 8 16 | :Manual group: g-sorcery 17 | 18 | 19 | SYNOPSIS 20 | ======== 21 | 22 | **g-sorcery** *BACKEND* **-o** *OVERLAY* [**-r** *REPO*] **sync** 23 | 24 | **g-sorcery** *BACKEND* **-o** *OVERLAY* [**-r** *REPO*] **list** 25 | 26 | **g-sorcery** *BACKEND* **-o** *OVERLAY* [**-r** *REPO*] **generate** *PACKAGE* 27 | 28 | **g-sorcery** *BACKEND* **-o** *OVERLAY* [**-r** *REPO*] **install** *PACKAGE* 29 | 30 | **g-sorcery** *BACKEND* **-o** *OVERLAY* [**-r** *REPO*] **generate-tree** [**-d**] 31 | 32 | DESCRIPTION 33 | =========== 34 | 35 | **g-sorcery** is aimed to provide you with easy way of integration of 3rd party software 36 | providers with Gentoo. 37 | 38 | 3rd party software provider is a software distribution like CTAN, CPAN or ELPA. 39 | Usualy there is a lot of software available in such a distribution and very few or no ebuilds 40 | for it. 41 | 42 | **g-sorcery** is a project aimed to implement a framework for ebuild generators (backends) 43 | for 3rd party software providers. The CLI tool g-sorcery is designed to be called rather 44 | by appropriate backends then by user. If you are not a backend developer and just want to 45 | manage your overlay see documentation for a backend you want to use. 46 | 47 | There are two ways of using **g-sorcery**: 48 | 49 | * use it with **layman** 50 | 51 | In this case all you need to do is install **layman-9999**, **g-sorcery** 52 | and appropriate backend. Then you should just run `layman -L` as 53 | root and find an overlay you want. Type of overlay will be 54 | displayed as *g-sorcery*. Then you add this overlay as 55 | usual. It's all you need to do and it's the recommended way of 56 | using **g-sorcery** and backends. 57 | 58 | * use it as stand-alone tool (not recommended) 59 | 60 | In this case you should create an overlay (see **portage** documentation), sync it and populate 61 | it with one or more ebuilds. Then ebuilds could be installed by emerge or by **g-sorcery** tool 62 | or backend. 63 | 64 | OPTIONS 65 | ======= 66 | 67 | *BACKEND* 68 | Backend to be used. 69 | 70 | **--overlay** *OVERLAY*, **-o** *OVERLAY* 71 | Overlay directory. This option is mandatory if there is no 72 | **default_overlay** entry in a backend config. 73 | 74 | **--repository** *REPO*, **-r** *REPO* 75 | Repository name. If there is more than one repository available 76 | for a given backend must be specified. 77 | 78 | COMMANDS 79 | ======== 80 | 81 | **sync** 82 | Synchronize a repository database. 83 | 84 | **list** 85 | List packages available in a repository. 86 | 87 | **generate** 88 | Generate a given ebuild and all its dependencies. 89 | 90 | **install** 91 | Generate and install an ebuild using your package mangler. 92 | 93 | **generate-tree** 94 | Generate entire overlay structure. Without option **-d** after 95 | this command sources are not fetched during generation and there 96 | are no entries for them in Manifest files. 97 | 98 | FILES 99 | ===== 100 | 101 | **/etc/g-sorcery/g-sorcery.cfg** 102 | Main g-sorcery config. 103 | 104 | **/etc/g-sorcery/\*.json** 105 | Backend configs. 106 | 107 | NOTES 108 | ===== 109 | 110 | 1. At the moment the only package mangler **g-sorcery** supports is **portage**. 111 | 112 | SEE ALSO 113 | ======== 114 | 115 | **g-sorcery.cfg**\(8), **gs-elpa**\(8), **gs-pypi**\(8), **portage**\(5), **emerge**\(1), **layman**\(8) 116 | -------------------------------------------------------------------------------- /gs_db_tool/gs_db_tool.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | gs_db_tool.py 6 | ~~~~~~~~~~~~~ 7 | 8 | CLI to manipulate package DB 9 | 10 | :copyright: (c) 2013-2015 by Jauhien Piatlicki 11 | :license: GPL-2, see LICENSE for more details. 12 | """ 13 | 14 | import argparse 15 | import sys 16 | 17 | from g_sorcery.package_db import PackageDB 18 | 19 | def main(): 20 | parser = \ 21 | argparse.ArgumentParser(description='Package DB manipulation tool') 22 | parser.add_argument('db_dir') 23 | 24 | subparsers = parser.add_subparsers() 25 | 26 | p_ebuild_data = subparsers.add_parser('ebuild_data') 27 | p_ebuild_data_subparsers = p_ebuild_data.add_subparsers() 28 | 29 | p_ebuild_data_rename = p_ebuild_data_subparsers.add_parser('add_var') 30 | p_ebuild_data_rename.set_defaults(func=add_var) 31 | p_ebuild_data_rename.add_argument('name') 32 | p_ebuild_data_rename.add_argument('-f', '--function') 33 | p_ebuild_data_rename.add_argument('-l', '--lambda_function') 34 | p_ebuild_data_rename.add_argument('-v', '--value') 35 | 36 | p_ebuild_data_rename = p_ebuild_data_subparsers.add_parser('rename_var') 37 | p_ebuild_data_rename.set_defaults(func=rename_var) 38 | p_ebuild_data_rename.add_argument('old_name') 39 | p_ebuild_data_rename.add_argument('new_name') 40 | 41 | p_ebuild_data_show_all = p_ebuild_data_subparsers.add_parser('show_all') 42 | p_ebuild_data_show_all.set_defaults(func=show_all) 43 | 44 | p_ebuild_data_for_all = p_ebuild_data_subparsers.add_parser('for_all') 45 | p_ebuild_data_for_all.add_argument('function') 46 | p_ebuild_data_for_all.set_defaults(func=for_all) 47 | 48 | p_sync = subparsers.add_parser('sync') 49 | p_sync.set_defaults(func=sync) 50 | p_sync.add_argument('uri') 51 | 52 | args = parser.parse_args() 53 | pkg_db = PackageDB(args.db_dir) 54 | return args.func(pkg_db, args) 55 | 56 | 57 | def transform_db(function): 58 | """ 59 | Decorator for functions that change database. 60 | """ 61 | def transformator(pkg_db, args): 62 | pkg_db.read() 63 | function(pkg_db, args) 64 | pkg_db.write() 65 | return transformator 66 | 67 | 68 | def read_db(function): 69 | """ 70 | Decorator for functions that read from database. 71 | """ 72 | def reader(pkg_db, args): 73 | pkg_db.read() 74 | function(pkg_db, args) 75 | return reader 76 | 77 | 78 | @read_db 79 | def for_all(pkg_db, args): 80 | """ 81 | Execute a given python code for all DB entries. 82 | """ 83 | for package, ebuild_data in pkg_db: 84 | exec(args.function) 85 | 86 | 87 | @transform_db 88 | def add_var(pkg_db, args): 89 | """ 90 | Add new variable to every entry. 91 | """ 92 | if args.function: 93 | for package, ebuild_data in pkg_db: 94 | exec(args.function) 95 | ebuild_data[args.name] = value 96 | pkg_db.add_package(package, ebuild_data) 97 | 98 | elif args.lambda_function: 99 | lmbd = "lambda package, ebuild_data: " + args.lambda_function 100 | f = eval(lmbd) 101 | for package, ebuild_data in pkg_db: 102 | value = f(package, ebuild_data) 103 | ebuild_data[args.name] = value 104 | pkg_db.add_package(package, ebuild_data) 105 | 106 | elif args.value: 107 | for package, ebuild_data in pkg_db: 108 | ebuild_data[args.name] = args.value 109 | pkg_db.add_package(package, ebuild_data) 110 | 111 | 112 | @read_db 113 | def show_all(pkg_db, args): 114 | """ 115 | Display all DB entries. 116 | """ 117 | for package, ebuild_data in pkg_db: 118 | print(package) 119 | print('-' * len(str(package))) 120 | for key, value in ebuild_data.items(): 121 | print(" " + key + ": " + repr(value)) 122 | print("") 123 | 124 | 125 | def sync(pkg_db, args): 126 | """ 127 | Synchronize database. 128 | """ 129 | pkg_db.sync(args.uri) 130 | 131 | 132 | @transform_db 133 | def rename_var(pkg_db, args): 134 | """ 135 | Rename variable in all entries. 136 | """ 137 | for package, ebuild_data in pkg_db: 138 | if args.old_name in ebuild_data: 139 | value = ebuild_data.pop(args.old_name) 140 | ebuild_data[args.new_name] = value 141 | pkg_db.add_package(package, ebuild_data) 142 | 143 | 144 | if __name__ == "__main__": 145 | sys.exit(main()) 146 | -------------------------------------------------------------------------------- /docs/g-sorcery.8: -------------------------------------------------------------------------------- 1 | .\" Man page generated from reStructuredText. 2 | . 3 | .TH G-SORCERY 8 "2015-04-20" "0.2.1" "g-sorcery" 4 | .SH NAME 5 | g-sorcery \- manage overlays for 3rd party software providers 6 | . 7 | .nr rst2man-indent-level 0 8 | . 9 | .de1 rstReportMargin 10 | \\$1 \\n[an-margin] 11 | level \\n[rst2man-indent-level] 12 | level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] 13 | - 14 | \\n[rst2man-indent0] 15 | \\n[rst2man-indent1] 16 | \\n[rst2man-indent2] 17 | .. 18 | .de1 INDENT 19 | .\" .rstReportMargin pre: 20 | . RS \\$1 21 | . nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] 22 | . nr rst2man-indent-level +1 23 | .\" .rstReportMargin post: 24 | .. 25 | .de UNINDENT 26 | . RE 27 | .\" indent \\n[an-margin] 28 | .\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] 29 | .nr rst2man-indent-level -1 30 | .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] 31 | .in \\n[rst2man-indent\\n[rst2man-indent-level]]u 32 | .. 33 | .SH SYNOPSIS 34 | .sp 35 | \fBg\-sorcery\fP \fIBACKEND\fP \fB\-o\fP \fIOVERLAY\fP [\fB\-r\fP \fIREPO\fP] \fBsync\fP 36 | .sp 37 | \fBg\-sorcery\fP \fIBACKEND\fP \fB\-o\fP \fIOVERLAY\fP [\fB\-r\fP \fIREPO\fP] \fBlist\fP 38 | .sp 39 | \fBg\-sorcery\fP \fIBACKEND\fP \fB\-o\fP \fIOVERLAY\fP [\fB\-r\fP \fIREPO\fP] \fBgenerate\fP \fIPACKAGE\fP 40 | .sp 41 | \fBg\-sorcery\fP \fIBACKEND\fP \fB\-o\fP \fIOVERLAY\fP [\fB\-r\fP \fIREPO\fP] \fBinstall\fP \fIPACKAGE\fP 42 | .sp 43 | \fBg\-sorcery\fP \fIBACKEND\fP \fB\-o\fP \fIOVERLAY\fP [\fB\-r\fP \fIREPO\fP] \fBgenerate\-tree\fP [\fB\-d\fP] 44 | .SH DESCRIPTION 45 | .sp 46 | \fBg\-sorcery\fP is aimed to provide you with easy way of integration of 3rd party software 47 | providers with Gentoo. 48 | .sp 49 | 3rd party software provider is a software distribution like CTAN, CPAN or ELPA. 50 | Usualy there is a lot of software available in such a distribution and very few or no ebuilds 51 | for it. 52 | .sp 53 | \fBg\-sorcery\fP is a project aimed to implement a framework for ebuild generators (backends) 54 | for 3rd party software providers. The CLI tool g\-sorcery is designed to be called rather 55 | by appropriate backends then by user. If you are not a backend developer and just want to 56 | manage your overlay see documentation for a backend you want to use. 57 | .sp 58 | There are two ways of using \fBg\-sorcery\fP: 59 | .INDENT 0.0 60 | .INDENT 3.5 61 | .INDENT 0.0 62 | .IP \(bu 2 63 | use it with \fBlayman\fP 64 | .sp 65 | In this case all you need to do is install \fBlayman\-9999\fP, \fBg\-sorcery\fP 66 | and appropriate backend. Then you should just run \fIlayman \-L\fP as 67 | root and find an overlay you want. Type of overlay will be 68 | displayed as \fIg\-sorcery\fP\&. Then you add this overlay as 69 | usual. It\(aqs all you need to do and it\(aqs the recommended way of 70 | using \fBg\-sorcery\fP and backends. 71 | .IP \(bu 2 72 | use it as stand\-alone tool (not recommended) 73 | .sp 74 | In this case you should create an overlay (see \fBportage\fP documentation), sync it and populate 75 | it with one or more ebuilds. Then ebuilds could be installed by emerge or by \fBg\-sorcery\fP tool 76 | or backend. 77 | .UNINDENT 78 | .UNINDENT 79 | .UNINDENT 80 | .SH OPTIONS 81 | .INDENT 0.0 82 | .TP 83 | .B \fIBACKEND\fP 84 | Backend to be used. 85 | .TP 86 | .B \fB\-\-overlay\fP \fIOVERLAY\fP, \fB\-o\fP \fIOVERLAY\fP 87 | Overlay directory. This option is mandatory if there is no 88 | \fBdefault_overlay\fP entry in a backend config. 89 | .TP 90 | .B \fB\-\-repository\fP \fIREPO\fP, \fB\-r\fP \fIREPO\fP 91 | Repository name. If there is more than one repository available 92 | for a given backend must be specified. 93 | .UNINDENT 94 | .SH COMMANDS 95 | .INDENT 0.0 96 | .TP 97 | .B \fBsync\fP 98 | Synchronize a repository database. 99 | .TP 100 | .B \fBlist\fP 101 | List packages available in a repository. 102 | .TP 103 | .B \fBgenerate\fP 104 | Generate a given ebuild and all its dependencies. 105 | .TP 106 | .B \fBinstall\fP 107 | Generate and install an ebuild using your package mangler. 108 | .TP 109 | .B \fBgenerate\-tree\fP 110 | Generate entire overlay structure. Without option \fB\-d\fP after 111 | this command sources are not fetched during generation and there 112 | are no entries for them in Manifest files. 113 | .UNINDENT 114 | .SH FILES 115 | .INDENT 0.0 116 | .TP 117 | .B \fB/etc/g\-sorcery/g\-sorcery.cfg\fP 118 | Main g\-sorcery config. 119 | .TP 120 | .B \fB/etc/g\-sorcery/*.json\fP 121 | Backend configs. 122 | .UNINDENT 123 | .SH NOTES 124 | .INDENT 0.0 125 | .IP 1. 3 126 | At the moment the only package mangler \fBg\-sorcery\fP supports is \fBportage\fP\&. 127 | .UNINDENT 128 | .SH SEE ALSO 129 | .sp 130 | \fBg\-sorcery.cfg\fP(8), \fBgs\-elpa\fP(8), \fBgs\-pypi\fP(8), \fBportage\fP(5), \fBemerge\fP(1), \fBlayman\fP(8) 131 | .SH AUTHOR 132 | Written by Jauhien Piatlicki . GSoC idea 133 | and mentorship by Rafael Martins. Lots of help and improvements 134 | by Brian Dolbec. Integration with layman based on work of Auke Booij. 135 | .SH COPYRIGHT 136 | Copyright (c) 2013-2015 Jauhien Piatlicki, License: GPL-2 137 | .\" Generated by docutils manpage writer. 138 | . 139 | -------------------------------------------------------------------------------- /g_sorcery/g_collections.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | g_collections.py 6 | ~~~~~~~~~~~~~~~~ 7 | 8 | Customized classes of standard python data types 9 | for use withing g-sorcery for custom formatted string output 10 | substitution in our ebuild templates and classes for storing 11 | information about packages and dependencies. 12 | 13 | :copyright: (c) 2013 by Brian Dolbec 14 | :copyright: (c) 2013-2015 by Jauhien Piatlicki 15 | :license: GPL-2, see LICENSE for more details. 16 | """ 17 | 18 | import portage 19 | 20 | class elist(list): 21 | '''Custom list type which adds a customized __str__() 22 | and takes an optional separator argument 23 | 24 | elist() -> new empty elist 25 | elist(iterable) -> new elist initialized from iterable's items 26 | elist(separator='\\n\\t') -> new empty elist with 27 | newline & tab indented str(x) output 28 | elist(iterable, ' ') -> new elist initialized from iterable's items 29 | with space separated str(x) output 30 | ''' 31 | 32 | __slots__ = ('_sep_') 33 | 34 | def __init__(self, iterable=None, separator=' '): 35 | ''' 36 | iterable: initialize from iterable's items 37 | separator: string used to join list members with for __str__() 38 | ''' 39 | list.__init__(self, iterable or []) 40 | self._sep_ = separator 41 | 42 | def __str__(self): 43 | '''Custom output function 44 | 'x.__str__() <==> str(separator.join(x))' 45 | ''' 46 | return self._sep_.join(map(str, self)) 47 | 48 | 49 | class serializable_elist(object): 50 | """ 51 | A JSON serializable version of elist. 52 | """ 53 | 54 | __slots__ = ('data') 55 | 56 | def __init__(self, iterable=None, separator=' '): 57 | ''' 58 | iterable: initialize from iterable's items 59 | separator: string used to join list members with for __str__() 60 | ''' 61 | self.data = elist(iterable or [], separator) 62 | 63 | def __eq__(self, other): 64 | return self.data == other.data 65 | 66 | def __iter__(self): 67 | return iter(self.data) 68 | 69 | def __str__(self): 70 | '''Custom output function 71 | ''' 72 | return str(self.data) 73 | 74 | def append(self, x): 75 | self.data.append(x) 76 | 77 | def serialize(self): 78 | return {"separator": self.data._sep_, "data" : self.data} 79 | 80 | @classmethod 81 | def deserialize(cls, value): 82 | return serializable_elist(value["data"], separator = value["separator"]) 83 | 84 | 85 | #todo: replace Package with something better 86 | 87 | class Package(object): 88 | """ 89 | Class to store full package name: category/package-version 90 | """ 91 | __slots__ = ('category', 'name', 'version') 92 | 93 | def __init__(self, category, package, version): 94 | self.category = category 95 | self.name = package 96 | self.version = version 97 | 98 | def __str__(self): 99 | return self.category + '/' + self.name + '-' + self.version 100 | 101 | def __eq__(self, other): 102 | return self.category == other.category and \ 103 | self.name == other.name and \ 104 | self.version == other.version 105 | 106 | def __hash__(self): 107 | return hash(self.category + self.name + self.version) 108 | 109 | def serialize(self): 110 | return [self.category, self.name, self.version] 111 | 112 | @classmethod 113 | def deserialize(cls, value): 114 | return Package(*value) 115 | 116 | 117 | #todo equality operator for Dependency, as it can be used in backend dependency solving algorithm 118 | 119 | class Dependency(object): 120 | """ 121 | Class to store a dependency. Uses portage Atom. 122 | """ 123 | 124 | __slots__ = ('atom', 'category', 'package', 'version', 'operator') 125 | 126 | def __init__(self, category, package, version="", operator=""): 127 | atom_str = operator + category + "/" + package 128 | if version: 129 | atom_str += "-" + str(version) 130 | object.__setattr__(self, "atom", portage.dep.Atom(atom_str)) 131 | object.__setattr__(self, "category", category) 132 | object.__setattr__(self, "package", package) 133 | object.__setattr__(self, "version", version) 134 | object.__setattr__(self, "operator", operator) 135 | 136 | def __setattr__(self, name, value): 137 | raise AttributeError("Dependency instances are immutable", 138 | self.__class__, name, value) 139 | 140 | def __str__(self): 141 | return str(self.atom) 142 | 143 | def serialize(self): 144 | return str(self) 145 | 146 | @classmethod 147 | def deserialize(cls, value): 148 | atom = portage.dep.Atom(value) 149 | operator = portage.dep.get_operator(atom) 150 | cpv = portage.dep.dep_getcpv(atom) 151 | category, rest = portage.catsplit(cpv) 152 | 153 | if operator: 154 | package, version, revision = portage.pkgsplit(rest) 155 | else: 156 | package = rest 157 | version = "" 158 | operator = "" 159 | 160 | return Dependency(category, package, version, operator) 161 | -------------------------------------------------------------------------------- /tests/test_PackageDB.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | test_PackageDB.py 6 | ~~~~~~~~~~~~~~~~ 7 | 8 | PackageDB test suite 9 | 10 | :copyright: (c) 2013-2015 by Jauhien Piatlicki 11 | :license: GPL-2, see LICENSE for more details. 12 | """ 13 | 14 | import os 15 | import time 16 | import unittest 17 | 18 | from g_sorcery.compatibility import TemporaryDirectory 19 | from g_sorcery.db_layout import JSON_FILE_SUFFIX, BSON_FILE_SUFFIX 20 | from g_sorcery.exceptions import IntegrityError, InvalidKeyError, SyncError 21 | from g_sorcery.g_collections import Package, serializable_elist 22 | from g_sorcery.package_db import PackageDB 23 | 24 | from tests.base import BaseTest 25 | from tests.serializable import DeserializableClass 26 | from tests.server import Server 27 | 28 | SUPPORTED_FILE_FORMATS = [JSON_FILE_SUFFIX] 29 | # bson module is optional, we should check if it is installed 30 | try: 31 | from g_sorcery.file_bson.file_bson import FileBSON 32 | SUPPORTED_FILE_FORMATS.append(BSON_FILE_SUFFIX) 33 | except ImportError as e: 34 | pass 35 | 36 | 37 | class TestPackageDB(BaseTest): 38 | 39 | def test_functionality(self): 40 | port = 8080 41 | for fmt in SUPPORTED_FILE_FORMATS: 42 | sync_address = "127.0.0.1:" + str(port) + "/dummy.tar.gz" 43 | orig_tempdir = TemporaryDirectory() 44 | orig_path = os.path.join(orig_tempdir.name, "db") 45 | os.makedirs(orig_path) 46 | orig_db = PackageDB(orig_path, preferred_category_format=fmt) 47 | orig_db.add_category("app-test1") 48 | orig_db.add_category("app-test2") 49 | ebuild_data = {"test1": "tst1", "test2": "tst2", 50 | "test3": serializable_elist([DeserializableClass("1", "2"), 51 | DeserializableClass("3", "4")])} 52 | common_data = {"common1": "cmn1", "common2": "cmn2", 53 | "common3": serializable_elist([DeserializableClass("c1", "c2"), 54 | DeserializableClass("c3", "c4")])} 55 | packages = [Package("app-test1", "test", "1"), Package("app-test1", "test", "2"), 56 | Package("app-test1", "test1", "1"), Package("app-test2", "test2", "1")] 57 | for package in packages: 58 | orig_db.add_package(package, ebuild_data) 59 | orig_db.set_common_data("app-test1", common_data) 60 | full_data = dict(ebuild_data) 61 | full_data.update(common_data) 62 | 63 | orig_db.write() 64 | os.system("cd " + orig_tempdir.name + " && tar cvzf good.tar.gz db") 65 | os.system("echo invalid >> " + orig_tempdir.name + "/db/app-test1/packages." + fmt) 66 | os.system("cd " + orig_tempdir.name + " && tar cvzf dummy.tar.gz db") 67 | 68 | test_db = PackageDB(self.tempdir.name) 69 | self.assertRaises(SyncError, test_db.sync, sync_address) 70 | 71 | srv = Server(orig_tempdir.name, port=port) 72 | srv.start() 73 | self.assertRaises(IntegrityError, test_db.sync, sync_address) 74 | os.system("cd " + orig_tempdir.name + " && mv good.tar.gz dummy.tar.gz") 75 | test_db.sync(sync_address) 76 | srv.shutdown() 77 | srv.join() 78 | test_db.read() 79 | self.assertEqual(orig_db.database, test_db.database) 80 | self.assertEqual(orig_db.get_common_data("app-test1"), test_db.get_common_data("app-test1")) 81 | self.assertEqual(orig_db.get_common_data("app-test2"), test_db.get_common_data("app-test2")) 82 | self.assertEqual(set(test_db.list_categories()), set(["app-test1", "app-test2"])) 83 | self.assertTrue(test_db.in_category("app-test1", "test")) 84 | self.assertFalse(test_db.in_category("app-test2", "test")) 85 | self.assertRaises(InvalidKeyError, test_db.in_category, "app-test3", "test") 86 | self.assertEqual(set(test_db.list_package_names("app-test1")), set(['test', 'test1'])) 87 | self.assertEqual(set(test_db.list_catpkg_names()),set(['app-test1/test', 'app-test1/test1', 'app-test2/test2'])) 88 | self.assertRaises(InvalidKeyError, test_db.list_package_versions, "invalid", "test") 89 | self.assertRaises(InvalidKeyError, test_db.list_package_versions, "app-test1", "invalid") 90 | self.assertEqual(set(test_db.list_package_versions("app-test1", "test")), set(['1', '2'])) 91 | self.assertEqual(set(test_db.list_all_packages()), set(packages)) 92 | self.assertEqual(test_db.get_package_description(packages[0]), full_data) 93 | self.assertRaises(KeyError, test_db.get_package_description, Package("invalid", "invalid", "1")) 94 | self.assertEqual(test_db.get_max_version("app-test1", "test"), "2") 95 | self.assertEqual(test_db.get_max_version("app-test1", "test1"), "1") 96 | self.assertRaises(InvalidKeyError, test_db.get_max_version, "invalid", "invalid") 97 | pkg_set = set(packages) 98 | for package, data in test_db: 99 | self.assertTrue(package in pkg_set) 100 | if package.category == "app-test1": 101 | self.assertEqual(data, full_data) 102 | else: 103 | self.assertEqual(data, ebuild_data) 104 | pkg_set.remove(package) 105 | self.assertTrue(not pkg_set) 106 | self.assertEqual(orig_db.database, test_db.database) 107 | port = port + 1 108 | 109 | def suite(): 110 | suite = unittest.TestSuite() 111 | suite.addTest(TestPackageDB('test_functionality')) 112 | return suite 113 | -------------------------------------------------------------------------------- /tests/test_DBGenerator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | test_DBGenerator.py 6 | ~~~~~~~~~~~~~~~~~~~ 7 | 8 | DBGenerator test suite 9 | 10 | :copyright: (c) 2013 by Jauhien Piatlicki 11 | :license: GPL-2, see LICENSE for more details. 12 | """ 13 | 14 | import os 15 | import unittest 16 | 17 | from g_sorcery.compatibility import TemporaryDirectory 18 | from g_sorcery.exceptions import InvalidKeyError 19 | from g_sorcery.g_collections import Package 20 | from g_sorcery.package_db import DBGenerator 21 | 22 | from tests.base import BaseTest 23 | from tests.server import Server 24 | 25 | 26 | class TestingDBGenerator(DBGenerator): 27 | def get_download_uries(self, common_config, config): 28 | return [config["repo_uri"] + "/repo.data"] 29 | 30 | def parse_data(self, data_f): 31 | content = data_f.read() 32 | content = content.split("packages\n") 33 | ebuild_data_lines = content[0].split("\n") 34 | packages_lines = content[1].split("\n") 35 | ebuild_data = {} 36 | packages = [] 37 | for line in ebuild_data_lines: 38 | if line: 39 | data = line.split(" ") 40 | ebuild_data[data[0]] = data[1] 41 | for line in packages_lines: 42 | if line: 43 | data = line.split(" ") 44 | packages.append(Package(data[0], data[1], data[2])) 45 | return {"ebuild_data": ebuild_data, "packages": packages} 46 | 47 | 48 | def process_data(self, pkg_db, data, common_config, config): 49 | data = data["repo.data"] 50 | ebuild_data = data["ebuild_data"] 51 | for package in data["packages"]: 52 | pkg_db.add_category(package.category) 53 | pkg_db.add_package(package, ebuild_data) 54 | 55 | def convert_internal_dependency(self, configs, dependency): 56 | return ("internal", dependency) 57 | 58 | def convert_external_dependency(self, configs, dependency): 59 | return ("external", dependency) 60 | 61 | 62 | class TestDBGenerator(BaseTest): 63 | 64 | def test_functionality(self): 65 | db_generator = TestingDBGenerator() 66 | common_config = {} 67 | config = {"repo_uri": "127.0.0.1:8080"} 68 | 69 | packages = [Package("app-test1", "test", "1"), Package("app-test1", "test", "2"), 70 | Package("app-test1", "test1", "1"), Package("app-test2", "test2", "1")] 71 | ebuild_data = {"test1": "test1", "test2": "test2"} 72 | 73 | orig_tempdir = TemporaryDirectory() 74 | with open(os.path.join(orig_tempdir.name, "repo.data"), "w") as f: 75 | for key, value in ebuild_data.items(): 76 | f.write(key + " " + value + "\n") 77 | f.write("packages\n") 78 | for package in packages: 79 | f.write(package.category + " " + package.name + " " + package.version + "\n") 80 | 81 | srv = Server(orig_tempdir.name) 82 | srv.start() 83 | 84 | pkg_db = db_generator(self.tempdir.name, "test_repo", 85 | common_config = common_config, config = config) 86 | 87 | srv.shutdown() 88 | srv.join() 89 | 90 | self.assertEqual(set(pkg_db.list_categories()), set(["app-test1", "app-test2"])) 91 | self.assertTrue(pkg_db.in_category("app-test1", "test")) 92 | self.assertFalse(pkg_db.in_category("app-test2", "test")) 93 | self.assertRaises(InvalidKeyError, pkg_db.in_category, "app-test3", "test") 94 | self.assertEqual(set(pkg_db.list_package_names("app-test1")), set(['test', 'test1'])) 95 | self.assertEqual(set(pkg_db.list_catpkg_names()),set(['app-test1/test', 'app-test1/test1', 'app-test2/test2'])) 96 | self.assertRaises(InvalidKeyError, pkg_db.list_package_versions, "invalid", "test") 97 | self.assertRaises(InvalidKeyError, pkg_db.list_package_versions, "app-test1", "invalid") 98 | self.assertEqual(set(pkg_db.list_package_versions("app-test1", "test")), set(['1', '2'])) 99 | self.assertEqual(set(pkg_db.list_all_packages()), set(packages)) 100 | self.assertEqual(pkg_db.get_package_description(packages[0]), ebuild_data) 101 | self.assertRaises(KeyError, pkg_db.get_package_description, Package("invalid", "invalid", "1")) 102 | self.assertEqual(pkg_db.get_max_version("app-test1", "test"), "2") 103 | self.assertEqual(pkg_db.get_max_version("app-test1", "test1"), "1") 104 | self.assertRaises(InvalidKeyError, pkg_db.get_max_version, "invalid", "invalid") 105 | pkg_set = set(packages) 106 | for package, data in pkg_db: 107 | self.assertTrue(package in pkg_set) 108 | self.assertEqual(data, ebuild_data) 109 | pkg_set.remove(package) 110 | self.assertTrue(not pkg_set) 111 | 112 | orig = "test" 113 | converted = "works" 114 | internal = "int" 115 | configs = [{}, {"converters": {orig:converted}, "external": {orig:converted}, "values": [orig, converted]}] 116 | 117 | self.assertEqual(db_generator.convert(configs, "converters", orig), converted) 118 | self.assertNotEqual(db_generator.convert(configs, "converters", "invalid"), converted) 119 | self.assertEqual(db_generator.convert_dependency(configs, orig), ("external", converted)) 120 | self.assertEqual(db_generator.convert_dependency(configs, orig, external = False), None) 121 | self.assertEqual(db_generator.convert_dependency(configs, internal), ("internal", internal)) 122 | self.assertTrue(db_generator.in_config(configs, "values", orig)) 123 | self.assertFalse(db_generator.in_config(configs, "values", "invalid")) 124 | 125 | 126 | def suite(): 127 | suite = unittest.TestSuite() 128 | suite.addTest(TestDBGenerator('test_functionality')) 129 | return suite 130 | -------------------------------------------------------------------------------- /g_sorcery/ebuild.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | ebuild.py 6 | ~~~~~~~~~~~~~ 7 | 8 | ebuild generation 9 | 10 | :copyright: (c) 2013-2015 by Jauhien Piatlicki 11 | :license: GPL-2, see LICENSE for more details. 12 | """ 13 | 14 | from .compatibility import basestring 15 | from .exceptions import DependencyError 16 | 17 | class EbuildGenerator(object): 18 | """ 19 | Ebuild generator. 20 | """ 21 | def __init__(self, package_db): 22 | """ 23 | Args: 24 | package_db: Package database. 25 | """ 26 | self.package_db = package_db 27 | 28 | def generate(self, package, ebuild_data=None): 29 | """ 30 | Generate an ebuild for a package. 31 | 32 | Args: 33 | package: g_collections.Package instance. 34 | ebuild_data: Dictionary with ebuild data. 35 | 36 | Returns: 37 | Ebuild source as a list of strings. 38 | """ 39 | #a possible exception should be catched in the caller 40 | if not ebuild_data: 41 | ebuild_data = self.package_db.get_package_description(package) 42 | ebuild_data = self.process_ebuild_data(ebuild_data) 43 | ebuild = self.get_template(package, ebuild_data) 44 | ebuild = self.process(ebuild, ebuild_data) 45 | ebuild = self.postprocess(ebuild, ebuild_data) 46 | return ebuild 47 | 48 | def process_ebuild_data(self, ebuild_data): 49 | """ 50 | A hook allowing changing ebuild_data before ebuild generation. 51 | 52 | Args: 53 | ebuild_data: Dictinary with ebuild data. 54 | 55 | Returns: 56 | Dictinary with ebuild data. 57 | """ 58 | return ebuild_data 59 | 60 | def process(self, ebuild, ebuild_data): 61 | """ 62 | Fill ebuild template with data. 63 | 64 | Args: 65 | ebuild: Ebuild template. 66 | ebuild_data: Dictionary with ebuild data. 67 | 68 | Returns: 69 | Ebuild source as a list of strings. 70 | """ 71 | result = [] 72 | for line in ebuild: 73 | error = "" 74 | try: 75 | line = line % ebuild_data 76 | except ValueError as e: 77 | error = str(e) 78 | if error: 79 | error = "substitution failed in line '" + line + "': " + error 80 | raise DependencyError(error) 81 | result.append(line) 82 | 83 | return result 84 | 85 | def get_template(self, package, ebuild_data): 86 | """ 87 | Generate ebuild template. Should be overriden. 88 | 89 | Args: 90 | package: g_collections.Package instance. 91 | ebuild_data: Dictionary with ebuild data. 92 | 93 | Returns: 94 | Ebuild template. 95 | """ 96 | ebuild = [] 97 | return ebuild 98 | 99 | def postprocess(self, ebuild, ebuild_data): 100 | """ 101 | A hook for changing of a generated ebuild. 102 | 103 | Args: 104 | ebuild: Ebuild source as a list of strings. 105 | ebuild_data: Dictionary with ebuild data. 106 | 107 | Returns: 108 | Ebuild source as a list of strings. 109 | """ 110 | return ebuild 111 | 112 | class EbuildGeneratorFromFile(EbuildGenerator): 113 | """ 114 | Ebuild generators that takes templates from files. 115 | """ 116 | def __init__(self, package_db, filename=""): 117 | super(EbuildGeneratorFromFile, self).__init__(package_db) 118 | self.filename = filename 119 | 120 | def get_template(self, package, ebuild_data): 121 | """ 122 | Generate ebuild template. 123 | 124 | Args: 125 | package: g_collections.Package instance. 126 | ebuild_data: Dictionary with ebuild data. 127 | 128 | Returns: 129 | Ebuild template. 130 | """ 131 | name = self.get_template_file(package, ebuild_data) 132 | with open(name, 'r') as f: 133 | ebuild = f.read().split('\n') 134 | if ebuild[-1] == '': 135 | ebuild = ebuild[:-1] 136 | return ebuild 137 | 138 | def get_template_file(self, package, ebuild_data): 139 | """ 140 | Get template filename for a package. Should be overriden. 141 | 142 | Args: 143 | package: g_collections.Package instance. 144 | ebuild_data: Dictionary with ebuild data. 145 | 146 | Returns: 147 | Template filename. 148 | """ 149 | return self.filename 150 | 151 | 152 | class DefaultEbuildGenerator(EbuildGenerator): 153 | """ 154 | Default ebuild generator. 155 | 156 | Takes a layout dictinary that describes ebuild structure and generates 157 | an ebuild template basing on it. 158 | 159 | Layout has entries for vars and inherited eclasses. Each entry is a list. 160 | Entries are processed in the following order: 161 | 162 | vars_before_inherit 163 | inherit 164 | vars_after_inherit 165 | vars_after_description 166 | vars_after_keywords 167 | 168 | inherit entry is just a list of eclass names. 169 | vars* entries are lists of variables in tw0 possible formats: 170 | 1. A string with variable name 171 | 2. A dictinary with entries: 172 | name: variable name 173 | value: variable value 174 | raw: if present, no quotation of value will be done 175 | Variable names are automatically transformed to the upper-case. 176 | """ 177 | def __init__(self, package_db, layout): 178 | super(DefaultEbuildGenerator, self).__init__(package_db) 179 | self.template = ["# automatically generated by g-sorcery", 180 | "# please do not edit this file", 181 | ""] 182 | 183 | if hasattr(layout, "eapi"): 184 | self.template.append("EAPI=%s" % layout.eapi) 185 | else: 186 | self.template.append("EAPI=5") 187 | self.template.append("") 188 | 189 | if hasattr(layout, "vars_before_inherit"): 190 | self._append_vars_to_template(layout.vars_before_inherit) 191 | self.template.append("") 192 | 193 | if hasattr(layout, "inherit"): 194 | self.template.append("inherit " + " ".join(layout.inherit)) 195 | self.template.append("") 196 | 197 | if hasattr(layout, "vars_after_inherit"): 198 | self._append_vars_to_template(layout.vars_after_inherit) 199 | self.template.append("") 200 | 201 | self.template.append('DESCRIPTION="%(description)s"') 202 | self.template.append("") 203 | 204 | if hasattr(layout, "vars_after_description"): 205 | self._append_vars_to_template(layout.vars_after_description) 206 | self.template.append("") 207 | 208 | self.template.append('SLOT="0"') 209 | self.template.append('KEYWORDS="~amd64 ~x86"') 210 | self.template.append("") 211 | 212 | if hasattr(layout, "vars_after_keywords"): 213 | self._append_vars_to_template(layout.vars_after_keywords) 214 | self.template.append("") 215 | 216 | 217 | def _append_vars_to_template(self, variables): 218 | """ 219 | Add a list of variables to the end of template. 220 | 221 | Args: 222 | variables: List of variables. 223 | """ 224 | for var in variables: 225 | if isinstance(var, basestring): 226 | self.template.append(var.upper() + '="%(' + var + ')s"') 227 | else: 228 | if "raw" in var: 229 | quote = '' 230 | else: 231 | quote = '"' 232 | if "value" in var: 233 | self.template.append(var["name"].upper() \ 234 | + '=' + quote + var["value"] + quote) 235 | else: 236 | self.template.append(var["name"].upper() + '=' + quote + '%(' + var["name"] + ')s' + quote) 237 | 238 | 239 | def get_template(self, package, ebuild_data): 240 | """ 241 | Generate ebuild template. 242 | 243 | Args: 244 | package: g_collections.Package instance. 245 | ebuild_data: Dictionary with ebuild data. 246 | 247 | Returns: 248 | Ebuild template. 249 | """ 250 | return self.template 251 | -------------------------------------------------------------------------------- /g_sorcery/db_layout.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | db_layout.py 6 | ~~~~~~~~~~~~ 7 | 8 | package database file layout 9 | 10 | :copyright: (c) 2013-2015 by Jauhien Piatlicki 11 | :license: GPL-2, see LICENSE for more details. 12 | """ 13 | 14 | import hashlib 15 | import os 16 | import shutil 17 | 18 | from .exceptions import DBLayoutError, DBStructureError, FileJSONError, IntegrityError 19 | from .fileutils import FileJSON, hash_file 20 | 21 | CATEGORIES_FILE_NAME = 'categories' 22 | MANIFEST_FILE_NAME = 'manifest' 23 | METADATA_FILE_NAME = 'metadata' 24 | PACKAGES_FILE_NAME = 'packages' 25 | 26 | JSON_FILE_SUFFIX = 'json' 27 | BSON_FILE_SUFFIX = 'bson' 28 | 29 | SUPPORTED_DB_LAYOUTS=[0, 1] 30 | 31 | class CategoryJSON(FileJSON): 32 | """ 33 | Category file in JSON format. 34 | """ 35 | def __init__(self, directory, category): 36 | super(CategoryJSON, self).__init__(os.path.join(os.path.abspath(directory), category), 37 | file_name(PACKAGES_FILE_NAME, JSON_FILE_SUFFIX)) 38 | 39 | 40 | SUPPORTED_FILE_FORMATS = {JSON_FILE_SUFFIX: CategoryJSON} 41 | 42 | 43 | # bson module is optional, we should check if it is installed 44 | try: 45 | from .file_bson.file_bson import FileBSON 46 | 47 | class CategoryBSON(FileBSON): 48 | """ 49 | Category file in BSON format. 50 | """ 51 | def __init__(self, directory, category): 52 | super(CategoryBSON, self).__init__(os.path.join(os.path.abspath(directory), category), 53 | file_name(PACKAGES_FILE_NAME, BSON_FILE_SUFFIX)) 54 | 55 | SUPPORTED_FILE_FORMATS[BSON_FILE_SUFFIX] = CategoryBSON 56 | 57 | except ImportError as e: 58 | pass 59 | 60 | 61 | def file_name(name, suffix=JSON_FILE_SUFFIX): 62 | """ 63 | Return file name based on name and suffix. 64 | """ 65 | return name + '.' + suffix 66 | 67 | 68 | class Manifest(FileJSON): 69 | """ 70 | Manifest file. 71 | """ 72 | 73 | def __init__(self, directory): 74 | super(Manifest, self).__init__(os.path.abspath(directory), file_name(MANIFEST_FILE_NAME)) 75 | 76 | def check(self): 77 | """ 78 | Check manifest. 79 | """ 80 | manifest = self.read() 81 | 82 | result = True 83 | errors = [] 84 | 85 | names = [file_name(CATEGORIES_FILE_NAME)] 86 | for name in names: 87 | if not name in manifest: 88 | raise DBLayoutError('Bad manifest: no ' + name + ' entry') 89 | 90 | for name, value in manifest.items(): 91 | if hash_file(os.path.join(self.directory, name), hashlib.md5()) != \ 92 | value: 93 | errors.append(name) 94 | 95 | if errors: 96 | result = False 97 | 98 | return (result, errors) 99 | 100 | def digest(self, mandatory_files): 101 | """ 102 | Generate manifest. 103 | """ 104 | if not file_name(CATEGORIES_FILE_NAME) in mandatory_files: 105 | raise DBLayoutError('Categories file: ' + file_name(CATEGORIES_FILE_NAME) \ 106 | + ' is not in the list of mandatory files') 107 | 108 | categories = Categories(self.directory) 109 | categories = categories.read() 110 | 111 | manifest = {} 112 | 113 | for name in mandatory_files: 114 | manifest[name] = hash_file(os.path.join(self.directory, name), 115 | hashlib.md5()) 116 | 117 | for category in categories: 118 | category_path = os.path.join(self.directory, category) 119 | if not os.path.isdir(category_path): 120 | raise DBStructureError('Empty category: ' + category) 121 | for root, _, files in os.walk(category_path): 122 | for f in files: 123 | manifest[os.path.join(root[len(self.directory)+1:], f)] = \ 124 | hash_file(os.path.join(root, f), hashlib.md5()) 125 | 126 | self.write(manifest) 127 | 128 | 129 | class Metadata(FileJSON): 130 | """ 131 | Metadata file. 132 | """ 133 | def __init__(self, directory): 134 | super(Metadata, self).__init__(os.path.abspath(directory), 135 | file_name(METADATA_FILE_NAME), 136 | ['db_version', 'layout_version', 'category_format']) 137 | 138 | def read(self): 139 | """ 140 | Read metadata file. 141 | 142 | If file doesn't exist, we have a legacy DB 143 | with DB layout v. 0. Fill metadata appropriately. 144 | """ 145 | if not os.path.exists(self.directory): 146 | os.makedirs(self.directory) 147 | content = {} 148 | if not os.path.isfile(self.path): 149 | content = {'db_version': 0, 'layout_version': 0, 'category_format': JSON_FILE_SUFFIX} 150 | else: 151 | content = self.read_content() 152 | for key in self.mandatories: 153 | if not key in content: 154 | raise FileJSONError('lack of mandatory key: ' + key) 155 | 156 | return content 157 | 158 | 159 | class Categories(FileJSON): 160 | """ 161 | Categories file. 162 | """ 163 | def __init__(self, directory): 164 | super(Categories, self).__init__(os.path.abspath(directory), 165 | file_name(CATEGORIES_FILE_NAME)) 166 | 167 | 168 | def get_layout(metadata): 169 | """ 170 | Get layout parameters based on metadata. 171 | """ 172 | layout_version = metadata['layout_version'] 173 | if layout_version == 0: 174 | return (CategoryJSON, [file_name(CATEGORIES_FILE_NAME)]) 175 | elif layout_version == 1: 176 | category_format = metadata['category_format'] 177 | wrong_fmt = True 178 | try: 179 | category_cls = SUPPORTED_FILE_FORMATS[category_format] 180 | wrong_fmt = False 181 | except KeyError: 182 | pass 183 | if wrong_fmt: 184 | raise DBLayoutError("unsupported packages file format: " + category_format) 185 | return (category_cls, [file_name(CATEGORIES_FILE_NAME), file_name(METADATA_FILE_NAME)]) 186 | else: 187 | raise DBLayoutError("unsupported DB layout version: " + str(layout_version)) 188 | 189 | 190 | class DBLayout(object): 191 | """ 192 | Filesystem DB layout. 193 | 194 | Directory layout. 195 | ~~~~~~~~~~~~~~~~~ 196 | 197 | For legacy DB layout v. 0: 198 | 199 | db dir 200 | manifest.json: database manifest 201 | categories.json: information about categories 202 | category1 203 | packages.json: information about available packages 204 | category2 205 | ... 206 | 207 | For DB layout v. 1: 208 | 209 | db dir 210 | manifest.json: database manifest 211 | categories.json: information about categories 212 | metadata.json: DB metadata 213 | category1 214 | packages.[b|j]son: information about available packages 215 | category2 216 | ... 217 | 218 | Packages file can be in json or bson formats. 219 | """ 220 | 221 | def __init__(self, directory): 222 | self.directory = os.path.abspath(directory) 223 | self.manifest = Manifest(self.directory) 224 | 225 | def check_manifest(self): 226 | """ 227 | Check manifest. 228 | """ 229 | sane, errors = self.manifest.check() 230 | if not sane: 231 | raise IntegrityError('Manifest error: ' + str(errors)) 232 | 233 | def clean(self): 234 | """ 235 | Remove DB files. 236 | """ 237 | if os.path.exists(self.directory): 238 | shutil.rmtree(self.directory) 239 | os.makedirs(self.directory) 240 | 241 | def read(self): 242 | """ 243 | Read DB files. 244 | 245 | Returns a tuple with metadata, list of categories 246 | and categories dictionary. 247 | """ 248 | self.check_manifest() 249 | 250 | metadata_f = Metadata(self.directory) 251 | metadata = metadata_f.read() 252 | 253 | category_cls, _ = get_layout(metadata) 254 | 255 | categories_f = Categories(self.directory) 256 | categories = categories_f.read() 257 | 258 | packages = {} 259 | for category in categories: 260 | category_path = os.path.join(self.directory, category) 261 | if not os.path.isdir(category_path): 262 | raise DBLayoutError('Empty category: ' + category) 263 | category_f = category_cls(self.directory, category) 264 | pkgs = category_f.read() 265 | if not pkgs: 266 | raise DBLayoutError('Empty category: ' + category) 267 | packages[category] = pkgs 268 | 269 | return (metadata, categories, packages) 270 | 271 | def write(self, metadata, categories, packages): 272 | """ 273 | Write DB files. 274 | """ 275 | category_cls, mandatory_files = get_layout(metadata) 276 | 277 | self.clean() 278 | 279 | if file_name(METADATA_FILE_NAME) in mandatory_files: 280 | metadata_f = Metadata(self.directory) 281 | metadata_f.write(metadata) 282 | 283 | categories_f = Categories(self.directory) 284 | categories_f.write(categories) 285 | 286 | for category in categories: 287 | category_f = category_cls(self.directory, category) 288 | category_f.write(packages[category]) 289 | 290 | self.manifest.digest(mandatory_files) 291 | -------------------------------------------------------------------------------- /g_sorcery/fileutils.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | fileutils.py 6 | ~~~~~~~~~~~~ 7 | 8 | file utilities 9 | 10 | :copyright: (c) 2013-2015 by Jauhien Piatlicki 11 | :license: GPL-2, see LICENSE for more details. 12 | """ 13 | 14 | import glob 15 | import json 16 | import hashlib 17 | import os 18 | import tarfile 19 | 20 | from .compatibility import TemporaryDirectory 21 | from .exceptions import FileJSONError, DownloadingError 22 | from .serialization import JSONSerializer, deserializeHook 23 | 24 | class FileJSONData(object): 25 | """ 26 | Class for files with JSON compatible data. 27 | """ 28 | def __init__(self, directory, name, mandatories=None): 29 | """ 30 | Args: 31 | directory: File directory. 32 | name: File name. 33 | mandatories: List of requiered keys. 34 | """ 35 | self.directory = os.path.abspath(directory) 36 | self.name = name 37 | self.path = os.path.join(directory, name) 38 | if not mandatories: 39 | self.mandatories = [] 40 | else: 41 | self.mandatories = mandatories 42 | 43 | def read(self): 44 | """ 45 | Read file. 46 | """ 47 | if not os.path.exists(self.directory): 48 | os.makedirs(self.directory) 49 | content = {} 50 | if not os.path.isfile(self.path): 51 | for key in self.mandatories: 52 | content[key] = "" 53 | self.write_content(content) 54 | else: 55 | content = self.read_content() 56 | for key in self.mandatories: 57 | if not key in content: 58 | raise FileJSONError('lack of mandatory key: ' + key) 59 | 60 | return content 61 | 62 | def read_content(self): 63 | """ 64 | Real read operation with deserialization. Should be overridden. 65 | """ 66 | return [] 67 | 68 | def write(self, content): 69 | """ 70 | Write file. 71 | """ 72 | for key in self.mandatories: 73 | if not key in content: 74 | raise FileJSONError('lack of mandatory key: ' + key) 75 | if not os.path.exists(self.directory): 76 | os.makedirs(self.directory) 77 | self.write_content(content) 78 | 79 | def write_content(self, content): 80 | """ 81 | Real write operation with serialization. Should be overridden. 82 | """ 83 | pass 84 | 85 | 86 | class FileJSON(FileJSONData): 87 | """ 88 | Class for JSON files. Supports custom JSON serialization 89 | provided by g_sorcery.serialization. 90 | """ 91 | 92 | def read_content(self): 93 | """ 94 | Read JSON file. 95 | """ 96 | content = {} 97 | with open(self.path, 'r') as f: 98 | content = json.load(f, object_hook=deserializeHook) 99 | return content 100 | 101 | def write_content(self, content): 102 | """ 103 | Write JSON file. 104 | """ 105 | with open(self.path, 'w') as f: 106 | json.dump(content, f, indent=2, sort_keys=True, cls=JSONSerializer) 107 | 108 | 109 | def hash_file(name, hasher, blocksize=65536): 110 | """ 111 | Get a file hash. 112 | 113 | Args: 114 | name: file name. 115 | hasher: Hasher. 116 | blocksize: Blocksize. 117 | 118 | Returns: 119 | Hash value. 120 | """ 121 | with open(name, 'rb') as f: 122 | buf = f.read(blocksize) 123 | while len(buf) > 0: 124 | hasher.update(buf) 125 | buf = f.read(blocksize) 126 | return hasher.hexdigest() 127 | 128 | def copy_all(src, dst): 129 | """ 130 | Copy entire tree. 131 | 132 | Args: 133 | src: Source. 134 | dst: Destination. 135 | """ 136 | os.system("cp -rv " + src + "/* " + dst) 137 | 138 | def wget(uri, directory, output="", timeout = None): 139 | """ 140 | Fetch a file. 141 | 142 | Args: 143 | uri: URI. 144 | directory: Directory where file should be saved. 145 | output: Name of output file. 146 | timeout: Timeout for wget. 147 | 148 | Returns: 149 | Nonzero in case of a failure. 150 | """ 151 | if timeout is None: 152 | timeout_str = ' ' 153 | else: 154 | timeout_str = ' -T ' + str(timeout) 155 | 156 | if output: 157 | ret = os.system('wget ' + uri + 158 | ' -O ' + os.path.join(directory, output) + timeout_str) 159 | else: 160 | ret = os.system('wget -P ' + directory + ' ' + uri + timeout_str) 161 | return ret 162 | 163 | def get_pkgpath(root = None): 164 | """ 165 | Get package path. 166 | 167 | Args: 168 | root: module file path 169 | 170 | Returns: 171 | Package path. 172 | """ 173 | if not root: 174 | root = __file__ 175 | if os.path.islink(root): 176 | root = os.path.realpath(root) 177 | return os.path.dirname(os.path.abspath(root)) 178 | 179 | class ManifestEntry(object): 180 | """ 181 | A manifest entry for a file. 182 | """ 183 | 184 | __slots__ = ('directory', 'name', 'ftype', 185 | 'size', 'sha256', 'sha512', 'whirlpool') 186 | 187 | def __init__(self, directory, name, ftype): 188 | self.directory = directory 189 | self.name = name 190 | self.ftype = ftype 191 | self.digest() 192 | 193 | def digest(self): 194 | """ 195 | Digest a file associated with a manifest entry. 196 | """ 197 | h_sha256 = hashlib.new('SHA256') 198 | h_sha512 = hashlib.new('SHA512') 199 | h_whirlpool = hashlib.new('whirlpool') 200 | with open(os.path.join(self.directory, self.name), 'rb') as f: 201 | src = f.read() 202 | h_sha256.update(src) 203 | h_sha512.update(src) 204 | h_whirlpool.update(src) 205 | self.size = str(len(src)) 206 | self.sha256 = h_sha256.hexdigest() 207 | self.sha512 = h_sha512.hexdigest() 208 | self.whirlpool = h_whirlpool.hexdigest() 209 | 210 | 211 | def fast_manifest(directory): 212 | """ 213 | Digest package directory. 214 | This function is intended to be used in place of repoman manifest, 215 | as it is to slow. 216 | 217 | Args: 218 | directory: Directory. 219 | """ 220 | manifest = [] 221 | metadata = os.path.join(directory, "metadata.xml") 222 | 223 | for aux in glob.glob(os.path.join(directory, "files/*")): 224 | manifest.append(ManifestEntry(os.path.dirname(aux), 225 | os.path.basename(aux), "AUX")) 226 | for ebuild in glob.glob(os.path.join(directory, "*.ebuild")): 227 | manifest.append(ManifestEntry(directory, 228 | os.path.basename(ebuild), "EBUILD")) 229 | if (os.path.isfile(metadata)): 230 | manifest.append(ManifestEntry(directory, "metadata.xml", "MISC")) 231 | 232 | manifest = [" ".join([m.ftype, m.name, m.size, 233 | "SHA256", m.sha256, "SHA512", m.sha512, 234 | "WHIRLPOOL", m.whirlpool]) 235 | for m in manifest] 236 | 237 | with open(os.path.join(directory, "Manifest"), 'w') as f: 238 | f.write('\n'.join(manifest) + '\n') 239 | 240 | 241 | def _call_parser(f_name, parser, open_file = True, open_mode = 'r'): 242 | """ 243 | Call parser on a given file. 244 | 245 | Args: 246 | f_name: File name. 247 | parser: Parser function. 248 | open_file: Whether parser accepts a file descriptor. 249 | open_mode: Open mode for a file. 250 | 251 | Returns: 252 | A dictionary with one entry. Key if a file name, content is 253 | content returned by parser. 254 | """ 255 | data = None 256 | if open_file: 257 | with open(f_name, open_mode) as f: 258 | data = parser(f) 259 | else: 260 | data = parser(f_name) 261 | return {os.path.basename(f_name): data} 262 | 263 | 264 | def load_remote_file(uri, parser, open_file = True, open_mode = 'r', output = "", timeout = None): 265 | """ 266 | Load files from an URI. 267 | 268 | Args: 269 | uri: URI. 270 | parser: Parser that will be applied to downloaded files. 271 | open_file: Whether parser accepts a file descriptor. 272 | open_mode: Open mode for a file. 273 | output: What output name should downloaded file have. 274 | timeout: URI access timeout. 275 | (it will be a key identifying data loaded from this file) 276 | 277 | Returns: 278 | Dictionary with a loaded data. Key is filename, content is data returned by parser. 279 | """ 280 | download_dir = TemporaryDirectory() 281 | loaded_data = {} 282 | if wget(uri, download_dir.name, output, timeout=timeout): 283 | raise DownloadingError("wget failed: " + uri) 284 | for f_name in glob.glob(os.path.join(download_dir.name, "*")): 285 | if tarfile.is_tarfile(f_name): 286 | unpack_dir = TemporaryDirectory() 287 | with tarfile.open(f_name) as f: 288 | f.extractall(unpack_dir.name) 289 | for uf_name in glob.glob(os.path.join(unpack_dir, "*")): 290 | loaded_data.update(_call_parser(uf_name, parser, 291 | open_file=open_file, open_mode=open_mode)) 292 | del unpack_dir 293 | else: 294 | name, extention = os.path.splitext(f_name) 295 | if extention in [".xz", ".lzma"]: 296 | if (os.system("xz -d " + f_name)): 297 | raise DownloadingError("xz failed: " 298 | + f_name + " from " + uri) 299 | f_name = name 300 | loaded_data.update(_call_parser(f_name, parser, 301 | open_file=open_file, open_mode=open_mode)) 302 | del download_dir 303 | return loaded_data 304 | -------------------------------------------------------------------------------- /g_sorcery/metadata.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | metadata.py 6 | ~~~~~~~~~~~ 7 | 8 | metadata generation 9 | 10 | :copyright: (c) 2013-2015 by Jauhien Piatlicki 11 | :license: GPL-2, see LICENSE for more details. 12 | """ 13 | 14 | from .exceptions import XMLGeneratorError 15 | 16 | import xml.etree.ElementTree as ET 17 | import xml.dom.minidom as minidom 18 | 19 | def prettify(tree): 20 | """ 21 | Convert XML tree to a string. 22 | 23 | Args: 24 | tree: xml.etree.ElementTree.Element instance 25 | 26 | Returns: 27 | A string with XML source. 28 | """ 29 | rough_str = ET.tostring(tree, "utf-8").decode("utf-8") 30 | reparsed = minidom.parseString(rough_str) 31 | return reparsed.toprettyxml(encoding="utf-8").decode("utf-8") 32 | 33 | class XMLGenerator(object): 34 | """ 35 | XML generator. Generates an XML tree according a given 36 | schema using a dict as a source of data. 37 | 38 | Schema format. 39 | ~~~~~~~~~~~~~~ 40 | Schema is a list of entries. Each entry describes one XML tag. 41 | Entry is a dict. dict keys are: 42 | name: Name of a tag 43 | multiple: Defines if a given tag can be used more 44 | then one time. It is a tuple. First element 45 | of a tuple is boolean. If it is set a tag can be 46 | repeated. Second element is a string. If it is not 47 | empty, it defines a name for an attribute 48 | that will distinguish different entries of a tag. 49 | required: Boolean that defines if a given tag is required. 50 | subtags: List of subtags. 51 | 52 | Data dictinonary format. 53 | ~~~~~~~~~~~~~~~~~~~~~~~~ 54 | Keys correspond to tags from a schema with the same name. 55 | If a tag is not multiple without subkeys value is just a 56 | string with text for the tag. 57 | If tag is multiple value is a list with entries 58 | corresponding to a single tag. 59 | If tag has subtags value is a dictionary with entries 60 | corresponding to subkeys and 'text' entry corresponding 61 | to text for the tag. 62 | If tag should have attributes value is a tuple or list with 63 | 0 element containing an attribute and 1 element containing 64 | a value for the tag as described previously. 65 | """ 66 | def __init__(self, external, schema): 67 | """ 68 | Args: 69 | external: Name of an outermost tag. 70 | schema: XML schema. 71 | """ 72 | self.external = external 73 | self.schema = schema 74 | 75 | def generate(self, values): 76 | """ 77 | Generate an XML tree filled with values from 78 | a given dictionary. 79 | 80 | Args: 81 | values: Data dictionary. 82 | 83 | Returns: 84 | XML tree being an istance of 85 | xml.etree.ElementTree.Element 86 | """ 87 | root = ET.Element(self.external) 88 | for tag in self.schema: 89 | self.add_tag(root, tag, values) 90 | return root 91 | 92 | def add_tag(self, root, tag, values): 93 | """ 94 | Add a tag. 95 | 96 | Args: 97 | root: A parent tag. 98 | tag: Tag from schema to be added. 99 | values: Data dictionary. 100 | """ 101 | name = tag['name'] 102 | if not name in values: 103 | if tag['required']: 104 | raise XMLGeneratorError('Required tag not found: ' + name) 105 | return 106 | value = values[name] 107 | multiple, attr = tag['multiple'] 108 | if multiple: 109 | for val in value: 110 | self.add_single_tag(root, name, tag, val, attr) 111 | else: 112 | self.add_single_tag(root, name, tag, value) 113 | 114 | def add_single_tag(self, root, name, tag, value, attr=None): 115 | """ 116 | Add a single tag. 117 | 118 | Args: 119 | root: A parent tag. 120 | name: Name of tag to be added. 121 | tag: Tag from schema to be added. 122 | value: Entry of a data dictionary 123 | corresponding to the tag. 124 | attr: An attribute of the tag. 125 | """ 126 | child = ET.SubElement(root, name) 127 | if attr: 128 | child.set(attr, value[0]) 129 | value = value[1] 130 | subtags = tag['subtags'] 131 | if subtags: 132 | if 'text' in value: 133 | child.text = value['text'] 134 | for child_tag in subtags: 135 | self.add_tag(child, child_tag, value) 136 | else: 137 | child.text = value 138 | 139 | 140 | # A default schema describing metadata.xml 141 | # See http://devmanual.gentoo.org/ebuild-writing/misc-files/metadata/ 142 | default_schema = [{'name' : 'herd', 143 | 'multiple' : (True, ""), 144 | 'required' : False, 145 | 'subtags' : []}, 146 | 147 | {'name' : 'maintainer', 148 | 'multiple' : (True, ""), 149 | 'required' : False, 150 | 'subtags' : [{'name' : 'email', 151 | 'multiple' : (False, ""), 152 | 'required' : True, 153 | 'subtags' : []}, 154 | {'name' : 'name', 155 | 'multiple' : (False, ""), 156 | 'required' : False, 157 | 'subtags' : []}, 158 | {'name' : 'description', 159 | 'multiple' : (False, ""), 160 | 'required' : False, 161 | 'subtags' : []}, 162 | ] 163 | }, 164 | 165 | {'name' : 'longdescription', 166 | 'multiple' : (False, ""), 167 | 'required' : False, 168 | 'subtags' : []}, 169 | 170 | {'name' : 'use', 171 | 'multiple' : (False, ""), 172 | 'required' : False, 173 | 'subtags' : [{'name' : 'flag', 174 | 'multiple' : (True, "name"), 175 | 'required' : True, 176 | 'subtags' : []}] 177 | }, 178 | 179 | {'name' : 'upstream', 180 | 'multiple' : (False, ""), 181 | 'required' : False, 182 | 'subtags' : [{'name' : 'maintainer', 183 | 'multiple' : (True, ""), 184 | 'required' : False, 185 | 'subtags' : [{'name' : 'name', 186 | 'multiple' : (False, ""), 187 | 'required' : True, 188 | 'subtags' : []}, 189 | {'name' : 'email', 190 | 'multiple' : (False, ""), 191 | 'required' : False, 192 | 'subtags' : []}]}, 193 | {'name' : 'changelog', 194 | 'multiple' : (False, ""), 195 | 'required' : False, 196 | 'subtags' : []}, 197 | {'name' : 'doc', 198 | 'multiple' : (False, ""), 199 | 'required' : False, 200 | 'subtags' : []}, 201 | {'name' : 'bugs-to', 202 | 'multiple' : (False, ""), 203 | 'required' : False, 204 | 'subtags' : []}, 205 | {'name' : 'remote-id', 206 | 'multiple' : (False, ""), 207 | 'required' : False, 208 | 'subtags' : []}, 209 | ] 210 | }, 211 | ] 212 | 213 | 214 | class MetadataGenerator(object): 215 | """ 216 | Metada generator. Generates metadata for a given package. 217 | """ 218 | def __init__(self, package_db, schema = None): 219 | """ 220 | Args: 221 | package_db: Package database. 222 | schema: Schema of an XML tree. 223 | """ 224 | if not schema: 225 | schema = default_schema 226 | self.package_db = package_db 227 | self.xmlg = XMLGenerator('pkgmetadata', schema) 228 | 229 | def generate(self, package): 230 | """ 231 | Generate metadata for a package. 232 | 233 | Args: 234 | package: g_collections.Package instance. 235 | 236 | Returns: 237 | Metadata source as a list of strings. 238 | """ 239 | description = self.package_db.get_package_description(package) 240 | metadata = self.process(package, description) 241 | metadata = self.postprocess(package, description, metadata) 242 | metadata = prettify(metadata) 243 | metadata = metadata.split('\n') 244 | if metadata[-1] == '': 245 | metadata = metadata[:-1] 246 | dtp = '' 247 | metadata.insert(1, dtp) 248 | return metadata 249 | 250 | def process(self, package, description): 251 | """ 252 | Generate metadata using values from a description. 253 | 254 | Args: 255 | package: g_collections.Package instance. 256 | description: Package description (see package_db module). 257 | 258 | Returns: 259 | Metadata source as a list of strings. 260 | DOCTYPE missing. 261 | """ 262 | metadata = self.xmlg.generate(description) 263 | return metadata 264 | 265 | def postprocess(self, package, description, metadata): 266 | """ 267 | Postprocess generated metadata. Can be overrided. 268 | 269 | Args: 270 | package: g_collections.Package instance. 271 | description: Package description (see package_db module). 272 | metadata: xml.etree.ElementTree.Element instance 273 | 274 | Returns: 275 | Metadata source as a list of strings. 276 | DOCTYPE missing. 277 | """ 278 | return metadata 279 | -------------------------------------------------------------------------------- /pylint.rc: -------------------------------------------------------------------------------- 1 | [MASTER] 2 | 3 | # Specify a configuration file. 4 | #rcfile= 5 | 6 | # Python code to execute, usually for sys.path manipulation such as 7 | # pygtk.require(). 8 | #init-hook= 9 | 10 | # Profiled execution. 11 | profile=no 12 | 13 | # Add files or directories to the blacklist. They should be base names, not 14 | # paths. 15 | ignore=CVS 16 | 17 | # Pickle collected data for later comparisons. 18 | persistent=yes 19 | 20 | # List of plugins (as comma separated values of python modules names) to load, 21 | # usually to register additional checkers. 22 | load-plugins= 23 | 24 | # DEPRECATED 25 | include-ids=no 26 | 27 | # DEPRECATED 28 | symbols=no 29 | 30 | 31 | [MESSAGES CONTROL] 32 | 33 | # Enable the message, report, category or checker with the given id(s). You can 34 | # either give multiple identifier separated by comma (,) or put this option 35 | # multiple time. See also the "--disable" option for examples. 36 | #enable= 37 | 38 | # Disable the message, report, category or checker with the given id(s). You 39 | # can either give multiple identifiers separated by comma (,) or put this 40 | # option multiple times (only on the command line, not in the configuration 41 | # file where it should appear only once).You can also use "--disable=all" to 42 | # disable everything first and then reenable specific checks. For example, if 43 | # you want to run only the similarities checker, you can use "--disable=all 44 | # --enable=similarities". If you want to run only the classes checker, but have 45 | # no Warning level messages displayed, use"--disable=all --enable=classes 46 | # --disable=W" 47 | #disable= 48 | 49 | 50 | [REPORTS] 51 | 52 | # Set the output format. Available formats are text, parseable, colorized, msvs 53 | # (visual studio) and html. You can also give a reporter class, eg 54 | # mypackage.mymodule.MyReporterClass. 55 | output-format=text 56 | 57 | # Put messages in a separate file for each module / package specified on the 58 | # command line instead of printing them on stdout. Reports (if any) will be 59 | # written in a file name "pylint_global.[txt|html]". 60 | files-output=no 61 | 62 | # Tells whether to display a full report or only the messages 63 | reports=yes 64 | 65 | # Python expression which should return a note less than 10 (10 is the highest 66 | # note). You have access to the variables errors warning, statement which 67 | # respectively contain the number of errors / warnings messages and the total 68 | # number of statements analyzed. This is used by the global evaluation report 69 | # (RP0004). 70 | evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) 71 | 72 | # Add a comment according to your evaluation note. This is used by the global 73 | # evaluation report (RP0004). 74 | comment=no 75 | 76 | # Template used to display messages. This is a python new-style format string 77 | # used to format the message information. See doc for all details 78 | #msg-template= 79 | 80 | 81 | [MISCELLANEOUS] 82 | 83 | # List of note tags to take in consideration, separated by a comma. 84 | notes=FIXME,XXX,TODO 85 | 86 | 87 | [BASIC] 88 | 89 | # Required attributes for module, separated by a comma 90 | required-attributes= 91 | 92 | # List of builtins function names that should not be used, separated by a comma 93 | bad-functions=map,filter,apply,input,file 94 | 95 | # Good variable names which should always be accepted, separated by a comma 96 | good-names=i,j,k,ex,Run,_ 97 | 98 | # Bad variable names which should always be refused, separated by a comma 99 | bad-names=foo,bar,baz,toto,tutu,tata 100 | 101 | # Colon-delimited sets of names that determine each other's naming style when 102 | # the name regexes allow several styles. 103 | name-group= 104 | 105 | # Include a hint for the correct naming format with invalid-name 106 | include-naming-hint=no 107 | 108 | # Regular expression matching correct function names 109 | function-rgx=[a-z_][a-z0-9_]{2,30}$ 110 | 111 | # Naming hint for function names 112 | function-name-hint=[a-z_][a-z0-9_]{2,30}$ 113 | 114 | # Regular expression matching correct variable names 115 | variable-rgx=[a-z_][a-z0-9_]{2,30}$ 116 | 117 | # Naming hint for variable names 118 | variable-name-hint=[a-z_][a-z0-9_]{2,30}$ 119 | 120 | # Regular expression matching correct constant names 121 | const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ 122 | 123 | # Naming hint for constant names 124 | const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ 125 | 126 | # Regular expression matching correct attribute names 127 | attr-rgx=[a-z_][a-z0-9_]{2,30}$ 128 | 129 | # Naming hint for attribute names 130 | attr-name-hint=[a-z_][a-z0-9_]{2,30}$ 131 | 132 | # Regular expression matching correct argument names 133 | argument-rgx=[a-z_][a-z0-9_]{2,30}$ 134 | 135 | # Naming hint for argument names 136 | argument-name-hint=[a-z_][a-z0-9_]{2,30}$ 137 | 138 | # Regular expression matching correct class attribute names 139 | class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ 140 | 141 | # Naming hint for class attribute names 142 | class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ 143 | 144 | # Regular expression matching correct inline iteration names 145 | inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ 146 | 147 | # Naming hint for inline iteration names 148 | inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ 149 | 150 | # Regular expression matching correct class names 151 | class-rgx=[A-Z_][a-zA-Z0-9]+$ 152 | 153 | # Naming hint for class names 154 | class-name-hint=[A-Z_][a-zA-Z0-9]+$ 155 | 156 | # Regular expression matching correct module names 157 | module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ 158 | 159 | # Naming hint for module names 160 | module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ 161 | 162 | # Regular expression matching correct method names 163 | method-rgx=[a-z_][a-z0-9_]{2,30}$ 164 | 165 | # Naming hint for method names 166 | method-name-hint=[a-z_][a-z0-9_]{2,30}$ 167 | 168 | # Regular expression which should only match function or class names that do 169 | # not require a docstring. 170 | no-docstring-rgx=__.*__ 171 | 172 | # Minimum line length for functions/classes that require docstrings, shorter 173 | # ones are exempt. 174 | docstring-min-length=-1 175 | 176 | 177 | [FORMAT] 178 | 179 | # Maximum number of characters on a single line. 180 | max-line-length=100 181 | 182 | # Regexp for a line that is allowed to be longer than the limit. 183 | ignore-long-lines=^\s*(# )??$ 184 | 185 | # Allow the body of an if to be on the same line as the test if there is no 186 | # else. 187 | single-line-if-stmt=no 188 | 189 | # List of optional constructs for which whitespace checking is disabled 190 | no-space-check=trailing-comma,dict-separator 191 | 192 | # Maximum number of lines in a module 193 | max-module-lines=1000 194 | 195 | # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 196 | # tab). 197 | indent-string=' ' 198 | 199 | # Number of spaces of indent required inside a hanging or continued line. 200 | indent-after-paren=4 201 | 202 | 203 | [SIMILARITIES] 204 | 205 | # Minimum lines number of a similarity. 206 | min-similarity-lines=4 207 | 208 | # Ignore comments when computing similarities. 209 | ignore-comments=yes 210 | 211 | # Ignore docstrings when computing similarities. 212 | ignore-docstrings=yes 213 | 214 | # Ignore imports when computing similarities. 215 | ignore-imports=no 216 | 217 | 218 | [LOGGING] 219 | 220 | # Logging modules to check that the string format arguments are in logging 221 | # function parameter format 222 | logging-modules=logging 223 | 224 | 225 | [TYPECHECK] 226 | 227 | # Tells whether missing members accessed in mixin class should be ignored. A 228 | # mixin class is detected if its name ends with "mixin" (case insensitive). 229 | ignore-mixin-members=yes 230 | 231 | # List of module names for which member attributes should not be checked 232 | # (useful for modules/projects where namespaces are manipulated during runtime 233 | # and thus extisting member attributes cannot be deduced by static analysis 234 | ignored-modules= 235 | 236 | # List of classes names for which member attributes should not be checked 237 | # (useful for classes with attributes dynamically set). 238 | ignored-classes=SQLObject 239 | 240 | # When zope mode is activated, add a predefined set of Zope acquired attributes 241 | # to generated-members. 242 | zope=no 243 | 244 | # List of members which are set dynamically and missed by pylint inference 245 | # system, and so shouldn't trigger E0201 when accessed. Python regular 246 | # expressions are accepted. 247 | generated-members=REQUEST,acl_users,aq_parent 248 | 249 | 250 | [VARIABLES] 251 | 252 | # Tells whether we should check for unused import in __init__ files. 253 | init-import=no 254 | 255 | # A regular expression matching the name of dummy variables (i.e. expectedly 256 | # not used). 257 | dummy-variables-rgx=_$|dummy 258 | 259 | # List of additional names supposed to be defined in builtins. Remember that 260 | # you should avoid to define new builtins when possible. 261 | additional-builtins= 262 | 263 | 264 | [CLASSES] 265 | 266 | # List of interface methods to ignore, separated by a comma. This is used for 267 | # instance to not check methods defines in Zope's Interface base class. 268 | ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by 269 | 270 | # List of method names used to declare (i.e. assign) instance attributes. 271 | defining-attr-methods=__init__,__new__,setUp 272 | 273 | # List of valid names for the first argument in a class method. 274 | valid-classmethod-first-arg=cls 275 | 276 | # List of valid names for the first argument in a metaclass class method. 277 | valid-metaclass-classmethod-first-arg=mcs 278 | 279 | 280 | [IMPORTS] 281 | 282 | # Deprecated modules which should not be used, separated by a comma 283 | deprecated-modules=regsub,TERMIOS,Bastion,rexec 284 | 285 | # Create a graph of every (i.e. internal and external) dependencies in the 286 | # given file (report RP0402 must not be disabled) 287 | import-graph= 288 | 289 | # Create a graph of external dependencies in the given file (report RP0402 must 290 | # not be disabled) 291 | ext-import-graph= 292 | 293 | # Create a graph of internal dependencies in the given file (report RP0402 must 294 | # not be disabled) 295 | int-import-graph= 296 | 297 | 298 | [DESIGN] 299 | 300 | # Maximum number of arguments for function / method 301 | max-args=5 302 | 303 | # Argument names that match this expression will be ignored. Default to name 304 | # with leading underscore 305 | ignored-argument-names=_.* 306 | 307 | # Maximum number of locals for function / method body 308 | max-locals=15 309 | 310 | # Maximum number of return / yield for function / method body 311 | max-returns=6 312 | 313 | # Maximum number of branch for function / method body 314 | max-branches=12 315 | 316 | # Maximum number of statements in function / method body 317 | max-statements=50 318 | 319 | # Maximum number of parents for a class (see R0901). 320 | max-parents=7 321 | 322 | # Maximum number of attributes for a class (see R0902). 323 | max-attributes=7 324 | 325 | # Minimum number of public methods for a class (see R0903). 326 | min-public-methods=2 327 | 328 | # Maximum number of public methods for a class (see R0904). 329 | max-public-methods=20 330 | 331 | 332 | [EXCEPTIONS] 333 | 334 | # Exceptions that will emit a warning when being caught. Defaults to 335 | # "Exception" 336 | overgeneral-exceptions=Exception 337 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 2, June 1991 3 | 4 | Copyright (C) 1989, 1991 Free Software Foundation, Inc., 5 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 6 | Everyone is permitted to copy and distribute verbatim copies 7 | of this license document, but changing it is not allowed. 8 | 9 | Preamble 10 | 11 | The licenses for most software are designed to take away your 12 | freedom to share and change it. By contrast, the GNU General Public 13 | License is intended to guarantee your freedom to share and change free 14 | software--to make sure the software is free for all its users. This 15 | General Public License applies to most of the Free Software 16 | Foundation's software and to any other program whose authors commit to 17 | using it. (Some other Free Software Foundation software is covered by 18 | the GNU Lesser General Public License instead.) You can apply it to 19 | your programs, too. 20 | 21 | When we speak of free software, we are referring to freedom, not 22 | price. Our General Public Licenses are designed to make sure that you 23 | have the freedom to distribute copies of free software (and charge for 24 | this service if you wish), that you receive source code or can get it 25 | if you want it, that you can change the software or use pieces of it 26 | in new free programs; and that you know you can do these things. 27 | 28 | To protect your rights, we need to make restrictions that forbid 29 | anyone to deny you these rights or to ask you to surrender the rights. 30 | These restrictions translate to certain responsibilities for you if you 31 | distribute copies of the software, or if you modify it. 32 | 33 | For example, if you distribute copies of such a program, whether 34 | gratis or for a fee, you must give the recipients all the rights that 35 | you have. You must make sure that they, too, receive or can get the 36 | source code. And you must show them these terms so they know their 37 | rights. 38 | 39 | We protect your rights with two steps: (1) copyright the software, and 40 | (2) offer you this license which gives you legal permission to copy, 41 | distribute and/or modify the software. 42 | 43 | Also, for each author's protection and ours, we want to make certain 44 | that everyone understands that there is no warranty for this free 45 | software. If the software is modified by someone else and passed on, we 46 | want its recipients to know that what they have is not the original, so 47 | that any problems introduced by others will not reflect on the original 48 | authors' reputations. 49 | 50 | Finally, any free program is threatened constantly by software 51 | patents. We wish to avoid the danger that redistributors of a free 52 | program will individually obtain patent licenses, in effect making the 53 | program proprietary. To prevent this, we have made it clear that any 54 | patent must be licensed for everyone's free use or not licensed at all. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | GNU GENERAL PUBLIC LICENSE 60 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 61 | 62 | 0. This License applies to any program or other work which contains 63 | a notice placed by the copyright holder saying it may be distributed 64 | under the terms of this General Public License. The "Program", below, 65 | refers to any such program or work, and a "work based on the Program" 66 | means either the Program or any derivative work under copyright law: 67 | that is to say, a work containing the Program or a portion of it, 68 | either verbatim or with modifications and/or translated into another 69 | language. (Hereinafter, translation is included without limitation in 70 | the term "modification".) Each licensee is addressed as "you". 71 | 72 | Activities other than copying, distribution and modification are not 73 | covered by this License; they are outside its scope. The act of 74 | running the Program is not restricted, and the output from the Program 75 | is covered only if its contents constitute a work based on the 76 | Program (independent of having been made by running the Program). 77 | Whether that is true depends on what the Program does. 78 | 79 | 1. You may copy and distribute verbatim copies of the Program's 80 | source code as you receive it, in any medium, provided that you 81 | conspicuously and appropriately publish on each copy an appropriate 82 | copyright notice and disclaimer of warranty; keep intact all the 83 | notices that refer to this License and to the absence of any warranty; 84 | and give any other recipients of the Program a copy of this License 85 | along with the Program. 86 | 87 | You may charge a fee for the physical act of transferring a copy, and 88 | you may at your option offer warranty protection in exchange for a fee. 89 | 90 | 2. You may modify your copy or copies of the Program or any portion 91 | of it, thus forming a work based on the Program, and copy and 92 | distribute such modifications or work under the terms of Section 1 93 | above, provided that you also meet all of these conditions: 94 | 95 | a) You must cause the modified files to carry prominent notices 96 | stating that you changed the files and the date of any change. 97 | 98 | b) You must cause any work that you distribute or publish, that in 99 | whole or in part contains or is derived from the Program or any 100 | part thereof, to be licensed as a whole at no charge to all third 101 | parties under the terms of this License. 102 | 103 | c) If the modified program normally reads commands interactively 104 | when run, you must cause it, when started running for such 105 | interactive use in the most ordinary way, to print or display an 106 | announcement including an appropriate copyright notice and a 107 | notice that there is no warranty (or else, saying that you provide 108 | a warranty) and that users may redistribute the program under 109 | these conditions, and telling the user how to view a copy of this 110 | License. (Exception: if the Program itself is interactive but 111 | does not normally print such an announcement, your work based on 112 | the Program is not required to print an announcement.) 113 | 114 | These requirements apply to the modified work as a whole. If 115 | identifiable sections of that work are not derived from the Program, 116 | and can be reasonably considered independent and separate works in 117 | themselves, then this License, and its terms, do not apply to those 118 | sections when you distribute them as separate works. But when you 119 | distribute the same sections as part of a whole which is a work based 120 | on the Program, the distribution of the whole must be on the terms of 121 | this License, whose permissions for other licensees extend to the 122 | entire whole, and thus to each and every part regardless of who wrote it. 123 | 124 | Thus, it is not the intent of this section to claim rights or contest 125 | your rights to work written entirely by you; rather, the intent is to 126 | exercise the right to control the distribution of derivative or 127 | collective works based on the Program. 128 | 129 | In addition, mere aggregation of another work not based on the Program 130 | with the Program (or with a work based on the Program) on a volume of 131 | a storage or distribution medium does not bring the other work under 132 | the scope of this License. 133 | 134 | 3. You may copy and distribute the Program (or a work based on it, 135 | under Section 2) in object code or executable form under the terms of 136 | Sections 1 and 2 above provided that you also do one of the following: 137 | 138 | a) Accompany it with the complete corresponding machine-readable 139 | source code, which must be distributed under the terms of Sections 140 | 1 and 2 above on a medium customarily used for software interchange; or, 141 | 142 | b) Accompany it with a written offer, valid for at least three 143 | years, to give any third party, for a charge no more than your 144 | cost of physically performing source distribution, a complete 145 | machine-readable copy of the corresponding source code, to be 146 | distributed under the terms of Sections 1 and 2 above on a medium 147 | customarily used for software interchange; or, 148 | 149 | c) Accompany it with the information you received as to the offer 150 | to distribute corresponding source code. (This alternative is 151 | allowed only for noncommercial distribution and only if you 152 | received the program in object code or executable form with such 153 | an offer, in accord with Subsection b above.) 154 | 155 | The source code for a work means the preferred form of the work for 156 | making modifications to it. For an executable work, complete source 157 | code means all the source code for all modules it contains, plus any 158 | associated interface definition files, plus the scripts used to 159 | control compilation and installation of the executable. However, as a 160 | special exception, the source code distributed need not include 161 | anything that is normally distributed (in either source or binary 162 | form) with the major components (compiler, kernel, and so on) of the 163 | operating system on which the executable runs, unless that component 164 | itself accompanies the executable. 165 | 166 | If distribution of executable or object code is made by offering 167 | access to copy from a designated place, then offering equivalent 168 | access to copy the source code from the same place counts as 169 | distribution of the source code, even though third parties are not 170 | compelled to copy the source along with the object code. 171 | 172 | 4. You may not copy, modify, sublicense, or distribute the Program 173 | except as expressly provided under this License. Any attempt 174 | otherwise to copy, modify, sublicense or distribute the Program is 175 | void, and will automatically terminate your rights under this License. 176 | However, parties who have received copies, or rights, from you under 177 | this License will not have their licenses terminated so long as such 178 | parties remain in full compliance. 179 | 180 | 5. You are not required to accept this License, since you have not 181 | signed it. However, nothing else grants you permission to modify or 182 | distribute the Program or its derivative works. These actions are 183 | prohibited by law if you do not accept this License. Therefore, by 184 | modifying or distributing the Program (or any work based on the 185 | Program), you indicate your acceptance of this License to do so, and 186 | all its terms and conditions for copying, distributing or modifying 187 | the Program or works based on it. 188 | 189 | 6. Each time you redistribute the Program (or any work based on the 190 | Program), the recipient automatically receives a license from the 191 | original licensor to copy, distribute or modify the Program subject to 192 | these terms and conditions. You may not impose any further 193 | restrictions on the recipients' exercise of the rights granted herein. 194 | You are not responsible for enforcing compliance by third parties to 195 | this License. 196 | 197 | 7. If, as a consequence of a court judgment or allegation of patent 198 | infringement or for any other reason (not limited to patent issues), 199 | conditions are imposed on you (whether by court order, agreement or 200 | otherwise) that contradict the conditions of this License, they do not 201 | excuse you from the conditions of this License. If you cannot 202 | distribute so as to satisfy simultaneously your obligations under this 203 | License and any other pertinent obligations, then as a consequence you 204 | may not distribute the Program at all. For example, if a patent 205 | license would not permit royalty-free redistribution of the Program by 206 | all those who receive copies directly or indirectly through you, then 207 | the only way you could satisfy both it and this License would be to 208 | refrain entirely from distribution of the Program. 209 | 210 | If any portion of this section is held invalid or unenforceable under 211 | any particular circumstance, the balance of the section is intended to 212 | apply and the section as a whole is intended to apply in other 213 | circumstances. 214 | 215 | It is not the purpose of this section to induce you to infringe any 216 | patents or other property right claims or to contest validity of any 217 | such claims; this section has the sole purpose of protecting the 218 | integrity of the free software distribution system, which is 219 | implemented by public license practices. Many people have made 220 | generous contributions to the wide range of software distributed 221 | through that system in reliance on consistent application of that 222 | system; it is up to the author/donor to decide if he or she is willing 223 | to distribute software through any other system and a licensee cannot 224 | impose that choice. 225 | 226 | This section is intended to make thoroughly clear what is believed to 227 | be a consequence of the rest of this License. 228 | 229 | 8. If the distribution and/or use of the Program is restricted in 230 | certain countries either by patents or by copyrighted interfaces, the 231 | original copyright holder who places the Program under this License 232 | may add an explicit geographical distribution limitation excluding 233 | those countries, so that distribution is permitted only in or among 234 | countries not thus excluded. In such case, this License incorporates 235 | the limitation as if written in the body of this License. 236 | 237 | 9. The Free Software Foundation may publish revised and/or new versions 238 | of the General Public License from time to time. Such new versions will 239 | be similar in spirit to the present version, but may differ in detail to 240 | address new problems or concerns. 241 | 242 | Each version is given a distinguishing version number. If the Program 243 | specifies a version number of this License which applies to it and "any 244 | later version", you have the option of following the terms and conditions 245 | either of that version or of any later version published by the Free 246 | Software Foundation. If the Program does not specify a version number of 247 | this License, you may choose any version ever published by the Free Software 248 | Foundation. 249 | 250 | 10. If you wish to incorporate parts of the Program into other free 251 | programs whose distribution conditions are different, write to the author 252 | to ask for permission. For software which is copyrighted by the Free 253 | Software Foundation, write to the Free Software Foundation; we sometimes 254 | make exceptions for this. Our decision will be guided by the two goals 255 | of preserving the free status of all derivatives of our free software and 256 | of promoting the sharing and reuse of software generally. 257 | 258 | NO WARRANTY 259 | 260 | 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY 261 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN 262 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES 263 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED 264 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 265 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS 266 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE 267 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, 268 | REPAIR OR CORRECTION. 269 | 270 | 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 271 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR 272 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, 273 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING 274 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED 275 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY 276 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER 277 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE 278 | POSSIBILITY OF SUCH DAMAGES. 279 | 280 | END OF TERMS AND CONDITIONS 281 | 282 | How to Apply These Terms to Your New Programs 283 | 284 | If you develop a new program, and you want it to be of the greatest 285 | possible use to the public, the best way to achieve this is to make it 286 | free software which everyone can redistribute and change under these terms. 287 | 288 | To do so, attach the following notices to the program. It is safest 289 | to attach them to the start of each source file to most effectively 290 | convey the exclusion of warranty; and each file should have at least 291 | the "copyright" line and a pointer to where the full notice is found. 292 | 293 | 294 | Copyright (C) 295 | 296 | This program is free software; you can redistribute it and/or modify 297 | it under the terms of the GNU General Public License as published by 298 | the Free Software Foundation; either version 2 of the License, or 299 | (at your option) any later version. 300 | 301 | This program is distributed in the hope that it will be useful, 302 | but WITHOUT ANY WARRANTY; without even the implied warranty of 303 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 304 | GNU General Public License for more details. 305 | 306 | You should have received a copy of the GNU General Public License along 307 | with this program; if not, write to the Free Software Foundation, Inc., 308 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 309 | 310 | Also add information on how to contact you by electronic and paper mail. 311 | 312 | If the program is interactive, make it output a short notice like this 313 | when it starts in an interactive mode: 314 | 315 | Gnomovision version 69, Copyright (C) year name of author 316 | Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 317 | This is free software, and you are welcome to redistribute it 318 | under certain conditions; type `show c' for details. 319 | 320 | The hypothetical commands `show w' and `show c' should show the appropriate 321 | parts of the General Public License. Of course, the commands you use may 322 | be called something other than `show w' and `show c'; they could even be 323 | mouse-clicks or menu items--whatever suits your program. 324 | 325 | You should also get your employer (if you work as a programmer) or your 326 | school, if any, to sign a "copyright disclaimer" for the program, if 327 | necessary. Here is a sample; alter the names: 328 | 329 | Yoyodyne, Inc., hereby disclaims all copyright interest in the program 330 | `Gnomovision' (which makes passes at compilers) written by James Hacker. 331 | 332 | , 1 April 1989 333 | Ty Coon, President of Vice 334 | 335 | This General Public License does not permit incorporating your program into 336 | proprietary programs. If your program is a subroutine library, you may 337 | consider it more useful to permit linking proprietary applications with the 338 | library. If this is what you want to do, use the GNU Lesser General 339 | Public License instead of this License. 340 | -------------------------------------------------------------------------------- /g_sorcery/backend.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | backend.py 6 | ~~~~~~~~~~ 7 | 8 | base class for backends 9 | 10 | :copyright: (c) 2013-2015 by Jauhien Piatlicki 11 | :license: GPL-2, see LICENSE for more details. 12 | """ 13 | 14 | import argparse 15 | import os 16 | 17 | import portage 18 | 19 | from .compatibility import configparser 20 | from .g_collections import Package, elist 21 | from .fileutils import fast_manifest, FileJSON 22 | from .exceptions import DependencyError, DigestError, InvalidKeyError 23 | from .logger import Logger 24 | from .mangler import package_managers 25 | from .package_db import PackageDB 26 | 27 | class Backend(object): 28 | """ 29 | Backend for a repository. 30 | 31 | Command format is as follows: 32 | g-backend [-o overlay_dir] [-r repository] command 33 | 34 | where command is one of the following: 35 | sync 36 | list 37 | search word 38 | generate package_name 39 | generate-tree [-d --digest] 40 | install package_name [portage flags] 41 | 42 | If no overlay directory is given the default one from backend config is used. 43 | """ 44 | 45 | def __init__(self, package_db_generator_class, 46 | ebuild_g_with_digest_class, ebuild_g_without_digest_class, 47 | eclass_g_class, metadata_g_class, 48 | package_db_class=PackageDB, sync_db=False): 49 | self.sorcery_dir = '.g-sorcery' 50 | self.sync_db = sync_db 51 | self.package_db_generator = package_db_generator_class(package_db_class) 52 | self.ebuild_g_with_digest_class = ebuild_g_with_digest_class 53 | self.ebuild_g_without_digest_class = ebuild_g_without_digest_class 54 | self.eclass_g_class = eclass_g_class 55 | self.metadata_g_class = metadata_g_class 56 | 57 | self.parser = \ 58 | argparse.ArgumentParser(description='Automatic ebuild generator.') 59 | self.parser.add_argument('-o', '--overlay') 60 | self.parser.add_argument('-r', '--repository') 61 | 62 | subparsers = self.parser.add_subparsers() 63 | 64 | p_sync = subparsers.add_parser('sync') 65 | p_sync.set_defaults(func=self.sync) 66 | 67 | p_list = subparsers.add_parser('list') 68 | p_list.set_defaults(func=self.list) 69 | 70 | p_generate = subparsers.add_parser('generate') 71 | p_generate.add_argument('pkgname') 72 | p_generate.set_defaults(func=self.generate) 73 | 74 | p_generate_tree = subparsers.add_parser('generate-tree') 75 | p_generate_tree.add_argument('-d', '--digest', action='store_true') 76 | p_generate_tree.set_defaults(func=self.generate_tree) 77 | 78 | p_install = subparsers.add_parser('install') 79 | p_install.add_argument('pkgname') 80 | p_install.add_argument('pkgmanager_flags', nargs=argparse.REMAINDER) 81 | p_install.set_defaults(func=self.install) 82 | 83 | self.logger = Logger() 84 | 85 | def _get_overlay(self, args, config, global_config): 86 | """ 87 | Get an overlay directory. 88 | 89 | Args: 90 | args: Command line arguments. 91 | config: Backend config. 92 | global_config: g-sorcery config. 93 | 94 | Returns: 95 | Overlay directory. 96 | """ 97 | overlay = args.overlay 98 | if not overlay: 99 | if not 'default_overlay' in config: 100 | self.logger.error("no overlay given, exiting.") 101 | return None 102 | else: 103 | overlay = config['default_overlay'] 104 | overlay = args.overlay 105 | overlay = os.path.abspath(overlay) 106 | return overlay 107 | 108 | def _get_package_db(self, args, config, global_config): 109 | """ 110 | Get package database object. 111 | 112 | Args: 113 | args: Command line arguments. 114 | config: Backend config. 115 | global_config: g-sorcery config. 116 | 117 | Returns: 118 | Package database object. 119 | """ 120 | overlay = self._get_overlay(args, config, global_config) 121 | backend_path = os.path.join(overlay, 122 | self.sorcery_dir, config["package"]) 123 | repository = args.repository 124 | pkg_db = self.package_db_generator(backend_path, 125 | repository, generate=False) 126 | return pkg_db 127 | 128 | def sync(self, args, config, global_config): 129 | """ 130 | Synchronize or generate local database. 131 | 132 | Args: 133 | args: Command line arguments. 134 | config: Backend config. 135 | global_config: g-sorcery config. 136 | 137 | Returns: 138 | Exit status. 139 | """ 140 | overlay = self._get_overlay(args, config, global_config) 141 | backend_path = os.path.join(overlay, 142 | self.sorcery_dir, config["package"]) 143 | repository = args.repository 144 | repository_config = {} 145 | 146 | if "common_config" in config: 147 | common_config = config["common_config"] 148 | else: 149 | common_config = {} 150 | 151 | if repository: 152 | if not "repositories" in config: 153 | self.logger.error("repository " + repository + 154 | " specified, but there is no repositories entry in config") 155 | return -1 156 | repositories = config["repositories"] 157 | if not repository in repositories: 158 | self.logger.error("repository " + repository + " not found") 159 | return -1 160 | repository_config = repositories[repository] 161 | else: 162 | self.logger.error('no repository given\n') 163 | return -1 164 | 165 | try: 166 | sync_method = repository_config["sync_method"] 167 | except KeyError: 168 | sync_method = "tgz" 169 | if self.sync_db: 170 | pkg_db = self.package_db_generator(backend_path, repository, 171 | common_config, repository_config, generate=False) 172 | pkg_db.sync(repository_config["db_uri"], repository_config=repository_config, sync_method=sync_method) 173 | else: 174 | pkg_db = self.package_db_generator(backend_path, 175 | repository, common_config, repository_config) 176 | return 0 177 | 178 | def list(self, args, config, global_config): 179 | """ 180 | List all available packages. 181 | 182 | Args: 183 | args: Command line arguments. 184 | config: Backend config. 185 | global_config: g-sorcery config. 186 | 187 | Returns: 188 | Exit status. 189 | """ 190 | pkg_db = self._get_package_db(args, config, global_config) 191 | pkg_db.read() 192 | try: 193 | categories = pkg_db.list_categories() 194 | for category in categories: 195 | print('Category ' + category + ':') 196 | print('\n') 197 | packages = pkg_db.list_package_names(category) 198 | for pkg in packages: 199 | max_ver = pkg_db.get_max_version(category, pkg) 200 | versions = pkg_db.list_package_versions(category, pkg) 201 | desc = pkg_db.get_package_description(Package(category, 202 | pkg, max_ver)) 203 | print(' ' + pkg + ': ' + desc['description']) 204 | print(' Available versions: ' + ' '.join(versions)) 205 | print('\n') 206 | except Exception as e: 207 | self.logger.error('list failed: ' + str(e) + '\n') 208 | return -1 209 | return 0 210 | 211 | def generate(self, args, config, global_config): 212 | """ 213 | Generate ebuilds for a given package and all its dependecies. 214 | 215 | Args: 216 | args: Command line arguments. 217 | config: Backend config. 218 | global_config: g-sorcery config. 219 | 220 | Returns: 221 | Exit status. 222 | """ 223 | overlay = self._get_overlay(args, config, global_config) 224 | pkg_db = self._get_package_db(args, config, global_config) 225 | pkg_db.read() 226 | 227 | pkgname = args.pkgname 228 | 229 | try: 230 | dependencies = self.get_dependencies(pkg_db, pkgname) 231 | except Exception as e: 232 | self.logger.error('dependency solving failed: ' + str(e) + '\n') 233 | return -1 234 | 235 | eclasses = [] 236 | for package in dependencies: 237 | eclasses += pkg_db.get_package_description(package)['eclasses'] 238 | eclasses = list(set(eclasses)) 239 | self.generate_eclasses(overlay, eclasses) 240 | self.generate_ebuilds(pkg_db, overlay, dependencies, True) 241 | self.generate_metadatas(pkg_db, overlay, dependencies) 242 | self.digest(overlay) 243 | return 0 244 | 245 | def generate_ebuilds(self, package_db, overlay, packages, digest=False): 246 | """ 247 | Generate ebuilds for given packages. 248 | 249 | Args: 250 | package_db: Package database 251 | overlay: Overlay directory. 252 | packages: List of packages. 253 | digest: whether sources should be digested in Manifest. 254 | """ 255 | 256 | self.logger.info("ebuild generation") 257 | if digest: 258 | ebuild_g = self.ebuild_g_with_digest_class(package_db) 259 | else: 260 | ebuild_g = self.ebuild_g_without_digest_class(package_db) 261 | for package in packages: 262 | category = package.category 263 | name = package.name 264 | version = package.version 265 | self.logger.info(" generating " + 266 | category + '/' + name + '-' + version) 267 | path = os.path.join(overlay, category, name) 268 | if not os.path.exists(path): 269 | os.makedirs(path) 270 | source = ebuild_g.generate(package) 271 | with open(os.path.join(path, 272 | name + '-' + version + '.ebuild'), 'w') as f: 273 | f.write('\n'.join(source)) 274 | 275 | 276 | def generate_metadatas(self, package_db, overlay, packages): 277 | """ 278 | Generate metada files for given packages. 279 | 280 | Args: 281 | package_db: Package database 282 | overlay: Overlay directory. 283 | packages: List of packages. 284 | """ 285 | self.logger.info("metadata generation") 286 | metadata_g = self.metadata_g_class(package_db) 287 | for package in packages: 288 | path = os.path.join(overlay, package.category, package.name) 289 | if not os.path.exists(path): 290 | os.makedirs(path) 291 | source = metadata_g.generate(package) 292 | with open(os.path.join(path, 'metadata.xml'), 'w') as f: 293 | f.write('\n'.join(source)) 294 | 295 | def generate_eclasses(self, overlay, eclasses): 296 | """ 297 | Generate given eclasses. 298 | 299 | Args: 300 | overlay: Overlay directory. 301 | eclasses: List of eclasses. 302 | """ 303 | self.logger.info("eclasses generation") 304 | eclass_g = self.eclass_g_class() 305 | path = os.path.join(overlay, 'eclass') 306 | if not os.path.exists(path): 307 | os.makedirs(path) 308 | for eclass in eclasses: 309 | self.logger.info(" generating " + eclass + " eclass") 310 | source = eclass_g.generate(eclass) 311 | with open(os.path.join(path, eclass + '.eclass'), 'w') as f: 312 | f.write('\n'.join(source)) 313 | 314 | 315 | def get_dependencies(self, package_db, pkgname): 316 | """ 317 | Get dependencies for a given package. 318 | 319 | Args: 320 | package_db: Database. 321 | pkgname: package name (string). 322 | 323 | Returns: 324 | A set containing dependencies (instances of Package). 325 | Package version is ignored currently and a returned set contains all 326 | the versions of packages pkgname depends on. 327 | """ 328 | parts = pkgname.split('/') 329 | category = None 330 | if len(parts) == 1: 331 | name = parts[0] 332 | elif len(parts) == 2: 333 | category = parts[0] 334 | name = parts[1] 335 | else: 336 | error = 'bad package name: ' + pkgname 337 | self.logger.error(error + '\n') 338 | raise DependencyError(error) 339 | 340 | if not category: 341 | all_categories = package_db.list_categories() 342 | categories = [] 343 | for cat in all_categories: 344 | if package_db.in_category(cat, name): 345 | categories.append(cat) 346 | 347 | if not len(categories): 348 | error = 'no package with name ' \ 349 | + pkgname + ' found' 350 | self.logger.error(error + '\n') 351 | raise DependencyError(error) 352 | 353 | if len(categories) > 1: 354 | self.logger.error('ambiguous packagename: ' + pkgname + '\n') 355 | self.logger.error('please select one of' \ 356 | + 'the following packages:\n') 357 | for cat in categories: 358 | self.logger.error(' ' + cat + '/' + pkgname + '\n') 359 | raise DependencyError("ambiguous packagename") 360 | 361 | category = categories[0] 362 | versions = package_db.list_package_versions(category, name) 363 | dependencies = set() 364 | for version in versions: 365 | dependencies |= self.solve_dependencies(package_db, 366 | Package(category, name, version))[0] 367 | return dependencies 368 | 369 | def solve_dependencies(self, package_db, package, 370 | solved_deps=None, unsolved_deps=None): 371 | """ 372 | Solve dependencies. 373 | 374 | Args: 375 | package_db: Package database. 376 | package: A package we want to solve dependencies for. 377 | solved_deps: List of solved dependencies. 378 | unsolved_deps: List of dependencies to be solved. 379 | 380 | Returns: 381 | A pair (solved_deps, unsolved_deps). 382 | 383 | Note: 384 | Each dependency is an object of class g_collections.Dependency. 385 | """ 386 | if not solved_deps: 387 | solved_deps = set() 388 | if not unsolved_deps: 389 | unsolved_deps = set() 390 | if package in solved_deps: 391 | return solved_deps 392 | if package in unsolved_deps: 393 | error = 'circular dependency for ' + package.category + '/' + \ 394 | package.name + '-' + package.version 395 | raise DependencyError(error) 396 | unsolved_deps.add(package) 397 | found = True 398 | try: 399 | desc = package_db.get_package_description(package) 400 | except KeyError: 401 | found = False 402 | if not found: 403 | error = "package " + package.category + '/' + \ 404 | package.name + '-' + package.version + " not found" 405 | self.logger.error(error) 406 | # at the moment ignore unsolved dependencies, as those deps can be in other repo 407 | # or can be external: portage will catch it 408 | unsolved_deps.remove(package) 409 | return (solved_deps, unsolved_deps) 410 | 411 | dependencies = desc["dependencies"] 412 | for pkg in dependencies: 413 | try: 414 | versions = package_db.list_package_versions(pkg.category, 415 | pkg.package) 416 | for version in versions: 417 | solved_deps, unsolved_deps = self.solve_dependencies(package_db, 418 | Package(pkg.category, pkg.package, version), 419 | solved_deps, unsolved_deps) 420 | except InvalidKeyError: 421 | # ignore non existing packages 422 | continue 423 | 424 | solved_deps.add(package) 425 | unsolved_deps.remove(package) 426 | 427 | return (solved_deps, unsolved_deps) 428 | 429 | 430 | def digest(self, overlay): 431 | """ 432 | Digest an overlay using repoman. 433 | 434 | Args: 435 | overlay: Overlay directory. 436 | """ 437 | self.logger.info("digesting overlay") 438 | prev = os.getcwd() 439 | os.chdir(overlay) 440 | if os.system("repoman manifest"): 441 | raise DigestError('repoman manifest failed') 442 | os.chdir(prev) 443 | 444 | def fast_digest(self, overlay, pkgnames): 445 | """ 446 | Digest an overlay using custom method faster than repoman. 447 | 448 | Args: 449 | overlay: Overlay directory. 450 | pkgnames: List of full package names (category/package). 451 | """ 452 | self.logger.info("fast digesting overlay") 453 | for pkgname in pkgnames: 454 | directory = os.path.join(overlay, pkgname) 455 | fast_manifest(directory) 456 | 457 | def generate_tree(self, args, config, global_config): 458 | """ 459 | Generate entire overlay. 460 | 461 | Args: 462 | args: Command line arguments. 463 | config: Backend config. 464 | global_config: g-sorcery config. 465 | 466 | Returns: 467 | Exit status. 468 | """ 469 | try: 470 | packages = global_config.get(config["backend"], args.repository + "_packages").split(" ") 471 | except Exception: 472 | packages = [] 473 | 474 | self.logger.info("tree generation") 475 | overlay = self._get_overlay(args, config, global_config) 476 | pkg_db = self._get_package_db(args, config, global_config) 477 | pkg_db.read() 478 | 479 | os.system('rm -rf ' + overlay + '/*') 480 | os.makedirs(os.path.join(overlay, 'profiles')) 481 | os.system("echo " + os.path.basename(overlay) + '>' + \ 482 | os.path.join(overlay, 'profiles', 'repo_name')) 483 | 484 | os.makedirs(os.path.join(overlay, 'metadata')) 485 | if not "masters" in config["repositories"][args.repository]: 486 | masters = elist(["gentoo"]) 487 | else: 488 | masters = elist(config["repositories"][args.repository]["masters"]) 489 | 490 | overlays = FileJSON("/var/lib/g-sorcery", "overlays.json", []) 491 | overlays_old_info = overlays.read() 492 | overlays_info = {} 493 | masters_overlays = elist() 494 | portage_overlays = [repo.location for repo in portage.settings.repositories] 495 | 496 | for repo, info in overlays_old_info.items(): 497 | if info["path"] in portage_overlays: 498 | overlays_info[repo] = info 499 | 500 | overlays.write(overlays_info) 501 | 502 | for repo in masters: 503 | if repo != "gentoo": 504 | if not repo in overlays_info: 505 | self.logger.error("Master repository " + repo + " not available on your system") 506 | self.logger.error("Please, add it with layman -a " + repo) 507 | return -1 508 | masters_overlays.append(overlays_info[repo]["repo-name"]) 509 | 510 | masters_overlays.append("gentoo") 511 | 512 | overlays_info[args.repository] = {"repo-name": os.path.basename(overlay), "path": overlay} 513 | with open(os.path.join(overlay, 'metadata', 'layout.conf'), 'w') as f: 514 | f.write("repo-name = %s\n" % os.path.basename(overlay)) 515 | f.write("masters = %s\n" % masters_overlays) 516 | 517 | if args.digest: 518 | ebuild_g = self.ebuild_g_with_digest_class(pkg_db) 519 | else: 520 | ebuild_g = self.ebuild_g_without_digest_class(pkg_db) 521 | metadata_g = self.metadata_g_class(pkg_db) 522 | 523 | packages_iter = pkg_db 524 | catpkg_names = pkg_db.list_catpkg_names() 525 | if packages: 526 | dependencies = set() 527 | catpkg_names = set() 528 | packages_dict = {} 529 | for pkg in packages: 530 | dependencies |= self.get_dependencies(pkg_db, pkg) 531 | 532 | for pkg in dependencies: 533 | catpkg_names |= set([pkg.category + '/' + pkg.name]) 534 | packages_dict[pkg] = pkg_db.get_package_description(pkg) 535 | packages_iter = packages_dict.items() 536 | 537 | for package, ebuild_data in packages_iter: 538 | category = package.category 539 | name = package.name 540 | version = package.version 541 | self.logger.info(" generating " + 542 | category + '/' + name + '-' + version) 543 | path = os.path.join(overlay, category, name) 544 | if not os.path.exists(path): 545 | os.makedirs(path) 546 | source = ebuild_g.generate(package, ebuild_data) 547 | with open(os.path.join(path, 548 | name + '-' + version + '.ebuild'), 549 | 'wb') as f: 550 | f.write('\n'.join(source).encode('utf-8')) 551 | 552 | source = metadata_g.generate(package) 553 | with open(os.path.join(path, 'metadata.xml'), 'wb') as f: 554 | f.write('\n'.join(source).encode('utf-8')) 555 | 556 | eclass_g = self.eclass_g_class() 557 | path = os.path.join(overlay, 'eclass') 558 | if not os.path.exists(path): 559 | os.makedirs(path) 560 | 561 | for eclass in eclass_g.list(): 562 | source = eclass_g.generate(eclass) 563 | with open(os.path.join(path, eclass + '.eclass'), 'w') as f: 564 | f.write('\n'.join(source)) 565 | 566 | if args.digest: 567 | self.digest(overlay) 568 | else: 569 | pkgnames = catpkg_names 570 | self.fast_digest(overlay, pkgnames) 571 | overlays.write(overlays_info) 572 | 573 | try: 574 | clean_db = config["repositories"][args.repository]["clean_db"] 575 | except KeyError: 576 | clean_db = False 577 | if clean_db: 578 | pkg_db.clean() 579 | 580 | def install(self, args, config, global_config): 581 | """ 582 | Install a package. 583 | 584 | Args: 585 | args: Command line arguments. 586 | config: Backend config. 587 | global_config: g-sorcery config. 588 | 589 | Returns: 590 | Exit status. 591 | """ 592 | self.generate(args, config, global_config) 593 | try: 594 | package_manager = global_config.get("main", "package_manager") 595 | except configparser.NoOptionError: 596 | package_manager_class = package_managers["portage"] 597 | package_manager = None 598 | if package_manager: 599 | if not package_manager in package_managers: 600 | self.logger.error('not supported package manager: ' \ 601 | + package_manager + '\n') 602 | return -1 603 | package_manager_class = package_managers[package_manager] 604 | package_manager = package_manager_class() 605 | package_manager.install(args.pkgname, *args.pkgmanager_flags) 606 | 607 | def __call__(self, args, config, global_config): 608 | """ 609 | Execute a command 610 | 611 | Args: 612 | args: Command line arguments. 613 | config: Backend config. 614 | global_config: g-sorcery config. 615 | 616 | Returns: 617 | Exit status. 618 | """ 619 | args = self.parser.parse_args(args) 620 | info_f = FileJSON(os.path.join(args.overlay, self.sorcery_dir), 621 | "info.json", ["repositories"]) 622 | self.info = info_f.read() 623 | repos = self.info["repositories"] 624 | if args.repository: 625 | if not repos: 626 | repos = {} 627 | back = config["package"] 628 | if back in repos: 629 | brepos = set(repos[back]) 630 | else: 631 | brepos = set() 632 | brepos.add(args.repository) 633 | repos[back] = list(brepos) 634 | self.info["repositories"] = repos 635 | info_f.write(self.info) 636 | else: 637 | back = config["package"] 638 | if back in repos: 639 | brepos = repos[back] 640 | if len(brepos) == 1: 641 | args.repository = brepos[0] 642 | else: 643 | self.logger.error("No repository specified," \ 644 | + " possible values:") 645 | for repo in brepos: 646 | print(" " + repo) 647 | return -1 648 | else: 649 | self.logger.error("No repository for backend " \ 650 | + back + " in overlay " + args.overlay) 651 | return -1 652 | return args.func(args, config, global_config) 653 | -------------------------------------------------------------------------------- /docs/developer_instructions.rst: -------------------------------------------------------------------------------- 1 | ====================== 2 | Developer Instructions 3 | ====================== 4 | 5 | g-sorcery overview 6 | ================== 7 | 8 | **g-sorcery** is a framework aimed to easy development of ebuild 9 | generators. 10 | 11 | Some terms used in this guide: 12 | 13 | * **3rd party software provider** or **repository** 14 | A system of software distribution like CTAN or CPAN that 15 | provides packages for some domain (e.g. TeX packages or elisp 16 | packages for emacs). 17 | 18 | * **backend** 19 | A tool developed using **g-sorcery** framework that provides 20 | support for repositories of a given type. 21 | 22 | * **overlay** 23 | Usual Gentoo overlay. 24 | 25 | **g-sorcery** consists of different parts: 26 | 27 | * **package_db.PackageDB** 28 | A package database. It holds information about all available 29 | packages in a given repository. 30 | 31 | * **package_db.DBGenerator** 32 | A fabric that creates PackageDB object and fills it with information. 33 | 34 | * **backend.Backend** 35 | Backend that processes user commands. 36 | 37 | * **ebuild** 38 | Module with different ebuild generators. 39 | 40 | * **eclass** 41 | Module with eclass generators. 42 | 43 | * **metadata.MetadataGenerator** 44 | Metadata generator. 45 | 46 | Also there are other modules and classes that will be described later. 47 | 48 | Usually repositories of a given type provide some kind of database. It can 49 | be just a plain ASCII file, xmlrpc interface or just a joint set of web-pages. 50 | This database describes what packages are available and how to install them. 51 | 52 | Also usually there is an upstream tool for repositories of a given type that 53 | allows installation of available packages. The main problem when using 54 | such tools is that package mangler you use is not aware of them and they are 55 | not aware of your package manager. 56 | 57 | The idea of **g-sorcery** is to convert a database provided by a repository 58 | into well defined format and then generate an overlay with ebuilds. 59 | Then available packages can be installed as usual **Gentoo** packages. 60 | 61 | So there are two phases of backend operation: 62 | 63 | - synchronize with repository database 64 | 65 | - populate overlay using obtained information 66 | 67 | There are two ways of using backend: 68 | 69 | - run it as a CLI tool manually (not recommended) 70 | 71 | - use its integration with layman 72 | 73 | 74 | Backend structure 75 | ================= 76 | 77 | The only mandatory module in a backend is called **backend**. It should contain 78 | at least one variable called **instance** that has a **__call__** method that 79 | takes 4 arguments. These arguments are: 80 | 81 | * self 82 | 83 | * command line arguments 84 | 85 | * backend config 86 | 87 | * g-sorcery config 88 | 89 | Usually **instance** variable should be an instance of a class g_sorcery.backend.Backend 90 | or derived class. 91 | 92 | g_sorcery.backend.Backend constructor takes 8 arguments. They are: 93 | 94 | * self 95 | 96 | * Package database generator class 97 | 98 | * Two ebuild generator classes 99 | 100 | * Eclass generator class 101 | 102 | * Metadata generator class 103 | 104 | * Package database class 105 | 106 | * Boolean variable that defines method of database generation 107 | 108 | There are two ebuild generator classes as there are two scenarios of using backend on user 109 | side: generate the entire overlay tree (possibly by layman) or generate a given ebuild 110 | and its dependencies. In a first case it would be very bad idea to have sources in ebuild's 111 | SRC_URI as during manifest generation for an overlay all the sources would be downloaded 112 | to the user's computer that inevitably would made user really happy. So one ebuild generator 113 | generates ebuild with empty SRC_URI. Note that a mechanism for downloading of sources during 114 | ebuild merging should be provided. For an example see **git-2** eclass from the main tree or 115 | any eclass from backends provided with g-sorcery if you want to implement such a mechanism or 116 | use eclass **g-sorcery** provided by standard eclass generator (can be found in data directory 117 | of **g_sorcery** package). 118 | 119 | Usually downloading and parsing of a database from a repository is an easy operation. But sometimes 120 | there could exist some problems. Hence exists the last parameter in Backend constructor that 121 | allows syncing with already generated database available somewhere in Internet (see **gs-pypi** 122 | for an example of using it). 123 | 124 | To do something usefull backend should customize any classes from g-sorcery it needs 125 | and define backend.instance variable using those classes. Other two things backend should do are: 126 | 127 | * install a binary that calls g-sorcery with appropriate backend name (see man g-sorcery) 128 | 129 | * install a config that allows g-sorcery find appropriate backend module 130 | 131 | A binary should just pass arguments to g-sorcery. For a backend named gs-elpa it could look like 132 | 133 | .. code-block:: 134 | 135 | #!/bin/bash 136 | 137 | g-sorcery g-elpa $@ 138 | 139 | Backend config 140 | ~~~~~~~~~~~~~~ 141 | 142 | Backend config is just a JSON file with a dictionary. There are two mandatory entries: 143 | 144 | * package 145 | Its value should be a string with a package containing backend. 146 | 147 | * repositories 148 | A dictionary describing available repositories. Should have at least one entry. 149 | 150 | Backend config should have a name BACKEND.js and should be installed under **/etc/g-sorcery** 151 | directory. BACKEND here is a backend name which was used in a g-sorcery call. 152 | 153 | An entry in repositories dictionary as key should have a repository name and should be a dictionary 154 | with repository properties. The only mandatory property is **repo_uri** in case database is 155 | generated using info downloaded from the repository or **db_uri** in case database is 156 | just synced with another already generated database. Also there can be a **masters** entry that 157 | contains a list of overlays this repository depends on. If present it should contain at least 158 | **gentoo** entry. 159 | 160 | A simple backend config: 161 | 162 | .. code-block:: 163 | 164 | { 165 | "package": "gs_elpa", 166 | "repositories": { 167 | "gnu-elpa": { 168 | "repo_uri": "http://elpa.gnu.org/packages/" 169 | }, 170 | "marmalade": { 171 | "repo_uri": "http://marmalade-repo.org/packages/", 172 | "masters": ["gentoo", "gnu-elpa"] 173 | }, 174 | "melpa": { 175 | "repo_uri": "http://melpa.milkbox.net/packages/", 176 | "masters": ["gentoo", "gnu-elpa"] 177 | } 178 | } 179 | } 180 | 181 | Package database 182 | ================ 183 | 184 | The package is an in memory structure that describes available 185 | packages and to this structure corresponding files layout. 186 | 187 | Directory layout versions 188 | ~~~~~~~~~~~~~~~~~~~~~~~~~ 189 | 190 | There are two directory layouts at the moment: 191 | 192 | * v.0 legacy layout 193 | * v.1 layout that supports different DB structure versions and 194 | different file formats. 195 | 196 | v.0 legacy layout 197 | ~~~~~~~~~~~~~~~~~ 198 | 199 | Package database is a directory tree with JSON files. The layout of this tree looks like: 200 | 201 | .. code-block:: 202 | 203 | db dir 204 | manifest.json: database manifest 205 | categories.json: information about categories 206 | category1 207 | packages.json: packages information 208 | category2 209 | ... 210 | 211 | v.1 layout 212 | ~~~~~~~~~~ 213 | 214 | Metadata file contains information about layout and DB versions as 215 | well as information about file format used to store packages 216 | information. At the moment JSON and BSON are supported. 217 | 218 | .. code-block:: 219 | 220 | db dir 221 | manifest.json: database manifest 222 | categories.json: information about categories 223 | metadata.json: DB metadata 224 | category1 225 | packages.[b|j]son: information about available packages 226 | category2 227 | ... 228 | 229 | Database structure versions 230 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~ 231 | 232 | Database structure has two versions: legacy v.0 and v.1. With 233 | directory layout v.0 only DB structure v.0 is supported. DB structure 234 | is internal and shouldn't be relied on by any external tools (including 235 | backends). PackageDB class API should be used instead. 236 | 237 | PackageDB class 238 | ~~~~~~~~~~~~~~~ 239 | 240 | PackageDB class is aimed for interaction with package database. It has methods that allow 241 | to add categories and packages and to do queries on them. Usually you do not want to customize this 242 | class. 243 | 244 | If you have a database that should be synced with another already generate database 245 | you can use **sync** method. Two sync methods are available 246 | currently: **tgz** and **git**. 247 | 248 | Note that before add any package you should add a category for it using **add_category**. 249 | Then packages can be added using **add_package**. PackageDB currently does not write changes 250 | automatically, so you should call **write** after changes are done. This is not relevant 251 | for database changing in **process_data** method of database generator as there all changes 252 | are written by other methods it calls internally after 253 | **process_data**. 254 | 255 | If you have some fields that are common to all ebuilds in a given 256 | category, it's better to split them to common data, that can be set for 257 | category. This data will be added to ebuild data in results of package 258 | queries automatically. 259 | 260 | Public API that should be used for manipulating packages data: 261 | 262 | * add_category(self, category, description=None) -- add new category. 263 | * set_common_data(self, category, common_data) -- set common ebuild 264 | data for a category. 265 | * get_common_data(self, category) -- get common ebuild data for a 266 | category. 267 | * add_package(self, package, ebuild_data=None) -- add new packages 268 | (characterized by category, package name and version) with given 269 | ebuild data. 270 | * list_categories(self) -- list categories. 271 | * in_category(self, category, name) -- test whether a package is in a 272 | given category. 273 | * list_package_names(self, category) -- list package names in a 274 | category. 275 | * list_catpkg_names(self) -- list category/package name. 276 | * list_package_versions(self, category, name) -- list package 277 | versions. 278 | * list_all_packages(self) -- list all packages. 279 | * get_package_description(self, package) -- get ebuild data (it 280 | returns a dict that contains both ebuild data for a given package 281 | and fields from common data for a given category). 282 | * get_max_version(self, category, name) -- get the recent available 283 | version of a package. 284 | * iterator -- PackageDB class defines an iterator that iterates 285 | through all available package/ebuild data pairs. 286 | 287 | To see description of these methods look in g_sorcery/package_db.py file. 288 | 289 | JSON serializable objects 290 | ~~~~~~~~~~~~~~~~~~~~~~~~~ 291 | 292 | If you need to store an object in a database it should be JSON serializable in terms of 293 | g_sorcery.serialization module. It means it should define two methods: 294 | 295 | * usual method **serialize** that returns a JSON serializable object in terms of standard Python 296 | json module 297 | 298 | * class method **deserialize** that takes a value returned by **serialize** and constructs new instance 299 | of your class using it 300 | 301 | This holds true for other supported file formats (BSON at the moment). 302 | 303 | Dependency handling 304 | ~~~~~~~~~~~~~~~~~~~ 305 | 306 | There is a special class g_sorcery.g_collections.Dependency aimed to handle dependencies. 307 | Its constructor takes two mandatory parameters: 308 | 309 | * category 310 | 311 | * package 312 | 313 | and two additional parameters: 314 | 315 | * version 316 | 317 | * operator 318 | 319 | These two are the same as version and operator used in the usual package atom. 320 | 321 | For storing dependency lists in a database you should use a collection 322 | g_sorcery.g_collections.serializable_elist. Its constructor takes an iterable and a 323 | separator that will be used to separate items when this collection is printed. In case of 324 | storing dependencies for using them in ebuild's DEPEND variable a separator should be "\n\t". 325 | 326 | Ebuild data for every package version must have a "dependencies" entry. This entry is used 327 | by backend during deciding which ebuilds should be generated. So make sure it does not have 328 | any external dependencies. 329 | 330 | 331 | Package database generator 332 | ========================== 333 | 334 | Customizing DBGenerator 335 | ~~~~~~~~~~~~~~~~~~~~~~~ 336 | 337 | To do something usefull you should customize package_db.DBGenerator class. 338 | With this aim you should subclass it and define some methods. Here they are: 339 | 340 | * get_download_uries 341 | Get a list with download URI entries. 342 | Each entry has one of the following formats: 343 | 344 | 1. String with URI. 345 | 346 | 2. A dictionary with entries: 347 | - uri: URI. 348 | 349 | - parser: Parser to be applied to downloaded data. 350 | 351 | - open_file: Whether parser accepts file objects. 352 | 353 | - open_mode: Open mode for a downloaded file. 354 | 355 | The only mandatory entry is uri. 356 | 357 | The default implementation returns [backend_config["repositories"][REPOSITORY]["repo_uri"]]. 358 | 359 | * parse_data 360 | This method parses a file downloaded from a repository 361 | and returns its content in any form you think useful. 362 | There is no useful default implementation of this method. 363 | 364 | * process_data 365 | This method should fill a package database with entries using 366 | already downloaded and parsed data. 367 | 368 | Generally speaking these are all the method you should implement. 369 | 370 | Both PackageDB and DBGenerator constructors accept these fields that 371 | are used to control preferred DB version/layout and file format (used 372 | during writing DB to disk): 373 | 374 | * preferred_layout_version, 1 by default 375 | * preferred_db_version, 1 by default 376 | * preferred_category_format, json by default 377 | 378 | To see how to use them look at the gs-pypi backend. 379 | 380 | Value convertion 381 | ~~~~~~~~~~~~~~~~ 382 | 383 | During database generation you may need to convert some values provided by repository 384 | (e.g license names that can not coincide with those used in Gentoo). With this aim 385 | you can use **convert** function. To understand how it works see its sources in 386 | g_sorcery.package_db.DBGenerator and as an example CTAN backend. 387 | 388 | Here is a very short example. If you want to convert licenses in the same way for all 389 | repositories of this type you just add **common_config** entry to backend config which 390 | looks like: 391 | 392 | .. code-block:: 393 | 394 | "common_config": { 395 | "licenses": { 396 | "apache2": "Apache-2.0", 397 | "artistic": "Artistic", 398 | "Artistic2": "Artistic-2", 399 | "gpl": "GPL-1", 400 | "gpl2": "GPL-2", 401 | "gpl3": "GPL-3", 402 | "knuth": "TeX", 403 | "lgpl": "LGPL-2", 404 | "lgpl2.1": "LGPL-2.1", 405 | "lppl": "LPPL-1.2", 406 | "lppl1": "LPPL-1.2", 407 | "lppl1.2": "LPPL-1.2", 408 | "lppl1.3": "LPPL-1.3c" 409 | } 410 | } 411 | 412 | And then call in your **process_data** method 413 | 414 | .. code-block:: 415 | 416 | license = self.convert([common_config, config], "licenses", repo_license) 417 | 418 | Where **common_config**, **config** are config provided as arguments to your **process_data** method 419 | and **repo_license** is a license name used by the repository. 420 | 421 | There is a special conversion function used for dependencies: **convert_dependency**. To use it you should 422 | usually redefine **convert_internal_dependency** and **convert_external_dependency**. To decide whether 423 | a dependency is external database generator uses **external** entry in config. 424 | 425 | You may want to test whether there is a given value in given entry in config. To do it use 426 | **in_config** function. 427 | 428 | Eclass generator 429 | ================ 430 | 431 | Usualy you do not want to modify eclass generator. Currently it is very simple: it just returns eclasses 432 | from a given directory. So all you should do is populating a directory with eclasses and then 433 | inheriting g_sorcery.eclass.EclassGenerator and defining a directory in constructor. It should look 434 | like 435 | 436 | .. code-block:: 437 | 438 | class ElpaEclassGenerator(EclassGenerator): 439 | """ 440 | Implementation of eclass generator. Only specifies a data directory. 441 | """ 442 | def __init__(self): 443 | super(ElpaEclassGenerator, self).__init__(os.path.join(get_pkgpath(__file__), 'data')) 444 | 445 | Eclass generator always provides **g-sorcery** eclass. It overrides *src_unpack* function 446 | so if *DIGEST_SOURCES* variable is not set sources are fetched during unpack from *${REPO_URI}${SOURCEFILE}*. 447 | If *DIGEST_SOURCES* variable is set usual unpack function is called. 448 | 449 | Ebuild generator 450 | ================ 451 | 452 | There is a number of ebuild generators in g_sorcery.ebuild module. The DefaultEbuildGenerator 453 | is a recommended one. To use it you should inherit it and define an ebuild layout in constructor. 454 | 455 | Layout has entries for vars and inherited eclasses. Each entry is a list. 456 | Entries are processed in the following order: 457 | 458 | * vars_before_inherit 459 | 460 | * inherit 461 | 462 | * vars_after_inherit 463 | 464 | * vars_after_description 465 | 466 | * vars_after_keywords 467 | 468 | **inherit** entry is just a list of eclass names. 469 | 470 | **vars*** entries are lists of variables in two possible formats: 471 | 472 | 1. A string with variable name 473 | 2. A dictinary with entries: 474 | * name: variable name 475 | * value: variable value 476 | * raw: if present, no quotation of value will be done 477 | 478 | Variable names are automatically transformed to the upper-case during ebuild generation. 479 | 480 | An example of ebuild generator: 481 | 482 | .. code-block:: 483 | 484 | Layout = collections.namedtuple("Layout", 485 | ["vars_before_inherit", "inherit", 486 | "vars_after_description", "vars_after_keywords"]) 487 | 488 | class ElpaEbuildWithoutDigestGenerator(DefaultEbuildGenerator): 489 | """ 490 | Implementation of ebuild generator without sources digesting. 491 | """ 492 | def __init__(self, package_db): 493 | 494 | vars_before_inherit = \ 495 | ["repo_uri", "source_type", "realname"] 496 | 497 | inherit = ["g-elpa"] 498 | 499 | vars_after_description = \ 500 | ["homepage"] 501 | 502 | vars_after_keywords = \ 503 | ["depend", "rdepend"] 504 | 505 | layout = Layout(vars_before_inherit, inherit, 506 | vars_after_description, vars_after_keywords) 507 | 508 | super(ElpaEbuildWithoutDigestGenerator, self).__init__(package_db, layout) 509 | 510 | Metadata generator 511 | ================== 512 | 513 | To use metadata generator you should just define some variables in ebuild data. 514 | 515 | XML schema format 516 | ~~~~~~~~~~~~~~~~~ 517 | 518 | Metadata generator uses a XML schema in format defined in g_sorcery.metadata module. 519 | Schema is a list of entries. Each entry describes one XML tag. 520 | Entry is a dictionary. Dictionary keys are: 521 | 522 | * **name** 523 | Name of a tag 524 | 525 | * **multiple** 526 | Defines if a given tag can be used more then one time. It is a tuple. First element 527 | of a tuple is boolean. If it is set a tag can be repeated. Second element is a string. 528 | If it is not empty, it defines a name for an attribute 529 | that will distinguish different entries of a tag. 530 | 531 | * **required** 532 | Boolean that defines if a given tag is required. 533 | 534 | * **subtags** 535 | List of subtags. 536 | 537 | Data dictionary format 538 | ~~~~~~~~~~~~~~~~~~~~~~~ 539 | 540 | The part of ebuild data used for metadata generation should have data dictionary format 541 | also defined in g_sorcery.metadata. 542 | 543 | Keys correspond to tags from a schema with the same name. 544 | If a tag is not multiple without subkeys value is just a 545 | string with text for the tag. 546 | If tag is multiple value is a list with entries 547 | corresponding to a single tag. 548 | If tag has subtags value is a dictionary with entries 549 | corresponding to subkeys and **text** entry corresponding 550 | to text for the tag. 551 | If tag should have attributes value is a tuple or list with 552 | 0 element containing an attribute and 1 element containing 553 | a value for the tag as described previously. 554 | 555 | Metadata XML schema 556 | ~~~~~~~~~~~~~~~~~~~ 557 | 558 | Metadata XML schema looks like 559 | 560 | .. code-block:: 561 | 562 | default_schema = [{'name' : 'herd', 563 | 'multiple' : (True, ""), 564 | 'required' : False, 565 | 'subtags' : []}, 566 | 567 | {'name' : 'maintainer', 568 | 'multiple' : (True, ""), 569 | 'required' : False, 570 | 'subtags' : [{'name' : 'email', 571 | 'multiple' : (False, ""), 572 | 'required' : True, 573 | 'subtags' : []}, 574 | {'name' : 'name', 575 | 'multiple' : (False, ""), 576 | 'required' : False, 577 | 'subtags' : []}, 578 | {'name' : 'description', 579 | 'multiple' : (False, ""), 580 | 'required' : False, 581 | 'subtags' : []}, 582 | ] 583 | }, 584 | 585 | {'name' : 'longdescription', 586 | 'multiple' : (False, ""), 587 | 'required' : False, 588 | 'subtags' : []}, 589 | 590 | {'name' : 'use', 591 | 'multiple' : (False, ""), 592 | 'required' : False, 593 | 'subtags' : [{'name' : 'flag', 594 | 'multiple' : (True, "name"), 595 | 'required' : True, 596 | 'subtags' : []}] 597 | }, 598 | 599 | {'name' : 'upstream', 600 | 'multiple' : (False, ""), 601 | 'required' : False, 602 | 'subtags' : [{'name' : 'maintainer', 603 | 'multiple' : (True, ""), 604 | 'required' : False, 605 | 'subtags' : [{'name' : 'name', 606 | 'multiple' : (False, ""), 607 | 'required' : True, 608 | 'subtags' : []}, 609 | {'name' : 'email', 610 | 'multiple' : (False, ""), 611 | 'required' : False, 612 | 'subtags' : []}]}, 613 | {'name' : 'changelog', 614 | 'multiple' : (False, ""), 615 | 'required' : False, 616 | 'subtags' : []}, 617 | {'name' : 'doc', 618 | 'multiple' : (False, ""), 619 | 'required' : False, 620 | 'subtags' : []}, 621 | {'name' : 'bugs-to', 622 | 'multiple' : (False, ""), 623 | 'required' : False, 624 | 'subtags' : []}, 625 | {'name' : 'remote-id', 626 | 'multiple' : (False, ""), 627 | 'required' : False, 628 | 'subtags' : []}, 629 | ] 630 | }, 631 | ] 632 | 633 | So to have metadata.xml filled with e.g. maintainer info you should add to ebuild data 634 | something like 635 | 636 | .. code-block:: 637 | 638 | {'maintainer' : [{'email' : 'piatlicki@gmail.com', 639 | 'name' : 'Jauhien Piatlicki'}]} 640 | 641 | Layman integration 642 | ================== 643 | 644 | There is a **layman** integration for **g-sorcery** (thanks to Brian Dolbec and Auke Booij here). 645 | To use it you just need to install an xml file describing your repositories in 646 | **/etc/layman/overlays** directory. For our example of backend config we could write an xml file 647 | that looks like 648 | 649 | .. code-block:: 650 | 651 | 652 | 653 | 654 | 655 | gnu-elpa 656 | packages for emacs 657 | http://elpa.gnu.org/ 658 | 659 | piatlicki@gmail.com 660 | Jauhien Piatlicki 661 | 662 | gs-elpa gnu-elpa 663 | 664 | 665 | marmalade 666 | packages for emacs 667 | http://marmalade-repo.org/ 668 | 669 | piatlicki@gmail.com 670 | Jauhien Piatlicki 671 | 672 | gs-elpa marmalade 673 | 674 | 675 | melpa 676 | packages for emacs 677 | http://melpa.milkbox.net 678 | 679 | piatlicki@gmail.com 680 | Jauhien Piatlicki 681 | 682 | gs-elpa melpa 683 | 684 | 685 | 686 | In entries **gs-elpa melpa** the source type 687 | should always be **g-sorcery**, **gs-elpa** is backend name and **melpa** is repository name. 688 | 689 | For full description of format of this file see **layman** documentation. 690 | 691 | Summary 692 | ======= 693 | 694 | So to create your own backend you should write a module named **backend** and define there 695 | a variable named **instance** that is an instance of g_sorcery.backend.Backend class. Or something 696 | that quacks like this class. 697 | 698 | Before doing it you should have defined classes you pass to it as parameters. They should be database 699 | generator, two ebuild generators, eclass and metadata generators. 700 | 701 | Also you should write an executable that calls g-sorcery and some configs. 702 | 703 | To have better understanding you can look at 704 | gs-elpa (https://github.com/jauhien/gs-elpa) and gs-pypi 705 | (https://github.com/jauhien/gs-pypi) backends. Also available tests 706 | could be usefull. 707 | 708 | Note that there is a tool for editing generated database named **gs-db-tool**. 709 | --------------------------------------------------------------------------------