├── __init__.py ├── tests ├── __init__.py └── test_communication.py ├── bioconductor ├── __init__.py ├── communication.py └── config.py ├── setup.py ├── bioconductor.properties ├── PIP-DEPENDENCIES--bioc-common-python.txt ├── .travis.yml ├── PIP-DEPENDENCIES--bioc-common-python_Original.txt ├── appveyor.yml ├── README.md ├── .gitignore └── bioc-common-python.properties /__init__.py: -------------------------------------------------------------------------------- 1 | # NOTE: This file can be empty, or nearly empty, but it must exist 2 | # as a part of Python's module loading system. 3 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # NOTE: This file can be empty, or nearly empty, but it must exist 2 | # as a part of Python's module loading system. 3 | -------------------------------------------------------------------------------- /bioconductor/__init__.py: -------------------------------------------------------------------------------- 1 | # NOTE: This file can be empty, or nearly empty, but it must exist 2 | # as a part of Python's module loading system. 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | setup( 3 | name = "bioc-common-python", 4 | version = "0.12", 5 | packages = find_packages(), 6 | ) 7 | -------------------------------------------------------------------------------- /bioconductor.properties: -------------------------------------------------------------------------------- 1 | # This is _like_ a Java properties file, except 2 | # you'll need a header (the next line in this file) 3 | [Environment] 4 | environment=bioc-common-python 5 | 6 | [UniversalProperties] 7 | bbs.bioc.version=3.3 8 | -------------------------------------------------------------------------------- /PIP-DEPENDENCIES--bioc-common-python.txt: -------------------------------------------------------------------------------- 1 | backports-abc 2 | backports.ssl-match-hostname 3 | certifi 4 | cffi 5 | enum34 6 | funcsigs 7 | idna 8 | ipaddress 9 | mechanize 10 | mock 11 | pbr 12 | pyasn1 13 | pycparser 14 | pyOpenSSL 15 | PyYAML 16 | singledispatch 17 | six 18 | SQLAlchemy 19 | stomp.py 20 | testify 21 | tornado 22 | wheel 23 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | # This configuration is only meant to do some 2 | # linting of the code. It's based in part, on 3 | # this blog: http://cramer.io/2012/05/03/using-travis-ci/ 4 | language: python 5 | python: 6 | - "2.7" 7 | before_install: 8 | - pip install pyflakes 9 | - pyflakes bioconductor/*.py 10 | script: 11 | - python -c 'print "Finished static analysis"' 12 | -------------------------------------------------------------------------------- /PIP-DEPENDENCIES--bioc-common-python_Original.txt: -------------------------------------------------------------------------------- 1 | backports-abc==0.4 2 | backports.ssl-match-hostname==3.4.0.2 3 | certifi==2017.4.17 4 | cffi==1.14.0 5 | cryptography==2.8 6 | enum34==1.1.6 7 | funcsigs==0.4 8 | idna==2.5 9 | ipaddress==1.0.15 10 | mechanize==0.4.4 11 | mock==1.3.0 12 | pbr==1.8.1 13 | pyasn1==0.1.9 14 | pycparser==2.14 15 | pyOpenSSL==19.1.0 16 | PyYAML==3.11 17 | singledispatch==3.4.0.3 18 | six==1.10.0 19 | SQLAlchemy==1.0.10 20 | stomp.py==4.1.9 21 | testify==0.7.2 22 | tornado==4.3 23 | wheel==0.24.0 24 | -------------------------------------------------------------------------------- /appveyor.yml: -------------------------------------------------------------------------------- 1 | build: false 2 | branches: 3 | only: 4 | - master 5 | - feature/use-sqs 6 | environment: 7 | matrix: 8 | - PYTHON: "C:/Python27" 9 | init: 10 | - "ECHO %PYTHON%" 11 | - ps: "ls C:/Python*" 12 | install: 13 | - "%PYTHON%/Scripts/pip.exe install pyflakes --no-use-wheel" 14 | # TODO: Lint files dynamically, rather than entering each individually 15 | test_script: 16 | - "set path=%PYTHON%/Scripts;%path%" 17 | - "%PYTHON%/python.exe --version" 18 | - "%PYTHON%/Scripts/pip.exe --version" 19 | - "pyflakes bioconductor/config.py" 20 | - "pyflakes bioconductor/communication.py" 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # bioc-common-python 2 | 3 | [![Build Status](https://travis-ci.org/Bioconductor/bioc-common-python.svg?branch=master)](https://travis-ci.org/Bioconductor/bioc-common-python) 4 | [![Build status](https://ci.appveyor.com/api/projects/status/ohoqikyl85l77db5?svg=true)](https://ci.appveyor.com/project/b-long/bioc-common-python) 5 | 6 | Purpose 7 | ------- 8 | This module provides common functionality to other Bioconductor python applications, such as 9 | [packagebuilder](https://github.com/Bioconductor/packagebuilder) and 10 | [spb_history](https://github.com/Bioconductor/spb_history) . 11 | 12 | It's important to note that in production, properties are required which are not exposed to 13 | the public. Those properties live under [Bioconductor/private](https://github.com/Bioconductor/private). 14 | 15 | Installation 16 | ------------ 17 | Typically, this module is not used by itself. Instead, it's a PIP dependency of other software. To include 18 | this as a dependency, create an entry in the PIP file, such as 19 | [this one](https://github.com/Bioconductor/spb_history/blob/9a687b1289185d37b40652291a706c3c076b006f/PIP-DEPENDENCIES--spb_history.txt#L1). 20 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Bioconductor .gitignore 2 | development.properties 3 | workers/jobs 4 | 5 | # The following was generated by 6 | # https://www.gitignore.io/api/python 7 | 8 | ### Python ### 9 | # Byte-compiled / optimized / DLL files 10 | __pycache__/ 11 | *.py[cod] 12 | *$py.class 13 | 14 | # C extensions 15 | *.so 16 | 17 | # Distribution / packaging 18 | .Python 19 | env/ 20 | build/ 21 | develop-eggs/ 22 | dist/ 23 | downloads/ 24 | eggs/ 25 | .eggs/ 26 | lib/ 27 | lib64/ 28 | parts/ 29 | sdist/ 30 | var/ 31 | *.egg-info/ 32 | .installed.cfg 33 | *.egg 34 | 35 | # PyInstaller 36 | # Usually these files are written by a python script from a template 37 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 38 | *.manifest 39 | *.spec 40 | 41 | # Installer logs 42 | pip-log.txt 43 | pip-delete-this-directory.txt 44 | 45 | # Unit test / coverage reports 46 | htmlcov/ 47 | .tox/ 48 | .coverage 49 | .coverage.* 50 | .cache 51 | nosetests.xml 52 | coverage.xml 53 | *,cover 54 | .hypothesis/ 55 | 56 | # Translations 57 | *.mo 58 | *.pot 59 | 60 | # Django stuff: 61 | *.log 62 | 63 | # Sphinx documentation 64 | docs/_build/ 65 | 66 | # PyBuilder 67 | target/ 68 | 69 | -------------------------------------------------------------------------------- /bioc-common-python.properties: -------------------------------------------------------------------------------- 1 | # This is _like_ a Java properties file, except 2 | # you'll need a header (the next line in this file) 3 | [Properties] 4 | 5 | # Assumes you're running a standard install (e.g. the recommended Docker container) and 6 | # have bound Stomp to you're machines localhost 7 | stomp.host=localhost 8 | stomp.port=61613 9 | 10 | # These are default ActiveMQ login credentials 11 | activemq.username=admin 12 | activemq.password=admin 13 | 14 | # The following values have not yet been needed 15 | bbs.rsa.key=foo 16 | spb.rsa.key=bar 17 | 18 | # Comma-separated list of nodes 19 | builders=localhost 20 | # Fill in with the absolute path to packagebuilder/workers 21 | packagebuilder.home=changeme 22 | 23 | # Previously environment variables 24 | bbs.home=changeme 25 | 26 | # The directory that contains R's `bin/`, `tools/`, `doc/`, etc. 27 | bbs.r.home=changeme 28 | # Same as bbs.r.home , but append "bin/R" or "bin/R.exe" 29 | bbs.r.cmd=changeme 30 | 31 | bbs.rsync.command=changeme 32 | 33 | # Fill in using your own SVN credentials 34 | svn.user=changeme 35 | svn.pass=changeme 36 | 37 | # Fill in using production credentials 38 | tracker.user=changeme 39 | tracker.pass=changeme 40 | -------------------------------------------------------------------------------- /bioconductor/communication.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | log = logging.getLogger("bioconductor.common") 4 | 5 | # log.basicConfig(format='%(levelname)s: %(asctime)s %(filename)s - %(message)s', 6 | # datefmt='%m/%d/%Y %I:%M:%S %p', 7 | # level=log.DEBUG) 8 | 9 | import stomp 10 | 11 | # Modules created by Bioconductor 12 | from bioconductor.config import BROKER 13 | 14 | stompHost = BROKER['host'] 15 | stompPort = BROKER['port'] 16 | 17 | def getNewStompConnection(listenerName, listenerObject): 18 | try: 19 | log.debug("Attempting to open connection to broker at '%s:%s'.", 20 | stompHost, stompPort) 21 | #stompClient = stomp.Connection([(stompHost, stompPort)]) 22 | stompClient = stomp.Connection( 23 | [(stompHost, stompPort)], 24 | heartbeats=(60000, 60000), # (client, server) ms 25 | reconnect_attempts_max=3 26 | ) 27 | 28 | stompClient.set_listener(listenerName, listenerObject) 29 | 30 | stompClient.connect(wait=True) 31 | log.debug("Stomp connection established.") 32 | except: 33 | log.error("Cannot connect to Stomp at '%s:%s'.", stompHost, stompPort) 34 | raise 35 | 36 | return stompClient 37 | -------------------------------------------------------------------------------- /tests/test_communication.py: -------------------------------------------------------------------------------- 1 | # Platform dependencies 2 | import sys 3 | import logging 4 | 5 | # Logging framework 6 | from testify import * 7 | 8 | # Stomp dependencies 9 | from stomp.listener import ConnectionListener 10 | 11 | # Bioconductor dependencies 12 | from bioconductor.communication import getNewStompConnection 13 | from bioconductor.config import BROKER 14 | 15 | # logging.basicConfig(stream=sys.stdout, level=logging.ERROR) 16 | log = logging.getLogger("CommunicationTestCase") 17 | log.addHandler(logging.StreamHandler()) 18 | log.propagate = False 19 | log.setLevel(logging.INFO) 20 | log.addHandler(logging.StreamHandler()) 21 | log.removeHandler(log.handlers[0]) 22 | 23 | rootLogger = logging.getLogger() 24 | fh = logging.FileHandler('TestingRoot.log') 25 | fh.setLevel(logging.INFO) 26 | rootLogger.addHandler(fh) 27 | rootLogger.disabled = True 28 | rootLogger.removeHandler(rootLogger.handlers[0]) 29 | 30 | 31 | class CommunicationTestCase(TestCase): 32 | 33 | @class_setup 34 | def init_the_variable(self): 35 | self.stompClient = None 36 | 37 | @setup 38 | def increment_the_variable(self): 39 | try: 40 | self.stompClient = getNewStompConnection('', ConnectionListener()) 41 | except: 42 | log.error(''' 43 | Cannot connect to Stomp; make sure you're running a local ActiveMQ instance. 44 | You can start a docker container with ActiveMQ using the following : 45 | `docker run --name='activemq' -d -p 8161:8161 -p 61616:61616 -p 61613:61613 rmohr/activemq:5.10.0` 46 | ''' 47 | ) 48 | sys.exit(1) 49 | 50 | def testHostIsString(self): 51 | assert_equal(type(BROKER['host']) is str, True) 52 | 53 | def testConnectionIsCreated(self): 54 | assert_not_equal(self.stompClient, None) 55 | 56 | def testSubscribing(self): 57 | self.stompClient.subscribe(destination="/topic/builderevents", id="StompUnitTest", 58 | ack='auto') 59 | log.info("Subscription is working") 60 | 61 | def testConnectionType(self): 62 | assert_equal(type("self.stompClient") is str, True) 63 | 64 | @teardown 65 | def decrement_the_variable(self): 66 | self.stompClient = None 67 | 68 | @class_teardown 69 | def get_rid_of_the_variable(self): 70 | log.info("Finished all tests") 71 | 72 | if __name__ == "__main__": 73 | run() 74 | -------------------------------------------------------------------------------- /bioconductor/config.py: -------------------------------------------------------------------------------- 1 | # A simple configuration module to reduce duplication. 2 | 3 | import os 4 | import os.path 5 | import platform 6 | import logging 7 | import configparser 8 | 9 | # log.basicConfig(format='%(levelname)s: %(asctime)s %(filename)s - %(message)s', 10 | # datefmt='%m/%d/%Y %I:%M:%S %p', 11 | # level=log.DEBUG) 12 | 13 | log = logging.getLogger("bioconductor.common") 14 | log.debug("Loading configuration") 15 | 16 | P_EXTENSION = '.properties' 17 | GLOBAL_PROPERTIES_FILE = os.path.join(os.getcwd(), 'bioconductor' + P_EXTENSION) 18 | 19 | def readFile(filename): 20 | if (os.path.isfile(filename) and os.access(filename, os.R_OK)): 21 | return True 22 | else: 23 | return False 24 | 25 | if not readFile(GLOBAL_PROPERTIES_FILE): 26 | errMsg = "Global properties file '{filename}' is missing or unreadable. " \ 27 | "Can not continue.".format(filename = GLOBAL_PROPERTIES_FILE) 28 | log.error(errMsg) 29 | raise Exception(errMsg) 30 | 31 | # Parse and read the file 32 | globalConfigParser = configparser.RawConfigParser() 33 | globalConfigParser.read(GLOBAL_PROPERTIES_FILE) 34 | 35 | environment = globalConfigParser.get('Environment', 'environment') 36 | ENVIRONMENT_PROPERTIES_FILE = os.path.join(os.getcwd(), environment + P_EXTENSION) 37 | # git clone https://github.com/Bioconductor/spb-properties in current directory 38 | # (that's a private repo to hold sensitive info) 39 | SENSITIVE_PROPERTIES_FILE = os.path.join(os.getcwd(), "spb-properties", "spb" + P_EXTENSION) 40 | 41 | if not readFile(ENVIRONMENT_PROPERTIES_FILE): 42 | errMsg = "A properties file '{filename}' is required to configure the environment. "\ 43 | "Can not continue.".format(filename = ENVIRONMENT_PROPERTIES_FILE) 44 | log.error(errMsg) 45 | raise Exception(errMsg) 46 | 47 | log.info("Environment is set to: '{env}'.".format(env = environment)) 48 | 49 | # Parse and read the environment specific configuration 50 | envConfig = configparser.RawConfigParser() 51 | envConfig.read(ENVIRONMENT_PROPERTIES_FILE) 52 | 53 | sensitiveConfigParser = configparser.RawConfigParser() 54 | sensitiveConfigParser.read(SENSITIVE_PROPERTIES_FILE) 55 | 56 | # FIXME: Rather than attempting to read the same properties in any environment, 57 | # it'd be much better if the config module's constructor (or factory) 58 | # offered a callback mechanism, to load the properties file and then 59 | # dispatch to environment specific functionality. 60 | 61 | # Only used in the packagebuilder for now (we need to adjust it's properties) 62 | BUILD_NODES = envConfig.get('Properties', 'builders').lower().split(",") 63 | BROKER = { 64 | "host": envConfig.get('Properties', 'stomp.host'), 65 | "port": int(envConfig.get('Properties', 'stomp.port')) 66 | } 67 | log.info("The following build nodes are enabled: %s.", BUILD_NODES) 68 | if envConfig.has_option('Properties', 'activemq.username'): 69 | ACTIVEMQ_USER = envConfig.get('Properties', 'activemq.username') 70 | else: 71 | ACTIVEMQ_USER = None 72 | if envConfig.has_option('Properties', 'activemq.password'): 73 | ACTIVEMQ_PASS = envConfig.get('Properties', 'activemq.password') 74 | else: 75 | ACTIVEMQ_PASS = None 76 | 77 | BIOC_VERSION = globalConfigParser.get('UniversalProperties', 'bbs.bioc.version') 78 | 79 | # TODO: Consider a better way to determine this 80 | BIOC_R_MAP = {"2.7": "2.12", "2.8": "2.13", "2.9": "2.14", 81 | "2.10": "2.15", "2.14": "3.1", "3.0": "3.1", 82 | "3.1": "3.2", "3.2": "3.2", "3.3": "3.3", 83 | "3.4": "3.3", "3.5": "3.4", "3.6": "3.4", 84 | "3.7": "3.5", "3.8": "3.5", "3.9": "3.6", 85 | "3.10": "3.6", "3.11": "4.0", "3.12": "4.0", 86 | "3.13": "4.1", "3.14": "4.1", "3.15": "4.2", 87 | "3.16": "4.2", "3.17": "4.3", "3.18": "4.3", 88 | "3.19": "4.4", "3.20": "4.4", "3.21": "4.5", 89 | "3.22": "4.5","3.23": "4.6","3.24": "4.6"} 90 | 91 | BUILDER_ID = platform.node().lower().replace(".fhcrc.org","") 92 | BUILDER_ID = BUILDER_ID.replace(".local", "") 93 | BUILDER_ID = BUILDER_ID.replace(".bioconductor.org", "") 94 | 95 | BBS_HOME = envConfig.get('Properties', 'bbs.home') 96 | 97 | ENVIR = { 98 | 'bbs_Bioc_version': BIOC_VERSION, 99 | 'bbs_RSA_key': envConfig.get('Properties', 'bbs.rsa.key'), 100 | 'bbs_R_cmd': envConfig.get('Properties', 'bbs.r.home') + "bin/R", 101 | 'bbs_R_home': envConfig.get('Properties', 'bbs.r.home'), 102 | 'bbs_central_rhost': envConfig.get('Properties','bbs.central.rhost'), 103 | 'bbs_central_ruser': envConfig.get('Properties','bbs.central.ruser'), 104 | 'bbs_home': BBS_HOME, 105 | 'bbs_mode': envConfig.get('Properties','bbs.mode'), 106 | 'bbs_node_hostname': BUILDER_ID, 107 | 'bbs_python_cmd': envConfig.get('Properties','bbs.python.cmd'), 108 | 'bbs_rsync_cmd': envConfig.get('Properties','bbs.rsync.cmd'), 109 | 'bbs_ssh_cmd': envConfig.get('Properties','bbs.ssh.cmd'), 110 | 'bbs_svn_cmd': envConfig.get('Properties','bbs.svn.cmd'), 111 | 'bbs_tar_cmd': envConfig.get('Properties','bbs.tar.cmd'), 112 | 'bbs_curl_cmd': envConfig.get('Properties','bbs.curl.cmd'), 113 | 'bbs_lang': envConfig.get('Properties','bbs.lang'), 114 | 115 | 'spb_RSA_key': envConfig.get('Properties', 'spb.rsa.key'), 116 | 'spb_home': envConfig.get('Properties', 'spb.home'), 117 | 'spb_staging_url': envConfig.get('Properties', 'spb.staging.url'), 118 | 119 | 'github_issue_repo': envConfig.get('Properties', 'github.issue.repo'), 120 | 121 | 'r_check_environ': envConfig.get('Properties', 'r.check.environ'), 122 | 123 | 'log_level': envConfig.get('Properties', 'log.level'), 124 | 'log_level_builder': envConfig.get('Properties', 'log.level.builder'), 125 | 'log_level_server': envConfig.get('Properties', 'log.level.server'), 126 | 'timeout_limit': envConfig.get('Properties', 'timeout.limit'), 127 | 128 | 'svn_pass': sensitiveConfigParser.get('Sensitive', 'svn.user'), 129 | 'svn_user': sensitiveConfigParser.get('Sensitive', 'svn.user'), 130 | 'github_token': sensitiveConfigParser.get('Sensitive', 'github.token'), 131 | 'bioc_devel_password': sensitiveConfigParser.get('Sensitive', 'bioc.devel.password') 132 | } 133 | 134 | TOPICS = { 135 | "jobs": "/topic/buildjobs", 136 | "events": "/topic/builderevents" 137 | } 138 | 139 | HOSTS = { 140 | 'svn': 'https://hedgehog.fhcrc.org', 141 | 'tracker': 'https://tracker.bioconductor.org', 142 | 'bioc': 'https://bioconductor.org' 143 | } 144 | 145 | 146 | log.info("Finished loading configuration.") 147 | --------------------------------------------------------------------------------