├── doc ├── crabclient │ ├── code │ │ └── .gitignore │ ├── userdoc │ │ ├── crab-library.rst │ │ ├── dependencies.rst │ │ ├── quickstart.rst │ │ ├── installation.rst │ │ └── introduction.rst │ ├── references.rst │ ├── index.rst │ ├── changelog.rst │ └── conf.py ├── references.rst ├── ExampleConfiguration.py ├── config │ ├── ExampleConfiguration.py │ └── FullConfiguration.py ├── FullConfiguration.py ├── Makefile └── generate_modules.py ├── src └── python │ ├── CRABClient │ ├── JobType │ │ ├── __init__.py │ │ ├── LumiMask.py │ │ ├── BasicJobType.py │ │ ├── ScramEnvironment.py │ │ └── PrivateMC.py │ ├── Commands │ │ ├── __init__.py │ │ ├── checkusername.py │ │ ├── request_type.py │ │ ├── getoutput.py │ │ ├── proceed.py │ │ ├── kill.py │ │ ├── createmyproxy.py │ │ ├── uploadlog.py │ │ ├── remake.py │ │ ├── getsandbox.py │ │ ├── setdatasetstatus.py │ │ ├── tasks.py │ │ ├── getlog.py │ │ ├── setfilestatus.py │ │ └── preparelocal.py │ ├── Emulator.py │ ├── SpellChecker.py │ ├── __init__.py │ ├── WMCoreConfigWrapper.py │ ├── ClientExceptions.py │ ├── CRABOptParser.py │ ├── UserUtilities.py │ └── ProxyInteractions.py │ └── CRABAPI │ ├── README.md │ ├── test_TopLevel.py │ ├── __init__.py │ ├── TestAPI.py │ ├── Abstractions.py │ ├── TopLevel.py │ ├── RawCommand.py │ └── test_Task.py ├── .gitignore ├── etc ├── init-light.csh ├── init-light-pre.sh └── init-light.sh ├── bin ├── rucio_env.sh ├── crab └── crab.py ├── test ├── python │ └── CRABClient_t │ │ ├── Commands_t │ │ ├── TestPlugin.py │ │ ├── FakeRESTServer.py │ │ └── CRABRESTModelMock.py │ │ ├── JobType_t │ │ ├── ScramEnvironment_t.py │ │ ├── CMSSWConfig_t.py │ │ ├── UserTarball_t.py │ │ └── CMSSW_t.py │ │ └── Client_t.py └── data │ ├── TestConfig.py │ └── mapper.py ├── README.md ├── scripts └── generate_completion.py └── setup.py /doc/crabclient/code/.gitignore: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/python/CRABClient/JobType/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | .project 3 | .pydevproject 4 | tags 5 | .idea 6 | -------------------------------------------------------------------------------- /etc/init-light.csh: -------------------------------------------------------------------------------- 1 | #!/bin/csh 2 | eval `sh /cvmfs/cms.cern.ch/crab3/crab.sh -csh` 3 | -------------------------------------------------------------------------------- /src/python/CRABClient/Commands/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | An init for the crab Commmand 3 | ''' 4 | -------------------------------------------------------------------------------- /doc/references.rst: -------------------------------------------------------------------------------- 1 | References 2 | ========== 3 | 4 | .. [CMS] http://cms.web.cern.ch/cms/ 5 | .. [LHC] http://public.web.cern.ch/public/en/LHC/LHC-en.html 6 | -------------------------------------------------------------------------------- /doc/crabclient/userdoc/crab-library.rst: -------------------------------------------------------------------------------- 1 | Client as library 2 | ================= 3 | This section describes how to use the CRAB Client as a pure Python library. 4 | 5 | -------------------------------------------------------------------------------- /doc/crabclient/references.rst: -------------------------------------------------------------------------------- 1 | References 2 | ========== 3 | 4 | .. [CMS] http://cms.web.cern.ch/cms/ 5 | .. [LHC] http://public.web.cern.ch/public/en/LHC/LHC-en.html 6 | -------------------------------------------------------------------------------- /src/python/CRABAPI/README.md: -------------------------------------------------------------------------------- 1 | CRABAPI 2 | ======= 3 | 4 | Read-only mirror of the [main](https://github.com/PerilousApricot/CRABAPI) 5 | repository. Please find the issue tracker and most up to date documentation 6 | there. 7 | -------------------------------------------------------------------------------- /bin/rucio_env.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # returns the two env. variables needed to make Rucio client work 3 | # on the current OS 4 | unset PYTHONPATH 5 | eval `scram unsetenv -sh` 6 | source /cvmfs/cms.cern.ch/rucio/setup-py3.sh > /dev/null 7 | echo $RUCIO_HOME $PYTHONPATH 8 | -------------------------------------------------------------------------------- /test/python/CRABClient_t/Commands_t/TestPlugin.py: -------------------------------------------------------------------------------- 1 | from CRABClient.JobType.BasicJobType import BasicJobType 2 | 3 | class TestPlugin(BasicJobType): 4 | def run(self, config): 5 | return [],'' 6 | 7 | def validateConfig(self,config): 8 | return (True, '') 9 | -------------------------------------------------------------------------------- /doc/crabclient/userdoc/dependencies.rst: -------------------------------------------------------------------------------- 1 | Dependencies 2 | ======================== 3 | CRAB Client is a Python application. 4 | The list of all CRAB Client dependencies is listed here: 5 | - *python*: link to Python 6 | - *pycurl*: what's? link it 7 | - *wmcore*: link to wmcore sphinx doc plus when available, alternatively link to WMCore in (t)wiki 8 | - and some more 9 | -------------------------------------------------------------------------------- /etc/init-light-pre.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | if [ -z "$CRAB_SOURCE_SCRIPT" ]; then 3 | CRAB_SOURCE_SCRIPT="/cvmfs/cms.cern.ch/crab3/crab_pre_standalone.sh" 4 | fi 5 | 6 | init_light_source=${BASH_SOURCE} 7 | readlink -q $init_light_source >/dev/null 2>&1 && init_light_source=$(readlink $init_light_source) 8 | init_light_source=${init_light_source%%-pre.sh}.sh 9 | 10 | source $init_light_source 11 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | **CRABClient** 2 | 3 | Command Line client for the CRABServer REST interface. 4 | 5 | BRANCHES as of August 30 2021 6 | * master 7 | * for porting to python3, no WMCore etc. 8 | * v3.210607patch (now deprecated) 9 | * forked from tag v3.210607 which is now in production 10 | * to push important, needed patched to production but aim is to keep stable while working on py3 11 | * py2only 12 | * created from v3.210607patch plus a couple fixes. Stable release which only works with python2 CMSSW (i.e. < CMSSW_12) 13 | -------------------------------------------------------------------------------- /src/python/CRABClient/Commands/checkusername.py: -------------------------------------------------------------------------------- 1 | from CRABClient.Commands.SubCommand import SubCommand 2 | from CRABClient.UserUtilities import getUsername 3 | 4 | class checkusername(SubCommand): 5 | """ 6 | Use to check extraction of username from DN 7 | """ 8 | 9 | name = 'checkusername' 10 | 11 | def __call__(self): 12 | username = getUsername(self.proxyfilename, logger=self.logger) 13 | self.logger.info("Username is: %s", username) 14 | if username: 15 | return {'commandStatus': 'SUCCESS', 'username': username} 16 | else: 17 | return{'commandStatus': 'FAILED', 'username': None} 18 | 19 | def terminate(self, exitcode): 20 | pass 21 | 22 | -------------------------------------------------------------------------------- /src/python/CRABClient/Emulator.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=global-statement 2 | """ 3 | Used to perform dependency injection necessary for testing 4 | """ 5 | 6 | defaultDict = None 7 | overrideDict = {} 8 | 9 | def clearEmulators(): 10 | global overrideDict 11 | overrideDict = {} 12 | 13 | def getEmulator(name): 14 | global overrideDict, defaultDict 15 | if name in overrideDict: 16 | return overrideDict[name] 17 | else: 18 | if not defaultDict: 19 | defaultDict = getDefaults() 20 | return defaultDict[name] 21 | 22 | def setEmulator(name, value): 23 | global overrideDict 24 | overrideDict[name] = value 25 | 26 | def getDefaults(): 27 | import CRABClient.RestInterfaces 28 | return {'rest' : CRABClient.RestInterfaces.CRABRest, 29 | 'ufc' : 'dummy_ufc'} 30 | -------------------------------------------------------------------------------- /doc/crabclient/index.rst: -------------------------------------------------------------------------------- 1 | .. CMS CRAB Client documentation master file, created by 2 | sphinx-quickstart on Fri Dec 2 11:03:47 2011. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | CMS CRAB Client's user documentation 7 | =========================================== 8 | 9 | * **Version**: |version| 10 | * **Last modified**: |today| 11 | 12 | 13 | Contents: 14 | --------- 15 | 16 | .. toctree:: 17 | :maxdepth: 2 18 | 19 | userdoc/introduction 20 | userdoc/quickstart 21 | userdoc/installation 22 | userdoc/configuration 23 | userdoc/running 24 | userdoc/crab-library 25 | userdoc/dependencies 26 | changelog 27 | code/modules 28 | references 29 | 30 | Indices and tables 31 | ================== 32 | 33 | * :ref:`genindex` 34 | * :ref:`modindex` 35 | * :ref:`search` 36 | 37 | -------------------------------------------------------------------------------- /src/python/CRABAPI/test_TopLevel.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=locally-disabled,missing-docstring,too-many-public-methods 2 | # The above is ONLY permissible in test suites 3 | import logging 4 | import unittest 5 | import CRABAPI 6 | class test_TopLevel(unittest.TestCase): 7 | def test_getTask_notimpl(self): 8 | self.assertRaises(NotImplementedError, CRABAPI.getTask, "") 9 | def test_setLogging(self): 10 | CRABAPI.setLogging(logging.DEBUG, logging.DEBUG, logging.DEBUG) 11 | log1, log2, log3 = CRABAPI.getAllLoggers('withsuffix') 12 | log1single = CRABAPI.getLogger('withsuffix') 13 | self.assertEqual(log1, log1single) 14 | log1, log2nosuffix, log3nosuffix = CRABAPI.getAllLoggers() 15 | log1single = CRABAPI.getLogger() 16 | self.assertEqual(log1, log1single) 17 | self.assertEqual(log2, log2nosuffix) 18 | self.assertEqual(log3, log3nosuffix) 19 | -------------------------------------------------------------------------------- /doc/crabclient/userdoc/quickstart.rst: -------------------------------------------------------------------------------- 1 | Quickstart 2 | ========== 3 | This section is intended for impatient users who are already confident with the work environment and tools. Please, read all the way through in any case! 4 | 5 | 6 | * source crab.(c)sh from the `CRAB `_ installation area, which have been setup either by you or by someone else for you; 7 | 8 | * modify the `CRAB `_ configuration file crab.cfg according to your need: see :ref:`basic-config` for an example; 9 | 10 | * submit the jobs:: 11 | 12 | $ crab submit 13 | 14 | 15 | * check the status of all the jobs:: 16 | 17 | $ crab status -d 18 | 19 | 20 | * retrieve the output of the jobs 1,2,3,5:: 21 | 22 | $ crab get-output -d -r 1-3,5 23 | 24 | 25 | * see the list of good lumis for your task:: 26 | 27 | $ crab report -d 28 | -------------------------------------------------------------------------------- /doc/ExampleConfiguration.py: -------------------------------------------------------------------------------- 1 | """ 2 | This is an example of a minimal CRAB3 configuration file. 3 | """ 4 | 5 | from WMCore.Configuration import Configuration 6 | import os 7 | 8 | config = Configuration() 9 | 10 | ## General options for the client 11 | config.section_("General") 12 | config.General.requestName = 'MyAnalysis' 13 | config.General.instance = 'prod' 14 | 15 | ## Specific option of the job type 16 | ## these options are directly readable from the job type plugin 17 | config.section_("JobType") 18 | config.JobType.pluginName = 'Analysis' 19 | config.JobType.psetName = 'pset.py' 20 | 21 | ## Specific data options 22 | config.section_("Data") 23 | config.Data.inputDataset = '/GenericTTbar/HC-CMSSW_5_3_1_START53_V5-v1/GEN-SIM-RECO' 24 | config.Data.splitting = 'LumiBased' 25 | config.Data.unitsPerJob = 20 26 | 27 | ## User options 28 | config.section_("User") 29 | config.User.email = '' 30 | 31 | config.section_("Site") 32 | config.Site.storageSite = 'T2_US_Nebraska' 33 | 34 | -------------------------------------------------------------------------------- /doc/config/ExampleConfiguration.py: -------------------------------------------------------------------------------- 1 | """ 2 | This is a test example of configuration file for CRAB-3 client 3 | """ 4 | 5 | from WMCore.Configuration import Configuration 6 | import os 7 | 8 | config = Configuration() 9 | 10 | ## General options for the client 11 | config.section_("General") 12 | config.General.requestName = 'MyAnalysis_1' 13 | #config.General.restHost = 'yourserver' 14 | #config.General.dbInstance = 'dev' 15 | #config.General.instance = 'other' 16 | 17 | ## Specific option of the job type 18 | ## these options are directly readable from the job type plugin 19 | config.section_("JobType") 20 | config.JobType.pluginName = 'Analysis' 21 | config.JobType.psetName = 'pset.py' 22 | 23 | ## Specific data options 24 | config.section_("Data") 25 | config.Data.inputDataset = '/cms/data/set' 26 | config.Data.splitting = 'LumiBased' 27 | config.Data.unitsPerJob = 20 28 | 29 | ## User options 30 | config.section_("User") 31 | config.User.email = '' 32 | 33 | config.section_("Site") 34 | config.Site.storageSite = 'T2_XX_XXX' 35 | -------------------------------------------------------------------------------- /doc/crabclient/changelog.rst: -------------------------------------------------------------------------------- 1 | CRABClient release notes 2 | ======================== 3 | This section contains the list of changes between each subsequent realeases of CRAB Client. Changes are grouped by release. 4 | .. Grouping can be also at greater level with release series. 5 | 6 | Release 3.0.6a 7 | ++++++++++++++ 8 | - If something fails print the log file path, #2776 9 | - Configuration vs CLI parameters, #2180 10 | - Client changes for user specified file upload, #2782 11 | - Example config changes 12 | - Automatically set team and group as Analysis, #2832 13 | - Replicate pycfg_params, #2482 14 | - Takes parameters from the ClientMapping and check if user config params are there, #2678 15 | - Handled intelligent ranges for black/white lists, #2561 16 | - Remove unused import and stronger check of server input parameter, #2881 17 | - Fix the initialization of proxy with a new group and role, #2912 18 | - Use group from request not from config, #2906 19 | - Better handling of user messaged, #2880, #2893, #2641 20 | -------------------------------------------------------------------------------- /test/data/TestConfig.py: -------------------------------------------------------------------------------- 1 | """ 2 | This is a test example of configuration file for CRAB-3 client 3 | """ 4 | 5 | from WMCore.Configuration import Configuration 6 | import os 7 | 8 | config = Configuration() 9 | 10 | ## General options for the client 11 | config.section_("General") 12 | config.General.serverUrl = '127.0.0.1:8518' 13 | config.General.requestName = 'MyAnalysis' 14 | 15 | ## Specific option of the job type 16 | ## these options are directly readable from the job type plugin 17 | config.section_("JobType") 18 | config.JobType.pluginName = 'TestPlugin' 19 | 20 | ## Specific data options 21 | config.section_("Data") 22 | config.Data.inputDataset = '/RelValProdTTbar/JobRobot-MC_3XY_V24_JobRobot-v1/GEN-SIM-DIGI-RECO' 23 | config.Data.splitting = 'FileBased' 24 | config.Data.processingVersion = 'v1' 25 | config.Data.unitsPerJob = 100 26 | 27 | 28 | ## User options 29 | config.section_("User") 30 | config.User.team = 'Analysis' 31 | config.User.group = 'Analysis' 32 | config.User.email = '' 33 | 34 | config.section_("Site") 35 | config.Site.storageSite = 'T2_IT_Pisa' 36 | config.Site.blacklist = ["T2_ES_CIEMAT"] 37 | -------------------------------------------------------------------------------- /src/python/CRABClient/Commands/request_type.py: -------------------------------------------------------------------------------- 1 | from CRABClient.Commands.SubCommand import SubCommand 2 | 3 | class request_type(SubCommand): 4 | """ 5 | Return the string of the workflow type identified by the -d/--dir option. 6 | """ 7 | visible = False 8 | 9 | def __call__(self): 10 | 11 | server = self.crabserver 12 | 13 | self.logger.debug('Looking type for task %s' % self.cachedinfo['RequestName']) 14 | dictresult, status, reason = server.get(api=self.defaultApi, # pylint: disable=unused-variable 15 | data={'workflow': self.cachedinfo['RequestName'], 'subresource': 'type'}) 16 | self.logger.debug('Task type %s' % dictresult['result'][0]) 17 | return dictresult['result'][0] 18 | 19 | def setOptions(self): 20 | """ 21 | __setOptions__ 22 | 23 | This allows to set specific command options 24 | """ 25 | self.parser.add_option("--proxyfile", 26 | dest="proxyfile", 27 | default=None, 28 | help="Proxy file to use.") 29 | -------------------------------------------------------------------------------- /src/python/CRABAPI/__init__.py: -------------------------------------------------------------------------------- 1 | """ CRABAPI - https://github.com/PerilousApricot/CRABAPI 2 | The outward-facing portion of the API 3 | """ 4 | 5 | from CRABAPI.TopLevel import getTask, setLogging, getAllLoggers, getLogger 6 | 7 | # Make sense of CRABClient's exceptions by making an exception tree 8 | class APIException(Exception): 9 | """ 10 | APIException - top of the CRABAPI exception tree 11 | """ 12 | pass 13 | 14 | class BadArgumentException(APIException): 15 | """ 16 | BadArgumentException - Arguments passed didn't pass optparse's muster 17 | """ 18 | pass 19 | 20 | 21 | def setUpPackage(): 22 | """ Need to make sure logging is initialized before any tests run. This 23 | should NOT be called by client functions, it is used by the testing 24 | suite """ 25 | import logging 26 | setLogging(logging.DEBUG, logging.DEBUG, logging.DEBUG) 27 | 28 | # Used if someone does an "import * from CRABAPI" 29 | from CRABAPI.Abstractions import Task 30 | from CRABAPI.RawCommand import execRaw 31 | __all__ = ["getTask", "setLogging", "getAllLoggers", "getLogger", "Task", \ 32 | "execRaw"] 33 | -------------------------------------------------------------------------------- /test/python/CRABClient_t/Commands_t/FakeRESTServer.py: -------------------------------------------------------------------------------- 1 | from WMQuality.WebTools.RESTBaseUnitTest import RESTBaseUnitTest 2 | from WMQuality.WebTools.RESTServerSetup import DefaultConfig 3 | import logging 4 | import os 5 | 6 | class FakeRESTServer(RESTBaseUnitTest): 7 | """ 8 | Loads a the CRABRESTModelMock REST interface which emulates the CRABRESTModel class. 9 | When testing a command which requires an interaction with the server wi will contact 10 | this class. 11 | """ 12 | 13 | def initialize(self): 14 | self.config = DefaultConfig('CRABRESTModelMock') 15 | self.config.Webtools.environment = 'development' 16 | self.config.Webtools.error_log_level = logging.ERROR 17 | self.config.Webtools.access_log_level = logging.ERROR 18 | self.config.Webtools.port = 8518 19 | self.config.Webtools.host = '127.0.0.1' 20 | self.config.UnitTests.object = 'CRABRESTModelMock' 21 | #self.config.UnitTests.views.active.rest.logLevel = 'DEBUG' 22 | 23 | self.urlbase = self.config.getServerUrl() 24 | 25 | 26 | def setUp(self): 27 | """ 28 | _setUp_ 29 | """ 30 | RESTBaseUnitTest.setUp(self) 31 | 32 | 33 | def tearDown(self): 34 | RESTBaseUnitTest.tearDown(self) 35 | -------------------------------------------------------------------------------- /src/python/CRABClient/SpellChecker.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on Nov 14, 2011 3 | 4 | See http://norvig.com/spell-correct.html for documentation 5 | ''' 6 | 7 | import re, collections 8 | import string 9 | 10 | 11 | def words(text): return re.findall('[a-z]+', text.lower()) 12 | 13 | def train(features): 14 | model = collections.defaultdict(lambda: 1) 15 | for f in features: 16 | model[f] += 1 17 | return model 18 | 19 | DICTIONARY = [] 20 | 21 | def edits1(word): 22 | splits = [(word[:i], word[i:]) for i in range(len(word) + 1)] 23 | deletes = [a + b[1:] for a, b in splits if b] 24 | transposes = [a + b[1] + b[0] + b[2:] for a, b in splits if len(b)>1] 25 | replaces = [a + c + b[1:] for a, b in splits for c in string.lowercase if b] 26 | inserts = [a + c + b for a, b in splits for c in string.lowercase] 27 | return set(deletes + transposes + replaces + inserts) 28 | 29 | def known_edits2(word): 30 | return set(e2 for e1 in edits1(word) for e2 in edits1(e1) if e2 in DICTIONARY) 31 | 32 | def known(words): return set(w for w in words if w in DICTIONARY) 33 | 34 | def correct(word): 35 | candidates = known([word]) or known(edits1(word)) or known_edits2(word) or [word] 36 | return max(candidates, key=DICTIONARY.get) 37 | 38 | def is_correct(word): 39 | return word in DICTIONARY 40 | 41 | -------------------------------------------------------------------------------- /bin/crab: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | script_dir=`dirname $0` 4 | python_cmd="python" 5 | [ -z "$CMSSW_VERSION" ] && echo "CMSSW is missing. You must do cmsenv first" && exit 6 | CMSSW_Major=$(echo $CMSSW_VERSION | cut -d_ -f2) # e.g. from CMSSW_13_1_x this is "13" 7 | CMSSW_Serie=$(echo $CMSSW_VERSION | cut -d_ -f3) # e.g. from CMSSW_10_6_x this is "6" 8 | 9 | # CRABClient can use python3 starting from CMSSW_10_2 10 | [ $CMSSW_Major -gt 10 ] && python_cmd="python3" 11 | [ $CMSSW_Major -eq 10 ] && [ $CMSSW_Serie -ge 2 ] && python_cmd="python3" 12 | 13 | # but for crab submit, need to stick with same python as cmsRun, to allow 14 | # proper processing of config files and proper PSet.pkl 15 | [ $CMSSW_Major -lt 12 ] && [ X$1 == Xsubmit ] && python_cmd="python" 16 | 17 | if [ $python_cmd = "python3" ] ; then 18 | # can also include Rucio client (we do not want to deal with old python2 client) 19 | RUCIO_ENV=`${script_dir}/rucio_env.sh` 20 | export RUCIO_HOME=`echo $RUCIO_ENV|cut -d' ' -f1` 21 | RUCIO_PYTHONPATH=`echo $RUCIO_ENV|cut -d' ' -f2` 22 | # alter PYTHONPATH here to keep flexibility to add or append as we find better 23 | export PYTHONPATH=$RUCIO_PYTHONPATH:$PYTHONPATH 24 | fi 25 | 26 | # avoid pythonpath contamination by user .local files 27 | export PYTHONNOUSERSITE=true 28 | 29 | # finally run crab client 30 | exec ${python_cmd} ${script_dir}/crab.py ${1+"$@"} 31 | -------------------------------------------------------------------------------- /etc/init-light.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | if [ -z "$CRAB_SOURCE_SCRIPT" ]; then 3 | CRAB_SOURCE_SCRIPT="/cvmfs/cms.cern.ch/crab3/crab_standalone.sh" 4 | fi 5 | 6 | function getVariableValue { 7 | VARNAME=$1 8 | SUBDIR=$2 9 | sh -c "source $CRAB_SOURCE_SCRIPT >/dev/null 2>/dev/null; if [ \$? -eq 0 ] && [ -d $VARNAME ]; \ 10 | then echo $VARNAME/$SUBDIR; else exit 1; fi" 11 | } 12 | 13 | CRAB3_BIN_ROOT=$(getVariableValue \$CRABCLIENT_ROOT bin) 14 | CRAB3_ETC_ROOT=$(getVariableValue \$CRABCLIENT_ROOT etc) 15 | CRAB3_PY_ROOT=$(getVariableValue \$CRABCLIENT_ROOT \$PYTHON_LIB_SITE_PACKAGES) 16 | DBS3_PY_ROOT=$(getVariableValue \$DBS3_CLIENT_ROOT \$PYTHON_LIB_SITE_PACKAGES) 17 | DBS3_PYCURL_ROOT=$(getVariableValue \$DBS3_PYCURL_CLIENT_ROOT \$PYTHON_LIB_SITE_PACKAGES) 18 | DBS3_CLIENT_ROOT=$(getVariableValue \$DBS3_CLIENT_ROOT) 19 | 20 | if [ $# -gt 0 ] && [ "$1" == "-csh" ]; then 21 | echo "setenv PYTHONPATH $CRAB3_PY_ROOT:$DBS3_PY_ROOT:$DBS3_PYCURL_ROOT:$PYTHONPATH; \ 22 | setenv PATH $CRAB3_BIN_ROOT:$PATH; setenv DBS3_CLIENT_ROOT $DBS3_CLIENT_ROOT" 23 | else 24 | export PYTHONPATH=$CRAB3_PY_ROOT:$DBS3_PY_ROOT:$DBS3_PYCURL_ROOT:$PYTHONPATH 25 | export PATH=$CRAB3_BIN_ROOT:$PATH 26 | export DBS3_CLIENT_ROOT 27 | if [ -n "$BASH" ]; then 28 | source $CRAB3_ETC_ROOT/crab-bash-completion.sh 29 | fi 30 | fi 31 | -------------------------------------------------------------------------------- /doc/crabclient/userdoc/installation.rst: -------------------------------------------------------------------------------- 1 | .. _installation-label: 2 | 3 | Installation 4 | ============ 5 | 6 | The setup of the local environment has the same logic than in previous `CRAB `_ versions. You need to setup the environment in the proper order, as listed here below: 7 | 8 | * source the LCG User Interface environment to get access to WLCG-affiliated resources in a fully transparent way; on LXPLUS users have it on AFS:: 9 | 10 | .. code-block:: console 11 | 12 | $ source /afs/cern.ch/cms/LCG/LCG-2/UI/cms_ui_env.csh 13 | 14 | 15 | * install CMSSW project and setup the relative environment:: 16 | 17 | .. code-block:: console 18 | 19 | $ mkdir MyTasks 20 | $ cd MyTasks 21 | $ cmsrel CMSSW_4_1_X 22 | $ cd CMSSW_4_1_X/src/ 23 | $ cmsenv 24 | 25 | 26 | * source the CRAB script file: in order to setup and use CRAB-3 from any directory, source the script /afs/cern.ch/user/g/grandi/public/CRAB3/client/CRABClient/crab.(c)sh, which points to the version of CRAB-3 maintained by the Integration team:: 27 | 28 | .. code-block:: console 29 | 30 | $ source /afs/cern.ch/user/g/grandi/public/CRAB3/client/CRABClient/crab.sh 31 | 32 | .. note:: The order in which are listed the points above it is important to avoid conflicts between the various software stacks involved. 33 | 34 | .. TODO Add RPM based installation 35 | -------------------------------------------------------------------------------- /src/python/CRABAPI/TestAPI.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | allow to test one CRAB command via the API 4 | usage: 5 | ./TestCommand.py 6 | examples: 7 | TestAPI.py checkusername 8 | TestAPI.py checkwrite --site T2_CH_CERN 9 | TestAPI.py checkwrite --site=T2_CH_CERN 10 | TestAPI.py checkdataset --dataset=/HIForward18/HIRun2023A-v1/RAW 11 | etc. 12 | """ 13 | 14 | import sys 15 | import CRABClient 16 | import pprint 17 | from CRABAPI.RawCommand import execRaw 18 | from CRABClient.ClientExceptions import ClientException 19 | 20 | def crab_cmd(command, arguments): 21 | 22 | try: 23 | output = crabCommand(command, arguments) 24 | return output 25 | except HTTPException as hte: 26 | print('Failed', command, ': %s' % (hte.headers)) 27 | except ClientException as cle: 28 | print('Failed', command, ': %s' % (cle)) 29 | 30 | def main(): 31 | 32 | if len(sys.argv) == 1: 33 | print('No command given') 34 | return 35 | command = sys.argv[1] 36 | print(f"testing: {command}") 37 | if len(sys.argv) > 2: 38 | arguments = sys.argv[2:] 39 | else: 40 | arguments = None 41 | 42 | result = execRaw(command, arguments) 43 | 44 | print (f"command returned:\n{pprint.pformat(result)}") 45 | 46 | 47 | if __name__ == '__main__': 48 | main() 49 | -------------------------------------------------------------------------------- /doc/crabclient/userdoc/introduction.rst: -------------------------------------------------------------------------------- 1 | Introduction 2 | =============== 3 | CRAB is an utility to create and submit CMSSW jobs to distributed computing resources. 4 | 5 | This twiki aims to help users already confident with `CRAB `_ submission tool to be able to access CMS distributed data through the next version of the tool, CRAB-3. This major version change of the tool has a relevant impact on user routines for job submission and job tracking since the whole architecture of the tool is changed, including the user interface. 6 | 7 | Important information 8 | +++++++++++++++++++++ 9 | 10 | * The CRAB-3 configuration is Python based; a script to translate old crab configuration to new one is available (:ref:`convert-config`). In this section you can find a description on how to create it manually with an example. 11 | 12 | * Client commands does not require anymore the ``-`` in front; as example ``crab -submit`` becomes ``crab submit``. 13 | 14 | * The ``crab (-)create`` command is not anymore available because the creation part is moved at server level; the workflow starts directly from the submission command (:ref:`submission`). 15 | 16 | * The ``--continue/-c`` is now replaced by ``--dir/-d`` option. 17 | 18 | * All outputs and logs are stored on a remote SE by default: ``get-output`` is no longer needed except to check your data. 19 | 20 | * Direct submission is not anymore supported. 21 | -------------------------------------------------------------------------------- /src/python/CRABAPI/Abstractions.py: -------------------------------------------------------------------------------- 1 | """ Task - top-level Task class """ 2 | from __future__ import print_function 3 | import CRABAPI.TopLevel 4 | import CRABClient.Commands.submit 5 | from WMCore.Configuration import Configuration 6 | class Task(object): 7 | """ 8 | Task - Wraps methods and attributes for a single analysis task 9 | """ 10 | def __init__(self, submitClass = CRABClient.Commands.submit.submit): 11 | self.config = Configuration() 12 | self.apiLog, self.clientLog, self.tracebackLog = \ 13 | CRABAPI.TopLevel.getAllLoggers() 14 | self.submitClass = submitClass 15 | 16 | def submit(self): 17 | """ 18 | submit - Sends the current task to the server. Returns requestID 19 | """ 20 | args = ['-c', self.config, '--proxy', '1'] 21 | submitCommand = self.submitClass(self.clientLog, args) 22 | retval = submitCommand() 23 | print("retval was %s" % retval) 24 | return retval['uniquerequestname'] 25 | 26 | def kill(self): 27 | """ 28 | kill - Tells the server to cancel the current task 29 | """ 30 | raise NotImplementedError 31 | 32 | def __getattr__(self, name): 33 | """ 34 | __getattr__ - expose certain values as attributes 35 | """ 36 | if name == 'jobs': 37 | raise NotImplementedError 38 | else: 39 | raise AttributeError 40 | -------------------------------------------------------------------------------- /src/python/CRABClient/Commands/getoutput.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | from __future__ import division 3 | 4 | from CRABClient.Commands.getcommand import getcommand 5 | 6 | class getoutput(getcommand): 7 | """ Retrieve the output files of a number of jobs specified by the -q/--quantity option. The task 8 | is identified by the -d/--dir option 9 | """ 10 | name = 'getoutput' 11 | shortnames = ['output', 'out'] 12 | visible = True #overwrite getcommand 13 | 14 | def __call__(self): 15 | returndict = getcommand.__call__(self, subresource = 'data2') 16 | 17 | # if something failed, getcommand raises exceptions, so if we got here it means OK 18 | returndict['commandStatus'] = 'SUCCESS' 19 | return returndict 20 | 21 | def setOptions(self): 22 | """ 23 | __setOptions__ 24 | 25 | This allows to set specific command options 26 | """ 27 | self.parser.add_option( '--quantity', 28 | dest = 'quantity', 29 | help = 'The number of output files you want to retrieve (or "all"). Ignored if --jobids is used.' ) 30 | self.parser.add_option( '--parallel', 31 | dest = 'nparallel', 32 | help = 'Number of parallel download, default is 10 parallel download.',) 33 | self.parser.add_option( '--wait', 34 | dest = 'waittime', 35 | help = 'Increase the sendreceive-timeout in second',) 36 | getcommand.setOptions(self) 37 | -------------------------------------------------------------------------------- /src/python/CRABAPI/TopLevel.py: -------------------------------------------------------------------------------- 1 | """ Module storing top-level functions that are exported to the CRABAPI 2 | package. These functions can also be accessed from CRABAPI.""" 3 | 4 | import logging 5 | 6 | API_LOGGER_NAME = 'CRAB3.CRABAPI' 7 | 8 | def getTask(taskName): 9 | """ Given a task name, initialize a task object""" 10 | taskName += "test" 11 | raise NotImplementedError("Need to implement this: Load %s" % taskName) 12 | 13 | def setLogging(apiLevel = logging.INFO, 14 | crabLevel = 100, 15 | crabTracebackLevel = 100): 16 | """Set logging parameters. Mutes the CRAB client by default. 17 | returns apiLogger""" 18 | crabLog = logging.getLogger('CRAB3') 19 | crabTracebackLog = logging.getLogger('CRAB3:traceback') 20 | apiLog = logging.getLogger(API_LOGGER_NAME) 21 | 22 | for oneLog, oneLevel in ( (apiLog, apiLevel), 23 | (crabLog, crabLevel), 24 | (crabTracebackLog, crabTracebackLevel) ): 25 | oneLog.setLevel(oneLevel) 26 | oneLog.logfile = "disabled_in_api" 27 | 28 | return apiLog 29 | 30 | def getLogger(suffix = ""): 31 | """ Helper function to get the logger back """ 32 | if suffix: 33 | suffix = "." + suffix 34 | return logging.getLogger(API_LOGGER_NAME + suffix) 35 | 36 | def getAllLoggers(suffix = ""): 37 | """ Helper function to get all the loggers - API, CRAB, CRABTraceback """ 38 | if suffix: 39 | suffix = "." + suffix 40 | return logging.getLogger(API_LOGGER_NAME + suffix), \ 41 | logging.getLogger('CRAB3'), \ 42 | logging.getLogger('CRAB3:traceback') 43 | -------------------------------------------------------------------------------- /src/python/CRABClient/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | CRAB Client modules 4 | """ 5 | __version__ = "development" 6 | 7 | #the __version__ will be automatically be change according to rpm for production 8 | 9 | import sys 10 | if (sys.version_info[0]*100+sys.version_info[1])>=312: 11 | import importlib 12 | def find_module(moduleName, moduleDir = None): 13 | if moduleDir: 14 | return importlib.machinery.PathFinder.find_spec(moduleName, [moduleDir]) 15 | return importlib.machinery.PathFinder.find_spec(moduleName) 16 | 17 | def load_module(moduleName, moduleSpec): 18 | moduleObj = importlib.util.module_from_spec(moduleSpec) 19 | sys.modules[moduleName] = moduleObj 20 | moduleSpec.loader.exec_module(moduleObj) 21 | return moduleObj 22 | 23 | def load_source(moduleName, moduleFile): 24 | moduleSpec = importlib.util.spec_from_file_location(moduleName, 25 | moduleFile) 26 | return load_module(moduleName, moduleSpec) 27 | 28 | def module_pathname(moduleObj): 29 | return moduleObj.origin 30 | else: 31 | import imp 32 | def find_module(moduleName, moduleDir = None): 33 | if moduleDir: 34 | return imp.find_module(moduleName, [moduleDir]) 35 | return imp.find_module(moduleName) 36 | 37 | def load_module(moduleName, moduleObj): 38 | return imp.load_module(moduleName, moduleObj[0], 39 | moduleObj[1], moduleObj[2]) 40 | 41 | def load_source(moduleName, moduleFile): 42 | return imp.load_source(moduleName, moduleFile) 43 | 44 | def module_pathname(moduleObj): 45 | return moduleObj[1] 46 | -------------------------------------------------------------------------------- /test/python/CRABClient_t/JobType_t/ScramEnvironment_t.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | 3 | """ 4 | _ScramEnvironment_t_ 5 | 6 | Unittests for ScramEnvironment module 7 | """ 8 | 9 | import logging 10 | import os 11 | import unittest 12 | 13 | from CRABClient.JobType.ScramEnvironment import ScramEnvironment 14 | 15 | class ScramEnvironmentTest(unittest.TestCase): 16 | """ 17 | unittest for ScramEnvironment class 18 | 19 | """ 20 | 21 | def setUp(self): 22 | """ 23 | Set up for unit tests 24 | """ 25 | 26 | self.testLogger = logging.getLogger('UNITTEST') 27 | self.testLogger.setLevel(logging.ERROR) 28 | ch = logging.StreamHandler() 29 | ch.setLevel(logging.DEBUG) 30 | self.testLogger.addHandler(ch) 31 | 32 | # set relevant variables 33 | 34 | self.arch = 'slc5_ia32_gcc434' 35 | self.version = 'CMSSW_3_8_7' 36 | self.base = '/tmp/CMSSW_3_8_7' 37 | 38 | os.environ['SCRAM_ARCH'] = self.arch 39 | os.environ['CMSSW_BASE'] = self.base 40 | os.environ['CMSSW_VERSION'] = self.version 41 | 42 | def tearDown(self): 43 | """ 44 | Do nothing 45 | """ 46 | return 47 | 48 | def testInit(self): 49 | """ 50 | Test constructor 51 | """ 52 | 53 | scram = ScramEnvironment(logger=self.testLogger) 54 | scram.getCmsswVersion() 55 | 56 | def testAccessors(self): 57 | """ 58 | Test various accessors 59 | """ 60 | 61 | scram = ScramEnvironment(logger=self.testLogger) 62 | 63 | self.assertEqual(scram.getCmsswVersion(), self.version) 64 | self.assertEqual(scram.getScramArch(), self.arch) 65 | self.assertEqual(scram.getCmsswBase(), self.base) 66 | 67 | 68 | 69 | if __name__ == '__main__': 70 | unittest.main() 71 | 72 | 73 | -------------------------------------------------------------------------------- /src/python/CRABClient/WMCoreConfigWrapper.py: -------------------------------------------------------------------------------- 1 | """ 2 | This is a temporary file used to help transitioning to a 3 | CRABClient which has no dependency on WMCore. Its only purpose 4 | is to allow users to keep having this line in crabConfig.py: 5 | from WMCore.Configuration import Configuration 6 | instead of : 7 | from CRABClient.Configuration import Configuration 8 | 9 | File is meant to be renamed and moved by the CRABClient build procedure 10 | (crab-build.file in cmsdist) or an ad hoc setup script in case a developer wants 11 | to run using source from GH, so that python discovers it as 12 | WMCore.Configuration. When executed, the code here prints a simple warning 13 | asking the user to update the configuration and calls 14 | CRABClient.Configuration. In CRABClient.Configuration we have 15 | a modified (and frozen) version of WMCore.Configuration which 16 | runs in both python2 and python3 without requiring external 17 | dependencies which are not available in CMSSW_8 or earlier, e.g. "future" 18 | """ 19 | 20 | from __future__ import division 21 | from CRABClient.Configuration import Configuration as Config 22 | from CRABClient.ClientUtilities import colors 23 | 24 | import logging 25 | logger = logging.getLogger("CRAB3.all") 26 | 27 | class Configuration(Config): 28 | def __init__(self): 29 | msg = '' 30 | msg += '%sWarning: CRABClient does not depend on WMCore anymore.\n' % (colors.RED) 31 | msg += 'Please update your config file to use configuration from CRABClient instead.\n' 32 | msg += 'Change inside that file from "from WMCore.Configuration import Configuration"\n' 33 | msg += 'to "from CRABClient.Configuration import Configuration%s\n"' % (colors.NORMAL) 34 | msg += 'Support for old style "from WMCore.Configuration import ..." will be remove in future versions of CRAB' 35 | logger.info(msg) 36 | Config.__init__(self) 37 | 38 | -------------------------------------------------------------------------------- /doc/crabclient/conf.py: -------------------------------------------------------------------------------- 1 | import sys, os 2 | 3 | from CRABClient import __version__ as cc_version 4 | 5 | extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 6 | 'sphinx.ext.intersphinx', 'sphinx.ext.todo', 7 | 'sphinx.ext.ifconfig', 'sphinx.ext.inheritance_diagram', 8 | 'sphinx.ext.viewcode'] 9 | templates_path = ['_templates'] 10 | source_suffix = '.rst' 11 | source_encoding = 'utf-8' 12 | master_doc = 'index' 13 | project = 'CMS CRAB Client' 14 | copyright = 'CERN, INFN and Fermilab' 15 | version = cc_version 16 | release = cc_version 17 | today_fmt = '%B %d, %Y' 18 | #unused_docs = [] 19 | #exclude_trees = [] 20 | #default_role = None 21 | add_function_parentheses = True 22 | add_module_names = True 23 | show_authors = True 24 | pygments_style = 'sphinx' 25 | #modindex_common_prefix = [] 26 | 27 | autoclass_content = 'both' 28 | 29 | html_theme = 'sphinxdoc' 30 | #html_theme_options = {} 31 | #html_theme_path = [] 32 | html_title = 'CRAB Client, version %s ' % version 33 | #html_short_title = None 34 | #html_logo = None 35 | #html_favicon = None 36 | #html_static_path = ['_static'] 37 | html_last_updated_fmt = '%b %d, %Y' 38 | #html_use_smartypants = True 39 | #html_sidebars = {} 40 | #html_additional_pages = {} 41 | #html_use_modindex = True 42 | #html_use_index = True 43 | #html_split_index = False 44 | html_show_sourcelink = False 45 | #html_use_opensearch = '' 46 | #html_file_suffix = '' 47 | htmlhelp_basename = 'CMSCRABClientdoc' 48 | 49 | latex_paper_size = 'a4' 50 | #latex_font_size = '10pt' 51 | # (source start file, target name, title, author, documentclass [howto/manual]). 52 | latex_documents = [ 53 | ('index', 'CMSCRABClient.tex', u'CMS CRAB Client Documentation', 54 | u'CMS DM/WM Project', 'manual'), 55 | ] 56 | #latex_logo = None 57 | #latex_use_parts = False 58 | #latex_preamble = '' 59 | #latex_appendices = [] 60 | #latex_use_modindex = True 61 | 62 | # Example configuration for intersphinx: refer to the Python standard library. 63 | intersphinx_mapping = {'http://docs.python.org/': None} 64 | -------------------------------------------------------------------------------- /src/python/CRABClient/Commands/proceed.py: -------------------------------------------------------------------------------- 1 | from __future__ import division 2 | from __future__ import print_function 3 | 4 | import sys 5 | 6 | if sys.version_info >= (3, 0): 7 | from urllib.parse import urlencode # pylint: disable=E0611 8 | if sys.version_info < (3, 0): 9 | from urllib import urlencode 10 | 11 | from CRABClient.Commands.SubCommand import SubCommand 12 | from CRABClient.ClientExceptions import RESTCommunicationException 13 | 14 | class proceed(SubCommand): 15 | """ 16 | Continue submission of a task which was initialized with 'crab submit --dryrun' 17 | """ 18 | 19 | def __init__(self, logger, cmdargs=None): 20 | SubCommand.__init__(self, logger, cmdargs) 21 | 22 | def __call__(self): 23 | server = self.crabserver 24 | 25 | msg = "Continuing submission of task %s" % (self.cachedinfo['RequestName']) 26 | self.logger.debug(msg) 27 | 28 | request = {'workflow': self.cachedinfo['RequestName'], 'subresource': 'proceed'} 29 | 30 | self.logger.info("Sending the request to the server") 31 | self.logger.debug("Submitting %s " % str(request)) 32 | result, status, reason = server.post(api=self.defaultApi, data=urlencode(request)) 33 | self.logger.debug("Result: %s" % (result)) 34 | if status != 200: 35 | msg = "Problem continuing task submission:\ninput:%s\noutput:%s\nreason:%s" \ 36 | % (str(request), str(result), str(reason)) 37 | raise RESTCommunicationException(msg) 38 | msg = "Task continuation request successfully sent to the CRAB3 server" 39 | if result['result'][0]['result'] != 'ok': 40 | msg += "\nServer responded with: '%s'" % (result['result'][0]['result']) 41 | resultDict = {'status': 'FAILED'} 42 | else: 43 | resultDict = {'status': 'SUCCESS'} 44 | self.logger.info("To check task progress, use 'crab status'") 45 | self.logger.info(msg) 46 | resultDict['commandStatus'] = resultDict['status'] # add, do not override status key, in case someone was using it 47 | 48 | return resultDict 49 | -------------------------------------------------------------------------------- /src/python/CRABClient/JobType/LumiMask.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module to handle lumiMask.json file 3 | """ 4 | import sys 5 | if sys.version_info >= (3, 0): 6 | from urllib.parse import urlparse # pylint: disable=E0611 7 | if sys.version_info < (3, 0): 8 | from urlparse import urlparse 9 | 10 | try: 11 | from FWCore.PythonUtilities.LumiList import LumiList 12 | except Exception: # pylint: disable=broad-except 13 | # if FWCore version is not py3 compatible, use our own 14 | from CRABClient.LumiList import LumiList 15 | 16 | from CRABClient.ClientExceptions import ConfigurationException 17 | 18 | 19 | def getLumiList(lumi_mask_name, logger = None): 20 | """ 21 | Takes a lumi-mask and returns a LumiList object. 22 | lumi-mask: either an http address or a json file on disk. 23 | """ 24 | lumi_list = None 25 | parts = urlparse(lumi_mask_name) 26 | if parts[0] in ['http', 'https']: 27 | if logger: 28 | logger.debug('Downloading lumi-mask from %s' % lumi_mask_name) 29 | try: 30 | lumi_list = LumiList(url = lumi_mask_name) 31 | except Exception as err: 32 | raise ConfigurationException("CMSSW failed to get lumimask from URL. Please try to download the lumimask yourself and point to it in crabConfig;\n%s" % str(err)) 33 | else: 34 | if logger: 35 | logger.debug('Reading lumi-mask from %s' % lumi_mask_name) 36 | try: 37 | lumi_list = LumiList(filename = lumi_mask_name) 38 | except IOError as err: 39 | raise ConfigurationException("Problem loading lumi-mask file; %s" % str(err)) 40 | 41 | return lumi_list 42 | 43 | 44 | def getRunList(myrange): 45 | """ 46 | Take a string like '1,2,5-8' and return a list of integers [1,2,5,6,7,8]. 47 | """ 48 | myrange = myrange.replace(' ','') 49 | if not myrange: 50 | return [] 51 | 52 | myrange = myrange.split(',') 53 | result = [] 54 | for element in myrange: 55 | if element.count('-') > 0: 56 | mySubRange = element.split('-') 57 | subInterval = range( int(mySubRange[0]), int(mySubRange[1])+1) 58 | result.extend(subInterval) 59 | else: 60 | result.append(int(element)) 61 | 62 | return result 63 | -------------------------------------------------------------------------------- /src/python/CRABClient/Commands/kill.py: -------------------------------------------------------------------------------- 1 | from __future__ import division 2 | from __future__ import print_function 3 | 4 | import sys 5 | 6 | if sys.version_info >= (3, 0): 7 | from urllib.parse import urlencode # pylint: disable=E0611 8 | if sys.version_info < (3, 0): 9 | from urllib import urlencode 10 | 11 | from CRABClient.Commands.SubCommand import SubCommand 12 | from CRABClient.ClientExceptions import RESTCommunicationException 13 | 14 | class kill(SubCommand): 15 | """ 16 | Command to kill submitted tasks. The user must give the crab project 17 | directory for the task he/she wants to kill. 18 | """ 19 | visible = True 20 | 21 | def __call__(self): 22 | server = self.crabserver 23 | 24 | self.logger.debug("Killing task %s" % self.cachedinfo['RequestName']) 25 | inputs = {'workflow' : self.cachedinfo['RequestName']} 26 | if self.options.killwarning: 27 | inputs.update({'killwarning' : self.options.killwarning}) 28 | 29 | dictresult, status, reason = server.delete(api=self.defaultApi, data=urlencode(inputs)) 30 | self.logger.debug("Result: %s" % dictresult) 31 | 32 | if status != 200: 33 | msg = "Problem killing task %s:\ninput:%s\noutput:%s\nreason:%s" % \ 34 | (self.cachedinfo['RequestName'], str(self.cachedinfo['RequestName']), str(dictresult), str(reason)) 35 | raise RESTCommunicationException(msg) 36 | 37 | self.logger.info("Kill request successfully sent") 38 | if dictresult['result'][0]['result'] != 'ok': 39 | resultdict = {'status' : 'FAILED'} 40 | self.logger.info(dictresult['result'][0]['result']) 41 | else: 42 | resultdict = {'status' : 'SUCCESS'} 43 | resultdict['commandStatus'] = resultdict['status'] # add, do not override status key, in case someone was using it 44 | 45 | return resultdict 46 | 47 | def setOptions(self): 48 | """ 49 | __setOptions__ 50 | 51 | This allows to set specific command options 52 | """ 53 | self.parser.add_option('--killwarning', 54 | dest='killwarning', 55 | default=None, 56 | help='A warning message to be appended to the warnings list shown by "crab status"') 57 | 58 | def validateOptions(self): 59 | SubCommand.validateOptions(self) 60 | -------------------------------------------------------------------------------- /test/python/CRABClient_t/Client_t.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | """ 4 | Client_t.py 5 | """ 6 | 7 | from WMCore.Configuration import Configuration 8 | import os 9 | import unittest 10 | import logging 11 | import socket 12 | from CRABClient import Handler 13 | 14 | class ClientTest(unittest.TestCase): 15 | # TODO: WOrk out how to mock the server for stand alone testing 16 | def setUp(self): 17 | self.client = Handler() 18 | logging.basicConfig(level=logging.DEBUG) 19 | self.client.logger = logging.getLogger('CRAB3 client tests') 20 | 21 | def tearDown(self): 22 | """ 23 | Standard tearDown 24 | 25 | """ 26 | 27 | #os.system('rm -rf crab_%s' % testConfig.General.requestName) 28 | pass 29 | 30 | def testBadCommand(self): 31 | """ 32 | Test executing a command that doesn't exist, make sure a KeyError 33 | is raised and that the return code is 1 34 | """ 35 | commando = 'foo' 36 | self.client.loadConfig(testConfig) 37 | 38 | self.assertRaises(KeyError, self.client, commando, {}) 39 | 40 | def testNoServer(self): 41 | """ 42 | For each command that interacts with the server make sure that a 43 | socket error is raised when the server isn't present. 44 | """ 45 | self.client.loadConfig(testConfig) 46 | # self.client.initialise('submit') 47 | 48 | self.assertRaises(socket.error, self.client, 'status', {}) 49 | #self.assertRaises(socket.error, self.client.runCommand, 'status', {'task': []}) 50 | 51 | 52 | testConfig = Configuration() 53 | 54 | ## General options for the client 55 | testConfig.section_("General") 56 | testConfig.General.serverUrl = 'fake.server:8080' 57 | testConfig.General.requestName = 'MyAnalysis' 58 | 59 | ## Specific option of the job type 60 | ## these options are directly readable from the job type plugin 61 | testConfig.section_("JobType") 62 | testConfig.JobType.pluginName = 'Example' 63 | testConfig.JobType.psetName = 'pset.py' 64 | testConfig.JobType.inputFiles = ['/tmp/input_file'] 65 | 66 | ## Specific data options 67 | testConfig.section_("Data") 68 | testConfig.Data.inputDatasetList = ['/cms/data/set'] 69 | testConfig.Data.lumiSectionFile = '/file/path/name' 70 | 71 | ## User options 72 | testConfig.section_("User") 73 | testConfig.User.role = '/cms/integration' 74 | 75 | 76 | if __name__ == "__main__": 77 | unittest.main() 78 | -------------------------------------------------------------------------------- /src/python/CRABClient/Commands/createmyproxy.py: -------------------------------------------------------------------------------- 1 | from datetime import timedelta 2 | 3 | from CRABClient.Commands.SubCommand import SubCommand 4 | 5 | from CRABClient.CredentialInteractions import CredentialInteractions 6 | from CRABClient.ClientUtilities import server_info 7 | 8 | class createmyproxy(SubCommand): 9 | """ 10 | creates a new credential in myproxy valid for --days days 11 | if existing credential already lasts longer, it is not changed 12 | """ 13 | 14 | name = 'createmyproxy' 15 | 16 | def __init__(self, logger, cmdargs=None): 17 | SubCommand.__init__(self, logger, cmdargs, disable_interspersed_args=True) 18 | 19 | self.configreq = None 20 | self.configreq_encoded = None 21 | 22 | def __call__(self): 23 | 24 | credentialHandler = CredentialInteractions(self.logger) 25 | days = self.options.days 26 | 27 | credentialHandler.setMyProxyValidity(int(days) * 24 * 60) # minutes 28 | # give a bit of slack to the threshold, avoid that repeating the c 29 | timeLeftThreshold = int(days-1) * 24 * 60 * 60 # seconds 30 | 31 | self.logger.info("Checking credentials") 32 | 33 | # need an X509 proxy in order to talk with CRABServer to get list of myproxy authorized retrievers 34 | credentialHandler.createNewVomsProxy(timeLeftThreshold=720) 35 | alldns = server_info(crabserver=self.crabserver, subresource='delegatedn') 36 | for authorizedDNs in alldns['services']: 37 | credentialHandler.setRetrievers(authorizedDNs) 38 | self.logger.info("Registering user credentials in myproxy") 39 | (credentialName, myproxyTimeleft) = credentialHandler.createNewMyProxy(timeleftthreshold=timeLeftThreshold) 40 | self.logger.info("Credential exists on myproxy: username: %s - validity: %s", credentialName, 41 | str(timedelta(seconds=myproxyTimeleft))) 42 | return {'commandStatus': 'SUCCESS'} 43 | 44 | 45 | def setOptions(self): 46 | """ 47 | __setOptions__ 48 | 49 | This allows to set specific command options. 50 | """ 51 | 52 | self.parser.add_option('--days', 53 | dest='days', 54 | default=30, 55 | type='int', 56 | help="Set the validity (in days) for the credential. Default is 30.") 57 | 58 | #def terminate(self, exitcode): 59 | # pass 60 | -------------------------------------------------------------------------------- /doc/config/FullConfiguration.py: -------------------------------------------------------------------------------- 1 | """ 2 | This is a test example of configuration file for CRAB-3 client 3 | """ 4 | 5 | from WMCore.Configuration import Configuration 6 | import os 7 | 8 | config = Configuration() 9 | 10 | ## General options for the client 11 | config.section_("General") 12 | #config.General.requestName = 'MyAnalysis_1' 13 | #config.General.workArea = '/path/to/workarea' 14 | # This identify which type of server you're using. If it is a private instance you must set 'other' 15 | # (other options are prod/preprod/dev/test) 16 | #config.General.instance = 'other' 17 | #config.General.restHost = 'yourserver.cern.ch' 18 | #config.General.dbInstance = 'dev' 19 | 20 | ## Specific option of the job type 21 | ## these options are directly readable from the job type plugin 22 | config.section_("JobType") 23 | #config.JobType.pluginName = 'Analysis' 24 | ## The plugin for MC Private Production 25 | #config.JobType.pluginName = 'PrivateMC' 26 | #config.JobType.psetName = 'pset.py' 27 | ## Does the job read any additional private file: 28 | #config.JobType.inputFiles = ['/tmp/input_file'] 29 | ## Does the job write any output files that need to be collected BESIDES those in output modules or TFileService 30 | #config.JobType.outputFiles = ['output_file'] 31 | 32 | 33 | ## Specific data options 34 | config.section_("Data") 35 | #config.Data.inputDataset = '/cms/data/set' 36 | #config.Data.outputDatasetTag = 'MyReskimForTwo' 37 | ## Splitting Algorithms 38 | #config.Data.splitting = 'LumiBased' 39 | #config.Data.splitting = 'EventBased' 40 | #config.Data.splitting = 'FileBased' 41 | #config.Data.unitsPerJob = 10 42 | 43 | ## For lumiMask http and https urls are also allowed 44 | #config.Data.lumiMask = 'lumi.json' 45 | 46 | ## If you are splitting a Private MC Production task 47 | ## you must specify the total amount of data to generate 48 | #config.Data.splitting = 'EventBased' 49 | #config.Data.unitsPerJob = 10 50 | #config.Data.totalUnits = 100 51 | 52 | 53 | ## To publish produced data there are 3 parameters to set: 54 | #config.Data.publication = True 55 | #config.Data.inputDBS = "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet" 56 | #config.Data.publishDBS = "https://cmsdbsprod.cern.ch:8443/cms_dbs_ph_analysis_02_writer/servlet/DBSServlet" 57 | 58 | ## User options 59 | config.section_("User") 60 | #config.User.voRole = 't1access' 61 | #config.User.voGroup = 'integration' 62 | #config.User.team = 'Analysis' 63 | #config.User.group = 'Analysis' 64 | #config.User.email = '' 65 | 66 | config.section_("Site") 67 | config.Site.storageSite = 'T2_XX_XXX' 68 | #config.Site.whitelist = ['T2_XY_XXY'] 69 | #config.Site.blacklist = ['T2_XZ_XXZ'] 70 | #config.Site.removeT1Blacklisting = False 71 | -------------------------------------------------------------------------------- /src/python/CRABAPI/RawCommand.py: -------------------------------------------------------------------------------- 1 | """ 2 | CRABAPI.RawCommand - wrapper if one wants to simply execute a CRAB command 3 | but doesn't want to subprocess.Popen() 4 | """ 5 | import CRABAPI 6 | 7 | import traceback 8 | 9 | from CRABClient.ClientUtilities import initLoggers, flushMemoryLogger, removeLoggerHandlers 10 | 11 | 12 | # NOTE: Not included in unittests 13 | def crabCommand(command, *args, **kwargs): 14 | """ crabComand - executes a given command with certain arguments and returns 15 | the raw result back from the client. Arguments are... 16 | """ 17 | #Converting all arguments to a list. Adding '--' and '=' 18 | arguments = [] 19 | for key, val in kwargs.items(): 20 | if isinstance(val, bool): 21 | if val: 22 | arguments.append('--'+str(key)) 23 | else: 24 | arguments.append('--'+str(key)) 25 | arguments.append(val) 26 | arguments.extend(list(args)) 27 | 28 | return execRaw(command, arguments) 29 | 30 | 31 | # NOTE: Not included in unittests 32 | def execRaw(command, args): 33 | """ 34 | execRaw - executes a given command with certain arguments and returns 35 | the raw result back from the client. args is a python list, 36 | the same python list parsed by the optparse module 37 | Every command returns a dictionary of the form 38 | {'commandStatus': status, key: val, key: val ....} 39 | where status can have the values 'SUCCESS' or 'FAILED' 40 | and the other keys and values are command dependent ! 41 | """ 42 | tblogger, logger, memhandler = initLoggers() 43 | 44 | try: 45 | mod = __import__('CRABClient.Commands.%s' % command, fromlist=command) 46 | except ImportError: 47 | raise CRABAPI.BadArgumentException( \ 48 | 'Could not find command "%s"' % command) 49 | 50 | try: 51 | cmdobj = getattr(mod, command)(logger, args) 52 | res = cmdobj() 53 | except SystemExit as se: 54 | # most likely an error from the OptionParser in Subcommand. 55 | # CRABClient #4283 should make this less ugly 56 | if se.code == 2: 57 | raise CRABAPI.BadArgumentException 58 | else: 59 | # We can reach here if the PSet raises a SystemExit exception 60 | # Without this, CRAB raises a confusing UnboundLocalError 61 | logger.error('PSet raised a SystemExit. Traceback follows:') 62 | logger.error(traceback.format_exc()) 63 | raise 64 | finally: 65 | flushMemoryLogger(tblogger, memhandler, logger.logfile) 66 | removeLoggerHandlers(tblogger) 67 | removeLoggerHandlers(logger) 68 | return res 69 | -------------------------------------------------------------------------------- /doc/FullConfiguration.py: -------------------------------------------------------------------------------- 1 | """ 2 | This is an example configuration file for CRAB3 client, covering 3 | most of the available configuration options. 4 | """ 5 | 6 | from WMCore.Configuration import Configuration 7 | import os 8 | 9 | config = Configuration() 10 | 11 | ## General options for the client 12 | config.section_("General") 13 | 14 | # Request name must be specified 15 | #config.General.requestName = 'MyAnalysis_1' 16 | #config.General.workArea = '/path/to/workarea' 17 | 18 | # Specify a custom server. This is almost never used; the defaults will point at the central server. 19 | ## https schema is used. If the port is not specified default 443 will be used 20 | #config.General.serverUrl = 'yourserver[:0000]' 21 | # This identify which type of server you're using. If it is a private instance you must set 'other' 22 | # (other options are prod/preprod/dev/test) 23 | #config.General.instance = 'other' 24 | #config.General.restHost = 'yourserver.cern.ch' 25 | #config.General.dbInstance = 'dev' 26 | # 27 | 28 | ## Specific option of the job type 29 | ## these options are directly readable from the job type plugin 30 | config.section_("JobType") 31 | #config.JobType.pluginName = 'Analysis' 32 | ## The plugin for MC Private Production 33 | #config.JobType.pluginName = 'PrivateMC' 34 | config.JobType.psetName = 'pset.py' 35 | ## Does the job read any additional private file: 36 | #config.JobType.inputFiles = ['/tmp/input_file'] 37 | ## Does the job write any output files that need to be collected BESIDES those in output modules or TFileService 38 | #config.JobType.outputFiles = ['output_file'] 39 | 40 | 41 | ## Specific data options 42 | config.section_("Data") 43 | #config.Data.inputDataset = '/cms/data/set' 44 | #config.Data.outputDatasetTag = 'MyReskimForTwo' 45 | ## Splitting Algorithms 46 | #config.Data.splitting = 'LumiBased' 47 | #config.Data.splitting = 'EventBased' 48 | #config.Data.splitting = 'FileBased' 49 | #config.Data.unitsPerJob = 10 50 | 51 | ## For lumiMask http and https urls are also allowed 52 | #config.Data.lumiMask = 'lumi.json' 53 | 54 | ## If you are splitting a Private MC Production task 55 | ## you must specify the total amount of data to generate 56 | #config.Data.splitting = 'EventBased' 57 | #config.Data.unitsPerJob = 10 58 | #config.Data.totalUnits = 100 59 | 60 | 61 | ## To publish produced data there are 3 parameters to set: 62 | #config.Data.publication = True 63 | #config.Data.inputDBS = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader" 64 | #config.Data.publishDBS = "https://cmsweb.cern.ch/dbs/prod/prod/phys03/DBSWriter" 65 | 66 | ## User options 67 | config.section_("User") 68 | #config.User.voRole = 't1access' 69 | #config.User.voGroup = 'integration' 70 | #config.User.team = 'Analysis' 71 | #config.User.group = 'Analysis' 72 | #config.User.email = '' 73 | 74 | config.section_("Site") 75 | config.Site.storageSite = 'T2_XX_XXX' 76 | #config.Site.whitelist = ['T2_XY_XXY'] 77 | #config.Site.blacklist = ['T2_XZ_XXZ'] 78 | #config.Site.removeT1Blacklisting = False 79 | 80 | -------------------------------------------------------------------------------- /src/python/CRABClient/Commands/uploadlog.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from CRABClient.Commands.SubCommand import SubCommand 4 | from CRABClient.ClientUtilities import colors, uploadlogfile 5 | from CRABClient.ClientExceptions import ConfigurationException, MissingOptionException 6 | 7 | 8 | class uploadlog(SubCommand): 9 | """ 10 | Upload the crab.log file (or any "log" file) to the CRAB User File Cache. 11 | The main purpose of this command is to make log files available to experts for 12 | debugging. The command accepts a path to a CRAB project directory via the 13 | -d/--dir option, in which case it will search for the crab.log file inside the 14 | directory. The log will be uplaoded in S3 in the task directory, if a task name 15 | is not available (crab submit fails in early stage) user will have to send the 16 | log in the mail. It will be a short log in that case. 17 | Usage: 18 | * crab uploadlog --dir= 19 | """ 20 | name = 'uploadlog' 21 | shortnames = ['uplog'] 22 | 23 | def __call__(self): 24 | self.logger.debug("uploadlog started") 25 | taskname = None 26 | #veryfing the log file exist 27 | if os.path.isfile(self.logfile): 28 | self.logger.debug("crab.log exists") 29 | try: 30 | taskname = self.cachedinfo['RequestName'] 31 | logfilename = str(taskname)+".log" 32 | except Exception: 33 | self.logger.info("Couldn't get information from .requestcache (file likely not created due to submission failure),\n" + 34 | "Please locate crab.log yourself and copy/paste into the mail to support if needed") 35 | return {'commandStatus': 'FAILED'} 36 | else: 37 | msg = "%sError%s: Could not locate log file." % (colors.RED, colors.NORMAL) 38 | self.logger.info(msg) 39 | raise ConfigurationException 40 | 41 | self.logger.info("Will upload file %s." % (self.logfile)) 42 | logfileurl = uploadlogfile(self.logger, self.proxyfilename, taskname=taskname, logfilename=logfilename, 43 | logpath=str(self.logfile), instance=self.instance, 44 | serverurl=self.serverurl) 45 | return {'commandStatus': 'SUCCESS', 46 | 'result': {'status': 'SUCCESS', 'logurl': logfileurl}} 47 | 48 | 49 | def setOptions(self): 50 | """ 51 | __setOptions__ 52 | 53 | This allows to set specific command options 54 | """ 55 | return 56 | 57 | 58 | def validateOptions(self): 59 | ## Do the options validation from SubCommand. 60 | try: 61 | SubCommand.validateOptions(self) 62 | except MissingOptionException as ex: 63 | if ex.missingOption == "task": 64 | msg = "%sError%s:" % (colors.RED, colors.NORMAL) 65 | msg += " Please provide a path to a CRAB project directory (use the -d/--dir option)." 66 | ex = MissingOptionException(msg) 67 | ex.missingOption = "task" 68 | raise ex 69 | -------------------------------------------------------------------------------- /src/python/CRABClient/ClientExceptions.py: -------------------------------------------------------------------------------- 1 | class ClientException(Exception): 2 | """ 3 | general client exception 4 | Each subclass must define the command line exit code associated with the exception 5 | exitcode > 3000 does not print logfile at the end of the command. 6 | """ 7 | exitcode = 3001 8 | pass 9 | 10 | 11 | class TaskNotFoundException(ClientException): 12 | """ 13 | Raised when the task directory is not found. 14 | """ 15 | exitcode = 3004 16 | 17 | class CachefileNotFoundException(ClientException): 18 | """ 19 | Raised when the .requestcache file is not found inside the Task directory. 20 | """ 21 | exitcode = 3005 22 | 23 | class ConfigException(ClientException): 24 | """ 25 | Raised when there are issues with the config cache. 26 | """ 27 | exitcode = 3006 28 | 29 | class InputFileNotFoundException(ClientException): 30 | """ 31 | Raised when a file in config.JobType.inputFiles cannot be found. 32 | """ 33 | exitcode = 3007 34 | 35 | class ConfigurationException(ClientException): 36 | """ 37 | Raised when there is an issue with configuration/command line parameters. 38 | """ 39 | exitcode = 3008 40 | 41 | class MissingOptionException(ConfigurationException): 42 | """ 43 | Raised when a mandatory option is not found in the command line. 44 | """ 45 | exitcode = 3008 46 | missingOption = None 47 | 48 | class RESTCommunicationException(ClientException): 49 | """ 50 | Raised when the REST does not answer the 200 HTTP code. 51 | """ 52 | exitcode = 3009 53 | 54 | class ProxyCreationException(ClientException): 55 | """ 56 | Raised when there is a problem in proxy creation. exitcode > 2000 prints log 57 | """ 58 | exitcode = 3010 59 | 60 | class EnvironmentException(ClientException): 61 | """ 62 | Raised when there is a problem in the environment where the client is executed. 63 | E.g.: if the X509_CERT_DIR variable is not set we raise this exception. 64 | """ 65 | exitcode = 3011 66 | 67 | class UsernameException(ClientException): 68 | """ 69 | Raised when there is a problem with the username (e.g. when retrieving it from SiteDB or CRIC). 70 | """ 71 | exitcode = 3012 72 | 73 | class ProxyException(ClientException): 74 | """ 75 | Raised when there is a problem with the proxy (e.g. proxy file not found). 76 | """ 77 | exitcode = 3013 78 | 79 | class UnknownOptionException(ClientException): 80 | """ 81 | Raised when an unknown option is specified in the command line. 82 | """ 83 | exitcode = 3014 84 | 85 | class SandboxTooBigException(ClientException): 86 | """ 87 | Raised when user ask client to send to server more bytes then we allow 88 | """ 89 | exitcode = 3015 90 | 91 | 92 | class CommandFailedException(ClientException): 93 | """ 94 | Command completed, but encountered a failure (e.g. a check failed, or 95 | some information could not be retrieved 96 | """ 97 | exitcode = 3100 98 | 99 | class RESTInterfaceException(ClientException): 100 | """ 101 | Errors coming from interaction with REST interface 102 | """ 103 | exitcode = 3016 104 | 105 | class RucioClientException(ClientException): 106 | """ 107 | Errors coming from interaction with REST interface 108 | """ 109 | exitcode = 3017 110 | -------------------------------------------------------------------------------- /doc/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | 3 | # You can set these variables from the command line. 4 | SPHINXOPTS = 5 | SPHINXBUILD = sphinx-build 6 | PAPER = 7 | BUILDDIR = build 8 | PROJECT = crabclient 9 | 10 | # Internal variables. 11 | PAPEROPT_a4 = -D latex_paper_size=a4 12 | PAPEROPT_letter = -D latex_paper_size=letter 13 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) $(PROJECT) 14 | 15 | .PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest 16 | 17 | help: 18 | @echo "Please use \`make ' where is one of" 19 | @echo " html to make standalone HTML files" 20 | @echo " dirhtml to make HTML files named index.html in directories" 21 | @echo " pickle to make pickle files" 22 | @echo " json to make JSON files" 23 | @echo " htmlhelp to make HTML files and a HTML help project" 24 | @echo " qthelp to make HTML files and a qthelp project" 25 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 26 | @echo " changes to make an overview of all changed/added/deprecated items" 27 | @echo " linkcheck to check all external links for integrity" 28 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 29 | 30 | clean: 31 | -rm -rf $(BUILDDIR)/* 32 | 33 | $(PROJECT): 34 | [ -d $(PROJECT) ] || cp -rp code $(PROJECT) 35 | 36 | apidoc: $(PROJECT) 37 | python generate_modules.py -f -d $(PROJECT)/code -s rst $${PYTHONPATH%%:*}/CRABClient 38 | cp references.rst $(PROJECT) 39 | 40 | html: apidoc 41 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 42 | @echo 43 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 44 | 45 | dirhtml: apidoc 46 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 47 | @echo 48 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 49 | 50 | pickle: apidoc 51 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 52 | @echo 53 | @echo "Build finished; now you can process the pickle files." 54 | 55 | json: apidoc 56 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 57 | @echo 58 | @echo "Build finished; now you can process the JSON files." 59 | 60 | htmlhelp: apidoc 61 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 62 | @echo 63 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 64 | ".hhp project file in $(BUILDDIR)/htmlhelp." 65 | 66 | qthelp: apidoc 67 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 68 | @echo 69 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 70 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 71 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/FileMoverService.qhcp" 72 | @echo "To view the help file:" 73 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/FileMoverService.qhc" 74 | 75 | latex: apidoc 76 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 77 | @echo 78 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 79 | @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ 80 | "run these through (pdf)latex." 81 | 82 | changes: apidoc 83 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 84 | @echo 85 | @echo "The overview file is in $(BUILDDIR)/changes." 86 | 87 | linkcheck: apidoc 88 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 89 | @echo 90 | @echo "Link check complete; look for any errors in the above output " \ 91 | "or in $(BUILDDIR)/linkcheck/output.txt." 92 | 93 | doctest: apidoc 94 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 95 | @echo "Testing of doctests in the sources finished, look at the " \ 96 | "results in $(BUILDDIR)/doctest/output.txt." 97 | -------------------------------------------------------------------------------- /src/python/CRABClient/Commands/remake.py: -------------------------------------------------------------------------------- 1 | """ 2 | re-creates the working directory for a task 3 | """ 4 | 5 | # avoid complains about things that we can not fix in python2 6 | # pylint: disable=consider-using-f-string, unspecified-encoding, raise-missing-from 7 | 8 | import re 9 | import pickle 10 | import os 11 | 12 | from CRABClient.Commands.SubCommand import SubCommand 13 | from CRABClient.ClientUtilities import colors, PKL_W_MODE 14 | from CRABClient.ClientUtilities import commandUsedInsideCrab 15 | from CRABClient.ClientExceptions import MissingOptionException,ConfigurationException 16 | 17 | 18 | class remake(SubCommand): 19 | """ 20 | Remake the .requestcache 21 | """ 22 | name = 'remake' 23 | shortnames = ['rmk'] 24 | 25 | def __call__(self): 26 | if not commandUsedInsideCrab(): 27 | msg = "ATTENTION return value for 'remake' has been changed to a dictionary" 28 | msg += "\n format is {'commandStatus': 'SUCCESS' or 'FAILED'," 29 | msg += "\n 'workDir': name of the work directory created}" 30 | self.logger.warning(msg) 31 | return self.remakecache(''.join(self.options.cmptask.split())) 32 | 33 | def remakecache(self,taskname): 34 | """ this does the actual work """ 35 | requestarea = taskname.split(":", 1)[1].split("_", 1)[1] 36 | cachepath = os.path.join(requestarea, '.requestcache') 37 | if os.path.exists(cachepath): 38 | self.logger.info("%sWarning%s: %s not created, because it already exists." % (colors.RED, colors.NORMAL, cachepath)) 39 | elif not os.path.exists(requestarea): 40 | self.logger.info('Remaking %s folder.' % (requestarea)) 41 | try: 42 | os.mkdir(requestarea) 43 | os.mkdir(os.path.join(requestarea, 'results')) 44 | os.mkdir(os.path.join(requestarea, 'inputs')) 45 | except IOError: 46 | self.logger.info("%sWarning%s: Failed to make a request area." % (colors.RED, colors.NORMAL)) 47 | self.logger.info("Remaking .requestcache file.") 48 | dumpfile = open(cachepath , PKL_W_MODE) 49 | pickle.dump({'voGroup': '', 'Server': self.serverurl , 'instance': self.instance, 50 | 'RequestName': taskname, 'voRole': '', 'Port': ''}, dumpfile, protocol=0) 51 | dumpfile.close() 52 | self.logger.info("%sSuccess%s: Finished remaking project directory %s" % (colors.GREEN, colors.NORMAL, requestarea)) 53 | 54 | returnDict = {'commandStatus': 'SUCCESS', 'workDir': requestarea} 55 | return returnDict 56 | 57 | def setOptions(self): 58 | """ 59 | __setOptions__ 60 | 61 | This allows to set specific command options 62 | """ 63 | self.parser.add_option("--task", 64 | dest = "cmptask", 65 | default = None, 66 | help = "The complete task name. Can be taken from 'crab status' output, or from dashboard.") 67 | 68 | 69 | def validateOptions(self): 70 | """ validate options """ 71 | if self.options.cmptask is None: 72 | msg = "%sError%s: Please specify the task name for which to remake a CRAB project directory." % (colors.RED, colors.NORMAL) 73 | msg += " Use the --task option." 74 | ex = MissingOptionException(msg) 75 | ex.missingOption = "cmptask" 76 | raise ex 77 | regex = r"^\d{6}_\d{6}_?([^\:]*)\:[a-zA-Z0-9-]+_(crab_)?.+" # pylint: disable=anomalous-backslash-in-string 78 | if not re.match(regex, self.options.cmptask): 79 | msg = "%sError%s: Task name does not match the regular expression '%s'." % (colors.RED, colors.NORMAL, regex) 80 | raise ConfigurationException(msg) 81 | -------------------------------------------------------------------------------- /src/python/CRABClient/JobType/BasicJobType.py: -------------------------------------------------------------------------------- 1 | """ 2 | Abstract class that should be inherited by each job type plug-in 3 | Conventions: 4 | 1) the plug-in file name has to be equal to the plug-in class 5 | 2) a plug-in needs to implement mainly the run method 6 | """ 7 | from ast import literal_eval 8 | 9 | try: 10 | from FWCore.PythonUtilities.LumiList import LumiList 11 | except Exception: # pylint: disable=broad-except 12 | # if FWCore version is not py3 compatible, use our own 13 | from CRABClient.LumiList import LumiList 14 | 15 | from CRABClient.ClientExceptions import ConfigurationException 16 | 17 | 18 | class BasicJobType(object): 19 | """ 20 | BasicJobType 21 | 22 | TODO: thinking on having a job type help here... 23 | """ 24 | 25 | def __init__(self, config, proxyfilename, logger, workingdir, crabserver, s3tester): 26 | self.logger = logger 27 | self.proxyfilename = proxyfilename 28 | self.automaticAvail = False 29 | self.crabserver = crabserver 30 | self.s3tester = s3tester 31 | ## Before everything, check if the config is ok. 32 | if config: 33 | valid, msg = self.validateConfig(config) 34 | if valid: 35 | self.config = config 36 | self.workdir = workingdir 37 | else: 38 | msg += "\nThe documentation about the CRAB configuration file can be found in" 39 | msg += " https://twiki.cern.ch/twiki/bin/view/CMSPublic/CRAB3ConfigurationFile" 40 | raise ConfigurationException(msg) 41 | 42 | 43 | def run(self): 44 | """ 45 | _run_ 46 | 47 | Here goes the job type algorithm 48 | """ 49 | raise NotImplementedError() 50 | 51 | 52 | def validateConfig(self, config): 53 | """ 54 | _validateConfig_ 55 | 56 | Allows to have a basic validation of the needed parameters 57 | """ 58 | ## (boolean with the result of the validation, eventual error message) 59 | return True, "Valid configuration" 60 | 61 | 62 | @staticmethod 63 | def mergeLumis(inputdata): 64 | """ 65 | Computes the processed lumis, merges if needed and returns the compacted list. 66 | """ 67 | mergedLumis = set() 68 | #merge the lumis from single files 69 | for reports in inputdata.values(): 70 | for report in reports: 71 | for run, lumis in literal_eval(report['runlumi']).items(): 72 | if isinstance(run, bytes): 73 | run = run.decode(encoding='UTF-8') 74 | for lumi in lumis: 75 | mergedLumis.add((run, int(lumi))) #lumi is str, but need int 76 | mergedLumis = LumiList(lumis=mergedLumis) 77 | return mergedLumis.getCompactList() 78 | 79 | 80 | @staticmethod 81 | def intersectLumis(lumisA, lumisB): 82 | result = LumiList(compactList=lumisA) & LumiList(compactList=lumisB) 83 | return result.getCompactList() 84 | 85 | 86 | @staticmethod 87 | def subtractLumis(lumisA, lumisB): 88 | result = LumiList(compactList=lumisA) - LumiList(compactList=lumisB) 89 | return result.getCompactList() 90 | 91 | 92 | @staticmethod 93 | def getDuplicateLumis(lumisDict): 94 | """ 95 | Get the run-lumis appearing more than once in the input 96 | dictionary of runs and lumis, which is assumed to have 97 | the following format: 98 | { 99 | '1': [1,2,3,4,6,7,8,9,10], 100 | '2': [1,4,5,20] 101 | } 102 | """ 103 | doubleLumis = set() 104 | for run, lumis in lumisDict.items(): 105 | seen = set() 106 | doubleLumis.update(set((run, lumi) for lumi in lumis if (run, lumi) in seen or seen.add((run, lumi)))) 107 | doubleLumis = LumiList(lumis=doubleLumis) 108 | return doubleLumis.getCompactList() 109 | -------------------------------------------------------------------------------- /src/python/CRABAPI/test_Task.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import os 3 | import os.path 4 | import shutil 5 | import tempfile 6 | import unittest 7 | import CRABAPI.Abstractions 8 | import CRABClient.Emulator 9 | class Task(unittest.TestCase): 10 | """ 11 | Test that our functionality is correct (stubbing out CRABClient 12 | completely) 13 | """ 14 | def setUp(self): 15 | self.myTask = CRABAPI.Abstractions.Task() 16 | 17 | def test_Task(self): 18 | self.assertIsInstance(self.myTask, CRABAPI.Abstractions.Task) 19 | 20 | def test_kill(self): 21 | self.assertRaises(NotImplementedError, self.myTask.kill) 22 | 23 | def test_getJob(self): 24 | self.assertRaises(NotImplementedError, getattr, self.myTask, 'jobs') 25 | 26 | def test_getNonexistant(self): 27 | self.assertRaises(AttributeError, getattr, self.myTask, 'doesntExist') 28 | 29 | def test_submit(self): 30 | class dummyClient: 31 | def __init__(*args, **kwargs): 32 | pass 33 | def __call__(*args, **kwargs): 34 | return {'uniquerequestname' :"TestingRequestID" } 35 | self.myTask.submitClass = dummyClient 36 | self.assertEqual(self.myTask.submit(), "TestingRequestID") 37 | 38 | class DeepTask(unittest.TestCase): 39 | """ 40 | Test that we actually get back what we want from CRABClient. Don't 41 | require too much from the internals, just inject enough fake 42 | dependecies to convince the client it's talking to something real 43 | """ 44 | def setUp(self): 45 | self.testDir = tempfile.mkdtemp() 46 | 47 | def tearDown(self): 48 | CRABClient.Emulator.clearEmulators() 49 | if os.path.exists(self.testDir): 50 | shutil.rmtree(self.testDir) 51 | 52 | def test_Submit(self): 53 | class dummyRest: 54 | def __init__(*args, **kwargs): 55 | pass 56 | def get(self, url, req): 57 | if url == '/crabserver/prod/info': 58 | if req == {'subresource': 'version'}: 59 | return {'result':["unittest"]},200,"" 60 | if req == {'subresource': 'backendurls'}: 61 | return {'result':["unittest.host"]},200,"" 62 | print("%s -> %s" % (url, req)) 63 | def put(self, url, data): 64 | if url == '/crabserver/prod/workflow': 65 | res = {'result':[{"RequestName" : "UnittestRequest"}]} 66 | return res, 200, "" 67 | print("%s -> %s" % (url, data)) 68 | @staticmethod 69 | def getCACertPath(): 70 | return "/tmp" 71 | class dummyUFC: 72 | def __init__(self, req): 73 | pass 74 | def upload(self, name): 75 | return {'hashkey':'unittest-dummy-tarball'} 76 | CRABClient.Emulator.setEmulator('rest', dummyRest) 77 | CRABClient.Emulator.setEmulator('ufc', dummyUFC) 78 | myTask = CRABAPI.Abstractions.Task() 79 | myTask.config.section_("General") 80 | myTask.config.General.requestName = 'test1' 81 | myTask.config.General.transferLogs = True 82 | myTask.config.General.workArea = os.path.join(self.testDir, "unit") 83 | myTask.config.section_("JobType") 84 | myTask.config.JobType.pluginName = 'PrivateMC' 85 | myTask.config.JobType.psetName = 'test_pset.py' 86 | myTask.config.section_("Data") 87 | myTask.config.Data.inputDataset = '/CrabTestSingleMu' 88 | myTask.config.Data.splitting = 'EventBased' 89 | myTask.config.Data.unitsPerJob = 100 90 | myTask.config.Data.totalUnits = 1000 91 | myTask.config.Data.publication = True 92 | myTask.config.Data.outputDatasetTag = 'CRABAPI-Unittest' 93 | myTask.config.section_("Site") 94 | myTask.config.Site.storageSite = 'T2_US_Nowhere' 95 | val = myTask.submit() 96 | print(val) 97 | self.assertEqual(val, 'UnittestRequest') 98 | 99 | 100 | -------------------------------------------------------------------------------- /src/python/CRABClient/Commands/getsandbox.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from CRABClient.Commands.SubCommand import SubCommand 4 | 5 | from CRABClient.UserUtilities import curlGetFileFromURL, getColumn 6 | 7 | from ServerUtilities import downloadFromS3, getProxiedWebDir 8 | 9 | 10 | class getsandbox(SubCommand): 11 | """ 12 | given a projdir, downloads locally the user sandbox. 13 | It will try s3 first, otherwise it will fall back to the schedd WEBDIR. 14 | """ 15 | 16 | name = "getsandbox" 17 | 18 | def __call__(self): 19 | 20 | # init. debug. print useful info 21 | self.logger.debug("requestarea: %s", self.requestarea) 22 | self.logger.debug("cachedinfo: %s", self.cachedinfo) 23 | 24 | # get information necessary for next steps 25 | # Get all of the columns from the database for a certain task 26 | self.taskname = self.cachedinfo['RequestName'] 27 | self.crabDBInfo, _, _ = self.crabserver.get(api='task', data={'subresource':'search', 'workflow':self.taskname}) 28 | self.logger.debug("Got information from server oracle database: %s", self.crabDBInfo) 29 | 30 | # arguments used by following functions 31 | self.downloadDir = os.path.join(self.requestarea, "taskconfig") 32 | 33 | # download files: user sandbox, debug sandbox 34 | filelist = [] 35 | # usersandbox = self.downloadUserSandbox() 36 | usersandbox = self.downloadSandbox( 37 | remotefile=getColumn(self.crabDBInfo, 'tm_user_sandbox'), 38 | localfile='sandbox.tar.gz') 39 | filelist.append(usersandbox) 40 | # debugfiles = self.downloadDebug() 41 | debugfiles = self.downloadSandbox( 42 | remotefile=getColumn(self.crabDBInfo, 'tm_debug_files'), 43 | localfile='debug_files.tar.gz') 44 | filelist.append(debugfiles) 45 | 46 | returnDict = {"commandStatus": "FAILED"} 47 | if filelist: 48 | returnDict = {"commandStatus": "SUCCESS", "sandbox_paths": filelist } 49 | 50 | return returnDict 51 | 52 | def downloadSandbox(self, remotefile, localfile): 53 | """ 54 | Copy remotefile from s3 to localfile on local disk. 55 | 56 | If remotefile is not s3, then as a fallback we look for the corresponding 57 | localfile in the schedd webdir. 58 | """ 59 | username = getColumn(self.crabDBInfo, 'tm_username') 60 | sandboxFilename = remotefile 61 | 62 | self.logger.debug("will download sandbox from s3: %s",sandboxFilename) 63 | 64 | if not os.path.isdir(self.downloadDir): 65 | os.mkdir(self.downloadDir) 66 | localSandboxPath = os.path.join(self.downloadDir, localfile) 67 | 68 | try: 69 | downloadFromS3(crabserver=self.crabserver, 70 | filepath=localSandboxPath, 71 | objecttype='sandbox', logger=self.logger, 72 | tarballname=sandboxFilename, 73 | username=username 74 | ) 75 | except Exception as e: 76 | self.logger.info("Sandbox download failed with %s", e) 77 | self.logger.info("We will look for the sandbox on the webdir of the schedd") 78 | 79 | webdir = getProxiedWebDir(crabserver=self.crabserver, task=self.taskname, 80 | logFunction=self.logger.debug) 81 | if not webdir: 82 | webdir = getColumn(self.crabDBInfo, 'tm_user_webdir') 83 | self.logger.debug("Downloading %s from %s", localfile, webdir) 84 | httpCode = curlGetFileFromURL(webdir + '/' + localfile, 85 | localSandboxPath, self.proxyfilename, 86 | logger=self.logger) 87 | if httpCode != 200: 88 | self.logger.error("Failed to download %s from %s", localfile, webdir) 89 | raise Exception("We could not locate the sandbox in the webdir neither.") 90 | # we should use 91 | # raise Exception("We could not locate the sandbox in the webdir neither.") from e 92 | # but that is not py2 compatible... 93 | 94 | return localSandboxPath 95 | 96 | -------------------------------------------------------------------------------- /src/python/CRABClient/JobType/ScramEnvironment.py: -------------------------------------------------------------------------------- 1 | """ 2 | ScramEnvironment class 3 | """ 4 | 5 | import os 6 | import json 7 | import logging 8 | import subprocess 9 | 10 | from CRABClient.ClientExceptions import EnvironmentException 11 | from CRABClient.ClientUtilities import BOOTSTRAP_ENVFILE, bootstrapDone, execute_command 12 | 13 | class ScramEnvironment(dict): 14 | 15 | """ 16 | _ScramEnvironment_, a class to determine and cache the user's scram environment. 17 | 18 | The class has two modes of work, depending on the existance of the CRAB3_BOOTSTRAP_DIR variable. 19 | If it is set loads the environemt from a file inside that deirectory 20 | Otherwise take the necessary information directly from the environment 21 | 22 | Raises: 23 | """ 24 | 25 | 26 | def __init__(self, logger=None): 27 | self.logger = logger if logger else logging 28 | 29 | if bootstrapDone(): 30 | self.logger.debug("Loading required information from the bootstrap environment file") 31 | try: 32 | self.initFromFile() 33 | except EnvironmentException as ee: 34 | self.logger.info(str(ee)) 35 | self.logger.info("Will try to find the necessary information from the environment") 36 | self.initFromEnv() 37 | else: 38 | self.logger.debug("Loading required information from the environment") 39 | self.initFromEnv() 40 | 41 | self.logger.debug("Found %s for %s with base %s" % (self.getCmsswVersion(), self.getScramArch(), self.getCmsswBase())) 42 | 43 | 44 | def initFromFile(self): 45 | """ Init the class taking the required information from the boostrap file 46 | """ 47 | 48 | bootFilename = os.path.join(os.environ['CRAB3_BOOTSTRAP_DIR'], BOOTSTRAP_ENVFILE) 49 | if not os.path.isfile(bootFilename): 50 | msg = "The CRAB3_BOOTSTRAP_DIR environment variable is set, but I could not find %s" % bootFilename 51 | raise EnvironmentException(msg) 52 | else: 53 | with open(bootFilename) as fd: 54 | self.update(json.load(fd)) 55 | 56 | 57 | def initFromEnv(self): 58 | """ Init the class taking the required information from the environment 59 | """ 60 | #self.command = 'scram' # SB I think this line is not needed 61 | self["SCRAM_ARCH"] = None 62 | 63 | if 'SCRAM_ARCH' in os.environ: 64 | self["SCRAM_ARCH"] = os.environ["SCRAM_ARCH"] 65 | else: 66 | stdout, _, _ = execute_command(command='scram arch') 67 | self["SCRAM_ARCH"] = stdout 68 | 69 | self["SCRAM_MIN_SUPPORTED_MICROARCH"] = os.environ.get("SCRAM_MIN_SUPPORTED_MICROARCH", "any") 70 | 71 | try: 72 | self["CMSSW_BASE"] = os.environ["CMSSW_BASE"] 73 | self["CMSSW_VERSION"] = os.environ["CMSSW_VERSION"] 74 | # Commenting these two out. I don't think they are really needed 75 | # self.cmsswReleaseBase = os.environ["CMSSW_RELEASE_BASE"] 76 | # self.localRT = os.environ["LOCALRT"] 77 | except KeyError as ke: 78 | self["CMSSW_BASE"] = None 79 | self["CMSSW_VERSION"] = None 80 | # self.cmsswReleaseBase = None 81 | # self.localRT = None 82 | msg = "Please make sure you have setup the CMS environment (cmsenv). Cannot find %s in your env" % str(ke) 83 | msg += "\nPlease refer to https://twiki.cern.ch/twiki/bin/view/CMSPublic/WorkBookCRAB3Tutorial#Setup_the_environment for how to setup the CMS environment." 84 | raise EnvironmentException(msg) 85 | 86 | 87 | def getCmsswBase(self): 88 | """ 89 | Determine the CMSSW base (user) directory 90 | """ 91 | return self["CMSSW_BASE"] 92 | 93 | 94 | def getCmsswVersion(self): 95 | """ 96 | Determine the CMSSW version number 97 | """ 98 | return self["CMSSW_VERSION"] 99 | 100 | 101 | def getScramArch(self): 102 | """ 103 | Determine the scram architecture 104 | """ 105 | return self["SCRAM_ARCH"] 106 | 107 | def getScramMicroArch(self): 108 | """ 109 | Determine the minimum required scram micro-architecture 110 | """ 111 | return self["SCRAM_MIN_SUPPORTED_MICROARCH"] 112 | -------------------------------------------------------------------------------- /test/data/mapper.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | _mapper_ 4 | 5 | parallel to CRABServer/configuration/ClientMapping.py 6 | """ 7 | 8 | import time 9 | 10 | defaulturi = { 11 | 'submit' : { 'uri': '/unittests/rest/task/', 12 | 'map': { 13 | "RequestType" : {"default": "Analysis", "config": None, "type": "StringType", "required": True }, 14 | "Group" : {"default": "Analysis", "config": 'User.group', "type": "StringType", "required": True }, 15 | "Team" : {"default": "Analysis", "config": 'User.group', "type": "StringType", "required": True }, 16 | "Requestor" : {"default": None, "config": None, "type": "StringType", "required": True }, 17 | "Username" : {"default": None, "config": None, "type": "StringType", "required": True }, 18 | "RequestName" : {"default": None, "config": None, "type": "StringType", "required": True }, 19 | "RequestorDN" : {"default": None, "config": None, "type": "StringType", "required": True }, 20 | "SaveLogs" : {"default": False, "config": 'General.transferLogs', "type": "BooleanType", "required": True }, 21 | "asyncDest" : {"default": None, "config": 'Site.storageSite', "type": "StringType", "required": True }, 22 | "PublishDataName" : {"default": str(time.time()), "config": 'Data.outputDatasetTag', "type": "StringType", "required": True }, 23 | "ProcessingVersion" : {"default": "v1", "config": 'Data.processingVersion', "type": "StringType", "required": True }, 24 | "DbsUrl" : {"default": "http://cmsdbsprod.cern.ch/cms_dbs_prod_global/servlet/DBSServlet", "config": 'Data.inputDBS', "type": "StringType", "required": True }, 25 | "SiteWhitelist" : {"default": None, "config": 'Site.whitelist', "type": "ListType", "required": False}, 26 | "SiteBlacklist" : {"default": None, "config": 'Site.blacklist', "type": "ListType", "required": False}, 27 | "RunWhitelist" : {"default": None, "config": 'Data.runWhitelist', "type": "StringType", "required": False}, 28 | "RunBlacklist" : {"default": None, "config": 'Data.runBlacklist', "type": "StringType", "required": False}, 29 | "BlockWhitelist" : {"default": None, "config": 'Data.blockWhitelist', "type": "ListType", "required": False}, 30 | "BlockBlacklist" : {"default": None, "config": 'Data.blockBlacklist', "type": "ListType", "required": False}, 31 | "JobSplitAlgo" : {"default": None, "config": 'Data.splitting', "type": "StringType", "required": False} 32 | #"JobSplitArgs" : {"default": None, "config": 'Data.filesPerJob', "type": IntType, "required": False}, 33 | #"JobSplitArgs" : {"default": None, "config": 'Data.eventPerJob', "type": IntType, "required": False}, 34 | }, 35 | 'other-config-params' : ['General.serverUrl', 'General.requestName', 'JobType.pluginName', 'JobType.externalPluginFile', 'Data.unitsPerJob', 'Data.splitting', \ 36 | "JobType.psetName", "JobType.inputFiles", "Data.inputDataset", "User.email", "Data.lumiMask", "General.workArea"] 37 | 38 | }, 39 | 'get-log' : {'uri': '/unittests/rest/log/'}, 40 | 'get-output' : {'uri': '/unittests/rest/data/'}, 41 | 'server_info' : {'uri': '/unittests/rest/info/'}, 42 | 'status' : {'uri': '/unittests/rest/task/'}, 43 | 'report' : {'uri': '/unittests/rest/goodLumis/'}, 44 | 'publish' : {'uri': '/unittests/rest/publish/'}, 45 | 'get_client_mapping': {'uri': '/unittests/rest/requestmapping/'}, 46 | 'get-error': {'uri': '/unittests/rest/jobErrors/'}, 47 | 'kill': {'uri': '/unittests/rest/task/'}, 48 | 'resubmit': {'uri': '/unittests/rest/resubmit/'}, 49 | } 50 | -------------------------------------------------------------------------------- /test/python/CRABClient_t/JobType_t/CMSSWConfig_t.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | 3 | """ 4 | _CMSSWConfig_t_ 5 | 6 | Unittests for CMSSW config files 7 | """ 8 | 9 | import logging 10 | import os 11 | import unittest 12 | 13 | from CRABClient.JobType.CMSSWConfig import CMSSWConfig 14 | from CRABClient.JobType.ScramEnvironment import ScramEnvironment 15 | from WMCore.Configuration import Configuration 16 | 17 | #### Test WMCore.Configuration 18 | 19 | testWMConfig = Configuration() 20 | 21 | testWMConfig.section_("JobType") 22 | testWMConfig.JobType.pluginName = 'CMSSW' 23 | testWMConfig.section_("Data") 24 | testWMConfig.Data.inputDataset = '/cms/data/set' 25 | testWMConfig.section_("General") 26 | testWMConfig.General.serverUrl = 'crabas.lnl.infn.it:8888' 27 | testWMConfig.section_("User") 28 | testWMConfig.User.group = 'Analysis' 29 | 30 | #### Test CMSSW python config 31 | 32 | testCMSSWConfig = """ 33 | import FWCore.ParameterSet.Config as cms 34 | 35 | process = cms.Process("ANALYSIS") 36 | process.load("FWCore.MessageLogger.MessageLogger_cfi") 37 | 38 | process.maxEvents = cms.untracked.PSet( 39 | input = cms.untracked.int32(5) 40 | ) 41 | process.source = cms.Source("PoolSource", 42 | fileNames = cms.untracked.vstring('/store/RelVal/2007/7/10/RelVal-RelVal152Z-MM-1184071556/0000/14BFDE39-1B2F-DC11-884F-000E0C3F0521.root') 43 | ) 44 | process.TFileService = cms.Service("TFileService", 45 | fileName = cms.string('histograms.root') 46 | ) 47 | process.copyAll = cms.OutputModule("PoolOutputModule", 48 | fileName = cms.untracked.string('output.root'), 49 | dataset = cms.untracked.PSet( 50 | filterName = cms.untracked.string('Out1'), 51 | ), 52 | ) 53 | process.copySome = cms.OutputModule("PoolOutputModule", 54 | fileName = cms.untracked.string('output2.root'), 55 | dataset = cms.untracked.PSet( 56 | filterName = cms.untracked.string('Out2'), 57 | ), 58 | ) 59 | 60 | process.out = cms.EndPath(process.copyAll+process.copySome) 61 | """ 62 | 63 | 64 | class CMSSWConfigTest(unittest.TestCase): 65 | """ 66 | unittest for ScramEnvironment class 67 | 68 | """ 69 | 70 | # Set up a dummy logger 71 | logger = logging.getLogger('UNITTEST') 72 | logger.setLevel(logging.ERROR) 73 | ch = logging.StreamHandler() 74 | ch.setLevel(logging.ERROR) 75 | logger.addHandler(ch) 76 | 77 | 78 | def setUp(self): 79 | """ 80 | Set up for unit tests 81 | """ 82 | 83 | # Write a test python config file to run tests on 84 | with open('unittest_cfg.py','w') as cfgFile: 85 | cfgFile.write(testCMSSWConfig) 86 | self.reqConfig = {} 87 | self.reqConfig['RequestorDN'] = "/DC=org/DC=doegrids/OU=People/CN=Eric Vaandering 768123" 88 | 89 | 90 | def tearDown(self): 91 | """ 92 | Clean up the files we've spewed all over 93 | """ 94 | os.unlink('unittest_cfg.py') 95 | try: 96 | os.unlink('unit_test_full.py') 97 | except OSError: 98 | pass 99 | 100 | return 101 | 102 | 103 | def testScram(self): 104 | """ 105 | Test Scram environment 106 | """ 107 | 108 | msg = "You must set up a CMSSW environment first" 109 | scram = ScramEnvironment(logger=self.logger) 110 | self.assertNotEqual(scram.getCmsswVersion(), None, msg) 111 | self.assertNotEqual(scram.getScramArch(), None, msg) 112 | self.assertNotEqual(scram.getCmsswBase(), None, msg) 113 | 114 | 115 | def testInit(self): 116 | """ 117 | Test constructor 118 | """ 119 | 120 | cmsConfig = CMSSWConfig(config=None, userConfig='unittest_cfg.py', logger=self.logger) 121 | self.assertNotEqual(cmsConfig.fullConfig, None) 122 | 123 | 124 | def testWrite(self): 125 | """ 126 | Test writing out to a file 127 | """ 128 | cmsConfig = CMSSWConfig(config=None, userConfig='unittest_cfg.py', logger=self.logger) 129 | cmsConfig.writeFile('unit_test_full.py') 130 | self.assertTrue(os.path.getsize('unit_test_full.py') > 0) 131 | 132 | 133 | def testOutputFiles(self): 134 | """ 135 | Test output file detection 136 | """ 137 | 138 | cmsConfig = CMSSWConfig(config=None, userConfig='unittest_cfg.py', logger=self.logger) 139 | self.assertEqual(cmsConfig.outputFiles()[0], ['output.root', 'output2.root']) 140 | self.assertEqual(cmsConfig.outputFiles()[1], ['histograms.root']) 141 | 142 | 143 | def testUpload(self): 144 | """ 145 | Test uploading of output file to CRABServer 146 | """ 147 | cmsConfig = CMSSWConfig(config=testWMConfig, userConfig='unittest_cfg.py', logger=self.logger) 148 | cmsConfig.writeFile('unit_test_full.py') 149 | result = cmsConfig.upload(self.reqConfig) 150 | 151 | self.assertTrue(result[0]['DocID']) 152 | 153 | 154 | 155 | if __name__ == '__main__': 156 | unittest.main() 157 | -------------------------------------------------------------------------------- /scripts/generate_completion.py: -------------------------------------------------------------------------------- 1 | """ 2 | 3 | This script is used to generate the file etc/crab-bash-completion.sh 4 | automatically from crab client code, without the need to manually edit it. 5 | The current crab client build process uses etc/crab-bash-completion.sh 6 | directly. 7 | 8 | When you change the interface of crab client, for example adding a new command 9 | or adding a new parameter to an existing command, you should run this script 10 | with: 11 | 12 | > cd CRABClient 13 | > python3 scripts/generate_completions.py 14 | 15 | This script will generate a new version of etc/crab-bash-completion.sh 16 | that will be used by the crab client build process. 17 | 18 | Known limitations: 19 | 20 | - sorting of the suggestions can be broken despite the use of "-o nosort". 21 | For example when using "set completion-ignore-case on" on bash 4.4.20, 22 | which is the version installed on lxplus8. 23 | see https://unix.stackexchange.com/a/567937 24 | 25 | """ 26 | 27 | import importlib.util 28 | import sys 29 | import argparse 30 | import logging 31 | 32 | from CRABClient.CRABOptParser import CRABCmdOptParser 33 | from CRABClient.ClientMapping import commandsConfiguration 34 | 35 | logging.basicConfig(level=logging.INFO) 36 | 37 | ###################################################### 38 | # Make sure that 'complete` command in template below meet following rules 39 | # - Starts with: complete\s+-F\s+\s+ 40 | # - Ends with: \s+crab 41 | # Otherwise suggest changes in crab-build.file in cms-sw/cmsdist repository 42 | ##################################################### 43 | 44 | template = """ 45 | _UseCrab () 46 | {{ 47 | local cur 48 | COMPRELPY=() 49 | cur=${{COMP_WORDS[COMP_CWORD]}} 50 | sub=this_is_none 51 | 52 | for i in $(seq 1 ${{#COMP_WORDS[*]}}); do 53 | if [ "${{COMP_WORDS[$i]#-}}" == "${{COMP_WORDS[$i]}}" ]; then 54 | sub=${{COMP_WORDS[$i]}} 55 | break 56 | fi 57 | done 58 | 59 | prev=${{COMP_WORDS[$((COMP_CWORD - 1))]}} 60 | 61 | if [ "x$sub" == "x$cur" ]; then 62 | sub="" 63 | fi 64 | 65 | case "$sub" in 66 | "") 67 | case "$cur" in 68 | "") 69 | COMPREPLY=( $(compgen -W '{topoptions} {topcommands}' -- $cur) ) 70 | ;; 71 | -*) 72 | COMPREPLY=( $(compgen -W '{topoptions}' -- $cur) ) 73 | ;; 74 | *) 75 | COMPREPLY=( $(compgen -W '{topcommands}' -- $cur) ) 76 | ;; 77 | esac 78 | ;; 79 | {commands} 80 | *) 81 | COMPREPLY=( $(compgen -W '{topcommands}' -- $cur) ) 82 | ;; 83 | esac 84 | 85 | return 0 86 | }} 87 | complete -F _UseCrab -o filenames crab 88 | """ 89 | 90 | template_cmd = """ 91 | "{cmd}") 92 | case "$cur" in 93 | -*) 94 | COMPREPLY=( $(compgen -W '{cmdflags} {cmdoptions}' -- $cur) ) 95 | ;; 96 | *) 97 | COMPREPLY=( $(compgen -f $cur) ) 98 | esac 99 | ;; 100 | """ 101 | 102 | class DummyLogger(object): 103 | def debug(self, *args, **kwargs): 104 | pass 105 | @property 106 | def logfile(self): 107 | return '' 108 | 109 | def main(): 110 | 111 | parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) 112 | parser.add_argument("-o", "--output-file", 113 | help="output completion file", type=str, 114 | default="etc/crab-bash-completion.sh" ) 115 | p_args = parser.parse_args() 116 | 117 | logging.info(p_args.output_file) 118 | 119 | # python "imp" is deprecated, migrated to "importlib" with the help of 120 | # https://stackoverflow.com/a/41595552 121 | spec = importlib.util.spec_from_file_location("crab", "bin/crab.py") 122 | crab = importlib.util.module_from_spec(spec) 123 | sys.modules["crab"] = crab 124 | spec.loader.exec_module(crab) 125 | 126 | client = crab.CRABClient() 127 | 128 | longnames = [] 129 | commands = {} 130 | options = [] 131 | 132 | for opt in client.parser.option_list: 133 | options.append(opt.get_opt_string()) 134 | options += opt._short_opts 135 | 136 | for _, v in client.subCommands.items(): 137 | class DummyCmd(v): 138 | def __init__(self): 139 | self.parser = CRABCmdOptParser(v.name, '', False) 140 | self.logger = DummyLogger() 141 | self.cmdconf = commandsConfiguration.get(v.name) 142 | 143 | cmd = DummyCmd() 144 | cmd.setSuperOptions() 145 | 146 | flags = [] 147 | opts = [] 148 | 149 | for opt in cmd.parser.option_list: 150 | args = opt.nargs if opt.nargs is not None else 0 151 | names = [opt.get_opt_string()] + opt._short_opts 152 | 153 | if args == 0: 154 | flags += names 155 | else: 156 | opts += names 157 | 158 | longnames.append(cmd.name) 159 | for c in [cmd.name] + cmd.shortnames: 160 | commands[c] = template_cmd.format( 161 | cmd=c, 162 | cmdflags=' '.join(flags), 163 | cmdoptions=' '.join(opts)) 164 | 165 | logging.info(longnames) 166 | 167 | with open(p_args.output_file, "w", encoding="utf-8") as f_: 168 | f_.write(template.format( 169 | topcommands=' '.join(longnames), 170 | topoptions=' '.join(options), 171 | commands=''.join(commands.values()))) 172 | 173 | if __name__ == "__main__": 174 | main() 175 | -------------------------------------------------------------------------------- /src/python/CRABClient/Commands/setdatasetstatus.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=consider-using-f-string, unspecified-encoding 2 | """ 3 | allow users to (in)validate their own DBS USER datasets 4 | """ 5 | 6 | import sys 7 | import json 8 | 9 | from CRABClient.Commands.SubCommand import SubCommand 10 | from CRABClient.ClientExceptions import MissingOptionException, ConfigurationException, CommandFailedException 11 | from CRABClient.ClientUtilities import colors 12 | from CRABClient.RestInterfaces import getDbsREST 13 | 14 | if sys.version_info >= (3, 0): 15 | from urllib.parse import urlencode # pylint: disable=E0611 16 | if sys.version_info < (3, 0): 17 | from urllib import urlencode 18 | 19 | 20 | class setdatasetstatus(SubCommand): 21 | """ 22 | Set status of a USER dataset in phys03 23 | """ 24 | 25 | name = 'setdatasetstatus' 26 | 27 | def __init__(self, logger, cmdargs=None): 28 | SubCommand.__init__(self, logger, cmdargs) 29 | 30 | def __call__(self): 31 | result = 'FAILED' # will change to 'SUCCESS' when all is OK 32 | 33 | dbsInstance = self.options.dbsInstance 34 | dataset = self.options.dataset 35 | status = self.options.status 36 | recursive = self.options.recursive 37 | self.logger.debug('dbsInstance = %s' % dbsInstance) 38 | self.logger.debug('dataset = %s' % dataset) 39 | self.logger.debug('status = %s' % status) 40 | self.logger.debug('recursive = %s' % recursive) 41 | 42 | if recursive: 43 | self.logger.warning("ATTENTION: recursive option is not implemented yet. Ignoring it") 44 | 45 | # from DBS instance, to DBS REST services 46 | dbsReader, dbsWriter = getDbsREST(instance=dbsInstance, logger=self.logger, 47 | cert=self.proxyfilename, key=self.proxyfilename) 48 | 49 | self.logger.info("looking up Dataset %s in DBS %s" % (dataset, dbsInstance)) 50 | datasetStatusQuery = {'dataset': dataset, 'dataset_access_type': '*', 'detail': True} 51 | ds, rc, msg = dbsReader.get(uri="datasets", data=urlencode(datasetStatusQuery)) 52 | self.logger.debug('exitcode= %s', rc) 53 | if not ds: 54 | self.logger.error("ERROR: dataset %s not found in DBS" % dataset) 55 | raise ConfigurationException 56 | self.logger.info("Dataset status in DBS is %s" % ds[0]['dataset_access_type']) 57 | self.logger.info("Will set it to %s" % status) 58 | data = {'dataset': dataset, 'dataset_access_type': status} 59 | jdata = json.dumps(data) 60 | out, rc, msg = dbsWriter.put(uri='datasets', data=jdata) 61 | if rc == 200 and msg == 'OK': 62 | self.logger.info("Dataset status changed successfully") 63 | result = 'SUCCESS' 64 | else: 65 | msg = "Dataset status change failed: %s" % out 66 | raise CommandFailedException(msg) 67 | 68 | ds, rc, msg = dbsReader.get(uri="datasets", data=urlencode(datasetStatusQuery)) 69 | self.logger.debug('exitcode= %s', rc) 70 | self.logger.info("Dataset status in DBS now is %s" % ds[0]['dataset_access_type']) 71 | 72 | self.logger.info("NOTE: status of files inside the dataset has NOT been changed") 73 | 74 | return {'commandStatus': result} 75 | 76 | def setOptions(self): 77 | """ 78 | __setOptions__ 79 | 80 | This allows to set specific command options 81 | """ 82 | self.parser.add_option('--dbs-instance', dest='dbsInstance', default='prod/phys03', 83 | help="DBS instance. e.g. prod/phys03 (default) or int/phys03 or full URL." 84 | + "\nUse at your own risk only if you really know what you are doing" 85 | ) 86 | self.parser.add_option('--dataset', dest='dataset', default=None, 87 | help='dataset name') 88 | self.parser.add_option('--status', dest='status', default=None, 89 | help="New status of the dataset: VALID/INVALID/DELETED/DEPRECATED", 90 | choices=['VALID', 'INVALID', 'DELETED', 'DEPRECATED'] 91 | ) 92 | self.parser.add_option('--recursive', dest='recursive', default=False, action="store_true", 93 | help="Apply status to children datasets and sets all files status in those" 94 | + "to VALID if status=VALID, INVALID otherwise" 95 | ) 96 | 97 | def validateOptions(self): 98 | SubCommand.validateOptions(self) 99 | 100 | if self.options.dataset is None: 101 | msg = "%sError%s: Please specify the dataset to check." % (colors.RED, colors.NORMAL) 102 | msg += " Use the --dataset option." 103 | ex = MissingOptionException(msg) 104 | ex.missingOption = "dataset" 105 | raise ex 106 | if self.options.status is None: 107 | msg = "%sError%s: Please specify the new dataset status." % (colors.RED, colors.NORMAL) 108 | msg += " Use the --status option." 109 | ex = MissingOptionException(msg) 110 | ex.missingOption = "status" 111 | raise ex 112 | # minimal sanity check 113 | dbsInstance = self.options.dbsInstance 114 | if '/' not in dbsInstance or len(dbsInstance.split('/')) > 2 and not dbsInstance.startswith('https://'): 115 | msg = "Bad DBS instance value %s. " % dbsInstance 116 | msg += "Use either server/db format or full URL" 117 | raise ConfigurationException(msg) 118 | -------------------------------------------------------------------------------- /src/python/CRABClient/Commands/tasks.py: -------------------------------------------------------------------------------- 1 | """ 2 | list tasks for the user 3 | """ 4 | # avoid complains about things that we can not fix in python2 5 | # pylint: disable=consider-using-f-string, unspecified-encoding, raise-missing-from 6 | from datetime import datetime, date, timedelta 7 | 8 | from ServerUtilities import TASKDBSTATUSES 9 | 10 | from CRABClient.Commands.SubCommand import SubCommand 11 | from CRABClient.ClientUtilities import commandUsedInsideCrab 12 | from CRABClient.ClientExceptions import ConfigurationException, RESTCommunicationException 13 | 14 | 15 | class tasks(SubCommand): 16 | """Give back all user tasks starting from a specific date. Default is last 30 days. 17 | Note that STATUS in here is task status from database which does not include grid jobs, 18 | task status does not progress beyond SUBMITTED unless the task is KILLED 19 | """ 20 | def __call__(self): 21 | server = self.crabserver 22 | dictresult, status, reason = server.get(api=self.defaultApi, data={'timestamp': self.date}) 23 | dictresult = dictresult['result'] #take just the significant part 24 | 25 | if status != 200: 26 | msg = "Problem retrieving tasks:\ninput:%s\noutput:%s\nreason:%s" % (str(self.date), str(dictresult), str(reason)) 27 | raise RESTCommunicationException(msg) 28 | 29 | dictresult.sort() 30 | dictresult.reverse() 31 | 32 | if self.options.status: 33 | dictresult = [item for item in dictresult if item[1] == self.options.status] 34 | 35 | result = [item[0:2] for item in dictresult] 36 | 37 | today = date.today() 38 | 39 | if not dictresult: 40 | msg = "No tasks found from %s until %s" % (self.date, today) 41 | if self.options.status: 42 | msg += " with status %s" % (self.options.status) 43 | self.logger.info(msg) 44 | returnDict = {'commandStatus': 'SUCCESS', 'taskList': []} 45 | return returnDict 46 | 47 | msg = "\nList of tasks from %s until %s" % (self.date, today) 48 | if self.options.status: 49 | msg += " with status %s" % (self.options.status) 50 | self.logger.info(msg) 51 | msg = "Beware that STATUS here does not include information from grid jobs" 52 | self.logger.info(msg) 53 | self.logger.info('='*80) 54 | self.logger.info('NAME\t\t\t\t\t\t\t\tSTATUS') 55 | self.logger.info('='*80) 56 | for item in dictresult: 57 | name, status = item[0:2] 58 | self.logger.info('%s\n\t\t\t\t\t\t\t\t%s' % (name, status)) 59 | self.logger.info('-'*80) 60 | self.logger.info('\n') 61 | 62 | if not commandUsedInsideCrab(): 63 | msg = "ATTENTION return value for 'tasks' has been changed to a dictionary" 64 | msg += "\n format is {'commandStatus': 'SUCCESS' or 'FAILED'," 65 | msg += "\n 'tasklist': list of [task, status] lists as before}" 66 | self.logger.warning(msg) 67 | returnDict = {'commandStatus': 'SUCCESS', 'taskList': result} 68 | return returnDict 69 | 70 | 71 | def setOptions(self): 72 | """ 73 | __setOptions__ 74 | 75 | This allows to set specific command options 76 | """ 77 | 78 | self.parser.add_option('--fromdate', 79 | dest='fromdate', 80 | default=None, 81 | help='Give the user tasks since YYYY-MM-DD.', 82 | metavar='YYYY-MM-DD') 83 | 84 | self.parser.add_option('--days', 85 | dest='days', 86 | default=None, 87 | type='int', 88 | help='Give the user tasks from the previous N days.', 89 | metavar='N') 90 | 91 | self.parser.add_option('--status', 92 | dest='status', 93 | default=None, 94 | help='Give the user tasks with the given STATUS.', 95 | metavar='STATUS') 96 | 97 | def validateOptions(self): 98 | 99 | if self.options.fromdate is not None and self.options.days is not None: 100 | msg = "Options --fromdate and --days cannot be used together. Please specify only one of them." 101 | raise ConfigurationException(msg) 102 | if self.options.fromdate is None and self.options.days is None: 103 | days = 30 104 | self.date = date.today() - timedelta(days=days) 105 | elif self.options.days is not None: 106 | days = self.options.days 107 | self.date = date.today() - timedelta(days=days) 108 | elif self.options.fromdate is not None: 109 | try: 110 | datetime.strptime(self.options.fromdate, '%Y-%m-%d') 111 | except ValueError: 112 | msg = "Please enter date with format 'YYYY-MM-DD'. Example: crab tasks --fromdate=2014-01-01" 113 | raise ConfigurationException(msg) 114 | if len(self.options.fromdate) != 10: 115 | msg = "Please enter date with format 'YYYY-MM-DD'. Example: crab tasks --fromdate=2014-01-01" 116 | raise ConfigurationException(msg) 117 | self.date = self.options.fromdate 118 | 119 | if self.options.status is not None: 120 | if self.options.status not in TASKDBSTATUSES: 121 | msg = "Please enter a valid task status. Valid task statuses are: %s" % (TASKDBSTATUSES) 122 | raise ConfigurationException(msg) 123 | -------------------------------------------------------------------------------- /src/python/CRABClient/CRABOptParser.py: -------------------------------------------------------------------------------- 1 | # silence pylint complaints about things we need for Python 2.6 compatibility 2 | # pylint: disable=unspecified-encoding, raise-missing-from, consider-using-f-string 3 | 4 | from optparse import OptionParser # pylint: disable=deprecated-module 5 | 6 | from ServerUtilities import SERVICE_INSTANCES 7 | from CRABClient import __version__ as client_version 8 | 9 | 10 | class CRABOptParser(OptionParser): 11 | """ 12 | Allows to make OptionParser behave how we prefer 13 | """ 14 | 15 | def __init__(self, subCommands=None): 16 | """ Initialize the option parser used in the the client. That's only the first step parsing 17 | which basically creates the help and looks for the --debug/--quiet options. Each command 18 | than has its own set of arguments (some are shared, see CRABCmdOptParser ). 19 | 20 | subCommands: if present used to prepare a nice help summary for all the commands 21 | """ 22 | usage = "usage: %prog [options] COMMAND [command-options] [args]" 23 | epilog = "" 24 | if subCommands: 25 | epilog = '\nValid commands are: \n' 26 | for k in sorted(subCommands.keys()): 27 | epilog += ' %s' % subCommands[k].name 28 | epilog += ''.join( [' (%s)' % name for name in subCommands[k].shortnames ] ) 29 | epilog += '\n' 30 | epilog += "To get single command help run:\n crab command --help|-h\n" 31 | 32 | epilog += '\nFor more information on how to run CRAB-3 please follow this link:\n' 33 | epilog += 'https://twiki.cern.ch/twiki/bin/view/CMSPublic/WorkBookCRAB3Tutorial\n' 34 | 35 | OptionParser.__init__(self, usage = usage, epilog = epilog, 36 | version = "CRAB client %s" % client_version 37 | ) 38 | 39 | # This is the important bit 40 | self.disable_interspersed_args() 41 | 42 | self.add_option( "--quiet", 43 | action = "store_true", 44 | dest = "quiet", 45 | default = False, 46 | help = "don't print any messages to stdout" ) 47 | 48 | self.add_option( "--debug", 49 | action = "store_true", 50 | dest = "debug", 51 | default = False, 52 | help = "print extra messages to stdout" ) 53 | 54 | 55 | def format_epilog(self, formatter): 56 | """ 57 | do not strip the new lines from the epilog 58 | """ 59 | return self.epilog 60 | 61 | 62 | 63 | class CRABCmdOptParser(OptionParser): 64 | """ A class that extract the pieces for parsing the command line arguments 65 | of the CRAB commands. 66 | 67 | """ 68 | 69 | def __init__(self, cmdname, doc, disable_interspersed_args): 70 | """ 71 | doc: the description of the command. Taken from self.__doc__ 72 | disable_interspersed_args: some commands (e.g.: submit) allow to overwrite configuration parameters 73 | """ 74 | usage = "usage: %prog " + cmdname + " [options] [args]" 75 | OptionParser.__init__(self, description = doc, usage = usage, add_help_option = True) 76 | if disable_interspersed_args: 77 | self.disable_interspersed_args() 78 | 79 | 80 | def addCommonOptions(self, cmdconf): 81 | """ 82 | cmdconf: the command configuration from the ClientMapping 83 | Note that default has to be None for most options in order not to 84 | override what is in the crab configuration file in case the option is 85 | not specified in the command line. E.g. the default instance in case 86 | a value is not indicated anywhere by the user is defined in ClientMapping.py 87 | """ 88 | if cmdconf['requiresDirOption']: 89 | self.add_option("-d", "--dir", 90 | dest = "projdir", 91 | default = None, 92 | help = "Path to the CRAB project directory for which the crab command should be executed.") 93 | self.add_option("--task", 94 | dest = "cmptask", 95 | default = None, 96 | help = "In alternative to -d, a complete task name. Can be taken from 'crab status' output, or from dashboard.") 97 | 98 | if cmdconf['requiresREST']: 99 | self.add_option("--instance", 100 | dest = "instance", 101 | type = "string", 102 | default = None, 103 | help = "Running instance of CRAB service." \ 104 | " Needed whenever --task is used." \ 105 | " Default value is 'prod'. " \ 106 | " Valid values are %s." \ 107 | % str(list(SERVICE_INSTANCES.keys())) ) 108 | 109 | if cmdconf['requiresProxyVOOptions']: 110 | self.add_option("--voRole", 111 | dest = "voRole", 112 | default = None) 113 | self.add_option("--voGroup", 114 | dest = "voGroup", 115 | default = None) 116 | 117 | self.add_option("--proxy", 118 | dest="proxy", 119 | default=False, 120 | help="Use the given proxy. Skip Grid proxy creation and myproxy delegation.") 121 | -------------------------------------------------------------------------------- /src/python/CRABClient/JobType/PrivateMC.py: -------------------------------------------------------------------------------- 1 | """ 2 | PrivateMC job type plug-in 3 | """ 4 | from __future__ import division 5 | from __future__ import print_function 6 | 7 | import os 8 | import math 9 | 10 | from CRABClient.ClientUtilities import colors 11 | from CRABClient.JobType.Analysis import Analysis 12 | from CRABClient.ClientExceptions import ConfigurationException 13 | from CRABClient.ClientMapping import getParamDefaultValue 14 | 15 | 16 | class PrivateMC(Analysis): 17 | """ 18 | PrivateMC job type plug-in 19 | """ 20 | 21 | def run(self, *args, **kwargs): 22 | """ 23 | Override run() for JobType 24 | """ 25 | tarFilename, configArguments = super(PrivateMC, self).run(*args, **kwargs) 26 | configArguments['jobtype'] = 'PrivateMC' 27 | 28 | lhe, nfiles = self.cmsswCfg.hasLHESource() 29 | pool = self.cmsswCfg.hasPoolSource() 30 | if lhe: 31 | self.logger.debug("LHESource found in the CMSSW configuration.") 32 | configArguments['generator'] = getattr(self.config.JobType, 'generator', 'lhe') 33 | 34 | major, minor, _ = os.environ['CMSSW_VERSION'].split('_')[1:4] 35 | warn = True 36 | if int(major) >= 7: 37 | if int(minor) >= 5: 38 | warn = False 39 | 40 | if nfiles > 1 and warn: 41 | msg = "{0}Warning{1}: Using an LHESource with ".format(colors.RED, colors.NORMAL) 42 | msg += "more than one input file may not be supported by the CMSSW version used. " 43 | msg += "Consider merging the LHE input files to guarantee complete processing." 44 | self.logger.warning(msg) 45 | 46 | if getattr(self.config.JobType, 'generator', '') == 'lhe' and not lhe: 47 | msg = "Generator set to 'lhe' but " 48 | if pool: 49 | msg += "'PoolSource' instead of 'LHESource' present in parameter set. If you " 50 | msg += "are processing files in EDMLHE format, please set 'JobType.pluginName' " 51 | msg += "to 'Analysis'." 52 | else: 53 | msg += "no 'LHESource' found in parameter set. If you are processing a gridpack " 54 | msg += "to produce EDMLHE files, please remove the parameter 'JobType.generator'." 55 | raise ConfigurationException(msg) 56 | elif pool: 57 | msg = "Found a 'PoolSource' in the parameter set. But that's not compatible with PrivateMC. " 58 | msg += "Please switch to either 'EmptySource' or 'LHESource' for event generation, " 59 | msg += "or set 'JobType.pluginName' to 'Analysis'." 60 | raise ConfigurationException(msg) 61 | 62 | configArguments['primarydataset'] = getattr(self.config.Data, 'outputPrimaryDataset', 'CRAB_PrivateMC') 63 | 64 | return tarFilename, configArguments 65 | 66 | 67 | def validateConfig(self, config): 68 | """ 69 | Validate the PrivateMC portion of the config file making sure 70 | required values are there and optional values don't conflict. Subclass to CMSSW for most of the work 71 | """ 72 | valid, reason = self.validateBasicConfig(config) 73 | if not valid: 74 | return valid, reason 75 | 76 | ## If publication is True, check that there is a primary dataset name specified. 77 | if getattr(config.Data, 'publication', getParamDefaultValue('Data.publication')): 78 | if not getattr(config.Data, 'outputPrimaryDataset', getParamDefaultValue('Data.outputPrimaryDataset')): 79 | msg = "Invalid CRAB configuration: Parameter Data.outputPrimaryDataset not specified." 80 | msg += "\nMC generation job type requires this parameter for publication." 81 | return False, msg 82 | 83 | if not hasattr(config.Data, 'totalUnits'): 84 | msg = "Invalid CRAB configuration: Parameter Data.totalUnits not specified." 85 | msg += "\nMC generation job type requires this parameter to know how many events to generate." 86 | return False, msg 87 | elif config.Data.totalUnits <= 0: 88 | msg = "Invalid CRAB configuration: Parameter Data.totalUnits has an invalid value (%s)." % (config.Data.totalUnits) 89 | msg += " It must be a natural number." 90 | return False, msg 91 | 92 | ## Make sure the splitting algorithm is valid. 93 | allowedSplitAlgos = ['EventBased'] 94 | if self.splitAlgo not in allowedSplitAlgos: 95 | msg = "Invalid CRAB configuration: Parameter Data.splitting has an invalid value ('%s')." % (self.splitAlgo) 96 | msg += "\nMC generation job type only supports the following splitting algorithms: %s." % (allowedSplitAlgos) 97 | return False, msg 98 | 99 | # Perform a check on the amount of lumis that the output files will have. 100 | # First, get the eventsPerLumi param. It will default to 100 if not specified in crabConfig: 101 | # https://github.com/dmwm/WMCore/blob/master/src/python/WMCore/JobSplitting/EventBased.py#L31 102 | eventsPerLumi = getattr(config.JobType, 'eventsPerLumi', 100) 103 | # Knowing eventsPerLumi and how many units the user wants to generate per job, we can tell 104 | # how many lumis each output file will have: 105 | lumisPerFile = math.ceil(config.Data.unitsPerJob / eventsPerLumi) 106 | if lumisPerFile > 1000: 107 | msg = "Given your input parameters, each output file will contain around %d lumis." % lumisPerFile 108 | msg += "\nPlease modify your configuration so that the output files contain at most 1000 lumis. " 109 | msg += "You can do so by increasing the 'config.JobType.eventsPerLumi' parameter " 110 | msg += "or by decreasing the 'config.Data.totalUnits', 'config.Data.unitsPerJob' parameters." 111 | return False, msg 112 | return True, "Valid configuration" 113 | -------------------------------------------------------------------------------- /src/python/CRABClient/Commands/getlog.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | from __future__ import division 3 | 4 | from CRABClient.ClientUtilities import colors, validateJobids, getColumn 5 | from CRABClient.UserUtilities import curlGetFileFromURL 6 | from CRABClient.Commands.getcommand import getcommand 7 | from CRABClient.ClientExceptions import RESTCommunicationException, MissingOptionException 8 | 9 | from ServerUtilities import getProxiedWebDir 10 | 11 | class getlog(getcommand): 12 | """ 13 | Retrieve the log files of a number of jobs specified by the -q/--quantity option. 14 | -q logfiles per exit code are returned if transferLogs = False; otherwise all the log files 15 | collected by the LogCollect job are returned. The task is identified by the -d/--dir option. 16 | """ 17 | name = 'getlog' 18 | shortnames = ['log'] 19 | visible = True #overwrite getcommand 20 | 21 | def __call__(self): # pylint: disable=arguments-differ 22 | if self.options.short: 23 | taskname = self.cachedinfo['RequestName'] 24 | inputlist = {'subresource': 'search', 'workflow': taskname} 25 | server = self.crabserver 26 | webdir = getProxiedWebDir(crabserver=self.crabserver, task=taskname, logFunction=self.logger.debug) 27 | dictresult, status, reason = server.get(api='task', data=inputlist) 28 | if not webdir: 29 | webdir = dictresult['result'][0] 30 | self.logger.info('Server result: %s' % webdir) 31 | if status != 200: 32 | msg = "Problem retrieving information from the server:\ninput:%s\noutput:%s\nreason:%s" % (str(inputlist), str(dictresult), str(reason)) 33 | raise RESTCommunicationException(msg) 34 | splitting = getColumn(dictresult, 'tm_split_algo') 35 | if getattr(self.options, 'jobids', None): 36 | self.options.jobids = validateJobids(self.options.jobids, splitting != 'Automatic') 37 | self.setDestination() 38 | self.logger.info("Setting the destination to %s " % self.dest) 39 | failed, success = self.retrieveShortLogs(webdir, self.proxyfilename) 40 | if failed: 41 | msg = "%sError%s: Failed to retrieve the following files: %s" % (colors.RED, colors.NORMAL, failed) 42 | self.logger.info(msg) 43 | else: 44 | self.logger.info("%sSuccess%s: All files successfully retrieved." % (colors.GREEN, colors.NORMAL)) 45 | returndict = {'success': success, 'failed': failed} 46 | else: 47 | # Different from the old getlog code: set 'logs2' as subresource so that 'getcommand' uses the new logic. 48 | returndict = getcommand.__call__(self, subresource='logs2') 49 | if ('success' in returndict and not returndict['success']) or \ 50 | ('failed' in returndict and returndict['failed']): 51 | msg = "You can use the --short option to retrieve a short version of the log files from the Grid scheduler." 52 | self.logger.info(msg) 53 | 54 | # if something failed, getcommand raises exceptions, so if we got here it means OK 55 | returndict['commandStatus'] = 'SUCCESS' 56 | return returndict 57 | 58 | 59 | def setOptions(self): 60 | """ 61 | __setOptions__ 62 | 63 | This allows to set specific command options 64 | """ 65 | self.parser.add_option('--quantity', 66 | dest='quantity', 67 | help='The number of logs you want to retrieve (or "all"). Ignored if --jobids is used.') 68 | self.parser.add_option('--parallel', 69 | dest='nparallel', 70 | help='Number of parallel download, default is 10 parallel download.',) 71 | self.parser.add_option('--wait', 72 | dest='waittime', 73 | help='Increase the sendreceive-timeout in second.',) 74 | self.parser.add_option('--short', 75 | dest='short', 76 | default=False, 77 | action='store_true', 78 | help='Get the short version of the log file. Use with --dir and --jobids.',) 79 | getcommand.setOptions(self) 80 | 81 | 82 | def validateOptions(self): 83 | getcommand.validateOptions(self) 84 | if self.options.short: 85 | if self.options.jobids is None: 86 | msg = "%sError%s: Please specify the job ids for which to retrieve the logs." % (colors.GREEN, colors.NORMAL) 87 | msg += " Use the --jobids option." 88 | ex = MissingOptionException(msg) 89 | ex.missingOption = "jobids" 90 | raise ex 91 | 92 | 93 | def retrieveShortLogs(self, webdir, proxyfilename): 94 | self.logger.info("Retrieving...") 95 | success = [] 96 | failed = [] 97 | for _, jobid in self.options.jobids: 98 | # We don't know a priori how many retries the job had. So we start with retry 0 99 | # and increase it by 1 until we are unable to retrieve a log file (interpreting 100 | # this as the fact that we reached the highest retry already). 101 | retry = 0 102 | succeded = True 103 | while succeded: 104 | filename = 'job_out.%s.%s.txt' % (jobid, retry) 105 | url = webdir + '/' + filename 106 | httpCode = curlGetFileFromURL(url, self.dest + '/' + filename, proxyfilename, logger=self.logger) 107 | if httpCode == 200: 108 | self.logger.info('Retrieved %s' % (filename)) 109 | success.append(filename) 110 | retry += 1 # To retrieve retried job log, if there is any. 111 | elif httpCode == 404: 112 | succeded = False 113 | # Ignore the exception if the HTTP status code is 404. Status 404 means file 114 | # not found (see http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html). File 115 | # not found error is expected, since we try all the job retries. 116 | else: 117 | # something went wront in trying to retrieve a file which was expected to be there 118 | succeded = False 119 | failed.append(filename) 120 | 121 | return failed, success 122 | 123 | -------------------------------------------------------------------------------- /test/python/CRABClient_t/JobType_t/UserTarball_t.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | 3 | """ 4 | _ScramEnvironment_t_ 5 | 6 | Unittests for ScramEnvironment module 7 | """ 8 | 9 | import logging 10 | import os 11 | import subprocess 12 | import tarfile 13 | import unittest 14 | 15 | from CRABClient.JobType.UserTarball import UserTarball 16 | from WMCore.Configuration import Configuration 17 | from CRABClient.ClientExceptions import InputFileNotFoundException 18 | 19 | testWMConfig = Configuration() 20 | 21 | testWMConfig.section_("JobType") 22 | testWMConfig.JobType.pluginName = 'CMSSW' 23 | testWMConfig.section_("Data") 24 | testWMConfig.Data.inputDataset = '/cms/data/set' 25 | testWMConfig.section_("General") 26 | testWMConfig.General.serverUrl = 'cms-xen39.fnal.gov:7723' 27 | testWMConfig.section_("User") 28 | testWMConfig.User.group = 'Analysis' 29 | 30 | class UserTarballTest(unittest.TestCase): 31 | """ 32 | unittest for ScramEnvironment class 33 | 34 | """ 35 | 36 | # Set up a dummy logger 37 | logger = logging.getLogger('UNITTEST') 38 | logger.setLevel(logging.DEBUG) 39 | ch = logging.StreamHandler() 40 | ch.setLevel(logging.ERROR) 41 | logger.addHandler(ch) 42 | 43 | 44 | def setUp(self): 45 | """ 46 | Set up for unit tests 47 | """ 48 | 49 | # Set relevant variables 50 | 51 | self.arch = 'slc5_ia32_gcc434' 52 | self.version = 'CMSSW_3_8_7' 53 | self.base = '/tmp/CMSSW_3_8_7' 54 | 55 | os.environ['SCRAM_ARCH'] = self.arch 56 | os.environ['CMSSW_BASE'] = self.base 57 | os.environ['CMSSW_RELEASE_BASE'] = 'DUMMY' 58 | os.environ['LOCALRT'] = 'DUMMY' 59 | os.environ['CMSSW_VERSION'] = self.version 60 | 61 | self.tarBalls = [] 62 | 63 | # Make a dummy CMSSW environment 64 | 65 | commands = [ 66 | 'rm -rf %s' % self.base, 67 | 'mkdir -p %s/lib/%s/' % (self.base, self.arch), 68 | 'touch %s/lib/%s/libSomething.so' % (self.base, self.arch), 69 | 'touch %s/lib/%s/libSomewhere.so' % (self.base, self.arch), 70 | 'mkdir -p %s/src/Module/Submodule/data/' % (self.base), 71 | 'touch %s/src/Module/Submodule/data/datafile.txt' % (self.base), 72 | 'touch %s/src/Module/Submodule/extra_file.txt' % (self.base), 73 | 'touch %s/src/Module/Submodule/extra_file2.txt' % (self.base), 74 | 'touch %s/src/Module/Submodule/additional_file.txt' % (self.base), 75 | ] 76 | 77 | for command in commands: 78 | self.logger.debug("Executing command %s" % command) 79 | subprocess.check_call(command.split(' ')) 80 | 81 | 82 | def tearDown(self): 83 | """ 84 | Clean up the files we've spewed all over 85 | """ 86 | subprocess.check_call(['rm', '-rf', self.base]) 87 | for filename in self.tarBalls: 88 | self.logger.debug('Deleting tarball %s' % filename) 89 | os.unlink(filename) 90 | 91 | return 92 | 93 | 94 | def testInit(self): 95 | """ 96 | Test constructor 97 | """ 98 | 99 | tb = UserTarball(name='default.tgz', logger=self.logger) 100 | self.assertEqual(os.path.basename(tb.name), 'default.tgz') 101 | self.tarBalls.append(tb.name) 102 | 103 | 104 | def testContext(self): 105 | """ 106 | Test the object out of context (after TarFile is closed) 107 | """ 108 | with UserTarball(name='default.tgz', logger=self.logger) as tb: 109 | self.tarBalls.append(tb.name) 110 | self.assertRaises(IOError, tb.addFiles) 111 | self.assertEqual(tarfile.GNU_FORMAT, tb.tarfile.format) 112 | 113 | 114 | def testAddFiles(self): 115 | """ 116 | Test the basic tarball, no userfiles 117 | """ 118 | members = ['lib', 'lib/slc5_ia32_gcc434', 'lib/slc5_ia32_gcc434/libSomewhere.so', 119 | 'lib/slc5_ia32_gcc434/libSomething.so', 'src/Module/Submodule/data', 120 | 'src/Module/Submodule/data/datafile.txt', ] 121 | with UserTarball(name='default.tgz', logger=self.logger) as tb: 122 | self.tarBalls.append(tb.name) 123 | tb.addFiles() 124 | self.assertEqual(sorted(tb.getnames()), sorted(members)) 125 | 126 | 127 | def testGlob(self): 128 | """ 129 | Test globbing and extra files 130 | """ 131 | userFiles = ['%s/src/Module/Submodule/extra_*.txt' % (self.base), 132 | '%s/src/Module/Submodule/additional_file.txt' % (self.base)] 133 | 134 | tb = UserTarball(name='default.tgz', logger=self.logger) 135 | tb.addFiles(userFiles=userFiles) 136 | 137 | members = ['lib', 'lib/slc5_ia32_gcc434', 'lib/slc5_ia32_gcc434/libSomewhere.so', 138 | 'lib/slc5_ia32_gcc434/libSomething.so', 'src/Module/Submodule/data', 139 | 'src/Module/Submodule/data/datafile.txt', 'extra_file2.txt', 140 | 'extra_file.txt', 'additional_file.txt'] 141 | 142 | self.assertEqual(sorted(tb.getnames()), sorted(members)) 143 | self.tarBalls.append(tb.name) 144 | 145 | 146 | def testMissingGlob(self): 147 | """ 148 | Test globbing and extra files 149 | """ 150 | userFiles = ['%s/src/Module/Submodule/extra_*.txt' % (self.base), 151 | '%s/src/Module/Submodule/missing_file.txt' % (self.base)] 152 | 153 | tb = UserTarball(name='default.tgz', logger=self.logger) 154 | 155 | self.assertRaises(InputFileNotFoundException, tb.addFiles, userFiles=userFiles) 156 | self.tarBalls.append(tb.name) 157 | 158 | 159 | def testAccess(self): 160 | """ 161 | Test accesses with __getattr__ to the underlying TarFile. 162 | This test really should be done with assertRaises as a context manager 163 | which is only available in python 2.7 164 | """ 165 | 166 | tb = UserTarball(name='default.tgz', logger=self.logger) 167 | 168 | try: 169 | tb.doesNotExist() 170 | self.fail('Did not raise AttributeError') 171 | except AttributeError: 172 | pass 173 | 174 | try: 175 | x = tb.doesNotExistEither 176 | self.fail('Did not raise AttributeError') 177 | except AttributeError: 178 | pass 179 | 180 | 181 | def testUpload(self): 182 | """ 183 | Test uploading to a crab server 184 | """ 185 | 186 | tb = UserTarball(name='default.tgz', logger=self.logger, config=testWMConfig) 187 | result = tb.upload() 188 | self.assertTrue(result['size'] > 0) 189 | self.assertTrue(len(result['hashkey']) > 0) 190 | 191 | 192 | if __name__ == '__main__': 193 | unittest.main() 194 | 195 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | Standard python setup.py file for CrabClient. 5 | To build : python setup.py build 6 | To install : python setup.py install --prefix= 7 | To clean : python setup.py clean 8 | To run tests: python setup.py test 9 | """ 10 | from __future__ import print_function 11 | 12 | import sys 13 | import os 14 | from unittest import TextTestRunner, TestLoader 15 | from glob import glob 16 | from os.path import splitext, basename, join as pjoin 17 | from distutils.core import setup 18 | from distutils.cmd import Command 19 | from distutils.command.install import INSTALL_SCHEMES 20 | 21 | sys.path.append(os.path.join(os.getcwd(), 'src/python')) 22 | from CRABClient import __version__ as cc_version 23 | 24 | required_python_version = '2.6' 25 | 26 | class TestCommand(Command): 27 | """ 28 | Class to handle unit tests 29 | """ 30 | user_options = [ ] 31 | 32 | def initialize_options(self): 33 | """Init method""" 34 | self._dir = os.getcwd() 35 | 36 | def finalize_options(self): 37 | """Finalize method""" 38 | pass 39 | 40 | def run(self): 41 | """ 42 | Finds all the tests modules in test/, and runs them. 43 | """ 44 | # list of files to exclude, 45 | # e.g. [pjoin(self._dir, 'test', 'exclude_t.py')] 46 | exclude = [] 47 | # list of test files 48 | testfiles = [] 49 | for tname in glob(pjoin(self._dir, 'test', '*_t.py')): 50 | if not tname.endswith('__init__.py') and \ 51 | tname not in exclude: 52 | testfiles.append('.'.join( 53 | ['test', splitext(basename(tname))[0]]) 54 | ) 55 | testfiles.sort() 56 | try: 57 | tests = TestLoader().loadTestsFromNames(testfiles) 58 | except: 59 | print("\nFail to load unit tests", testfiles) 60 | raise 61 | test = TextTestRunner(verbosity = 2) 62 | test.run(tests) 63 | 64 | class CleanCommand(Command): 65 | """ 66 | Class which clean-up all pyc files 67 | """ 68 | user_options = [ ] 69 | 70 | def initialize_options(self): 71 | """Init method""" 72 | self._clean_me = [ ] 73 | for root, dirs, files in os.walk('.'): 74 | for fname in files: 75 | if fname.endswith('.pyc'): 76 | self._clean_me.append(pjoin(root, fname)) 77 | 78 | def finalize_options(self): 79 | """Finalize method""" 80 | pass 81 | 82 | def run(self): 83 | """Run method""" 84 | for clean_me in self._clean_me: 85 | try: 86 | os.unlink(clean_me) 87 | except: 88 | pass 89 | 90 | def dirwalk(relativedir): 91 | """ 92 | Walk a directory tree and look-up for __init__.py files. 93 | If found yield those dirs. Code based on 94 | http://code.activestate.com/recipes/105873-walk-a-directory-tree-using-a-generator/ 95 | """ 96 | idir = os.path.join(os.getcwd(), relativedir) 97 | for fname in os.listdir(idir): 98 | fullpath = os.path.join(idir, fname) 99 | if os.path.isdir(fullpath) and not os.path.islink(fullpath): 100 | for subdir in dirwalk(fullpath): # recurse into subdir 101 | yield subdir 102 | else: 103 | initdir, initfile = os.path.split(fullpath) 104 | if initfile == '__init__.py': 105 | yield initdir 106 | 107 | def find_packages(relativedir): 108 | "Find list of packages in a given dir" 109 | packages = [] 110 | for idir in dirwalk(relativedir): 111 | package = idir.replace(os.getcwd() + '/', '') 112 | package = package.replace(relativedir + '/', '') 113 | package = package.replace('/', '.') 114 | packages.append(package) 115 | return packages 116 | 117 | def datafiles(idir): 118 | """Return list of data files in provided relative dir""" 119 | files = [] 120 | for dirname, dirnames, filenames in os.walk(idir): 121 | for subdirname in dirnames: 122 | files.append(os.path.join(dirname, subdirname)) 123 | for filename in filenames: 124 | if filename[-1] == '~': 125 | continue 126 | files.append(os.path.join(dirname, filename)) 127 | return files 128 | 129 | def main(): 130 | "Main function" 131 | version = cc_version 132 | name = "CRABClient" 133 | description = "CMS CRAB Client" 134 | url = \ 135 | "https://twiki.cern.ch/twiki/bin/viewauth/CMS/RunningCRAB3" 136 | readme = "CRAB Client %s" % url 137 | author = "", 138 | author_email = "", 139 | keywords = ["CRABClient"] 140 | package_dir = \ 141 | {"CRABClient": "src/python/CRABClient", "CRABAPI": "src/python/CRABAPI"} 142 | packages = find_packages('src/python') 143 | data_files = [('etc', ['doc/FullConfiguration.py', 'doc/ExampleConfiguration.py', 'etc/crab-bash-completion.sh', 'etc/init-light.sh', 'etc/init-light.csh', 'etc/init-light-pre.sh'])] # list of tuples whose entries are (dir, [data_files]) 144 | cms_license = "CMS experiment software" 145 | classifiers = [ 146 | "Development Status :: 3 - Production/Beta", 147 | "Intended Audience :: Developers", 148 | "License :: OSI Approved :: CMS/CERN Software License", 149 | "Operating System :: MacOS :: MacOS X", 150 | "Operating System :: Microsoft :: Windows", 151 | "Operating System :: POSIX", 152 | "Programming Language :: Python", 153 | "Topic :: Scientific/Engineering" 154 | ] 155 | 156 | if sys.version < required_python_version: 157 | msg = "I'm sorry, but %s %s requires Python %s or later." 158 | print(msg % (name, version, required_python_version)) 159 | sys.exit(1) 160 | 161 | # set default location for "data_files" to 162 | # platform specific "site-packages" location 163 | for scheme in INSTALL_SCHEMES.values(): 164 | scheme['data'] = scheme['purelib'] 165 | 166 | setup( 167 | name = name, 168 | version = version, 169 | description = description, 170 | long_description = readme, 171 | keywords = keywords, 172 | packages = packages, 173 | package_dir = package_dir, 174 | data_files = data_files, 175 | scripts = datafiles('bin'), 176 | requires = ['python (>=2.6)'], 177 | classifiers = classifiers, 178 | cmdclass = {'test': TestCommand, 'clean': CleanCommand}, 179 | author = author, 180 | author_email = author_email, 181 | url = url, 182 | license = cms_license, 183 | ) 184 | 185 | if __name__ == "__main__": 186 | main() 187 | -------------------------------------------------------------------------------- /test/python/CRABClient_t/JobType_t/CMSSW_t.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | 3 | """ 4 | _CMSSW_t_ 5 | 6 | Unittests for CMSSW JobType. These tests need to be pointed at a CRAB Server 7 | with functioning UserFileCache (sandbox) to function (see testWMConfig.General.serverUrl) below. 8 | """ 9 | 10 | import copy 11 | import logging 12 | import os 13 | import unittest 14 | 15 | from CRABClient.JobType.CMSSW import CMSSW 16 | from CRABClient.JobType.ScramEnvironment import ScramEnvironment 17 | 18 | # Re-use and extend simple configs from CMSSWConfig_t 19 | 20 | from CMSSWConfig_t import testWMConfig, testCMSSWConfig 21 | 22 | class CMSSWTest(unittest.TestCase): 23 | """ 24 | unittest for CMSSW JobType class 25 | 26 | """ 27 | 28 | # Set up a dummy logger 29 | level = logging.ERROR 30 | logger = logging.getLogger('UNITTEST') 31 | logger.setLevel(level) 32 | ch = logging.StreamHandler() 33 | ch.setLevel(level) 34 | logger.addHandler(ch) 35 | 36 | def setUp(self): 37 | """ 38 | Set up for unit tests 39 | """ 40 | # Extend simple config 41 | testWMConfig.JobType.inputFiles = [] 42 | testWMConfig.JobType.psetName = 'unittest_cfg.py' 43 | testWMConfig.Data.processingVersion = 'v1' 44 | testWMConfig.General.serverUrl = 'cms-xen39.fnal.gov:7723' # Set your server URL here if needed 45 | self.reqConfig = {} 46 | self.reqConfig['RequestorDN'] = "/DC=org/DC=doegrids/OU=People/CN=Eric Vaandering 768123" 47 | 48 | # Write a test python config file to run tests on 49 | with open(testWMConfig.JobType.psetName,'w') as f: 50 | f.write(testCMSSWConfig) 51 | 52 | 53 | def tearDown(self): 54 | """ 55 | Clean up the files we've spewed all over 56 | """ 57 | os.unlink('unittest_cfg.py') 58 | try: 59 | os.unlink('unit_test_full.py') 60 | except OSError: 61 | pass 62 | 63 | return 64 | 65 | 66 | def testScram(self): 67 | """ 68 | Test Scram environment 69 | """ 70 | 71 | msg = "You must set up a CMSSW environment first" 72 | scram = ScramEnvironment(logger=self.logger) 73 | self.assertNotEqual(scram.getCmsswVersion(), None, msg) 74 | self.assertNotEqual(scram.getScramArch(), None, msg) 75 | self.assertNotEqual(scram.getCmsswBase(), None, msg) 76 | 77 | 78 | def testInit(self): 79 | """ 80 | Test constructor 81 | """ 82 | 83 | cmssw = CMSSW(config=testWMConfig, logger=self.logger, workingdir=None) 84 | cmssw.run(self.reqConfig) 85 | 86 | 87 | def testOutputFiles(self): 88 | """ 89 | Make sure return arguments are modified to reflect files 90 | to be written 91 | """ 92 | outputFiles = ['histograms.root', 'output.root', 'output2.root'] 93 | cmssw = CMSSW(config=testWMConfig, logger=self.logger, workingdir=None) 94 | _dummy, configArguments = cmssw.run(self.reqConfig) 95 | self.assertEqual(configArguments['outputFiles'], outputFiles) 96 | 97 | 98 | def testSandbox(self): 99 | """ 100 | Make sure userSandbox is set and it creates a sandbox 101 | """ 102 | cmssw = CMSSW(config=testWMConfig, logger=self.logger, workingdir=None) 103 | tarFileName, configArguments = cmssw.run(self.reqConfig) 104 | self.assertTrue(configArguments['userSandbox']) 105 | self.assertTrue(os.path.getsize(tarFileName) > 0) 106 | 107 | 108 | def testNoInputFiles(self): 109 | """ 110 | Make sure userSandbox is set and it creates a sandbox even if inputFiles are not set 111 | """ 112 | del testWMConfig.JobType.inputFiles 113 | cmssw = CMSSW(config=testWMConfig, logger=self.logger, workingdir=None) 114 | tarFileName, configArguments = cmssw.run(self.reqConfig) 115 | self.assertTrue(configArguments['userSandbox']) 116 | self.assertTrue(os.path.getsize(tarFileName) > 0) 117 | 118 | 119 | def testScramOut(self): 120 | """ 121 | Make sure return arguments contain SCRAM info 122 | """ 123 | cmssw = CMSSW(config=testWMConfig, logger=self.logger, workingdir=None) 124 | _dummy, configArguments = cmssw.run(self.reqConfig) 125 | self.assertEqual(configArguments['ScramArch'], os.environ['SCRAM_ARCH']) 126 | self.assertEqual(configArguments['CMSSWVersion'], os.environ['CMSSW_VERSION']) 127 | 128 | 129 | def testSpecKeys(self): 130 | """ 131 | Make sure return arguments contain other stuff eventually in WMSpec 132 | """ 133 | cmssw = CMSSW(config=testWMConfig, logger=self.logger, workingdir=None) 134 | _dummy, configArguments = cmssw.run(self.reqConfig) 135 | self.assertTrue(len(configArguments['InputDataset']) > 0) 136 | self.assertTrue('ProcessingVersion' in configArguments) 137 | self.assertTrue('AnalysisConfigCacheDoc' in configArguments) 138 | 139 | 140 | def testValidateConfig(self): 141 | """ 142 | Validate config, done as part of the constructor 143 | """ 144 | origConfig = copy.deepcopy(testWMConfig) 145 | 146 | # Make sure the original config works 147 | cmssw = CMSSW(config=origConfig, logger=self.logger, workingdir=None) 148 | valid, reason = cmssw.validateConfig(config=testWMConfig) 149 | self.assertTrue(valid) 150 | self.assertEqual(reason, '') 151 | 152 | # Test a couple of ways of screwing up the processing version 153 | testConfig = copy.deepcopy(testWMConfig) 154 | testConfig.Data.processingVersion = '' 155 | self.assertRaises(Exception, CMSSW, config=testConfig, logger=self.logger, workingdir=None) 156 | del testConfig.Data.processingVersion 157 | self.assertRaises(Exception, CMSSW, config=testConfig, logger=self.logger, workingdir=None) 158 | 159 | # Test a bad input dataset 160 | testConfig = copy.deepcopy(testWMConfig) 161 | testConfig.Data.inputDataset = '' 162 | self.assertRaises(Exception, CMSSW, config=testConfig, logger=self.logger, workingdir=None) 163 | 164 | # Test a bad psetName 165 | testConfig = copy.deepcopy(testWMConfig) 166 | del testConfig.JobType.psetName 167 | self.assertRaises(Exception, CMSSW, config=testConfig, logger=self.logger, workingdir=None) 168 | 169 | # Test several errors, make sure the reason message catches them all. 170 | cmssw = CMSSW(config=origConfig, logger=self.logger, workingdir=None) 171 | testConfig = copy.deepcopy(testWMConfig) 172 | testConfig.Data.processingVersion = '' 173 | testConfig.Data.inputDataset = '' 174 | del testConfig.JobType.psetName 175 | valid, reason = cmssw.validateConfig(config=testConfig) 176 | self.assertFalse(valid) 177 | self.assertEqual(reason.count('.'), 3) 178 | 179 | 180 | 181 | if __name__ == '__main__': 182 | unittest.main() 183 | -------------------------------------------------------------------------------- /src/python/CRABClient/Commands/setfilestatus.py: -------------------------------------------------------------------------------- 1 | # pylint: disable=consider-using-f-string, unspecified-encoding 2 | """ 3 | allow users to (in)validate some files in their USER datasets in phys03 4 | """ 5 | 6 | import json 7 | 8 | from CRABClient.Commands.SubCommand import SubCommand 9 | from CRABClient.ClientExceptions import MissingOptionException, ConfigurationException, CommandFailedException 10 | from CRABClient.ClientUtilities import colors 11 | from CRABClient.RestInterfaces import getDbsREST 12 | 13 | 14 | class setfilestatus(SubCommand): 15 | """ 16 | Set status of a USER file in phys03 17 | """ 18 | 19 | name = 'setfilestatus' 20 | 21 | def __init__(self, logger, cmdargs=None): 22 | SubCommand.__init__(self, logger, cmdargs) 23 | 24 | def __call__(self): 25 | 26 | result = 'FAILED' # will change to 'SUCCESS' when all is OK 27 | 28 | # intitalize, and validate args 29 | dbsInstance = self.options.dbsInstance 30 | dataset = self.options.dataset 31 | files = self.options.files 32 | status = self.options.status 33 | self.logger.debug('dbsInstance = %s' % dbsInstance) 34 | self.logger.debug('dataset = %s' % dataset) 35 | self.logger.debug('files = %s' % files) 36 | self.logger.debug('status = %s' % status) 37 | 38 | statusToSet = 1 if status == 'VALID' else 0 39 | 40 | filesToChange = None 41 | if files: 42 | # did the user specify the name of a file containing a list of LFN's ? 43 | try: 44 | with open(files, 'r') as f: 45 | flist = [lfn.strip() for lfn in f] 46 | filesToChange = ','.join(flist) 47 | except IOError: 48 | # no. Assume we have a comma separated list of LFN's (a single LFN is also OK) 49 | filesToChange = files.strip(",").strip() 50 | finally: 51 | # files and dataset options are mutually exclusive 52 | dataset = None 53 | if ',' in filesToChange: 54 | raise NotImplementedError('list of LFNs is not supported yet') 55 | 56 | # from DBS instance, to DBS REST services 57 | dbsReader, dbsWriter = getDbsREST(instance=dbsInstance, logger=self.logger, 58 | cert=self.proxyfilename, key=self.proxyfilename) 59 | # we will need the dataset name 60 | if dataset: 61 | datasetName = dataset 62 | else: 63 | # get it from DBS 64 | lfn = filesToChange.split(',')[0] 65 | query = {'logical_file_name': lfn} 66 | out, rc, msg = dbsReader.get(uri='datasets', data=query) 67 | if not out: 68 | self.logger.error("ERROR: file %s not found in DBS" % lfn) 69 | raise ConfigurationException 70 | datasetName = out[0]['dataset'] 71 | self.logger.info('LFN to be changed belongs to dataset %s' % datasetName) 72 | 73 | # when acting on a list of LFN's, can't print status of all files before/after 74 | # best we can do is to print the number of valid/invalid file in the dataset 75 | # before/after. 76 | 77 | self.logFilesTally(dataset=datasetName, dbs=dbsReader) 78 | 79 | if filesToChange: 80 | data = {'logical_file_name': filesToChange, 'is_file_valid': statusToSet} 81 | if dataset: 82 | data = {'dataset': dataset, 'is_file_valid': statusToSet} 83 | jdata = json.dumps(data) # PUT requires data in JSON format 84 | out, rc, msg = dbsWriter.put(uri='files', data=jdata) 85 | if rc == 200 and msg == 'OK': 86 | self.logger.info("File(s) status changed successfully") 87 | result = 'SUCCESS' 88 | else: 89 | msg = "File(s) status change failed: %s" % out 90 | raise CommandFailedException(msg) 91 | 92 | self.logFilesTally(dataset=datasetName, dbs=dbsReader) 93 | 94 | return {'commandStatus': result} 95 | 96 | def logFilesTally(self, dataset=None, dbs=None): 97 | """ prints total/valid/invalid files in dataset """ 98 | query = {'dataset': dataset, 'validFileOnly': 1} 99 | out, _, _ = dbs.get(uri='files', data=query) 100 | valid = len(out) 101 | query = {'dataset': dataset, 'validFileOnly': 0} 102 | out, _, _ = dbs.get(uri='files', data=query) 103 | total = len(out) 104 | invalid = total - valid 105 | self.logger.info("Dataset file count total/valid/invalid = %d/%d/%d" % (total, valid, invalid)) 106 | 107 | def setOptions(self): 108 | """ 109 | __setOptions__ 110 | 111 | This allows to set specific command options 112 | """ 113 | self.parser.add_option('--dbs-instance', dest='dbsInstance', default='prod/phys03', 114 | help="DBS instance. e.g. prod/phys03 (default) or int/phys03 or full URL." 115 | + "\nUse at your own risk only if you really know what you are doing" 116 | ) 117 | self.parser.add_option('-d', '--dataset', dest='dataset', default=None, 118 | help='Will apply status to all files in this dataset.' 119 | + ' Use either --files or--dataset', 120 | metavar='') 121 | self.parser.add_option('-s', '--status', dest='status', default=None, 122 | help='New status of the file(s): VALID/INVALID', 123 | choices=['VALID', 'INVALID'] 124 | ) 125 | self.parser.add_option('-f', '--files', dest='files', default=None, 126 | help='List of files to be validated/invalidated.' 127 | + ' Can be either a simple LFN or a file containg LFNs or' 128 | + ' a comma separated list of LFNs. Use either --files or --dataset', 129 | metavar="") 130 | 131 | def validateOptions(self): 132 | SubCommand.validateOptions(self) 133 | 134 | if not self.options.files and not self.options.dataset: 135 | msg = "%sError%s: Please specify the files to change." % (colors.RED, colors.NORMAL) 136 | msg += " Use either the --files or the --dataset option." 137 | ex = MissingOptionException(msg) 138 | ex.missingOption = "files" 139 | raise ex 140 | if self.options.files and self.options.dataset: 141 | msg = "%sError%s: You can not use both --files and --dataset at same time" % (colors.RED, colors.NORMAL) 142 | raise ConfigurationException(msg) 143 | if self.options.status is None: 144 | msg = "%sError%s: Please specify the new file(s) status." % (colors.RED, colors.NORMAL) 145 | msg += " Use the --status option." 146 | ex = MissingOptionException(msg) 147 | ex.missingOption = "status" 148 | raise ex 149 | dbsInstance = self.options.dbsInstance 150 | if '/' not in dbsInstance or len(dbsInstance.split('/')) > 2 and not dbsInstance.startswith('https://'): 151 | msg = "Bad DBS instance value %s. " % dbsInstance 152 | msg += "Use either server/db format or full URL" 153 | raise ConfigurationException(msg) 154 | -------------------------------------------------------------------------------- /src/python/CRABClient/Commands/preparelocal.py: -------------------------------------------------------------------------------- 1 | # tell pylint to accept some old style which is needed for python2 2 | # pylint: disable=unspecified-encoding, raise-missing-from 3 | """ 4 | The commands prepares a directory and the relative scripts to execute the jobs locally. 5 | It can also execute a specific job if the jobid option is passed 6 | """ 7 | import os 8 | import shutil 9 | import tarfile 10 | import tempfile 11 | 12 | from ServerUtilities import getColumn, downloadFromS3 13 | 14 | from CRABClient.ClientUtilities import execute_command 15 | from CRABClient.Commands.SubCommand import SubCommand 16 | from CRABClient.ClientExceptions import ClientException 17 | 18 | 19 | class preparelocal(SubCommand): 20 | """ the preparelocal command instance """ 21 | 22 | def __init__(self, logger, cmdargs=None): 23 | SubCommand.__init__(self, logger, cmdargs) 24 | self.destination = None #Save the ASO destintion from he DB when we download input files 25 | 26 | def __call__(self): 27 | #Creating dest directory if needed 28 | if self.options.destdir is None: 29 | self.options.destdir = os.path.join(self.requestarea, 'local') 30 | if not os.path.isdir(self.options.destdir): 31 | os.makedirs(self.options.destdir) 32 | self.options.destdir = os.path.abspath(self.options.destdir) 33 | cwd = os.getcwd() 34 | try: 35 | tmpDir = tempfile.mkdtemp() 36 | os.chdir(tmpDir) 37 | 38 | self.logger.info("Getting input files into tmp dir %s" % tmpDir) 39 | self.getInputFiles() 40 | 41 | if self.options.jobid: 42 | self.logger.info("Executing job %s locally" % self.options.jobid) 43 | self.prepareDir(self.options.destdir) 44 | self.executeTestRun(self.options.destdir, self.options.jobid) 45 | self.logger.info("Job execution terminated") 46 | else: 47 | self.logger.info("Copying and preparing files for local execution in %s" % self.options.destdir) 48 | self.prepareDir(self.options.destdir) 49 | self.logger.info("go to that directory IN A CLEAN SHELL and use 'sh run_job.sh NUMJOB' to execute the job") 50 | finally: 51 | os.chdir(cwd) 52 | shutil.rmtree(tmpDir) 53 | 54 | # all methods called before raise if something goes wrong. Getting here means success 55 | return {'commandStatus': 'SUCCESS'} 56 | 57 | def getInputFiles(self): 58 | """ 59 | Get the InputFiles.tar.gz and extract the necessary files 60 | """ 61 | taskname = self.cachedinfo['RequestName'] 62 | 63 | #Get task status from the task DB 64 | self.logger.debug("Getting status from he DB") 65 | server = self.crabserver 66 | crabDBInfo, _, _ = server.get(api='task', data={'subresource': 'search', 'workflow': taskname}) 67 | status = getColumn(crabDBInfo, 'tm_task_status') 68 | self.destination = getColumn(crabDBInfo, 'tm_asyncdest') 69 | username = getColumn(crabDBInfo, 'tm_username') 70 | sandboxName = getColumn(crabDBInfo, 'tm_user_sandbox') 71 | inputsFilename = os.path.join(os.getcwd(), 'InputFiles.tar.gz') 72 | 73 | uploadedStatus = getColumn(crabDBInfo, 'tm_uploaded') 74 | if uploadedStatus == 'F': 75 | raise ClientException('This only works for tasks submitted or uploaded. Current status is %s' % status) 76 | inputsFilename = os.path.join(os.getcwd(), 'InputFiles.tar.gz') 77 | sandboxFilename = os.path.join(os.getcwd(), 'sandbox.tar.gz') 78 | downloadFromS3(crabserver=self.crabserver, filepath=inputsFilename, 79 | objecttype='runtimefiles', taskname=taskname, logger=self.logger) 80 | downloadFromS3(crabserver=self.crabserver, filepath=sandboxFilename, 81 | objecttype='sandbox', logger=self.logger, 82 | tarballname=sandboxName, username=username) 83 | 84 | jobWrapperTarball = 'CMSRunAnalysis.tar.gz' 85 | twScriptsTarball = 'TaskManagerRun.tar.gz' 86 | with tarfile.open(inputsFilename) as tf: 87 | # this contains jobWrapperTarball and twScriptsTarball 88 | # various needed files are inside those in new version of TW, or present 89 | # at top level of inputFilename for older TW. Follosing code works for both 90 | tf.extractall() 91 | with tarfile.open(jobWrapperTarball) as tf: 92 | tf.extractall() 93 | with tarfile.open(twScriptsTarball) as tf: 94 | tf.extractall() 95 | 96 | def executeTestRun(self, destDir, jobnr): 97 | """ 98 | Execute a test run calling CMSRunAnalysis.sh 99 | """ 100 | os.chdir(destDir) 101 | cmd = 'eval `scram unsetenv -sh`;'\ 102 | ' bash run_job.sh %s' % str(jobnr) 103 | execute_command(cmd, logger=self.logger, redirect=False) 104 | 105 | def prepareDir(self, targetDir): 106 | """ Prepare a directory with just the necessary files: 107 | """ 108 | 109 | for f in ["gWMS-CMSRunAnalysis.sh", "CMSRunAnalysis.sh", "cmscp.py", "CMSRunAnalysis.tar.gz", 110 | "sandbox.tar.gz", "run_and_lumis.tar.gz", "input_files.tar.gz", "Job.submit", 111 | "submit_env.sh", "splitting-summary.json", "input_args.json" 112 | ]: 113 | try: # for backward compatibility with TW v3.241017 where splitting-summary.json is missing 114 | shutil.copy2(f, targetDir) 115 | except FileNotFoundError: 116 | pass 117 | 118 | cmd = "cd %s; tar xf CMSRunAnalysis.tar.gz" % targetDir 119 | execute_command(command=cmd, logger=self.logger) 120 | 121 | self.logger.debug("Creating run_job.sh file") 122 | # Few observations about the wrapper: 123 | # All the export are done because normally this env is set by condor (see the Environment classad) 124 | # Exception is CRAB3_RUNTIME_DEBUG that is set to avoid the dashboard code to blows up since come classad are not there 125 | # We check the X509_USER_PROXY variable is set otherwise stageout fails 126 | # The "tar xzmf CMSRunAnalysis.tar.gz" is needed because in CRAB3_RUNTIME_DEBUG mode the file is not unpacked (why?) 127 | # Job.submit is also modified to set some things that are condor macro expanded during submission (needed by cmscp) 128 | bashWrapper = """#!/bin/bash 129 | 130 | [[ "$1" =~ ^[1-9][0-9]*$ ]] || { echo "Usage: $0 "; exit 1; } 131 | 132 | . ./submit_env.sh && save_env && setup_local_env 133 | 134 | export _CONDOR_JOB_AD=Job.${1}.submit 135 | # leading '+' signs must be removed to use JDL as classAd file 136 | sed -e 's/^+//' Job.submit > Job.${1}.submit 137 | 138 | sh ./CMSRunAnalysis.sh --jobId ${1} 139 | """ 140 | 141 | with open(os.path.join(targetDir, "run_job.sh"), "w") as fd: 142 | fd.write(bashWrapper) 143 | 144 | def setOptions(self): 145 | """ 146 | __setOptions__ 147 | 148 | This allows to set specific command options 149 | """ 150 | self.parser.add_option("--jobid", 151 | dest="jobid", 152 | default=None, 153 | type="int", 154 | help="Optional id of the job you want to execute locally") 155 | 156 | self.parser.add_option("--destdir", 157 | dest="destdir", 158 | default=None, 159 | help="Optional name of the directory to use, defaults to /local") 160 | 161 | def validateOptions(self): 162 | SubCommand.validateOptions(self) 163 | 164 | if self.options.jobid is not None: 165 | try: 166 | int(self.options.jobid) 167 | except ValueError: 168 | raise ClientException("The --jobid option has to be an integer") 169 | -------------------------------------------------------------------------------- /test/python/CRABClient_t/Commands_t/CRABRESTModelMock.py: -------------------------------------------------------------------------------- 1 | from CRABClient import load_source 2 | from WMCore.WebTools.RESTModel import RESTModel 3 | import WMCore 4 | 5 | import threading 6 | import cherrypy 7 | import os 8 | import uuid 9 | import tempfile 10 | 11 | SI_RESULT = {} 12 | SI_RESULT['server_dn'] = '' 13 | SI_RESULT['my_proxy'] = 'myproxy.cern.ch' 14 | 15 | FILE_NAME = 'src_output.root' 16 | goodLumisResult = '{"1":[ [1,15], [30,50] ], "3":[ [10,15], [30,50] ]}' 17 | publishResult = {u'status': True, u'message': 'Publication completed for campaign ewv_crab_something_1_111229_140959', u'summary': {u'/primary/secondary-out1-v1/USER': {u'files': 10, u'blocks': 1, u'existingFiles': 10}, u'/primary/secondary-out2-v1/USER': {u'files': 10, u'blocks': 1, u'existingFiles': 10}}} 18 | 19 | 20 | class CRABRESTModelMock(RESTModel): 21 | def __init__(self, config={}): 22 | RESTModel.__init__(self, config) 23 | 24 | self.mapme = load_source('', os.path.join( os.path.dirname(__file__), "../../../data/mapper.py")) 25 | 26 | self.defaulturi = self.mapme.defaulturi 27 | 28 | self._addMethod('POST', 'user', self.addNewUser, 29 | args=[], 30 | validation=[self.isalnum]) 31 | 32 | self._addMethod('POST', 'task', self.postRequest, 33 | args=['requestName'], 34 | validation=[self.isalnum]) 35 | 36 | self._addMethod('DELETE', 'task', self.deleteRequest, 37 | args=['requestID'], 38 | validation=[self.isalnum]) 39 | 40 | self._addMethod('GET', 'task', self.getTaskStatus, 41 | args=['requestID'], 42 | validation=[self.isalnum]) 43 | #/data 44 | self._addMethod('GET', 'data', self.getDataLocation, 45 | args=['requestID','jobRange'], validation=[self.isalnum]) 46 | 47 | self._addMethod('POST', 'publish', self.publish, 48 | args=['requestName'], 49 | validation=[self.isalnum]) 50 | 51 | #/goodLumis 52 | self._addMethod('GET', 'goodLumis', self.getGoodLumis, 53 | args=['requestID'], validation=[self.isalnum]) 54 | # 55 | self._addMethod('POST', 'lumiMask', self.postLumiMask, 56 | args=[], validation=[self.isalnum]) 57 | 58 | # Server 59 | self._addMethod('GET', 'info', self.getServerInfo, 60 | args=[], 61 | validation=[self.isalnum]) 62 | 63 | self._addMethod('GET', 'requestmapping', self.getClientMapping, 64 | args=[], 65 | validation=[self.isalnum]) 66 | 67 | self._addMethod('GET', 'jobErrors', self.getJobErrors, 68 | args=['requestID'], 69 | validation=[self.isalnum]) 70 | 71 | self._addMethod('POST', 'resubmit', self.reSubmit, 72 | args=['requestID'], 73 | validation=[self.isalnum]) 74 | 75 | cherrypy.engine.subscribe('start_thread', self.initThread) 76 | 77 | 78 | #not sure if we really need to validate input. 79 | def isalnum(self, call_input): 80 | """ 81 | Validates that all input is alphanumeric, with spaces and underscores 82 | tolerated. 83 | """ 84 | for v in call_input.values(): 85 | WMCore.Lexicon.identifier(v) 86 | return call_input 87 | 88 | 89 | 90 | def initThread(self, thread_index): 91 | """ 92 | The ReqMgr expects the DBI to be contained in the Thread 93 | """ 94 | myThread = threading.currentThread() 95 | #myThread = cherrypy.thread_data 96 | # Get it from the DBFormatter superclass 97 | myThread.dbi = self.dbi 98 | 99 | 100 | def getServerInfo(self): 101 | """ 102 | Return information to allow client operations 103 | """ 104 | 105 | return SI_RESULT 106 | 107 | 108 | def getTaskStatus(self, requestID): 109 | return {u'workflows': [{u'request': u'cinquilli.nocern_crab_TESTME_1_111025_181202', 110 | u'requestDetails': {u'RequestMessages': [], u'RequestStatus': u'aborted'}, 111 | u'states': {u'/cinquilli.nocern_crab_TESTME_1_111025_181202/Analysis': {u'success': {u'count': 9, u'jobIDs': [117, 118, 119, 120, 121, 122, 123, 124, 125], 112 | u'jobs': [1, 2, 3, 4, 5, 6, 7, 8, 9]}}, 113 | u'/cinquilli.nocern_crab_TESTME_1_111025_181202/Analysis/LogCollect': {u'success': {u'count': 1, u'jobIDs': [126], u'jobs': [10]}}}, 114 | u'subOrder': 1}, 115 | {u'request': u'cinquilli.nocern_crab_TESTME_1_resubmit_111028_000117', 116 | u'requestDetails': {u'RequestMessages': [['request failed']], u'RequestStatus': u'failed'}, 117 | u'states': {}, 118 | u'subOrder': 2}]} 119 | 120 | 121 | def getDataLocation(self, requestID, jobRange): 122 | f = open(FILE_NAME, 'w') 123 | f.close() 124 | return {u'data': [{u'output': {u'1': {u'pfn': unicode(FILE_NAME)}}, 125 | u'request': u'cinquilli.nocern_crab_TESTME_1_111025_181202', 126 | u'subOrder': 1}, 127 | {u'output': {}, 128 | u'request': u'cinquilli.nocern_crab_TESTME_1_resubmit_111028_000117', 129 | u'subOrder': 2}]} 130 | 131 | 132 | def getGoodLumis(self, requestID): 133 | """ 134 | Mockup to return the list of good lumis processed as generated 135 | by CouchDB 136 | """ 137 | return goodLumisResult 138 | 139 | 140 | def publish(self, requestName): 141 | """ 142 | Mockup to return the publication summary 143 | """ 144 | return publishResult 145 | 146 | 147 | def getClientMapping(self): 148 | """ 149 | Return the dictionary that allows the client to map the client configuration to the server request 150 | It also returns the URI for each API 151 | """ 152 | 153 | return self.defaulturi 154 | 155 | def deleteRequest(self, requestID): 156 | return {"result": "ok"} 157 | 158 | 159 | def addNewUser(self): 160 | return { "hn_name" : "mmascher" } 161 | 162 | 163 | def postRequest(self, requestName): 164 | return {'ID': 'mmascher_crab_MyAnalysis26_110707_164957'} 165 | 166 | def postLumiMask(self): 167 | """ 168 | Mock version of result of ACDC upload 169 | """ 170 | 171 | result = {} 172 | result['DocID'] = uuid.uuid4().hex 173 | result['DocRev'] = uuid.uuid4().hex 174 | result['Name'] = "%s-cmsRun1" % params['RequestName'] 175 | 176 | return result 177 | 178 | 179 | def getJobErrors(self, requestID): 180 | failed = {'1': 181 | {'0': { 182 | 'step1': [ { "details": "Error in StageOut: 99109\n'x.z.root' does not match regular expression /store/temp/([a-zA-Z0-9\\-_]+).root", 183 | "type":"Misc. StageOut error: 99109\n", 184 | "exitCode":99109 } 185 | ], 186 | 'step2': [ { "details": "Cannot find file in jobReport path: /x/y/z/job_134/Report.1.pkl", 187 | "type":"99999", 188 | "exitCode":84 } 189 | ] 190 | } 191 | }, 192 | '2': 193 | {'0': { 194 | 'step1': [ { "details": "Error in StageOut: 99109\n'x.z.root' does not match regular expression /store/temp/([a-zA-Z0-9\\-_]+).root", 195 | "type":"Misc. StageOut error: 99109\n", 196 | "exitCode":99109 } 197 | ] 198 | }, 199 | '1': { 200 | 'step1': [ { "details": "Error in StageOut: 99109\n'x.z.root' does not match regular expression /store/temp/([a-zA-Z0-9\\-_]+).root", "type":"Misc. StageOut error: 99109\n", 201 | "exitCode":99109 } 202 | ] 203 | } 204 | } 205 | } 206 | return {u'errors': [{u'details': failed, u'request': u'cinquilli.nocern_crab_TESTME_1_111025_181202', u'subOrder': 1}, 207 | {u'details': {}, u'request': u'cinquilli.nocern_crab_TESTME_1_resubmit_111028_000117', u'subOrder': 2}]} 208 | 209 | def reSubmit(self, requestID): 210 | return {"result": "ok"} 211 | -------------------------------------------------------------------------------- /src/python/CRABClient/UserUtilities.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module contains the utility methods available for users. 3 | """ 4 | 5 | # avoid complains about things that we can not fix in python2 6 | # pylint: disable=consider-using-f-string, unspecified-encoding, raise-missing-from 7 | 8 | import os 9 | import logging 10 | import json 11 | 12 | try: 13 | from FWCore.PythonUtilities.LumiList import LumiList 14 | except Exception: # pylint: disable=broad-except 15 | # if FWCore version is not py3 compatible, use our own 16 | from CRABClient.LumiList import LumiList 17 | 18 | ## CRAB dependencies 19 | from CRABClient.ClientUtilities import LOGLEVEL_MUTE, colors 20 | from CRABClient.ClientUtilities import execute_command 21 | from CRABClient.ClientExceptions import ClientException 22 | from CRABClient.ClientUtilities import getUsernameFromCRIC_wrapped 23 | from WMCore.Configuration import Configuration 24 | 25 | def config(): 26 | """ 27 | Return a Configuration object containing all the sections that CRAB recognizes. 28 | """ 29 | config = Configuration() # pylint: disable=redefined-outer-name 30 | config.section_("General") 31 | config.section_("JobType") 32 | config.section_("Data") 33 | config.section_("Site") 34 | config.section_("User") 35 | config.section_("Debug") 36 | return config 37 | 38 | def getUsername(proxyFile=None, logger=None): 39 | """ 40 | get globally unique username to be used for this CRAB work 41 | this is a generic high level function which can be called even w/o arguments 42 | and will figure out the username from the current authentication credential 43 | found in the environment. 44 | Yet it allows the called to guide it via optional argument to be quicker 45 | and easier to tune to different authentication systemd (X509 now, tokens later e.g.) 46 | :param proxyFile: the full path of the file containing the X509 VOMS proxy, if missing 47 | :param logger: a logger object to use for messages, if missing, it will report to standard logger 48 | :return: username : a string 49 | """ 50 | 51 | if not logger: logger=logging.getLogger() 52 | logger.debug("Retrieving username ...") 53 | 54 | if not proxyFile: 55 | proxyFile = '/tmp/x509up_u%d'%os.getuid() if 'X509_USER_PROXY' not in os.environ else os.environ['X509_USER_PROXY'] 56 | username = getUsernameFromCRIC_wrapped(logger, proxyFile, quiet=True) 57 | if username: 58 | logger.debug("username is %s", username) 59 | else: 60 | msg = "%sERROR:%s CRIC could not resolve the DN in the user proxy into a user name" \ 61 | % (colors.RED, colors.NORMAL) 62 | msg += "\n Please find below details of failures for investigation:" 63 | logger.error(msg) 64 | username = getUsernameFromCRIC_wrapped(logger, proxyFile, quiet=False) 65 | 66 | return username 67 | 68 | 69 | def curlGetFileFromURL(url, filename = None, proxyfilename = None, logger=None): 70 | """ 71 | Read the content of a URL into a file via curl 72 | 73 | url: the link you would like to retrieve 74 | filename: the local filename where the url is saved to. Defaults to the filename in the url 75 | proxyfilename: the x509 proxy certificate to be used in case auth is required 76 | returns: the exit code of the command if command failed, otherwise the HTTP code of the call 77 | note that curl exits with status 0 if the HTTP calls fail, 78 | """ 79 | 80 | ## Path to certificates. 81 | capath = os.environ['X509_CERT_DIR'] if 'X509_CERT_DIR' in os.environ else "/etc/grid-security/certificates" 82 | 83 | # send curl output to file and http_code to stdout 84 | downloadCommand = 'curl -sS --capath %s --cert %s --key %s -o %s -w %%"{http_code}"' %\ 85 | (capath, proxyfilename, proxyfilename, filename) 86 | downloadCommand += ' "%s"' % url 87 | if logger: 88 | logger.debug("Will execute:\n%s", downloadCommand) 89 | stdout, stderr, rc = execute_command(downloadCommand, logger=logger) 90 | errorDetails = '' 91 | 92 | if rc != 0: 93 | os.unlink(filename) 94 | httpCode = 503 95 | else: 96 | httpCode = int(stdout) 97 | if httpCode != 200: 98 | with open(filename) as f: 99 | errorDetails = f.read() 100 | os.unlink(filename) 101 | if logger: 102 | logger.debug('exitcode: %s\nstdout: %s\nstderr: %s\nerror details: %s', rc, stdout, stderr, errorDetails) 103 | 104 | return httpCode 105 | 106 | 107 | def getLumiListInValidFiles(dataset, dbsurl='phys03'): 108 | """ 109 | Get the runs/lumis in the valid files of a given dataset via dasgoclient 110 | 111 | dataset: the dataset name as published in DBS 112 | dbsurl: the DBS URL or DBS prod instance 113 | 114 | Returns a LumiList object. 115 | """ 116 | 117 | def complain(cmd, stdout, stderr, returncode): 118 | """ factor out a bit or distracting code """ 119 | msg = 'Failed executing %s. Exitcode is %s' % (cmd, returncode) 120 | if stdout: 121 | msg += '\n Stdout:\n %s' % str(stdout).replace('\n', '\n ') 122 | if stderr: 123 | msg += '\n Stderr:\n %s' % str(stderr).replace('\n', '\n ') 124 | raise ClientException(msg) 125 | 126 | # prepare a dasgoclient command line where only the query is missing 127 | instance = "prod/" + dbsurl 128 | dasCmd = "dasgoclient --query " + " '%s instance=" + instance + "' --json" 129 | 130 | # note that dasgpoclient offern the handy query "file,run,lumi dataset=... status=valid" 131 | # but the output has one entry per file (ok) and that has one list of run numbers and one 132 | # uncorrelated list of lumis. We need to stick to the query "lumi file=..." to get 133 | # one entry per lumi with lumi number and corresponding run number. Sigh 134 | 135 | # get the list of valid files 136 | validFiles = [] 137 | query = 'file dataset=%s' % dataset 138 | cmd = dasCmd % query 139 | stdout, stderr, returncode = execute_command(command=cmd) 140 | if returncode or not stdout: 141 | complain(cmd, stdout, stderr, returncode) 142 | else: 143 | result = json.loads(stdout) 144 | # returns a list of dictionaries, one per file 145 | # each dictionary has the keys 'das', 'qhash' and 'file'. 146 | # value of 'file' key is a list of dictionaries with only 1 element and 147 | # the usual DBS fields for a file 148 | for record in result: 149 | file = record['file'][0] 150 | if file['is_file_valid']: 151 | validFiles.append(file['name']) 152 | 153 | # get (run,lumi) pair list from each valid file 154 | runLumiPairs = [] 155 | for file in validFiles: 156 | query = "lumi file=%s" % file 157 | cmd = dasCmd % query 158 | stdout, stderr, returncode = execute_command(command=cmd) 159 | if returncode or not stdout: 160 | complain(cmd, stdout, stderr, returncode) 161 | else: 162 | result = json.loads(stdout) 163 | # returns a list of dictionaries, one per lumi, with keys 'das', 'qhash' and 'lumi' 164 | # valud of 'lumi' is a list of dictionaries with only 1 element and 165 | # keys: 'event_count', 'file', 'lumi_section_num', 'nevents', 'number', 'run.run_number', 'run_number' 166 | # upon inspection run.run_number is always 0 167 | for lumiInfo in result: 168 | lumiDict = lumiInfo['lumi'][0] 169 | run = lumiDict['run_number'] 170 | lumi = lumiDict['lumi_section_num'] 171 | runLumiPairs.append((run, lumi)) 172 | 173 | # transform into a LumiList object 174 | lumiList = LumiList(lumis=runLumiPairs) 175 | 176 | return lumiList 177 | 178 | 179 | def getLoggers(): 180 | from CRABClient.ClientUtilities import LOGGERS 181 | return LOGGERS 182 | 183 | 184 | def getConsoleLogLevel(): 185 | from CRABClient.ClientUtilities import CONSOLE_LOGLEVEL 186 | return CONSOLE_LOGLEVEL 187 | 188 | 189 | def setConsoleLogLevel(lvl): 190 | from CRABClient.ClientUtilities import setConsoleLogLevelVar 191 | setConsoleLogLevelVar(lvl) 192 | if 'CRAB3.all' in logging.getLogger().manager.loggerDict: 193 | for h in logging.getLogger('CRAB3.all').handlers: 194 | h.setLevel(lvl) 195 | 196 | def getMutedStatusInfo(logger=None, proxy=None, projdir=None): 197 | """ 198 | Mute the status console output before calling status and change it back to normal afterwards. 199 | """ 200 | mod = __import__('CRABClient.Commands.status', fromlist='status') 201 | cmdargs = [] 202 | if proxy: 203 | cmdargs.append("--proxy") 204 | cmdargs.append(proxy) 205 | if projdir: 206 | cmdargs.append("-d") 207 | cmdargs.append(projdir) 208 | cmdobj = getattr(mod, 'status')(logger=logger, cmdargs=cmdargs) 209 | loglevel = getConsoleLogLevel() 210 | setConsoleLogLevel(LOGLEVEL_MUTE) 211 | statusDict = cmdobj.__call__() 212 | setConsoleLogLevel(loglevel) 213 | 214 | if statusDict['statusFailureMsg']: 215 | # If something happens during status execution we still want to print it 216 | logger.error("Error while getting status information. Got:\n%s " % 217 | statusDict['statusFailureMsg']) 218 | 219 | return statusDict 220 | 221 | def getColumn(dictresult, columnName): 222 | columnIndex = dictresult['desc']['columns'].index(columnName) 223 | value = dictresult['result'][columnIndex] 224 | if value=='None': 225 | return None 226 | else: 227 | return value 228 | -------------------------------------------------------------------------------- /bin/crab.py: -------------------------------------------------------------------------------- 1 | """ 2 | This contains the hooks to call the different command plug-ins. 3 | It is not intended to contain any of the CRAB-3 client logic, 4 | it simply: 5 | - intercepts the CLI options and command 6 | - loads and calls the specified command 7 | - exit with the proper exit codes 8 | """ 9 | from __future__ import print_function 10 | from __future__ import division 11 | import sys 12 | import os 13 | import logging 14 | import logging.handlers 15 | import re 16 | 17 | from ServerUtilities import FEEDBACKMAIL 18 | from CRABClient.CRABOptParser import CRABOptParser 19 | from CRABClient import __version__ as client_version 20 | from CRABClient.ClientMapping import parametersMapping 21 | from CRABClient.ClientExceptions import ClientException, RESTInterfaceException, CommandFailedException 22 | from CRABClient.ClientUtilities import getAvailCommands, initLoggers, setConsoleLogLevelVar, StopExecution, flushMemoryLogger, LOGFORMATTER 23 | 24 | if not os.environ.get('CMSSW_VERSION',None): 25 | print('\nError: $CMSSW_VERSION is not defined. Make sure you do cmsenv first. Exiting...') 26 | sys.exit() 27 | 28 | if sys.version_info < (2, 6): 29 | print('\nError: using a version of python < 2.6. Exiting...\n') 30 | sys.exit() 31 | 32 | if 'crab-dev' in __file__: 33 | print('BEWARE: this is the development version of CRAB Client.\nBe sure to have a good reason for using it\n') 34 | 35 | 36 | class MyNullHandler(logging.Handler): 37 | """ 38 | TODO: Python 2.7 supplies a null handler that will replace this. 39 | """ 40 | def emit(self, record): 41 | """ 42 | TODO: Python 2.7 supplies a null handler that will replace this. 43 | """ 44 | pass # pylint: disable=unnecessary-pass 45 | 46 | 47 | class CRABClient(object): 48 | def __init__( self ): 49 | """ 50 | Get the command to run, the options to pass it and a logger instance 51 | at appropriate level 52 | """ 53 | 54 | self.subCommands = getAvailCommands() 55 | self.parser = CRABOptParser(self.subCommands) 56 | self.tblogger = None 57 | self.logger = None 58 | self.memhandler = None 59 | self.cmd = None 60 | 61 | def __call__(self): 62 | 63 | (options, args) = self.parser.parse_args() 64 | 65 | ## The default logfile destination is ./crab.log. It will be changed once we 66 | ## know/create the CRAB project directory. 67 | if options.quiet: 68 | setConsoleLogLevelVar(logging.WARNING) 69 | elif options.debug: 70 | setConsoleLogLevelVar(logging.DEBUG) 71 | self.tblogger, self.logger, self.memhandler = initLoggers() 72 | 73 | #Instructions needed in case of early failures: sometimes the traceback logger 74 | #has not been set yet. 75 | 76 | ## Will replace Python's sys.excepthook default function with the next function. 77 | ## This function is used for handling uncaught exceptions (in a Python program 78 | ## this happens just before the program exits). 79 | ## In this function: 80 | ## - make sure everything is logged to the crab.log file; 81 | ## However, we already have a `finally' clause where we make sure everything is 82 | ## logged to the crab log file. 83 | def log_exception(exc_type, exc_value, tback): 84 | """ 85 | Send a short version of the exception to the console, 86 | a long version to the log 87 | 88 | Adapted from Doug Hellmann 89 | 90 | This might help sometimes: 91 | import traceback,pprint; 92 | pprint.pprint(traceback.format_tb(tback)) 93 | """ 94 | 95 | ## Add to the CRAB3 logger a file handler to the log file (if it doesn't have it 96 | ## already). 97 | tbLogger = logging.getLogger('CRAB3') 98 | hasFileHandler = False 99 | for h in tbLogger.handlers: 100 | if isinstance(h, logging.FileHandler) and h.stream.name == client.logger.logfile: 101 | hasFileHandler = True 102 | if not hasFileHandler: 103 | filehandler = logging.FileHandler(client.logger.logfile) 104 | filehandler.setFormatter(LOGFORMATTER) 105 | tbLogger.addHandler(filehandler) 106 | ## This goes to the log file. 107 | tbLogger.error("Unhandled Exception!") 108 | tbLogger.error(exc_value, exc_info = (exc_type, exc_value, tback)) 109 | 110 | ## This goes to the console (via the CRAB3.all logger) and to the log file (via 111 | ## the parent CRAB3 logger). 112 | logger = logging.getLogger('CRAB3.all') 113 | logger.error("ERROR: %s: %s", exc_type.__name__, exc_value) 114 | logger.error("\n Look for more details in crab.log file %s", client.logger.logfile) 115 | logger.error(" If you can't figure out the problem, please send email to") 116 | logger.error("\t%s with the crab.log file or crab.log URL.", FEEDBACKMAIL) 117 | logger.error(" Please use 'crab uploadlog' to upload the log file %s to the CRAB cache.", client.logger.logfile) 118 | 119 | sys.excepthook = log_exception 120 | 121 | # check that the command is valid 122 | if len(args) == 0: 123 | print("You have not specified a command.") 124 | # Described the valid commands in epilog, reuse here 125 | print(self.parser.epilog) 126 | sys.exit(-1) 127 | 128 | sub_cmd = None 129 | try: 130 | sub_cmd = next( v for k,v in self.subCommands.items() if args[0] in v.shortnames or args[0]==v.name) 131 | except StopIteration: 132 | print("'" + str(args[0]) + "' is not a valid command.") 133 | self.parser.print_help() 134 | sys.exit(-1) 135 | self.cmd = sub_cmd(self.logger, args[1:]) # the crab command to be executed 136 | 137 | # Every command returns a dictionary which MUST contain the "commandStatus" key. 138 | # Any value other then "SUCCESS" for this key indicates command failure 139 | # and will result in 'crab' terminating with non-zero exit code 140 | # Additional keys may be present in the dictionary, depending on the specific command, 141 | # which are used to pass information to caller when CRABAPI is used 142 | returnDict = self.cmd() 143 | if returnDict['commandStatus'] != 'SUCCESS': 144 | raise CommandFailedException("Command %s failed" % str(args[0])) 145 | 146 | 147 | if __name__ == "__main__": 148 | # Create the crab object and start it 149 | # Handled in a try/except to run in a controlled environment 150 | # - do not want to expose known exception to the outside 151 | # - exceptions thrown in the client should exit and set an approprate 152 | # exit code, this is a safety net 153 | exitcode = 1 154 | client = CRABClient() 155 | schedInterv = "It seems the CMSWEB frontend is not responding. Please check: https://twiki.cern.ch/twiki/bin/viewauth/CMS/ScheduledInterventions?sortcol=3;table=1;up=2#sorted_table" 156 | try: 157 | client() 158 | exitcode = 0 #no exceptions no errors 159 | except RESTInterfaceException as err: 160 | exitcode=err.exitcode 161 | client.logger.info("The server answered with an error.") 162 | client.logger.debug("") 163 | err = str(err) 164 | if ("CMSWEB Error: Service unavailable") in err: 165 | client.logger.info(schedInterv) 166 | if 'X-Error-Detail' in err: 167 | errorDetail = re.search(r'(?<=X-Error-Detail:\s)[^\n]*', err).group(0) 168 | client.logger.info('Server answered with: %s', errorDetail) 169 | if 'X-Error-Info' in err: 170 | reason = re.search(r'(?<=X-Error-Info:\s)[^\n]*', err).group(0) 171 | for parname in parametersMapping['on-server']: 172 | tmpmsg = "'%s'" % (parname) 173 | if tmpmsg in reason and parametersMapping['on-server'][parname]['config']: 174 | reason = reason.replace(tmpmsg, tmpmsg.replace(parname, ' or '.join(parametersMapping['on-server'][parname]['config']))) 175 | client.logger.info('Reason is: %s', reason) 176 | if 'X-Error-Id' in err: 177 | errorId = re.search(r'(?<=X-Error-Id:\s)[^\n]*', err).group(0) 178 | client.logger.info('Error Id: %s', errorId) 179 | logging.getLogger('CRAB3').exception('Caught RESTInterfaceException exception') 180 | except CommandFailedException as ce: 181 | client.logger.warning(ce) 182 | logging.getLogger('CRAB3').exception('Caught CommandFailedException exception') 183 | exitcode = 1 184 | except ClientException as ce: 185 | client.logger.error(ce) 186 | logging.getLogger('CRAB3').exception('Caught ClientException exception') 187 | exitcode = ce.exitcode 188 | except KeyboardInterrupt: 189 | client.logger.error('Keyboard Interrupted') 190 | exitcode = 1 191 | except StopExecution: 192 | exitcode = 0 193 | finally: 194 | # the command crab --version does not have a logger instance 195 | if getattr(client, 'tblogger', None) and getattr(client, 'memhandler', None) and getattr(client, 'logger', None): 196 | flushMemoryLogger(client.tblogger, client.memhandler, client.logger.logfile) 197 | 198 | if getattr(client, 'cmd', None): 199 | # this will also print out location of log file 200 | client.cmd.terminate( exitcode ) 201 | else: 202 | client.logger.info('Log file is %s', client.logger.logfile) 203 | 204 | sys.exit( exitcode ) 205 | -------------------------------------------------------------------------------- /src/python/CRABClient/ProxyInteractions.py: -------------------------------------------------------------------------------- 1 | """ 2 | Provides utilities do deal with voms-proxy* and myproy-* 3 | Meant to replace use of WMCore/Credential/Proxy 4 | But differently from that, use two different objects to interact with 5 | voms-proxy-* or myproxy-* since, this will make things more clear and 6 | coder more readable 7 | """ 8 | import os 9 | from datetime import datetime 10 | 11 | from CRABClient.ClientUtilities import execute_command 12 | from CRABClient.ClientExceptions import ProxyCreationException 13 | 14 | class VomsProxy(object): 15 | def __init__(self, logger=None): 16 | """ 17 | Constructor, sets sensible defaults for everything 18 | """ 19 | vomsDesiredValidDays = 8 20 | self.proxyChanged = False 21 | self.certLocation = '~/.globus/usercert.pem' if 'X509_USER_CERT' not in os.environ else os.environ['X509_USER_CERT'] 22 | self.keyLocation = '~/.globus/userkey.pem' if 'X509_USER_KEY' not in os.environ else os.environ['X509_USER_KEY'] 23 | self.proxyFile = '/tmp/x509up_u%d' % os.getuid() if 'X509_USER_PROXY' not in os.environ else os.environ['X509_USER_PROXY'] 24 | self.logger = logger 25 | self.DN = '' 26 | self.desiredValidity = '%i:00' % (vomsDesiredValidDays*24) # from days to hh:mm 27 | self.timeleft = 0 28 | self.group = '' 29 | self.role = 'NULL' 30 | 31 | def create(self, timeLeftThreshold=720): 32 | # is there a proxy already ? 33 | # does it have correct group and role ? 34 | # is it valid long enough ? 35 | # all OK, do nothing 36 | # need a new proxy 37 | cmd = 'voms-proxy-init --rfc' 38 | cmd += ' --cert %s' % self.certLocation 39 | cmd += ' --key %s' % self.keyLocation 40 | cmd += ' --out %s' % self.proxyFile 41 | cmd += ' --valid %s' % self.desiredValidity 42 | vomsString = 'cms:/cms' 43 | if self.group: 44 | vomsString += '/%s' % self.group 45 | if self.role and self.role != 'NULL': 46 | vomsString += '/Role=%s' % self.role 47 | cmd += ' --voms %s' % vomsString 48 | stdout, stderr, rc = execute_command(cmd, logger=self.logger, redirect=False) 49 | if rc != 0: 50 | self.logger.error(stdout+'\n'+stderr) 51 | msg = "\n".join(['Error executing %s:' % cmd, stdout, stderr]) 52 | raise ProxyCreationException(msg) 53 | 54 | def setVOGroupVORole(self, group, role): 55 | self.group = group 56 | self.role = role if role != '' else 'NULL' 57 | 58 | def getFilename(self): 59 | return self.proxyFile 60 | 61 | def validateVO(self): 62 | # make sure that proxy has a VOMS extension for CMS VirtualOrganization 63 | cmd = 'voms-proxy-info --vo --file %s' % self.proxyFile 64 | stdout, stderr, rc = execute_command(cmd, logger=self.logger) 65 | if rc != 0 or 'cms' not in stdout: 66 | msg = "\n".join(['Error executing %s:' % cmd, stdout, stderr]) 67 | self.logger.error(msg) 68 | msg = 'proxy %s is not a valid proxy file or has no valid VOMS extension.\n' % self.proxyFile 69 | stdout, stderr, rc = execute_command('voms-proxy-info -all', logger=self.logger) 70 | msg += 'output of voms-proxy-info -all is\n%s' % (stdout+'\n'+stderr) 71 | msg += '\n**** Make sure you do voms-proxy-init -voms cms ****\n' 72 | raise ProxyCreationException(msg) 73 | 74 | def getTimeLeft(self): 75 | cmd = 'voms-proxy-info --actimeleft --timeleft --file %s' % self.proxyFile 76 | stdout, stderr, rc = execute_command(cmd, logger=self.logger) 77 | if rc != 0: 78 | self.logger.error(stdout+'\n'+stderr) 79 | msg = "\n".join(['Error executing %s:' % cmd, stdout, stderr]) 80 | raise ProxyCreationException(msg) 81 | 82 | # pick the shorter between actimeleft and timeleft 83 | times = stdout.split('\n') 84 | timeLeft = min(int(times[0]), int(times[1])) 85 | return timeLeft 86 | 87 | def getGroupAndRole(self): 88 | cmd = 'voms-proxy-info --fqan --file %s' % self.proxyFile 89 | stdout, stderr, rc = execute_command(cmd, logger=self.logger) 90 | if rc != 0: 91 | self.logger.error(stdout+'\n'+stderr) 92 | msg = "\n".join(['Error executing %s:' % cmd, stdout, stderr]) 93 | raise ProxyCreationException(msg) 94 | fqans = str(stdout) 95 | primaryFqan = fqans.split('\n')[0] 96 | attributes = primaryFqan.split('/') 97 | if len(attributes) > 4: 98 | group = attributes[2] 99 | role = attributes[3].split('=')[1] 100 | else: 101 | group = '' 102 | role = attributes[2].split('=')[1] 103 | return group, role 104 | 105 | def getSubject(self): 106 | cmd = 'voms-proxy-info --identity --file %s' % self.proxyFile 107 | stdout, stderr, rc = execute_command(cmd, logger=self.logger) 108 | if rc != 0: 109 | self.logger.error(stdout+'\n'+stderr) 110 | msg = "\n".join(['Error executing %s:' % cmd, stdout, stderr]) 111 | raise ProxyCreationException(msg) 112 | return stdout.rstrip() 113 | 114 | 115 | class MyProxy(object): 116 | """ 117 | an object to interact with myproxy-* commands 118 | """ 119 | 120 | def __init__(self, username=None, logger=None): 121 | """ 122 | Constructor, sets sensible defaults for everything 123 | """ 124 | self.certLocation = '~/.globus/usercert.pem' if 'X509_USER_CERT' not in os.environ else os.environ['X509_USER_CERT'] 125 | self.keyLocation = '~/.globus/userkey.pem' if 'X509_USER_KEY' not in os.environ else os.environ['X509_USER_KEY'] 126 | self.logger = logger 127 | self.timeleft = 0 128 | self.username = username 129 | 130 | def create(self, username=None, retrievers=None, validity=720): 131 | """ 132 | creates a new credential in myproxy.cern.ch 133 | args: username: string: the username of the credential, usually the has of the user DN 134 | args: retrievers: string: regexp indicating list of DN's authorized to retrieve this credential 135 | args: validity: integer: how long this credential will be valid for in hours, default is 30 days 136 | example of the command we want : 137 | command : export GT_PROXY_MODE=rfc 138 | myproxy-init -d -n -s myproxy.cern.ch 139 | -x -R '/DC=ch/DC=cern/OU=computers/CN=vocms0105.cern.ch|/DC=ch/DC=cern/OU=computers/CN=crab-(preprod|prod|dev)-tw(02|01).cern.ch|/DC=ch/DC=cern/OU=computers/CN=(ddi|ddidk|mytw).cern.ch|/DC=ch/DC=cern/OU=computers/CN=stefanov(m|m2).cern.ch' 140 | -x -Z '/DC=ch/DC=cern/OU=computers/CN=vocms0105.cern.ch|/DC=ch/DC=cern/OU=computers/CN=crab-(preprod|prod|dev)-tw(02|01).cern.ch|/DC=ch/DC=cern/OU=computers/CN=(ddi|ddidk|mytw).cern.ch|/DC=ch/DC=cern/OU=computers/CN=stefanov(m|m2).cern.ch' 141 | -l 'be1f4dc5be8664cbd145bf008f5399adf42b086f' 142 | -t 168:00 -c 3600:00 143 | """ 144 | cmd = 'export GT_PROXY_MODE=rfc ; myproxy-init -d -n -s myproxy.cern.ch' 145 | cmd += ' -C %s' % self.certLocation 146 | cmd += ' -y %s' % self.keyLocation 147 | cmd += ' -x -R \'%s\'' % retrievers 148 | cmd += ' -x -Z \'%s\'' % retrievers 149 | cmd += ' -l %s' % username 150 | cmd += ' -t 168 -c %s' % validity # validity of the retrieved proxy: 7 days = 168 hours 151 | stdout, stderr, rc = execute_command(cmd, logger=self.logger) 152 | if rc != 0: 153 | self.logger.error(stdout+'\n'+stderr) 154 | msg = "\n".join(['Error executing %s:' % cmd, stdout, stderr]) 155 | raise ProxyCreationException(msg) 156 | 157 | def getInfo(self, username=None): 158 | """ 159 | returns information about a credential stored in myproxy.cern.ch 160 | args: username: string: the username of the credential, usually the has of the user DN 161 | """ 162 | cmd = 'myproxy-info -s myproxy.cern.ch -l %s' % username 163 | stdout, stderr, rc = execute_command(cmd, logger=self.logger) 164 | if rc != 0: 165 | self.logger.error(stdout+'\n'+stderr) 166 | if rc > 0 or not stdout: # if there's no credential myproxy-info returns rc=1 167 | return 0, '' 168 | olines = stdout.rstrip().split('\n') 169 | trustedRetrievalPolicy = olines[-2] 170 | # allow for ':' in the trustedRetrievers DN's (as for robot cert !) 171 | # by taking everything after the first ':' in myproxy-info output 172 | # split(':', maxsplit=1) would be more clear, but it is not allowed in python2 173 | trustedRetrievers = trustedRetrievalPolicy.split(':', 1)[1].strip() 174 | times = olines[-1].split(':') 175 | hours = int(times[1]) 176 | mins = int(times[2]) 177 | timeLeft = hours*3600 + mins*60 # let's ignore seconds 178 | return timeLeft, trustedRetrievers 179 | 180 | def getUserCertEndDate(self): 181 | """ 182 | Return the number of seconds until the expiration of the user cert 183 | in .globus/usercert.pem or $X509_USER_CERT if set 184 | """ 185 | cmd = 'openssl x509 -noout -dates -in %s' % self.certLocation 186 | stdout, stderr, rc = execute_command(cmd, logger=self.logger) 187 | if rc != 0: 188 | self.logger.error(stdout+'\n'+stderr) 189 | msg = "\n".join(['Error executing %s:' % cmd, stdout, stderr]) 190 | raise ProxyCreationException(msg) 191 | out = stdout.rstrip().split('notAfter=')[1] 192 | 193 | possibleFormats = ['%b %d %H:%M:%S %Y %Z', 194 | '%b %d %H:%M:%S %Y %Z'] 195 | exptime = None 196 | for frmt in possibleFormats: 197 | try: 198 | exptime = datetime.strptime(out, frmt) 199 | except ValueError: 200 | pass # try next format 201 | if not exptime: 202 | # If we cannot decode the output in any way print 203 | # a message and fallback to voms-proxy-info command 204 | self.logger.warning( 205 | 'Cannot decode "openssl x509 -noout -in %s -dates" date format.' % self.certLocation) 206 | timeleft = 0 207 | else: 208 | # if everything is fine then we are ready to return!! 209 | timeleft = (exptime - datetime.utcnow()).total_seconds() 210 | daystoexp = int(timeleft // (60. * 60 * 24)) 211 | return daystoexp 212 | 213 | -------------------------------------------------------------------------------- /doc/generate_modules.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | """ 4 | sphinx-autopackage-script 5 | 6 | This script parses a directory tree looking for python modules and packages and 7 | creates ReST files appropriately to create code documentation with Sphinx. 8 | It also creates a modules index (named modules.). 9 | """ 10 | from __future__ import print_function 11 | 12 | # Copyright 2008 Société des arts technologiques (SAT), http://www.sat.qc.ca/ 13 | # Copyright 2010 Thomas Waldmann 14 | # All rights reserved. 15 | # 16 | # This program is free software: you can redistribute it and/or modify 17 | # it under the terms of the GNU General Public License as published by 18 | # the Free Software Foundation, either version 2 of the License, or 19 | # (at your option) any later version. 20 | # 21 | # This program is distributed in the hope that it will be useful, 22 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 23 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 24 | # GNU General Public License for more details. 25 | # 26 | # You should have received a copy of the GNU General Public License 27 | # along with this program. If not, see . 28 | 29 | 30 | import os 31 | import optparse 32 | 33 | 34 | # automodule options 35 | OPTIONS = ['members', 36 | 'undoc-members', 37 | # 'inherited-members', # disabled because there's a bug in sphinx 38 | 'show-inheritance', 39 | ] 40 | 41 | INIT = '__init__.py' 42 | 43 | def makename(package, module): 44 | """Join package and module with a dot.""" 45 | # Both package and module can be None/empty. 46 | if package: 47 | name = package 48 | if module: 49 | name += '.' + module 50 | else: 51 | name = module 52 | return name 53 | 54 | def write_file(name, text, opts): 55 | """Write the output file for module/package .""" 56 | if opts.dryrun: 57 | return 58 | fname = os.path.join(opts.destdir, "%s.%s" % (name, opts.suffix)) 59 | if not opts.force and os.path.isfile(fname): 60 | print ('File %s already exists, skipping.' % fname) 61 | else: 62 | print ('Creating file %s.' % fname) 63 | f = open(fname, 'w') 64 | f.write(text) 65 | f.close() 66 | 67 | def format_heading(level, text): 68 | """Create a heading of [1, 2 or 3 supported].""" 69 | underlining = ['=', '-', '~', ][level-1] * len(text) 70 | return '%s\n%s\n\n' % (text, underlining) 71 | 72 | def format_directive(module, package=None): 73 | """Create the automodule directive and add the options.""" 74 | directive = '.. automodule:: %s\n' % makename(package, module) 75 | for option in OPTIONS: 76 | directive += ' :%s:\n' % option 77 | return directive 78 | 79 | def create_module_file(package, module, opts): 80 | """Build the text of the file and write the file.""" 81 | text = format_heading(1, '%s Module' % module) 82 | text += format_heading(2, ':mod:`%s` Module' % module) 83 | text += format_directive(module, package) 84 | write_file(makename(package, module), text, opts) 85 | 86 | def create_package_file(root, master_package, subroot, py_files, opts, subs): 87 | """Build the text of the file and write the file.""" 88 | package = os.path.split(root)[-1] 89 | text = format_heading(1, '%s Package' % package) 90 | # add each package's module 91 | for py_file in py_files: 92 | if shall_skip(os.path.join(root, py_file)): 93 | continue 94 | is_package = py_file == INIT 95 | py_file = os.path.splitext(py_file)[0] 96 | py_path = makename(subroot, py_file) 97 | if is_package: 98 | heading = ':mod:`%s` Package' % package 99 | else: 100 | heading = ':mod:`%s` Module' % py_file 101 | text += format_heading(2, heading) 102 | text += format_directive(is_package and subroot or py_path, master_package) 103 | text += '\n' 104 | 105 | # build a list of directories that are packages (they contain an INIT file) 106 | subs = [sub for sub in subs if os.path.isfile(os.path.join(root, sub, INIT))] 107 | # if there are some package directories, add a TOC for theses subpackages 108 | if subs: 109 | text += format_heading(2, 'Subpackages') 110 | text += '.. toctree::\n\n' 111 | for sub in subs: 112 | text += ' %s.%s\n' % (makename(master_package, subroot), sub) 113 | text += '\n' 114 | 115 | write_file(makename(master_package, subroot), text, opts) 116 | 117 | def create_modules_toc_file(master_package, modules, opts, name='modules'): 118 | """ 119 | Create the module's index. 120 | """ 121 | text = format_heading(1, '%s Modules' % opts.header) 122 | text += '.. toctree::\n' 123 | text += ' :maxdepth: %s\n\n' % opts.maxdepth 124 | 125 | modules.sort() 126 | prev_module = '' 127 | for module in modules: 128 | # look if the module is a subpackage and, if yes, ignore it 129 | if module.startswith(prev_module + '.'): 130 | continue 131 | prev_module = module 132 | text += ' %s\n' % module 133 | 134 | write_file(name, text, opts) 135 | 136 | def shall_skip(module): 137 | """ 138 | Check if we want to skip this module. 139 | """ 140 | # skip it, if there is nothing (or just \n or \r\n) in the file 141 | return os.path.getsize(module) < 3 142 | 143 | def recurse_tree(path, excludes, opts): 144 | """ 145 | Look for every file in the directory tree and create the corresponding 146 | ReST files. 147 | """ 148 | # use absolute path for root, as relative paths like '../../foo' cause 149 | # 'if "/." in root ...' to filter out *all* modules otherwise 150 | path = os.path.abspath(path) 151 | # check if the base directory is a package and get is name 152 | if INIT in os.listdir(path): 153 | package_name = path.split(os.path.sep)[-1] 154 | else: 155 | package_name = None 156 | 157 | toc = [] 158 | tree = os.walk(path, False) 159 | for root, subs, files in tree: 160 | # keep only the Python script files 161 | py_files = sorted([f for f in files if os.path.splitext(f)[1] == '.py']) 162 | if INIT in py_files: 163 | py_files.remove(INIT) 164 | py_files.insert(0, INIT) 165 | # remove hidden ('.') and private ('_') directories 166 | subs = sorted([sub for sub in subs if sub[0] not in ['.', '_']]) 167 | # check if there are valid files to process 168 | # TODO: could add check for windows hidden files 169 | if "/." in root or "/_" in root \ 170 | or not py_files \ 171 | or is_excluded(root, excludes): 172 | continue 173 | if INIT in py_files: 174 | # we are in package ... 175 | if (# ... with subpackage(s) 176 | subs 177 | or 178 | # ... with some module(s) 179 | len(py_files) > 1 180 | or 181 | # ... with a not-to-be-skipped INIT file 182 | not shall_skip(os.path.join(root, INIT)) 183 | ): 184 | subroot = root[len(path):].lstrip(os.path.sep).replace(os.path.sep, '.') 185 | create_package_file(root, package_name, subroot, py_files, opts, subs) 186 | toc.append(makename(package_name, subroot)) 187 | elif root == path: 188 | # if we are at the root level, we don't require it to be a package 189 | for py_file in py_files: 190 | if not shall_skip(os.path.join(path, py_file)): 191 | module = os.path.splitext(py_file)[0] 192 | create_module_file(package_name, module, opts) 193 | toc.append(makename(package_name, module)) 194 | 195 | # create the module's index 196 | if not opts.notoc: 197 | create_modules_toc_file(package_name, toc, opts) 198 | 199 | def normalize_excludes(rootpath, excludes): 200 | """ 201 | Normalize the excluded directory list: 202 | * must be either an absolute path or start with rootpath, 203 | * otherwise it is joined with rootpath 204 | * with trailing slash 205 | """ 206 | sep = os.path.sep 207 | f_excludes = [] 208 | for exclude in excludes: 209 | if not os.path.isabs(exclude) and not exclude.startswith(rootpath): 210 | exclude = os.path.join(rootpath, exclude) 211 | if not exclude.endswith(sep): 212 | exclude += sep 213 | f_excludes.append(exclude) 214 | return f_excludes 215 | 216 | def is_excluded(root, excludes): 217 | """ 218 | Check if the directory is in the exclude list. 219 | 220 | Note: by having trailing slashes, we avoid common prefix issues, like 221 | e.g. an exlude "foo" also accidentally excluding "foobar". 222 | """ 223 | sep = os.path.sep 224 | if not root.endswith(sep): 225 | root += sep 226 | for exclude in excludes: 227 | if root.startswith(exclude): 228 | return True 229 | return False 230 | 231 | def main(): 232 | """ 233 | Parse and check the command line arguments. 234 | """ 235 | parser = optparse.OptionParser(usage="""usage: %prog [options] [exclude paths, ...] 236 | 237 | Note: By default this script will not overwrite already created files.""") 238 | parser.add_option("-n", "--doc-header", action="store", dest="header", help="Documentation Header (default=Project)", default="Project") 239 | parser.add_option("-d", "--dest-dir", action="store", dest="destdir", help="Output destination directory", default="") 240 | parser.add_option("-s", "--suffix", action="store", dest="suffix", help="module suffix (default=txt)", default="txt") 241 | parser.add_option("-m", "--maxdepth", action="store", dest="maxdepth", help="Maximum depth of submodules to show in the TOC (default=4)", type="int", default=4) 242 | parser.add_option("-r", "--dry-run", action="store_true", dest="dryrun", help="Run the script without creating the files") 243 | parser.add_option("-f", "--force", action="store_true", dest="force", help="Overwrite all the files") 244 | parser.add_option("-t", "--no-toc", action="store_true", dest="notoc", help="Don't create the table of content file") 245 | (opts, args) = parser.parse_args() 246 | if not args: 247 | parser.error("package path is required.") 248 | else: 249 | rootpath, excludes = args[0], args[1:] 250 | if os.path.isdir(rootpath): 251 | # check if the output destination is a valid directory 252 | if opts.destdir and os.path.isdir(opts.destdir): 253 | excludes = normalize_excludes(rootpath, excludes) 254 | recurse_tree(rootpath, excludes, opts) 255 | else: 256 | print ('%s is not a valid output destination directory.' % opts.destdir) 257 | else: 258 | print ('%s is not a valid directory.' % rootpath) 259 | 260 | 261 | if __name__ == '__main__': 262 | main() 263 | 264 | --------------------------------------------------------------------------------