├── pandaclient ├── __init__.py ├── PandaToolsPkgInfo.py ├── PcontainerScript.py ├── test_client_file.py ├── time_utils.py ├── test_client_submit_job.py ├── test_client_job.py ├── test_client_misc.py ├── PLogger.py ├── example_task.py ├── idds_api.py ├── BookConfig.py ├── test_client_task.py ├── FileSpec.py ├── localSpecs.py ├── Group_argparse.py ├── panda_jupyter.py ├── queryPandaMonUtils.py ├── panda_gui.py ├── pcontainer_core.py ├── openidc_utils.py ├── panda_api.py ├── LocalJobsetSpec.py ├── MiscUtils.py ├── PchainScript.py ├── LocalJobSpec.py ├── MyproxyUtils.py └── ParseJobXML.py ├── icons ├── red.png ├── back.png ├── green.png ├── kill.png ├── retry.png ├── sync.png ├── config.png ├── forward.png ├── orange.png ├── pandamon.png ├── savannah.png ├── update.png └── yellow.png ├── scripts ├── pathena ├── pcontainer ├── phpo ├── prun ├── pbook └── pchain ├── MANIFEST.in ├── glade └── pbook.gladep ├── setup.cfg ├── templates ├── panda_setup.sh.template ├── site_path.sh.template ├── panda_setup.csh.template ├── conda_meta.yaml.template └── panda_setup.example.cfg.template ├── .pre-commit-config.yaml ├── README.md ├── .github └── workflows │ ├── pythonpublish.yml │ └── condapublish.yml ├── packages ├── light │ └── pyproject.toml ├── full │ └── pyproject.toml └── hatch_build.py ├── share ├── functions.sh └── FakeAppMgr.py ├── setup.py └── LICENSE /pandaclient/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pandaclient/PandaToolsPkgInfo.py: -------------------------------------------------------------------------------- 1 | release_version = "1.6.5" 2 | -------------------------------------------------------------------------------- /icons/red.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PanDAWMS/panda-client/HEAD/icons/red.png -------------------------------------------------------------------------------- /icons/back.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PanDAWMS/panda-client/HEAD/icons/back.png -------------------------------------------------------------------------------- /icons/green.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PanDAWMS/panda-client/HEAD/icons/green.png -------------------------------------------------------------------------------- /icons/kill.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PanDAWMS/panda-client/HEAD/icons/kill.png -------------------------------------------------------------------------------- /icons/retry.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PanDAWMS/panda-client/HEAD/icons/retry.png -------------------------------------------------------------------------------- /icons/sync.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PanDAWMS/panda-client/HEAD/icons/sync.png -------------------------------------------------------------------------------- /icons/config.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PanDAWMS/panda-client/HEAD/icons/config.png -------------------------------------------------------------------------------- /icons/forward.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PanDAWMS/panda-client/HEAD/icons/forward.png -------------------------------------------------------------------------------- /icons/orange.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PanDAWMS/panda-client/HEAD/icons/orange.png -------------------------------------------------------------------------------- /icons/pandamon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PanDAWMS/panda-client/HEAD/icons/pandamon.png -------------------------------------------------------------------------------- /icons/savannah.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PanDAWMS/panda-client/HEAD/icons/savannah.png -------------------------------------------------------------------------------- /icons/update.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PanDAWMS/panda-client/HEAD/icons/update.png -------------------------------------------------------------------------------- /icons/yellow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PanDAWMS/panda-client/HEAD/icons/yellow.png -------------------------------------------------------------------------------- /scripts/pathena: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source ${PANDA_SYS}/etc/panda/share/functions.sh 4 | 5 | exec_p_command "import pandaclient.PathenaScript" "$@" 6 | -------------------------------------------------------------------------------- /scripts/pcontainer: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source ${PANDA_SYS}/etc/panda/share/functions.sh 4 | 5 | exec_p_command "import pandaclient.PcontainerScript" "$@" 6 | -------------------------------------------------------------------------------- /scripts/phpo: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source ${PANDA_SYS}/etc/panda/share/functions.sh 4 | 5 | exec_p_command "from pandaclient.PhpoScript import main; main()" "$@" 6 | -------------------------------------------------------------------------------- /scripts/prun: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source ${PANDA_SYS}/etc/panda/share/functions.sh 4 | 5 | exec_p_command "from pandaclient.PrunScript import main; main()" "$@" 6 | -------------------------------------------------------------------------------- /scripts/pbook: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source ${PANDA_SYS}/etc/panda/share/functions.sh 4 | 5 | exec_p_command "import pandaclient.PBookScript as pbook; pbook.main()" "$@" 6 | -------------------------------------------------------------------------------- /scripts/pchain: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source ${PANDA_SYS}/etc/panda/share/functions.sh 4 | 5 | exec_p_command "from pandaclient.PchainScript import main; main()" "$@" 6 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.txt *.py *.cfg 2 | recursive-include pandaclient *.py 3 | recursive-include share * 4 | recursive-include scripts * 5 | recursive-include glade * 6 | recursive-include icons *.png 7 | recursive-include templates *.template 8 | 9 | -------------------------------------------------------------------------------- /glade/pbook.gladep: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Pbook 6 | pbook 7 | 8 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [global] 2 | 3 | [bdist_rpm] 4 | provides = panda-client 5 | release = 1 6 | packager = Panda Team 7 | build_requires = python-devel 8 | requires = python 9 | 10 | [bdist_wheel] 11 | universal=1 12 | 13 | [install] 14 | record = install_record.txt 15 | -------------------------------------------------------------------------------- /templates/panda_setup.sh.template: -------------------------------------------------------------------------------- 1 | export PATH=@@install_scripts@@:$PATH 2 | 3 | export PANDA_CONFIG_ROOT=~/.pathena 4 | export PANDA_SYS=@@install_dir@@ 5 | 6 | export PANDA_PYTHONPATH=`bash ${PANDA_SYS}/etc/panda/site_path.sh` 7 | 8 | export PYTHONPATH=${PANDA_PYTHONPATH}${PYTHONPATH:+:$PYTHONPATH} 9 | -------------------------------------------------------------------------------- /pandaclient/PcontainerScript.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | from pandaclient import pcontainer_core 5 | 6 | os.environ['PANDA_EXEC_STRING'] = 'pcontainer' 7 | 8 | optP = pcontainer_core.make_arg_parse() 9 | 10 | options = optP.parse_args() 11 | 12 | status, output = pcontainer_core.submit(options) 13 | sys.exit(status) 14 | -------------------------------------------------------------------------------- /templates/site_path.sh.template: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source ${PANDA_SYS}/etc/panda/share/functions.sh 4 | 5 | exec_p_command \ 6 | "import sys; import os.path; "\ 7 | "s_path='@@install_dir@@/lib/python{0}.{1}/site-packages'.format(*sys.version_info); "\ 8 | "s_path = s_path if os.path.exists(s_path) else '@@install_purelib@@'; "\ 9 | "print(s_path)" 10 | -------------------------------------------------------------------------------- /templates/panda_setup.csh.template: -------------------------------------------------------------------------------- 1 | setenv PATH @@install_scripts@@:$PATH 2 | 3 | setenv PANDA_CONFIG_ROOT ~/.pathena 4 | setenv PANDA_SYS @@install_dir@@ 5 | setenv PANDA_PYTHONPATH `bash ${PANDA_SYS}/etc/panda/site_path.sh` 6 | 7 | if ($?PYTHONPATH) then 8 | setenv PYTHONPATH ${PANDA_PYTHONPATH}:$PYTHONPATH 9 | else 10 | setenv PYTHONPATH ${PANDA_PYTHONPATH} 11 | endif 12 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | 3 | - repo: https://github.com/psf/black 4 | rev: 23.9.1 5 | hooks: 6 | - id: black 7 | types: [python] 8 | args: ["--config", "packages/light/pyproject.toml"] 9 | 10 | - repo: https://github.com/pycqa/isort 11 | rev: 5.12.0 12 | hooks: 13 | - id: isort 14 | name: isort (python) 15 | args: ["--settings-path", "packages/light/pyproject.toml"] -------------------------------------------------------------------------------- /pandaclient/test_client_file.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from pandaclient.Client import putFile 3 | 4 | # Parse command-line arguments 5 | parser = argparse.ArgumentParser(description="Process panda IDs.") 6 | parser.add_argument("panda_ids", nargs="*", type=int, help="List of panda IDs") 7 | args = parser.parse_args() 8 | panda_ids = args.panda_ids 9 | 10 | print("=============================================================") 11 | 12 | file_ret = putFile("/root/test/a.py", verbose=True) 13 | print("putFile returned: {0}".format(file_ret)) 14 | -------------------------------------------------------------------------------- /pandaclient/time_utils.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | 3 | def aware_utcnow(): 4 | """ 5 | Return the current UTC date and time, with tzinfo timezone.utc 6 | 7 | Returns: 8 | datetime: current UTC date and time, with tzinfo timezone.utc 9 | """ 10 | return datetime.datetime.now(datetime.timezone.utc) 11 | 12 | 13 | def naive_utcnow(): 14 | """ 15 | Return the current UTC date and time, without tzinfo 16 | 17 | Returns: 18 | datetime: current UTC date and time, without tzinfo 19 | """ 20 | return aware_utcnow().replace(tzinfo=None) 21 | 22 | -------------------------------------------------------------------------------- /pandaclient/test_client_submit_job.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from pandaclient.Client import getJobStatus, submitJobs 3 | 4 | # Parse command-line arguments 5 | parser = argparse.ArgumentParser(description="Process panda IDs.") 6 | parser.add_argument("panda_ids", nargs="*", type=int, help="List of panda IDs") 7 | args = parser.parse_args() 8 | panda_ids = args.panda_ids 9 | 10 | if not panda_ids: 11 | print("No panda IDs provided. Please provide at least one panda ID.") 12 | exit(1) 13 | 14 | print("=============================================================") 15 | jobs_old = getJobStatus(ids=panda_ids, verbose=True) 16 | job_specs = jobs_old[1] 17 | 18 | submit_old = submitJobs(job_specs) 19 | print("submitJobs returned {0}".format(submit_old)) -------------------------------------------------------------------------------- /pandaclient/test_client_job.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from pandaclient.Client import getJobStatus, getFullJobStatus, killJobs 3 | 4 | # Parse command-line arguments 5 | parser = argparse.ArgumentParser(description="Process panda IDs.") 6 | parser.add_argument("panda_ids", nargs="*", type=int, help="List of panda IDs") 7 | args = parser.parse_args() 8 | panda_ids = args.panda_ids 9 | 10 | print("=============================================================") 11 | job_status_ret_old = getJobStatus(ids=panda_ids, verbose=True) 12 | job_full_status_ret_old = getFullJobStatus(ids=panda_ids, verbose=True) 13 | kill_ret_old = killJobs(ids=panda_ids, verbose=True) 14 | 15 | print("getJobStatus returned: {0}".format(job_status_ret_old)) 16 | print("getFullJobStatus returned: {0}".format(job_full_status_ret_old)) 17 | print("killJobs returned: {0}".format(kill_ret_old)) -------------------------------------------------------------------------------- /templates/conda_meta.yaml.template: -------------------------------------------------------------------------------- 1 | {% set name = "panda-client" %} 2 | {% set version = "___PACKAGE_VERSION___" %} 3 | 4 | 5 | package: 6 | name: {{ name|lower }} 7 | version: {{ version }} 8 | 9 | source: 10 | url: https://github.com/PanDAWMS/{{ name }}/archive/refs/tags/{{ version }}.tar.gz 11 | sha256: ___SHA256SUM___ 12 | 13 | build: 14 | number: 0 15 | noarch: python 16 | script: {{ PYTHON }} -m pip install . -vv 17 | 18 | requirements: 19 | host: 20 | - pip 21 | - python >=3.6 22 | run: 23 | - python >=3.6 24 | 25 | test: 26 | imports: 27 | - pandatools 28 | 29 | about: 30 | home: https://panda-wms.readthedocs.io/en/latest/ 31 | summary: PanDA Client Package 32 | license: Apache-2.0 33 | license_file: LICENSE 34 | 35 | extra: 36 | recipe-maintainers: 37 | - tmaeno 38 | - wguanicedew 39 | - yesw2000 40 | -------------------------------------------------------------------------------- /pandaclient/test_client_misc.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from pandaclient.Client import set_user_secret, get_user_secrets, get_events_status 3 | 4 | # Parse command-line arguments 5 | parser = argparse.ArgumentParser(description="Process panda IDs.") 6 | parser.add_argument("panda_ids", nargs="*", type=int, help="List of panda IDs") 7 | args = parser.parse_args() 8 | panda_ids = args.panda_ids 9 | 10 | print("=============================================================") 11 | set_secret_ret_old = set_user_secret('my_key', 'my_value') 12 | get_secret_ret_old = get_user_secrets() 13 | events_status_ret_old = get_events_status([{"task_id": 4004040, "job_id": 4674379348}]) 14 | 15 | print("set_user_secret returned: {0}".format(set_secret_ret_old)) 16 | print("get_user_secrets returned: {0}".format(get_secret_ret_old)) 17 | print("get_events_status returned: {0}".format(events_status_ret_old)) -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # How to install 2 | 3 | Download the code 4 | ``` 5 | git clone https://github.com/PanDAWMS/panda-client.git 6 | cd panda-client 7 | ``` 8 | and install it 9 | ``` 10 | python setup.py install --prefix=/path/to/install/dir 11 | ``` 12 | or create the tar ball 13 | ``` 14 | python setup.py sdist 15 | pip install dist/panda-client-*.tar.gz 16 | ``` 17 | 18 | # How to use 19 | ``` 20 | source /path/to/install/dir/etc/panda/panda_setup.[c]sh 21 | prun -h 22 | pathena -h 23 | ``` 24 | 25 | # Release Notes 26 | 27 | https://github.com/PanDAWMS/panda-client/releases 28 | 29 | # CVMFS deployment 30 | Request atlas-adc-tier3sw-install to install the new specific version on CVMFS. They will download the package from the github release page. 31 | 32 | # Uploading to pip 33 | ``` 34 | python setup.py sdist upload 35 | ``` 36 | Uploading source so that wheel generates setup files locally. 37 | -------------------------------------------------------------------------------- /templates/panda_setup.example.cfg.template: -------------------------------------------------------------------------------- 1 | [main] 2 | 3 | # base HTTPS URL of PanDA server 4 | #PANDA_URL_SSL = https://ai-idds-01.cern.ch:25443/server/panda 5 | 6 | # base HTTP URL of PanDA server 7 | #PANDA_URL = http://ai-idds-01.cern.ch:25080/server/panda 8 | 9 | # URL of PanDA monitor 10 | #PANDAMON_URL = https://panda-doma.cern.ch 11 | 12 | # Virtual organization name 13 | #PANDA_AUTH_VO = sphenix 14 | 15 | # Use native httplib instead of curl for HTTP operations 16 | #PANDA_USE_NATIVE_HTTPLIB = 1 17 | 18 | # Authentication mechanism (oidc, x509, or x509_no_grid) 19 | #PANDA_AUTH = oidc 20 | 21 | # Grid proxy file path (required only when PANDA_AUTH = x509_no_grid) 22 | #X509_USER_PROXY = /Users/hage/x509up_u12345 23 | 24 | # Grid nickname (required only when PANDA_AUTH = x509_no_grid) 25 | #PANDA_NICKNAME = your_nickname 26 | 27 | 28 | # !!!!! DO NOT CHANGE THE CODE BELOW !!!!! 29 | PANDA_INSTALL_SCRIPTS = @@install_scripts@@ 30 | PANDA_INSTALL_PURELIB = @@install_purelib@@ 31 | PANDA_INSTALL_DIR = @@install_dir@@ 32 | -------------------------------------------------------------------------------- /.github/workflows/pythonpublish.yml: -------------------------------------------------------------------------------- 1 | name: Upload Python Package 2 | 3 | on: 4 | release: 5 | types: [published] 6 | workflow_dispatch: 7 | 8 | jobs: 9 | deploy: 10 | runs-on: ubuntu-latest 11 | permissions: 12 | id-token: write 13 | 14 | steps: 15 | - uses: actions/checkout@v3 16 | with: 17 | fetch-depth: 0 18 | 19 | - name: Set up Python 20 | uses: actions/setup-python@v4 21 | with: 22 | python-version: '3.x' 23 | 24 | - name: Install dependencies 25 | run: | 26 | python -m pip install --upgrade pip setuptools wheel 27 | python -m pip install hatch twine build 28 | python -m pip list 29 | 30 | - name: Build a sdist 31 | run: | 32 | python -m build -s 33 | cp packages/light/pyproject.toml . 34 | hatch build -t wheel 35 | 36 | - name: Verify the distribution 37 | run: twine check dist/* 38 | 39 | - name: Publish distribution to PyPI 40 | if: github.event_name == 'release' && github.event.action == 'published' && github.repository == 'PanDAWMS/panda-client' 41 | uses: pypa/gh-action-pypi-publish@release/v1 42 | -------------------------------------------------------------------------------- /pandaclient/PLogger.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import logging 4 | 5 | 6 | rootLog = None 7 | 8 | # set logger 9 | def setLogger(tmpLog): 10 | global rootLog 11 | rootLog = tmpLog 12 | 13 | 14 | # return logger 15 | def getPandaLogger(use_stdout=True): 16 | # use root logger 17 | global rootLog 18 | if rootLog is None: 19 | rootLog = logging.getLogger('panda-client') 20 | # add StreamHandler if no handler 21 | if rootLog.handlers == []: 22 | rootLog.setLevel(logging.DEBUG) 23 | if use_stdout: 24 | console = logging.StreamHandler(sys.stdout) 25 | else: 26 | console = logging.StreamHandler(sys.stderr) 27 | formatter = logging.Formatter('%(levelname)s : %(message)s') 28 | console.setFormatter(formatter) 29 | rootLog.addHandler(console) 30 | # return 31 | return rootLog 32 | 33 | 34 | # disable logging 35 | def disable_logging(): 36 | global rootLog 37 | if not rootLog: 38 | rootLog = logging.getLogger('') 39 | rootLog.disabled = True 40 | # keep orignal stdout mainly for jupyter 41 | sys.__stdout__ = sys.stdout 42 | sys.stdout = open(os.devnull, 'w') 43 | 44 | 45 | # enable logging 46 | def enable_logging(): 47 | global rootLog 48 | if rootLog: 49 | rootLog.disabled = False 50 | sys.stdout.close() 51 | sys.stdout = sys.__stdout__ 52 | -------------------------------------------------------------------------------- /packages/light/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["hatchling"] 3 | build-backend = "hatchling.build" 4 | 5 | 6 | [project] 7 | name = "panda-client-light" 8 | dynamic = ["version"] 9 | description = "Lightweight PanDA Client Package installed on PanDA server" 10 | readme = "README.md" 11 | license = "Apache-2.0" 12 | requires-python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" 13 | authors = [ 14 | { name = "PanDA Team", email = "panda-support@cern.ch" }, 15 | ] 16 | classifiers = [ 17 | "Programming Language :: Python", 18 | "Programming Language :: Python :: 2", 19 | "Programming Language :: Python :: 2.7", 20 | "Programming Language :: Python :: 3", 21 | "Programming Language :: Python :: 3.5", 22 | "Programming Language :: Python :: 3.6", 23 | "Programming Language :: Python :: 3.7", 24 | "Programming Language :: Python :: 3.8", 25 | "Programming Language :: Python :: 3.9", 26 | "Programming Language :: Python :: 3.10", 27 | "Programming Language :: Python :: 3.11", 28 | ] 29 | 30 | 31 | [project.urls] 32 | Homepage = "https://panda-wms.readthedocs.io/en/latest/" 33 | 34 | 35 | [tool.hatch.version] 36 | path = "pandaclient/PandaToolsPkgInfo.py" 37 | pattern = "release_version = \"(?P[^\"]+)\"" 38 | 39 | 40 | [tool.hatch.build] 41 | directory = "dist" 42 | 43 | 44 | [tool.hatch.build.targets.wheel] 45 | packages = ["pandaclient"] 46 | 47 | 48 | [tool.black] 49 | line-length=160 50 | 51 | [tool.autopep8] 52 | # https://pypi.org/project/autopep8/#pyproject-toml 53 | max_line_length = 160 54 | ignore = ["E501", "W6"] 55 | in-place = true 56 | recursive = true 57 | aggressive = 3 58 | 59 | [tool.isort] 60 | profile = "black" 61 | -------------------------------------------------------------------------------- /packages/full/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["hatchling"] 3 | build-backend = "hatchling.build" 4 | 5 | 6 | [project] 7 | name = "panda-client" 8 | dynamic = ["version"] 9 | description = "PanDA Client Package" 10 | readme = "README.md" 11 | license = "Apache-2.0" 12 | requires-python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" 13 | authors = [ 14 | { name = "PanDA Team", email = "panda-support@cern.ch" }, 15 | ] 16 | classifiers = [ 17 | "Programming Language :: Python", 18 | "Programming Language :: Python :: 2", 19 | "Programming Language :: Python :: 2.7", 20 | "Programming Language :: Python :: 3", 21 | "Programming Language :: Python :: 3.5", 22 | "Programming Language :: Python :: 3.6", 23 | "Programming Language :: Python :: 3.7", 24 | "Programming Language :: Python :: 3.8", 25 | "Programming Language :: Python :: 3.9", 26 | "Programming Language :: Python :: 3.10", 27 | "Programming Language :: Python :: 3.11", 28 | ] 29 | 30 | 31 | [project.optional-dependencies] 32 | jupyter = [ 33 | "jupyter-dash", 34 | "pandas", 35 | ] 36 | 37 | 38 | [project.urls] 39 | Homepage = "https://panda-wms.readthedocs.io/en/latest/" 40 | 41 | 42 | [tool.hatch.version] 43 | path = "pandaclient/PandaToolsPkgInfo.py" 44 | pattern = "release_version = \"(?P[^\"]+)\"" 45 | 46 | 47 | [tool.hatch.build] 48 | directory = "dist" 49 | 50 | 51 | [tool.hatch.build.targets.wheel] 52 | exclude = ["*.template"] 53 | packages = ["pandaclient", "pandatools", "pandaserver", "taskbuffer"] 54 | 55 | 56 | [tool.hatch.build.targets.wheel.shared-data] 57 | "templates" = "etc/panda" 58 | "scripts" = "bin" 59 | "share" = "etc/panda/share" 60 | 61 | [tool.hatch.build.targets.wheel.hooks.custom] 62 | path = "packages/hatch_build.py" 63 | 64 | 65 | [tool.hatch.build.targets.sdist] 66 | exclude = [ 67 | ".github", 68 | ".idea" 69 | ] 70 | -------------------------------------------------------------------------------- /pandaclient/example_task.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | 3 | inFileList = ['file1','file2','file3'] 4 | 5 | logDatasetName = 'panda.jeditest.log.{0}'.format(uuid.uuid4()) 6 | 7 | taskParamMap = {} 8 | 9 | taskParamMap['nFilesPerJob'] = 1 10 | taskParamMap['nFiles'] = len(inFileList) 11 | #taskParamMap['nEventsPerInputFile'] = 10000 12 | #taskParamMap['nEventsPerJob'] = 10000 13 | #taskParamMap['nEvents'] = 25000 14 | taskParamMap['noInput'] = True 15 | taskParamMap['pfnList'] = inFileList 16 | #taskParamMap['mergeOutput'] = True 17 | taskParamMap['taskName'] = str(uuid.uuid4()) 18 | taskParamMap['userName'] = 'someone' 19 | taskParamMap['vo'] = 'wlcg' 20 | taskParamMap['taskPriority'] = 900 21 | #taskParamMap['reqID'] = reqIdx 22 | taskParamMap['architecture'] = 'power9' 23 | taskParamMap['transUses'] = 'A' 24 | taskParamMap['transHome'] = 'B' 25 | taskParamMap['transPath'] = 'executable' 26 | taskParamMap['processingType'] = 'step1' 27 | taskParamMap['prodSourceLabel'] = 'test' 28 | taskParamMap['taskType'] = 'test' 29 | taskParamMap['workingGroup'] = 'groupA' 30 | #taskParamMap['coreCount'] = 1 31 | #taskParamMap['walltime'] = 1 32 | taskParamMap['cloud'] = 'NA' 33 | taskParamMap['site'] = 'TEST_PQ' 34 | taskParamMap['log'] = {'dataset': logDatasetName, 35 | 'type':'template', 36 | 'param_type':'log', 37 | 'token':'local', 38 | 'destination':'local', 39 | 'value':'{0}.${{SN}}.log.tgz'.format(logDatasetName)} 40 | outDatasetName = 'panda.jeditest.{0}'.format(uuid.uuid4()) 41 | 42 | 43 | taskParamMap['jobParameters'] = [ 44 | {'type':'constant', 45 | 'value':'-i "${IN/T}"', 46 | }, 47 | {'type':'constant', 48 | 'value': 'ecmEnergy=8000 runNumber=12345' 49 | }, 50 | {'type':'template', 51 | 'param_type':'output', 52 | 'token':'local', 53 | 'destination':'local', 54 | 'value':'outputEVNTFile={0}.${{SN}}.root'.format(outDatasetName), 55 | 'dataset':outDatasetName, 56 | 'offset':1000, 57 | }, 58 | {'type':'constant', 59 | 'value':'aaaa', 60 | }, 61 | ] 62 | -------------------------------------------------------------------------------- /pandaclient/idds_api.py: -------------------------------------------------------------------------------- 1 | from . import Client 2 | 3 | 4 | # API call class 5 | class IddsApi(object): 6 | 7 | def __init__(self, name, dumper, verbose, idds_host, compress, manager, loader): 8 | self.name = name 9 | if idds_host is not None: 10 | self.name += '+{}'.format(idds_host) 11 | self.dumper = dumper 12 | self.verbose = verbose 13 | self.compress = compress 14 | self.manager = manager 15 | self.loader = loader 16 | 17 | def __call__(self, *args, **kwargs): 18 | return Client.call_idds_command(self.name, args, kwargs, self.dumper, self.verbose, self.compress, 19 | self.manager, self.loader) 20 | 21 | 22 | # interface to API 23 | class IddsApiInteface(object): 24 | def __init__(self): 25 | self.dumper = None 26 | self.loader = None 27 | self.json_outputs = False 28 | 29 | def __getattr__(self, item): 30 | return IddsApi(item, self.dumper, self.verbose, self.idds_host, self.compress, self.manager, 31 | self.loader) 32 | 33 | def setup(self, dumper, verbose, idds_host, compress, manager, loader, json_outputs): 34 | self.dumper = dumper 35 | self.verbose = verbose 36 | self.idds_host = idds_host 37 | self.compress = compress 38 | self.manager = manager 39 | self.loader = loader 40 | self.json_outputs = json_outputs 41 | 42 | 43 | # entry for API 44 | api = IddsApiInteface() 45 | del IddsApiInteface 46 | 47 | 48 | def get_api(dumper=None, verbose=False, idds_host=None, compress=True, manager=False, loader=None, 49 | json_outputs=False): 50 | """Get an API object to access iDDS through PanDA 51 | 52 | args: 53 | dumper: function object to dump json-serialized data 54 | verbose: True to see verbose messages 55 | idds_host: iDDS hostname 56 | compress: True to compress request body 57 | manager: True to use ClientManager API 58 | loader: function object to load json-serialized data 59 | json_outputs: True to use json outputs 60 | return: 61 | an API object 62 | """ 63 | api.setup(dumper, verbose, idds_host, compress, manager, loader, json_outputs) 64 | return api 65 | -------------------------------------------------------------------------------- /pandaclient/BookConfig.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | try: 4 | import ConfigParser 5 | except ImportError: 6 | import configparser as ConfigParser 7 | 8 | sectionName = 'book' 9 | confFile = os.path.expanduser('%s/panda.cfg' % os.environ['PANDA_CONFIG_ROOT']) 10 | 11 | 12 | # create config or add section when missing 13 | parser=ConfigParser.ConfigParser() 14 | newFlag = False 15 | if not os.path.exists(confFile): 16 | # create new config 17 | newFlag = True 18 | # make dir 19 | try: 20 | os.makedirs(os.environ['PANDA_CONFIG_ROOT']) 21 | except Exception: 22 | pass 23 | else: 24 | # add section 25 | parser.read(confFile) 26 | if not parser.has_section(sectionName): 27 | newFlag = True 28 | # new file or missing section 29 | if newFlag: 30 | # add section 31 | parser.add_section(sectionName) 32 | # set dummy time 33 | parser.set(sectionName,'last_synctime','') 34 | # keep old config just in case 35 | try: 36 | os.rename(confFile, '%s.back' % confFile) 37 | except Exception: 38 | pass 39 | # write 40 | confFH = open(confFile,'w') 41 | parser.write(confFH) 42 | confFH.close() 43 | 44 | 45 | # get config 46 | def getConfig(): 47 | # instantiate parser 48 | parser=ConfigParser.ConfigParser() 49 | parser.read(confFile) 50 | # config class 51 | class _bookConfig: 52 | pass 53 | bookConf = _bookConfig() 54 | # expand sequencer section 55 | for key,val in parser.items(sectionName): 56 | # convert int/bool 57 | if re.search('^\d+$',val) is not None: 58 | val = int(val) 59 | elif re.search('true',val,re.I) is not None: 60 | val = True 61 | elif re.search('false',val,re.I) is not None: 62 | val = False 63 | # set attributes 64 | setattr(bookConf,key,val) 65 | # return 66 | return bookConf 67 | 68 | 69 | # update 70 | def updateConfig(bookConf): 71 | # instantiate parser 72 | parser=ConfigParser.ConfigParser() 73 | parser.read(confFile) 74 | # set new values 75 | for attr in dir(bookConf): 76 | if not attr.startswith('_'): 77 | val = getattr(bookConf,attr) 78 | if val is not None: 79 | parser.set(sectionName,attr,val) 80 | # keep old config 81 | try: 82 | os.rename(confFile, '%s.back' % confFile) 83 | except Exception as e: 84 | print("WARNING : cannot make backup for %s with %s" % (confFile, str(e))) 85 | return 86 | # update conf 87 | conFH = open(confFile,'w') 88 | parser.write(conFH) 89 | # close 90 | conFH.close() 91 | -------------------------------------------------------------------------------- /share/functions.sh: -------------------------------------------------------------------------------- 1 | # execute panda-client command 2 | function exec_p_command () { 3 | export LD_LIBRARY_PATH_ORIG=${LD_LIBRARY_PATH} 4 | export LD_LIBRARY_PATH= 5 | export PYTHONPATH_ORIG=${PYTHONPATH} 6 | export PYTHONPATH=${PANDA_PYTHONPATH} 7 | export PYTHONHOME_ORIG=${PYTHONHOME} 8 | unset PYTHONHOME 9 | 10 | # look for option for python3 11 | for i in "$@" 12 | do 13 | case $i in 14 | -3) 15 | PANDA_PY3=1 16 | ;; 17 | *) 18 | ;; 19 | esac 20 | done 21 | 22 | # check virtual env 23 | if [[ -z "$PANDA_PYTHON_EXEC" ]]; then 24 | if [[ -n "$VIRTUAL_ENV" ]]; then 25 | if [[ -z "$PANDA_PY3" ]]; then 26 | PANDA_PYTHON_EXEC=${VIRTUAL_ENV}/bin/python 27 | if [ ! -f "$PANDA_PYTHON_EXEC" ]; then 28 | unset PANDA_PYTHON_EXEC 29 | fi 30 | fi 31 | if [[ -z "$PANDA_PYTHON_EXEC" ]]; then 32 | PANDA_PYTHON_EXEC=${VIRTUAL_ENV}/bin/python3 33 | if [ ! -f "$PANDA_PYTHON_EXEC" ]; then 34 | unset PANDA_PYTHON_EXEC 35 | fi 36 | fi 37 | fi 38 | fi 39 | 40 | # check conda 41 | if [[ -z "$PANDA_PYTHON_EXEC" ]]; then 42 | if [[ -n "$CONDA_PREFIX" ]]; then 43 | if [[ -z "$PANDA_PY3" ]]; then 44 | PANDA_PYTHON_EXEC=${CONDA_PREFIX}/bin/python 45 | if [ ! -f "$PANDA_PYTHON_EXEC" ]; then 46 | unset PANDA_PYTHON_EXEC 47 | fi 48 | fi 49 | if [[ -z "$PANDA_PYTHON_EXEC" ]]; then 50 | PANDA_PYTHON_EXEC=${CONDA_PREFIX}/bin/python3 51 | if [ ! -f "$PANDA_PYTHON_EXEC" ]; then 52 | unset PANDA_PYTHON_EXEC 53 | fi 54 | fi 55 | fi 56 | fi 57 | 58 | # system python 59 | if [[ -z "$PANDA_PYTHON_EXEC" ]]; then 60 | if [[ -z "$PANDA_PY3" ]]; then 61 | PANDA_PYTHON_EXEC=/usr/bin/python 62 | if [ ! -f "$PANDA_PYTHON_EXEC" ]; then 63 | unset PANDA_PYTHON_EXEC 64 | fi 65 | fi 66 | if [[ -z "$PANDA_PYTHON_EXEC" ]]; then 67 | PANDA_PYTHON_EXEC=/usr/bin/python3 68 | if [ ! -f "$PANDA_PYTHON_EXEC" ]; then 69 | unset PANDA_PYTHON_EXEC 70 | fi 71 | fi 72 | fi 73 | 74 | # no interpreter 75 | if [[ -z "$PANDA_PYTHON_EXEC" ]]; then 76 | echo "ERROR: No python interpreter found in \$VIRTUAL_ENV/bin, \$CONDA_PREFIX/bin, or /usr/bin. You may set \$PANDA_PYTHON_EXEC if python is available in another location" 77 | exit 1 78 | fi 79 | 80 | # execute 81 | local exec_string=$1 82 | shift 83 | $PANDA_PYTHON_EXEC -u -W ignore -c "${exec_string}" "$@" 84 | } -------------------------------------------------------------------------------- /packages/hatch_build.py: -------------------------------------------------------------------------------- 1 | from hatchling.builders.hooks.plugin.interface import BuildHookInterface 2 | 3 | import os 4 | import re 5 | import sys 6 | import stat 7 | import glob 8 | import sysconfig 9 | import distutils 10 | 11 | 12 | class CustomBuildHook(BuildHookInterface): 13 | def initialize(self, version, build_data): 14 | # chmod +x 15 | for f in glob.glob("./scripts/*"): 16 | st = os.stat(f) 17 | os.chmod(f, st.st_mode | stat.S_IEXEC) 18 | 19 | # parameters to be resolved 20 | self.params = {} 21 | self.params['install_dir'] = os.environ.get('PANDA_INSTALL_TARGET') 22 | if self.params['install_dir']: 23 | # non-standard installation path 24 | self.params['install_purelib'] = self.params['install_dir'] 25 | self.params['install_scripts'] = os.path.join(self.params['install_dir'], 'bin') 26 | else: 27 | self.params['install_dir'] = sys.prefix 28 | try: 29 | # python3.2 or higher 30 | self.params['install_purelib'] = sysconfig.get_path('purelib') 31 | self.params['install_scripts'] = sysconfig.get_path('scripts') 32 | except Exception: 33 | # old python 34 | self.params['install_purelib'] = distutils.sysconfig.get_python_lib() 35 | self.params['install_scripts'] = os.path.join(sys.prefix, 'bin') 36 | for k in self.params: 37 | path = self.params[k] 38 | self.params[k] = os.path.abspath(os.path.expanduser(path)) 39 | 40 | # instantiate templates 41 | for in_f in glob.glob("./templates/*"): 42 | if not in_f.endswith('.template'): 43 | continue 44 | with open(in_f) as in_fh: 45 | file_data = in_fh.read() 46 | # replace patterns 47 | for item in re.findall(r'@@([^@]+)@@', file_data): 48 | if item not in self.params: 49 | raise RuntimeError('unknown pattern %s in %s' % (item, in_f)) 50 | # get pattern 51 | patt = self.params[item] 52 | # convert to absolute path 53 | if item.startswith('install'): 54 | patt = os.path.abspath(patt) 55 | # remove build/*/dump for bdist 56 | patt = re.sub('build/[^/]+/dumb', '', patt) 57 | # remove /var/tmp/*-buildroot for bdist_rpm 58 | patt = re.sub('/var/tmp/.*-buildroot', '', patt) 59 | # replace 60 | file_data = file_data.replace('@@%s@@' % item, patt) 61 | out_f = re.sub(r'\.template$', '', in_f) 62 | with open(out_f, 'w') as out_fh: 63 | out_fh.write(file_data) 64 | 65 | # post install only for client installation 66 | if not os.path.exists(os.path.join(self.params['install_purelib'], 'pandacommon')): 67 | target = 'pandatools' 68 | if not os.path.exists(os.path.join(self.params['install_purelib'], target)): 69 | os.symlink('pandaclient', target) 70 | -------------------------------------------------------------------------------- /.github/workflows/condapublish.yml: -------------------------------------------------------------------------------- 1 | # This is a basic workflow to help you get started with Actions 2 | 3 | name: Publish on Conda-forge 4 | 5 | # Controls when the workflow will run 6 | on: 7 | 8 | release: 9 | types: [ published ] 10 | 11 | workflow_dispatch: 12 | 13 | # A workflow run is made up of one or more jobs that can run sequentially or in parallel 14 | jobs: 15 | # This workflow contains a single job called "build" 16 | publish: 17 | # The type of runner that the job will run on 18 | runs-on: ubuntu-latest 19 | 20 | # variables 21 | env: 22 | REPO_NAME: ${{ github.event.repository.name }} 23 | OWNER_NAME: ${{ github.repository_owner }} 24 | 25 | # Steps represent a sequence of tasks that will be executed as part of the job 26 | steps: 27 | # Check-out src repository 28 | - uses: actions/checkout@v3 29 | with: 30 | path: src 31 | 32 | # Check-out feedstock 33 | - uses: actions/checkout@v2 34 | with: 35 | token: ${{ secrets.PAT_GITHUB }} 36 | repository: ${{ github.repository_owner }}/${{ github.event.repository.name }}-feedstock 37 | path: dst 38 | 39 | # Re-sync 40 | - name: Re-sync 41 | run: | 42 | cd dst 43 | git remote add upstream https://github.com/conda-forge/${REPO_NAME}-feedstock.git 44 | git fetch upstream 45 | git checkout main 46 | git merge upstream/main 47 | 48 | # Generate meta.yaml 49 | - name: Generate and push meta.yaml 50 | run: | 51 | PACKAGE_NAME=`echo $REPO_NAME | sed -e 's/-//g'` 52 | cd src/${PACKAGE_NAME} 53 | VERSION=`python -c 'exec(open("PandaToolsPkgInfo.py").read());print (release_version)'` 54 | cd - 55 | echo REPO_NAME=$REPO_NAME 56 | echo "REPO_NAME=$REPO_NAME" >> $GITHUB_ENV 57 | echo PACKAGE_NAME=$PACKAGE_NAME 58 | echo VERSION=$VERSION 59 | echo "VERSION=$VERSION" >> $GITHUB_ENV 60 | wget https://github.com/${OWNER_NAME}/${REPO_NAME}/archive/refs/tags/${VERSION}.tar.gz -q -O dummy.tar.gz 61 | SHA256SUM=`sha256sum dummy.tar.gz` 62 | SHA256SUM=${SHA256SUM% *} 63 | echo SHA256SUM=$SHA256SUM 64 | sed -e "s/___PACKAGE_VERSION___/${VERSION}/g" src/templates/conda_meta.yaml.template \ 65 | | sed -e "s/___SHA256SUM___/${SHA256SUM}/g" > dst/recipe/meta.yaml 66 | 67 | - name: Push the change 68 | run: | 69 | cd dst 70 | # use personal info since github-actions/github-actions@github.com doesn't work for forked repos 71 | git config --global user.name 'Tadashi Maeno' 72 | git config --global user.email 'tmaeno@bnl.gov' 73 | git diff --quiet && git diff --staged --quiet || git commit -am "${VERSION} github action" 74 | git push 75 | 76 | - name: Request pull request 77 | env: 78 | # use PAT instead of GITHUB_TOKEN since the latter cannot submit a PR 79 | GITHUB_TOKEN: ${{ secrets.PAT_GITHUB }} 80 | run: | 81 | cd dst 82 | gh pr create -t "${REPO_NAME} ${VERSION} github action" -b "automatic pull request" 83 | -------------------------------------------------------------------------------- /share/FakeAppMgr.py: -------------------------------------------------------------------------------- 1 | """ 2 | replace AppMgr with a fake mgr to disable DLL loading 3 | 4 | """ 5 | 6 | 7 | # fake property 8 | class fakeProperty(list): 9 | def __init__(self, name): 10 | self.name = name 11 | 12 | def __getattribute__(self, name): 13 | try: 14 | return object.__getattribute__(self, name) 15 | except Exception: 16 | setattr(self, name, fakeProperty(name)) 17 | return object.__getattribute__(self, name) 18 | 19 | def __getstate__(self): 20 | return self.__dict__ 21 | 22 | def __reduce_ex__(self, proto=0): 23 | if proto >= 2: 24 | proto = 1 25 | return super(fakeProperty, self).__reduce_ex__(proto) 26 | 27 | def properties(self): 28 | prp = fakeProperty("properties") 29 | for attr in dir(self): 30 | prp.append(attr) 31 | return prp 32 | 33 | def get(self, name): 34 | return self.__getattribute__(name) 35 | 36 | def value(self): 37 | return self 38 | 39 | 40 | # fake application manager 41 | class fakeAppMgr(fakeProperty): 42 | def __init__(self, origTheApp): 43 | self.origTheApp = origTheApp 44 | fakeProperty.__init__(self, "AppMgr") 45 | # streams 46 | try: 47 | self._streams = self.origTheApp._streams 48 | except Exception: 49 | self._streams = [] 50 | # for https://savannah.cern.ch/bugs/index.php?66675 51 | try: 52 | self.allConfigurables = self.origTheApp.allConfigurables 53 | except Exception: 54 | pass 55 | 56 | def service(self, name): 57 | return fakeProperty(name) 58 | 59 | def createSvc(self, name): 60 | return fakeProperty(name) 61 | 62 | def algorithm(self, name): 63 | return fakeProperty(name) 64 | 65 | def setup(self, var): 66 | pass 67 | 68 | def exit(self): 69 | import sys 70 | 71 | sys.exit(0) 72 | 73 | def serviceMgr(self): 74 | try: 75 | return self.origTheApp.serviceMgr() 76 | except Exception: 77 | return self._serviceMgr 78 | 79 | def toolSvc(self): 80 | try: 81 | return self.origTheApp.toolSvc() 82 | except Exception: 83 | return self._toolSvc 84 | 85 | def addOutputStream(self, stream): 86 | self._streams += stream 87 | 88 | def initialize(self): 89 | include("%s/etc/panda/share/ConfigExtractor.py" % os.environ["PANDA_SYS"]) 90 | import sys 91 | 92 | sys.exit(0) 93 | 94 | def run(self): 95 | import sys 96 | 97 | sys.exit(0) 98 | 99 | 100 | # replace AppMgr with the fake mgr to disable DLL loading 101 | _theApp = theApp 102 | theApp = fakeAppMgr(_theApp) 103 | 104 | # for 13.X.0 or higher 105 | try: 106 | import AthenaCommon.AppMgr 107 | 108 | AthenaCommon.AppMgr.theApp = theApp 109 | except Exception: 110 | pass 111 | 112 | # for boot strap 113 | theApp.EventLoop = _theApp.EventLoop 114 | if hasattr(_theApp, "OutStreamType"): 115 | theApp.OutStreamType = _theApp.OutStreamType 116 | del _theApp 117 | -------------------------------------------------------------------------------- /pandaclient/test_client_task.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import subprocess, re, uuid, tempfile, sys, os 3 | 4 | from pandaclient.Client import insertTaskParams, getTaskParamsMap, killTask, pauseTask, resumeTask, getTaskStatus, finishTask, retryTask, reactivateTask, increase_attempt_nr, reload_input, getJediTaskDetails, get_files_in_datasets, getJobIDsJediTasksInTimeRange, getPandaIDsWithTaskID, getUserJobMetadata 5 | 6 | def main(task_id): 7 | outds = "user.pandasv2.{0}".format(uuid.uuid4()) 8 | with tempfile.TemporaryDirectory() as tmpdir: 9 | # go into temp dir 10 | cwd = os.getcwd() 11 | os.chdir(tmpdir) 12 | 13 | cmd = ( 14 | '''prun --exec "pwd; ls; echo Hello-world > myout.txt" ''' 15 | '''--outDS {outds} --nJobs 3 --output myout.txt'''.format(outds=outds) 16 | ) 17 | 18 | res = subprocess.run(cmd, shell=True, text=True, capture_output=True) 19 | out = (res.stdout or "") + "\n" + (res.stderr or "") 20 | print(out) 21 | 22 | os.chdir(cwd) # back to original dir 23 | 24 | m = re.search(r"new jediTaskID=(\d+)", out) 25 | if not m: 26 | print("Failed to find task ID in output of prun command:") 27 | print(out.strip()) 28 | sys.exit(1) 29 | 30 | print("=============================================================") 31 | status_ret_old = getTaskStatus(task_id) 32 | params_ret_old = getTaskParamsMap(task_id) 33 | details_ret_old = getJediTaskDetails({"jediTaskID": task_id}, True, True) 34 | pause_ret_old = pauseTask(task_id) 35 | resume_ret_old = resumeTask(task_id) 36 | kill_ret_old = killTask(task_id) 37 | finish_ret_old = finishTask(task_id) 38 | retry_ret_old = retryTask(task_id) 39 | reactivate_ret_old = reactivateTask(task_id) 40 | get_jobs_old = getJobIDsJediTasksInTimeRange('2025-08-01 14:30:45') 41 | get_ids_old = getPandaIDsWithTaskID(task_id) 42 | increase_ret_old = increase_attempt_nr(task_id) 43 | reload_ret_old = reload_input(task_id) 44 | files_ret_old = get_files_in_datasets(task_id) 45 | metadata_old = getUserJobMetadata(task_id, verbose=True) 46 | 47 | print("getTaskStatus returned: {0}".format(status_ret_old)) 48 | print("getTaskParams returned: {0}".format(params_ret_old)) 49 | print("getJediTaskDetails returned: {0}".format(details_ret_old)) 50 | print("pauseTask returned: {0}".format(pause_ret_old)) 51 | print("resumeTask returned: {0}".format(resume_ret_old)) 52 | print("killTask returned: {0}".format(kill_ret_old)) 53 | print("finishTask returned: {0}".format(finish_ret_old)) 54 | print("retryTask returned: {0}".format(retry_ret_old)) 55 | print("reactivateTask returned: {0}".format(reactivate_ret_old)) 56 | print("getJobIDsJediTasksInTimeRange returned: {0}".format(get_jobs_old)) 57 | print("getPandaIDsWithTaskID returned: {0}".format(get_ids_old)) 58 | print("increaseAttemptNr returned: {0}".format(increase_ret_old)) 59 | print("reloadInput returned: {0}".format(reload_ret_old)) 60 | print("get_files_in_datasets returned: {0}".format(files_ret_old)) 61 | print("getUserJobMetadata returned: {0}".format(metadata_old)) 62 | 63 | if __name__ == "__main__": 64 | parser = argparse.ArgumentParser() 65 | parser.add_argument("task_id", type=int, help="The task ID to process.") 66 | args = parser.parse_args() 67 | 68 | main(args.task_id) -------------------------------------------------------------------------------- /pandaclient/FileSpec.py: -------------------------------------------------------------------------------- 1 | """ 2 | file specification 3 | 4 | """ 5 | 6 | class FileSpec(object): 7 | # attributes 8 | _attributes = ('rowID','PandaID','GUID','lfn','type','dataset','status','prodDBlock', 9 | 'prodDBlockToken','dispatchDBlock','dispatchDBlockToken','destinationDBlock', 10 | 'destinationDBlockToken','destinationSE','fsize','md5sum','checksum','scope') 11 | # slots 12 | __slots__ = _attributes+('_owner',) 13 | 14 | 15 | # constructor 16 | def __init__(self): 17 | # install attributes 18 | for attr in self._attributes: 19 | setattr(self,attr,None) 20 | # set owner to synchronize PandaID 21 | self._owner = None 22 | 23 | 24 | # override __getattribute__ for SQL and PandaID 25 | def __getattribute__(self,name): 26 | # PandaID 27 | if name == 'PandaID': 28 | if self._owner is None: 29 | return 'NULL' 30 | return self._owner.PandaID 31 | # others 32 | ret = object.__getattribute__(self,name) 33 | if ret is None: 34 | return "NULL" 35 | return ret 36 | 37 | 38 | # set owner 39 | def setOwner(self,owner): 40 | self._owner = owner 41 | 42 | 43 | # return a tuple of values 44 | def values(self): 45 | ret = [] 46 | for attr in self._attributes: 47 | val = getattr(self,attr) 48 | ret.append(val) 49 | return tuple(ret) 50 | 51 | 52 | # pack tuple into FileSpec 53 | def pack(self,values): 54 | for i in range(len(self._attributes)): 55 | attr= self._attributes[i] 56 | val = values[i] 57 | setattr(self,attr,val) 58 | 59 | 60 | # return state values to be pickled 61 | def __getstate__(self): 62 | state = [] 63 | for attr in self._attributes: 64 | val = getattr(self,attr) 65 | state.append(val) 66 | # append owner info 67 | state.append(self._owner) 68 | return state 69 | 70 | 71 | # restore state from the unpickled state values 72 | def __setstate__(self,state): 73 | for i in range(len(self._attributes)): 74 | if i+1 < len(state): 75 | setattr(self,self._attributes[i],state[i]) 76 | else: 77 | setattr(self,self._attributes[i],'NULL') 78 | self._owner = state[-1] 79 | 80 | 81 | # return column names for INSERT 82 | def columnNames(cls): 83 | ret = "" 84 | for attr in cls._attributes: 85 | if ret != "": 86 | ret += ',' 87 | ret += attr 88 | return ret 89 | columnNames = classmethod(columnNames) 90 | 91 | 92 | # return expression of values for INSERT 93 | def valuesExpression(cls): 94 | ret = "VALUES(" 95 | for attr in cls._attributes: 96 | ret += "%s" 97 | if attr != cls._attributes[len(cls._attributes)-1]: 98 | ret += "," 99 | ret += ")" 100 | return ret 101 | valuesExpression = classmethod(valuesExpression) 102 | 103 | 104 | # return an expression for UPDATE 105 | def updateExpression(cls): 106 | ret = "" 107 | for attr in cls._attributes: 108 | ret = ret + attr + "=%s" 109 | if attr != cls._attributes[len(cls._attributes)-1]: 110 | ret += "," 111 | return ret 112 | updateExpression = classmethod(updateExpression) 113 | 114 | # dump to be json-serializable 115 | def dump_to_json_serializable(self): 116 | stat = self.__getstate__()[:-1] 117 | # set None as _owner 118 | stat.append(None) 119 | return stat 120 | -------------------------------------------------------------------------------- /pandaclient/localSpecs.py: -------------------------------------------------------------------------------- 1 | import copy 2 | 3 | 4 | task_active_superstatus_list = ['running', 'submitting', 'registered', 'ready'] 5 | task_final_superstatus_list = ['finished', 'failed', 'done', 'broken', 'aborted'] 6 | 7 | 8 | class LocalTaskSpec(object): 9 | 10 | _attributes_hidden = ( 11 | '_pandaserver', 12 | '_timestamp', 13 | '_sourceurl', 14 | '_weburl', 15 | '_fulldict', 16 | ) 17 | 18 | _attributes_direct = ( 19 | 'jeditaskid', 20 | 'reqid', 21 | 'taskname', 22 | 'username', 23 | 'creationdate', 24 | 'modificationtime', 25 | 'superstatus', 26 | 'status', 27 | ) 28 | 29 | _attributes_dsinfo = ( 30 | 'pctfinished', 31 | 'pctfailed', 32 | 'nfiles', 33 | 'nfilesfinished', 34 | 'nfilesfailed', 35 | ) 36 | 37 | __slots__ = _attributes_hidden + _attributes_direct + _attributes_dsinfo 38 | 39 | # stdout string format 40 | strf_dict = {} 41 | strf_dict['standard'] = '{jtid:>10} {reqid:>8} {st:>10} {pctf:>5} {tname}' 42 | strf_dict['long'] = ( '{jtid:>10} {st:>10} {cret:20} {modt:20} ({filesprog})\n' 43 | '{reqid:>10} {pctf:>10} {tname}\n' 44 | ' {weburl}\n' ) + '_'*78 45 | 46 | # header row 47 | head_dict = {} 48 | head_dict['standard'] = strf_dict['standard'].format(st='Status', jtid='JediTaskID', 49 | reqid='ReqID', pctf='Fin%', tname='TaskName') \ 50 | + '\n' + '_'*64 51 | head_dict['long'] = strf_dict['long'].format(st='Status', jtid='JediTaskID', reqid='ReqID', 52 | tname='TaskName', weburl='Webpage', filesprog='finished| failed| total NInputFiles', 53 | pctf='Finished_%', cret='CreationDate', modt='ModificationTime') 54 | 55 | def __init__(self, task_dict, source_url=None, timestamp=None, 56 | pandaserver='https://pandaserver.cern.ch:25443/server/panda'): 57 | self._timestamp = timestamp 58 | self._sourceurl = source_url 59 | self._pandaserver = pandaserver 60 | self._fulldict = copy.deepcopy(task_dict) 61 | for aname in self._attributes_direct: 62 | setattr(self, aname, self._fulldict.get(aname)) 63 | for aname in self._attributes_dsinfo: 64 | if aname.startswith('pct'): 65 | setattr(self, aname, '{0}%'.format(self._fulldict['dsinfo'][aname])) 66 | else: 67 | setattr(self, aname, '{0}'.format(self._fulldict['dsinfo'][aname])) 68 | self._weburl = 'https://bigpanda.cern.ch/tasknew/{0}/'.format(self.jeditaskid) 69 | 70 | def is_terminated(self): 71 | if self.superstatus in task_final_superstatus_list: 72 | return True 73 | else: 74 | return False 75 | 76 | def print_plain(self): 77 | print('_'*64) 78 | str_format = '{attr:18} : \t{value}' 79 | for aname in self.__slots__: 80 | if aname in ['_fulldict']: 81 | continue 82 | print(str_format.format(attr=aname, value= getattr(self, aname))) 83 | 84 | def print_long(self): 85 | str_format = self.strf_dict['long'] 86 | print(str_format.format(sst=self.superstatus, st=self.status, jtid=self.jeditaskid, reqid=self.reqid, 87 | tname=self.taskname, weburl=self._weburl, pctf=self.pctfinished, 88 | cret=self.creationdate, modt=self.modificationtime, 89 | filesprog='{nff:>8}|{nfb:>8}|{nf:>8}'.format( 90 | nf=self.nfiles, nff=self.nfilesfinished, nfb=self.nfilesfailed) 91 | )) 92 | 93 | def print_standard(self): 94 | str_format = self.strf_dict['standard'] 95 | print(str_format.format(sst=self.superstatus, st=self.status, jtid=self.jeditaskid, 96 | reqid=self.reqid, pctf=self.pctfinished, tname=self.taskname)) 97 | -------------------------------------------------------------------------------- /pandaclient/Group_argparse.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import sys 3 | from collections import OrderedDict 4 | try: 5 | unicode 6 | except Exception: 7 | unicode = str 8 | 9 | 10 | # check string args if they can be encoded with utf-8 11 | class ActionWithUnicodeCheck(argparse.Action): 12 | def __call__(self, parser, namespace, values, option_string=None): 13 | if isinstance(values, (unicode, str)): 14 | try: 15 | values.encode() 16 | except Exception: 17 | parser.exit(1, message="ERROR: argument --{0} cannot be encoded with utf-8: '{1}'\n".format(self.dest, values)) 18 | setattr(namespace, self.dest, values) 19 | 20 | 21 | class GroupArgParser(argparse.ArgumentParser): 22 | def __init__(self, usage, conflict_handler): 23 | self.groups_dict = OrderedDict() 24 | self.briefHelp = None 25 | self.examples = "" 26 | super(GroupArgParser, self).__init__(usage=usage, conflict_handler=conflict_handler) 27 | 28 | def set_examples(self, examples): 29 | self.examples = examples 30 | 31 | def add_group(self, name, desc=None, usage=None): 32 | # group = argparse._ArgumentGroup(self, name, desc) 33 | group = self.MyArgGroup(self, name, desc) 34 | group.usage = usage 35 | self.groups_dict[name.upper()] = group 36 | return group 37 | 38 | def update_action_groups(self): 39 | for group in self.groups_dict.values(): 40 | self._action_groups.append(group) 41 | 42 | def add_helpGroup(self, addHelp=None): 43 | help='Print individual group help (the group name is not case-sensitive), where "ALL" will print all groups together.' 44 | if addHelp: 45 | help += ' ' + addHelp 46 | choices_m = self.MyList(list(self.groups_dict.keys()) + ['ALL']) 47 | self.add_argument('--helpGroup', choices=choices_m, action=self.print_groupHelp, help=help) 48 | 49 | try: 50 | from cStringIO import StringIO 51 | except Exception: 52 | from io import StringIO 53 | old_stdout = sys.stdout 54 | sys.stdout = self.briefHelp = StringIO() 55 | self.print_help() 56 | sys.stdout = old_stdout 57 | 58 | self.update_action_groups() 59 | self.add_argument('-h', '--help', action=self.print_briefHelp, nargs=0, help="Print this help") 60 | 61 | def shareWithGroup(self, action, group): 62 | # share option action to another group 63 | if action and group: 64 | if action not in group._group_actions: 65 | group._group_actions.append(action) 66 | 67 | class MyArgGroup(argparse._ArgumentGroup): 68 | def shareWithMe(self, action): 69 | self._group_actions.append(action) 70 | 71 | class MyList(list): 72 | # list subclass that uses upper() when testing for 'in' 73 | def __contains__(self, other): 74 | return super(GroupArgParser.MyList,self).__contains__(other.upper()) 75 | 76 | class print_briefHelp(argparse.Action): 77 | def __call__(self, parser, namespace, values, option_string=None): 78 | parser.print_help() 79 | sys.exit(0) 80 | 81 | class print_groupHelp(argparse.Action): 82 | def __init__(self, option_strings, dest, nargs=None, **kwargs): 83 | if nargs is not None: 84 | raise ValueError("nargs not allowed") 85 | super(GroupArgParser.print_groupHelp, self).__init__(option_strings, dest, **kwargs) 86 | 87 | def __call__(self, parser, namespace, values, option_string=None): 88 | values = values.upper() 89 | groups = parser.groups_dict 90 | if values == 'ALL': 91 | parser.print_help() 92 | elif values in groups.keys(): 93 | group = groups[values] 94 | formatter = parser._get_formatter() 95 | formatter.start_section(group.title) 96 | formatter.add_text(group.description) 97 | formatter.add_arguments(group._group_actions) 98 | formatter.end_section() 99 | print(formatter.format_help()) 100 | if group.usage: 101 | print(group.usage) 102 | else: 103 | raise Exception("!!!ERROR!!! Unknown group name=%s" % values) 104 | sys.exit(0) 105 | 106 | 107 | # factory method 108 | def get_parser(usage, conflict_handler): 109 | p = GroupArgParser(usage, conflict_handler) 110 | p.register('action', 'store', ActionWithUnicodeCheck) 111 | return p 112 | -------------------------------------------------------------------------------- /pandaclient/panda_jupyter.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import getpass 4 | import subprocess 5 | from IPython.core.magic import register_line_magic 6 | 7 | try: 8 | import ConfigParser 9 | except ImportError: 10 | import configparser as ConfigParser 11 | from . import PLogger 12 | 13 | 14 | # setup 15 | def setup(): 16 | tmp_log = PLogger.getPandaLogger() 17 | # parse config file 18 | conf_file = os.path.expanduser('~/.panda/panda_setup.cfg') 19 | if not os.path.exists(conf_file): 20 | tmp_log.error('panda conifg file is missing at {}'.format(conf_file)) 21 | return False 22 | parser = ConfigParser.ConfigParser() 23 | parser.read(conf_file) 24 | section = parser['main'] 25 | 26 | # variables 27 | panda_install_scripts = section['PANDA_INSTALL_SCRIPTS'] 28 | panda_install_purelib = section['PANDA_INSTALL_PURELIB'] 29 | panda_install_dir = section['PANDA_INSTALL_DIR'] 30 | 31 | # PATH 32 | paths = os.environ['PATH'].split(':') 33 | if not panda_install_scripts in paths: 34 | paths.insert(0, panda_install_scripts) 35 | os.environ['PATH'] = ':'.join(paths) 36 | 37 | # PYTHONPATH 38 | if 'PYTHONPATH' not in os.environ: 39 | os.environ['PYTHONPATH'] = panda_install_purelib 40 | else: 41 | paths = os.environ['PYTHONPATH'].split(':') 42 | if panda_install_purelib not in paths: 43 | paths.insert(0, panda_install_scripts) 44 | os.environ['PYTHONPATH'] = ':'.join(paths) 45 | 46 | # env 47 | panda_env = {'PANDA_CONFIG_ROOT': '~/.pathena', 48 | 'PANDA_SYS': panda_install_dir, 49 | "PANDA_PYTHONPATH": panda_install_purelib, 50 | "PANDA_VERIFY_HOST": "off", 51 | "PANDA_JUPYTER": "1", 52 | } 53 | for i in ['PANDA_AUTH', 54 | 'PANDA_AUTH_VO', 55 | 'PANDA_URL_SSL', 56 | 'PANDA_URL', 57 | 'PANDAMON_URL', 58 | 'X509_USER_PROXY', 59 | 'PANDA_USE_NATIVE_HTTPLIB', 60 | 'PANDA_NICKNAME', 61 | ]: 62 | try: 63 | panda_env[i] = section[i] 64 | except Exception: 65 | pass 66 | os.environ.update(panda_env) 67 | 68 | 69 | # magic commands 70 | 71 | GETPASS_STRINGS = ['Enter GRID pass phrase for this identity:'] 72 | RAWINPUT_STRINGS = ['>>> \n', "[y/n] \n"] 73 | 74 | 75 | def _execute(command): 76 | with subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, 77 | stdin=subprocess.PIPE, universal_newlines=True) as p: 78 | while True: 79 | nextline = p.stdout.readline() 80 | if nextline == '' and p.poll() is not None: 81 | break 82 | # check if uses getpass or raw_input 83 | is_getpass = False 84 | is_raw_input = False 85 | for one_str in GETPASS_STRINGS: 86 | if one_str in nextline: 87 | is_getpass = True 88 | break 89 | for one_str in RAWINPUT_STRINGS: 90 | if one_str == nextline: 91 | is_raw_input = True 92 | break 93 | if not is_raw_input: 94 | sys.stdout.write(nextline) 95 | sys.stdout.flush() 96 | # need to call getpass or input since jupyter notebook doesn't pass stdin from subprocess 97 | st = None 98 | if is_getpass: 99 | st = getpass.getpass() 100 | p.stdin.write(st) 101 | p.stdin.flush() 102 | elif is_raw_input: 103 | st = input('\n' + one_str.strip()) 104 | # feed stdin 105 | if st is not None: 106 | p.stdin.write(st + '\n') 107 | p.stdin.flush() 108 | 109 | output = p.communicate()[0] 110 | exit_code = p.returncode 111 | if exit_code == 0: 112 | if output: 113 | print(output) 114 | return exit_code 115 | 116 | 117 | @register_line_magic 118 | def pathena(line): 119 | _execute('pathena ' + line + ' -3') 120 | return 121 | 122 | 123 | @register_line_magic 124 | def prun(line): 125 | _execute('prun ' + line + ' -3') 126 | return 127 | 128 | 129 | @register_line_magic 130 | def phpo(line): 131 | _execute('phpo ' + line + ' -3') 132 | return 133 | 134 | 135 | @register_line_magic 136 | def pbook(line): 137 | _execute('pbook ' + line + ' --prompt_with_newline') 138 | return 139 | 140 | 141 | del pathena, prun, phpo, pbook 142 | -------------------------------------------------------------------------------- /pandaclient/queryPandaMonUtils.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import json 3 | import os 4 | import re 5 | import ssl 6 | import sys 7 | import time 8 | 9 | try: 10 | from urllib.error import HTTPError, URLError 11 | from urllib.parse import urlencode 12 | from urllib.request import Request, urlopen 13 | except ImportError: 14 | from urllib import urlencode 15 | 16 | from urllib2 import HTTPError, Request, URLError, urlopen 17 | 18 | try: 19 | baseMonURL = os.environ["PANDAMON_URL"] 20 | except Exception: 21 | baseMonURL = "https://bigpanda.cern.ch" 22 | 23 | HEADERS = {"Accept": "application/json", "Content-Type": "application/json"} 24 | 25 | 26 | def query_tasks( 27 | jeditaskid=None, username=None, limit=10000, taskname=None, status=None, superstatus=None, reqid=None, days=None, metadata=False, sync=False, verbose=False 28 | ): 29 | timestamp = int(time.time()) 30 | parmas = { 31 | "json": 1, 32 | "datasets": True, 33 | "limit": limit, 34 | } 35 | if jeditaskid: 36 | parmas["jeditaskid"] = jeditaskid 37 | if username: 38 | parmas["username"] = username 39 | if taskname: 40 | parmas["taskname"] = taskname 41 | if status: 42 | parmas["status"] = status 43 | if superstatus: 44 | parmas["superstatus"] = superstatus 45 | if reqid: 46 | parmas["reqid"] = reqid 47 | if days is not None: 48 | parmas["days"] = days 49 | if metadata: 50 | parmas["extra"] = "metastruct" 51 | if sync: 52 | parmas["timestamp"] = timestamp 53 | url = baseMonURL + "/tasks/?{0}".format(urlencode(parmas)) 54 | if verbose: 55 | sys.stderr.write("query url = {0}\n".format(url)) 56 | sys.stderr.write("headers = {0}\n".format(json.dumps(HEADERS))) 57 | try: 58 | req = Request(url, headers=HEADERS) 59 | try: 60 | # Skip SSL verification 61 | context = ssl._create_unverified_context() 62 | except AttributeError: 63 | # Legacy Python that doesn't verify HTTPS certificates by default 64 | rep = urlopen(req) 65 | else: 66 | rep = urlopen(req, context=context) 67 | if verbose: 68 | sys.stderr.write("time UTC = {0}\n".format(datetime.datetime.utcnow())) 69 | rec = rep.getcode() 70 | if verbose: 71 | sys.stderr.write("resp code = {0}\n".format(rec)) 72 | res = rep.read().decode("utf-8") 73 | if verbose: 74 | sys.stderr.write("data = {0}\n".format(res)) 75 | ret = json.loads(res) 76 | return timestamp, url, ret 77 | except Exception as e: 78 | err_str = "{0} : {1}".format(e.__class__.__name__, e) 79 | sys.stderr.write("{0}\n".format(err_str)) 80 | raise 81 | 82 | 83 | def datetime_parser(d): 84 | for k, v in d.items(): 85 | if isinstance(v, str) and re.search("^\d{4}-\d{2}-\d{2}(\W|T)\d{2}:\d{2}:\d{2}", v): 86 | try: 87 | d[k] = datetime.datetime.strptime(v, "%Y-%m-%dT%H:%M:%S") 88 | except Exception: 89 | try: 90 | d[k] = datetime.datetime.strptime(v, "%Y-%m-%d %H:%M:%S") 91 | except Exception: 92 | pass 93 | return d 94 | 95 | 96 | def query_jobs(jeditaskid, limit=10000, drop=True, verbose=False): 97 | parmas = { 98 | "json": 1, 99 | "limit": limit, 100 | } 101 | parmas["jeditaskid"] = jeditaskid 102 | if not drop: 103 | parmas["mode"] = "nodrop" 104 | url = baseMonURL + "/jobs/?{0}".format(urlencode(parmas)) 105 | if verbose: 106 | sys.stderr.write("query url = {0}\n".format(url)) 107 | sys.stderr.write("headers = {0}\n".format(json.dumps(HEADERS))) 108 | try: 109 | req = Request(url, headers=HEADERS) 110 | try: 111 | # Skip SSL verification 112 | context = ssl._create_unverified_context() 113 | except AttributeError: 114 | # Legacy Python that doesn't verify HTTPS certificates by default 115 | rep = urlopen(req) 116 | else: 117 | rep = urlopen(req, context=context) 118 | if verbose: 119 | sys.stderr.write("time UTC = {0}\n".format(datetime.datetime.utcnow())) 120 | rec = rep.getcode() 121 | if verbose: 122 | sys.stderr.write("resp code = {0}\n".format(rec)) 123 | res = rep.read().decode("utf-8") 124 | if verbose: 125 | sys.stderr.write("data = {0}\n".format(res)) 126 | ret = json.loads(res, object_hook=datetime_parser) 127 | return url, ret 128 | except Exception as e: 129 | err_str = "{0} : {1}".format(e.__class__.__name__, e) 130 | sys.stderr.write("{0}\n".format(err_str)) 131 | raise 132 | -------------------------------------------------------------------------------- /pandaclient/panda_gui.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import plotly.express as px 3 | from jupyter_dash import JupyterDash 4 | import dash_core_components as dcc 5 | import dash_html_components as html 6 | import dash_table 7 | import plotly.graph_objects as go 8 | from dash.dependencies import Input, Output 9 | 10 | from . import queryPandaMonUtils 11 | 12 | 13 | def show_task(jeditaskid, verbose=False, mode='inline'): 14 | # get task 15 | task = queryPandaMonUtils.query_tasks(23518002, verbose=False)[-1][0] 16 | # get tasks of the user 17 | tasks = queryPandaMonUtils.query_tasks(username=task['username'], verbose=False)[-1] 18 | tids = set([x['jeditaskid'] for x in tasks]) 19 | tids.add(jeditaskid) 20 | 21 | # Build App 22 | app = JupyterDash(__name__) 23 | app.layout = html.Div([ 24 | html.Div([ 25 | html.H2("TaskID: "), 26 | dcc.Dropdown( 27 | id='dropdown_taskid', 28 | options=[{'label': i, 'value': i} for i in tids], 29 | value=jeditaskid 30 | ),], 31 | style={'display': 'inline-block', 'width': '20%'} 32 | ), 33 | html.Div([ 34 | html.Div([ 35 | html.H2('Task Attributes'), 36 | dash_table.DataTable(id='00_table', 37 | columns=[{'id': 'attribute', 'name': 'attribute'}, 38 | {'id': 'value', 'name': 'value'}], 39 | page_action='none', 40 | style_table={'height': '330px', 'overflowY': 'auto'}, 41 | style_cell_conditional=[ 42 | { 43 | 'if': {'column_id': 'value'}, 44 | 'textAlign': 'left' 45 | }, 46 | ]),], 47 | style={'display': 'inline-block', 'width': '49%', 'float': 'left', 'padding-top': '30px'} 48 | ), 49 | html.Div([ 50 | dcc.Graph(id='01_graph'),], 51 | style={'display': 'inline-block', 'width': '49%'} 52 | ), 53 | ],), 54 | html.Div([ 55 | html.Div([ 56 | dcc.Graph(id='10_graph')], 57 | style={'display': 'inline-block', 'width': '49%'}), 58 | html.Div([ 59 | dcc.Graph(id='11_graph')], 60 | style={'display': 'inline-block', 'width': '49%'}) 61 | ],), 62 | ]) 63 | 64 | # Run app and display result inline in the notebook 65 | app.run_server(mode=mode) 66 | 67 | 68 | @app.callback( 69 | Output('00_table', 'data'), 70 | Output('01_graph', 'figure'), 71 | Output('10_graph', 'figure'), 72 | Output('11_graph', 'figure'), 73 | Input('dropdown_taskid', "value") 74 | ) 75 | def make_elements(jeditaskid): 76 | verbose = False 77 | task = queryPandaMonUtils.query_tasks(jeditaskid, verbose=verbose)[-1][0] 78 | jobs = queryPandaMonUtils.query_jobs(jeditaskid, drop=False, verbose=verbose)[-1]['jobs'] 79 | jobs = pd.DataFrame(jobs) 80 | 81 | # task data 82 | task_data = [{'attribute': k, 'value': task[k]} for k in task if isinstance(task[k], (str, type(None)))] 83 | 84 | # figures 85 | site_fig = px.histogram(jobs, x="computingsite", color="jobstatus") 86 | ram_fig = px.histogram(jobs, x="maxrss") 87 | 88 | exectime_fig = go.Figure() 89 | legend_set = set() 90 | for d in jobs.itertuples(index=False): 91 | if d.jobstatus == 'finished': 92 | t_color = 'green' 93 | elif d.jobstatus == 'failed': 94 | t_color = 'red' 95 | else: 96 | t_color = 'orange' 97 | if d.jobstatus not in legend_set: 98 | show_legend = True 99 | legend_set.add(d.jobstatus) 100 | exectime_fig.add_trace( 101 | go.Scatter( 102 | x=[d.creationtime, d.creationtime], 103 | y=[d.pandaid, d.pandaid], 104 | mode="lines", 105 | line=go.scatter.Line(color=t_color), 106 | showlegend=True, 107 | legendgroup=d.jobstatus, 108 | name=d.jobstatus, 109 | hoverinfo='skip' 110 | ) 111 | ) 112 | exectime_fig.add_trace( 113 | go.Scatter( 114 | x=[d.creationtime, d.endtime], 115 | y=[d.pandaid, d.pandaid], 116 | mode="lines", 117 | line=go.scatter.Line(color=t_color), 118 | showlegend=False, 119 | legendgroup=d.jobstatus, 120 | name="", 121 | hovertemplate="PandaID: %{y:d}") 122 | ) 123 | exectime_fig.update_xaxes(range=[jobs['creationtime'].min(), jobs['endtime'].max()], 124 | title_text='Job Lifetime') 125 | exectime_fig.update_yaxes(range=[jobs['pandaid'].min() * 0.999, jobs['pandaid'].max() * 1.001], 126 | title_text='PandaID') 127 | 128 | return task_data, site_fig, ram_fig, exectime_fig 129 | -------------------------------------------------------------------------------- /pandaclient/pcontainer_core.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import argparse 4 | import tempfile 5 | import subprocess 6 | 7 | # make arg parse with option definitions 8 | def make_arg_parse(): 9 | usage = 'pcontainer [options]\n' \ 10 | ' HowTo is available at https://twiki.cern.ch/twiki/bin/view/PanDA/PandaContainer' 11 | optP = argparse.ArgumentParser(usage=usage, conflict_handler="resolve") 12 | optP.add_argument('--version',action='store_const', const=True, dest='version', default=None, 13 | help='Displays version') 14 | optP.add_argument('--loadJson', action='store', dest='loadJson', default=None, 15 | help='Read command-line parameters from a json file which contains a dict of {parameter: value}') 16 | optP.add_argument('--dumpJson', action='store', dest='dumpJson', default=None, 17 | help='Dump all command-line parameters and submission result such as returnCode, returnOut, ' 18 | 'jediTaskID, etc to a json file') 19 | optP.add_argument('--cvmfs', action='store_const', const=True, dest='cvmfs', default=False, 20 | help="Bind /cvmfs to the container, bool, default False") 21 | optP.add_argument('--noX509', action='store_const', const=True, dest='noX509', default=False, 22 | help="Unset X509 environment in the container, bool, default False") 23 | optP.add_argument('--datadir', action='store', dest='datadir', default='', 24 | help="Binds the job directory to datadir for I/O operations, string, default /ctrdata") 25 | optP.add_argument('--workdir', action='store', dest='workdir', default='', 26 | help="chdir to workdir in the container, string, default /ctrdata") 27 | optP.add_argument('--debug', action='store_const', const=True, dest='debug', default=False, 28 | help="Enable more verbose output from runcontainer, bool, default False") 29 | optP.add_argument('--containerImage', action='store', dest='containerImage', default=None, 30 | help="Name of a container image") 31 | optP.add_argument('--useCentralRegistry', action='store_const', const=True, 32 | dest='useCentralRegistry', default=False, 33 | help="Use the central container registry when --containerImage is used") 34 | optP.add_argument('--excludedSite', action='append', dest='excludedSite', default=None, 35 | help="list of sites which are not used for site section, e.g., ANALY_ABC,ANALY_XYZ") 36 | optP.add_argument('--site', action='store', dest='site', default=None, 37 | help='Site name where jobs are sent. If omitted, jobs are automatically sent to sites ' 38 | 'where input is available. A comma-separated list of sites can be specified ' 39 | '(e.g. siteA,siteB,siteC), so that best sites are chosen from the given site list') 40 | optP.add_argument('--architecture', action='store', dest='architecture', default='', 41 | help="CPU and/or GPU requirements. #CPU_spec&GPU_spec where CPU or GPU spec can be " 42 | "omitted. CPU_spec = architecture<-vendor<-instruction set>>, " 43 | "GPU_spec = vendor<-model>. A wildcards can be used if there is no special " 44 | "requirement for the attribute. E.g., #x86_64-*-avx2&nvidia to ask for x86_64 " 45 | "CPU with avx2 support and nvidia GPU") 46 | optP.add_argument('--noSubmit', action='store_const', const=True, dest='noSubmit', default=None, 47 | help=argparse.SUPPRESS) 48 | optP.add_argument('--outDS', action='store', dest='outDS', default=None, 49 | help='Base name of the output dataset container. Actual output dataset name is defined ' 50 | 'for each output file type') 51 | optP.add_argument('--outputs', action='store', dest='outputs', default=None, 52 | help='Output file names. Comma separated. e.g., --outputs out1.dat,out2.txt. You can specify ' 53 | 'a suffix for each output container like :. ' 54 | 'e.g., --outputs AAA:out1.dat,BBB:out2.txt, so that output container names are outDS_AAA/ ' 55 | 'and outDS_BBB/ instead of outDS_out1.dat/ and outDS_out2.txt/') 56 | optP.add_argument('--intrSrv', action='store_const', const=True, dest='intrSrv', default=None, 57 | help=argparse.SUPPRESS) 58 | optP.add_argument('--exec', action='store', dest='exec', default=None, 59 | help='execution string. e.g., --exec "./myscript arg1 arg2"') 60 | optP.add_argument('-v', '--verbose', action='store_const', const=True, dest='verbose', default=None, 61 | help='Verbose') 62 | optP.add_argument('--priority', action='store', dest='priority', default=None, type=int, 63 | help='Set priority of the task (1000 by default). The value must be between 900 and 1100. ' \ 64 | 'Note that priorities of tasks are relevant only in ' \ 65 | "each user's share, i.e., your tasks cannot jump over other user's tasks " \ 66 | 'even if you give higher priorities.') 67 | optP.add_argument('--useSandbox', action='store_const', const=True, dest='useSandbox', default=False, 68 | help='To send files in the run directory to remote sites which are not sent out by default ' \ 69 | 'when --containerImage is used') 70 | optP.add_argument("-3", action="store_true", dest="python3", default=False, 71 | help="Use python3") 72 | 73 | return optP 74 | 75 | # construct command-line options from arg parse 76 | def construct_cli_options(options): 77 | options = vars(options) 78 | if 'loadJson' in options and options['loadJson'] is not None: 79 | newOpts = json.load(open(options['loadJson'])) 80 | else: 81 | newOpts = dict() 82 | for key in options: 83 | val = options[key] 84 | if key in ['loadJson']: 85 | continue 86 | if key == 'architecture': 87 | key = 'cmtConfig' 88 | if key == 'cvmfs': 89 | key = 'ctrCvmfs' 90 | if key == 'noX509': 91 | key = 'ctrNoX509' 92 | if key == 'datadir': 93 | key = 'ctrDatadir' 94 | if key == 'workdir': 95 | key = 'ctrWorkdir' 96 | if key == 'debug': 97 | key = 'ctrDebug' 98 | if val is None: 99 | continue 100 | newOpts[key] = val 101 | newOpts['noBuild'] = True 102 | tmpLoadJson = tempfile.NamedTemporaryFile(delete=False, mode='w') 103 | json.dump(newOpts, tmpLoadJson) 104 | tmpLoadJson.close() 105 | return tmpLoadJson.name 106 | 107 | # submit 108 | def submit(options): 109 | tmpDumpJson = None 110 | if 'dumpJson' not in options: 111 | tmpDumpJson = tempfile.mkstemp()[-1] 112 | options['dumpJson'] = tmpDumpJson 113 | tmpLoadJson = construct_cli_options(options) 114 | com = ['prun'] 115 | com += ['--loadJson={0}'.format(tmpLoadJson)] 116 | ret_val = subprocess.call(com) 117 | ret_dict = None 118 | if ret_val == 0: 119 | try: 120 | ret_dict = json.load(open(options['dumpJson'])) 121 | if tmpDumpJson is not None: 122 | os.remove(tmpDumpJson) 123 | except Exception: 124 | pass 125 | os.remove(tmpLoadJson) 126 | return (ret_val, ret_dict) 127 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # set PYTHONPATH to use the current directory first 2 | import sys 3 | import os 4 | import re 5 | import site 6 | from io import open 7 | 8 | sys.path.insert(0,'.') 9 | 10 | # get release version 11 | from pandaclient import PandaToolsPkgInfo 12 | release_version = PandaToolsPkgInfo.release_version 13 | 14 | from setuptools import setup 15 | from setuptools.command.install import install as install_org 16 | # import distutils after setuptools to tweak sys.modules so that the distutils module in setuptools is used 17 | import distutils 18 | from distutils.command.install_data import install_data as install_data_org 19 | 20 | 21 | # custom install to disable egg 22 | class install_panda (install_org): 23 | def finalize_options (self): 24 | install_org.finalize_options(self) 25 | self.single_version_externally_managed = True 26 | 27 | 28 | # generates files using templates and install them 29 | class install_data_panda (install_data_org): 30 | def initialize_options (self): 31 | install_data_org.initialize_options (self) 32 | self.prefix = None 33 | self.root = None 34 | self.install_purelib = None 35 | self.install_scripts = None 36 | 37 | def finalize_options (self): 38 | # set install_purelib 39 | self.set_undefined_options('install', 40 | ('prefix','prefix')) 41 | self.set_undefined_options('install', 42 | ('root','root')) 43 | self.set_undefined_options('install', 44 | ('install_purelib','install_purelib')) 45 | self.set_undefined_options('install', 46 | ('install_scripts','install_scripts')) 47 | 48 | def run (self): 49 | rpmInstall = False 50 | # set install_dir 51 | if not self.install_dir: 52 | if self.root: 53 | # rpm or wheel 54 | self.install_dir = self.prefix 55 | self.install_purelib = distutils.sysconfig.get_python_lib() 56 | self.install_scripts = os.path.join(self.prefix, 'bin') 57 | rpmInstall = True 58 | else: 59 | # sdist 60 | if not self.prefix: 61 | if '--user' in self.distribution.script_args: 62 | self.install_dir = site.USER_BASE 63 | else: 64 | self.install_dir = site.PREFIXES[0] 65 | else: 66 | self.install_dir = self.prefix 67 | #raise Exception, (self.install_dir, self.prefix, self.install_purelib) 68 | self.install_dir = os.path.expanduser(self.install_dir) 69 | self.install_dir = os.path.abspath(self.install_dir) 70 | # remove /usr for bdist/bdist_rpm 71 | match = re.search('(build/[^/]+/dumb)/usr',self.install_dir) 72 | if match is not None: 73 | self.install_dir = re.sub(match.group(0),match.group(1),self.install_dir) 74 | # remove /var/tmp/*-buildroot for bdist_rpm 75 | match = re.search('(/var/tmp/.*-buildroot)/usr',self.install_dir) 76 | if match is not None: 77 | self.install_dir = re.sub(match.group(0),match.group(1),self.install_dir) 78 | # create tmp area 79 | tmpDir = 'build/tmp' 80 | self.mkpath(tmpDir) 81 | new_data_files = [] 82 | autoGenFiles = [] 83 | for destDir,dataFiles in self.data_files: 84 | newFilesList = [] 85 | for srcFile in dataFiles: 86 | # dest filename 87 | destFile = re.sub('\.template$','',srcFile) 88 | # append 89 | newFilesList.append(destFile) 90 | if destFile == srcFile: 91 | continue 92 | autoGenFiles.append(destFile) 93 | # open src 94 | inFile = open(srcFile) 95 | # read 96 | filedata=inFile.read() 97 | # close 98 | inFile.close() 99 | # replace patterns 100 | for item in re.findall('@@([^@]+)@@',filedata): 101 | if not hasattr(self,item): 102 | raise RuntimeError('unknown pattern %s in %s' % (item,srcFile)) 103 | # get pattern 104 | patt = getattr(self,item) 105 | # convert to absolute path 106 | if item.startswith('install'): 107 | patt = os.path.abspath(patt) 108 | # remove build/*/dump for bdist 109 | patt = re.sub('build/[^/]+/dumb','',patt) 110 | # remove /var/tmp/*-buildroot for bdist_rpm 111 | patt = re.sub('/var/tmp/.*-buildroot','',patt) 112 | # replace 113 | filedata = filedata.replace('@@%s@@' % item, patt) 114 | # write to dest 115 | oFile = open(destFile,'w') 116 | oFile.write(filedata) 117 | oFile.close() 118 | # replace dataFiles to install generated file 119 | new_data_files.append((destDir,newFilesList)) 120 | # install 121 | self.data_files = new_data_files 122 | install_data_org.run(self) 123 | # post install only for client installation 124 | if not os.path.exists(os.path.join(self.install_purelib, 'pandacommon')): 125 | target = os.path.join(self.install_purelib, 'pandatools') 126 | if not os.path.exists(target): 127 | os.symlink('pandaclient', target) 128 | # delete 129 | for autoGenFile in autoGenFiles: 130 | try: 131 | os.remove(autoGenFile) 132 | except Exception: 133 | pass 134 | 135 | with open('README.md', 'r', encoding='utf-8') as description_file: 136 | long_description = description_file.read() 137 | 138 | setup( 139 | name="panda-client", 140 | version=release_version, 141 | description='PanDA Client Package', 142 | long_description=long_description, 143 | long_description_content_type='text/markdown', 144 | license='GPL', 145 | author='PanDA Team', 146 | author_email='atlas-adc-panda@cern.ch', 147 | url='https://panda-wms.readthedocs.io/en/latest/', 148 | python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*', 149 | classifiers=[ 150 | 'Programming Language :: Python', 151 | 'Programming Language :: Python :: 2', 152 | 'Programming Language :: Python :: 2.7', 153 | 'Programming Language :: Python :: 3', 154 | 'Programming Language :: Python :: 3.5', 155 | 'Programming Language :: Python :: 3.6', 156 | 'Programming Language :: Python :: 3.7', 157 | 'Programming Language :: Python :: 3.8', 158 | 'Programming Language :: Python :: 3.9', 159 | 'Programming Language :: Python :: 3.10', 160 | ], 161 | 162 | # optional pip dependencies 163 | extras_require={ 164 | 'jupyter': ['pandas', 'jupyter-dash'], 165 | }, 166 | 167 | packages = [ 'pandaclient', 168 | ], 169 | scripts = [ 'scripts/prun', 170 | 'scripts/pcontainer', 171 | 'scripts/pbook', 172 | 'scripts/pathena', 173 | 'scripts/phpo', 174 | 'scripts/pchain', 175 | ], 176 | data_files = [ ('etc/panda', ['templates/panda_setup.sh.template', 177 | 'templates/panda_setup.csh.template', 178 | 'templates/panda_setup.example.cfg.template', 179 | 'templates/site_path.sh.template', 180 | 'glade/pbook.glade', 181 | ] 182 | ), 183 | ('etc/panda/icons', ['icons/retry.png', 184 | 'icons/update.png', 185 | 'icons/kill.png', 186 | 'icons/pandamon.png', 187 | 'icons/savannah.png', 188 | 'icons/config.png', 189 | 'icons/back.png', 190 | 'icons/sync.png', 191 | 'icons/forward.png', 192 | 'icons/red.png', 193 | 'icons/green.png', 194 | 'icons/yellow.png', 195 | 'icons/orange.png', 196 | ] 197 | ), 198 | ('etc/panda/share', ['share/FakeAppMgr.py', 199 | 'share/ConfigExtractor.py', 200 | 'share/functions.sh' 201 | ] 202 | ), 203 | ], 204 | cmdclass={ 205 | 'install': install_panda, 206 | 'install_data': install_data_panda 207 | } 208 | ) 209 | -------------------------------------------------------------------------------- /pandaclient/openidc_utils.py: -------------------------------------------------------------------------------- 1 | import base64 2 | import datetime 3 | import glob 4 | import json 5 | import os 6 | import ssl 7 | import sys 8 | import time 9 | import uuid 10 | 11 | try: 12 | from urllib import urlencode 13 | 14 | from urllib2 import HTTPError, Request, urlopen 15 | except ImportError: 16 | from urllib.error import HTTPError 17 | from urllib.parse import urlencode 18 | from urllib.request import Request, urlopen 19 | 20 | raw_input = input 21 | 22 | 23 | TOKEN_BASENAME = ".token" 24 | CACHE_PREFIX = ".page_cache_" 25 | 26 | 27 | # decode ID token 28 | def decode_id_token(enc): 29 | enc = enc.split(".")[1] 30 | enc += "=" * (-len(enc) % 4) 31 | dec = json.loads(base64.urlsafe_b64decode(enc.encode())) 32 | return dec 33 | 34 | 35 | # utility class 36 | class OpenIdConnect_Utils: 37 | # constructor 38 | def __init__(self, auth_config_url, token_dir=None, log_stream=None, verbose=False): 39 | self.auth_config_url = auth_config_url 40 | if token_dir is None: 41 | token_dir = os.environ["PANDA_CONFIG_ROOT"] 42 | self.token_dir = os.path.expanduser(token_dir) 43 | if not os.path.exists(token_dir): 44 | os.makedirs(token_dir) 45 | self.log_stream = log_stream 46 | self.verbose = verbose 47 | 48 | # get token path 49 | def get_token_path(self): 50 | return os.path.join(self.token_dir, TOKEN_BASENAME) 51 | 52 | # get device code 53 | def get_device_code(self, device_auth_endpoint, client_id, audience): 54 | if self.verbose: 55 | self.log_stream.debug("getting device code") 56 | data = {"client_id": client_id, "scope": "openid profile email offline_access ", "audience": audience} # iam", 57 | rdata = urlencode(data).encode() 58 | req = Request(device_auth_endpoint, rdata) 59 | req.add_header("content-type", "application/x-www-form-urlencoded") 60 | try: 61 | conn = urlopen(req) 62 | text = conn.read() 63 | if self.verbose: 64 | self.log_stream.debug(text) 65 | return True, json.loads(text) 66 | except HTTPError as e: 67 | return False, "code={0}. reason={1}. description={2}".format(e.code, e.reason, e.read()) 68 | except Exception as e: 69 | return False, str(e) 70 | 71 | # get ID token 72 | def get_id_token(self, token_endpoint, client_id, client_secret, device_code, interval, expires_in): 73 | self.log_stream.info("Ready to get ID token?") 74 | while True: 75 | sys.stdout.write("[y/n] \n") 76 | choice = raw_input().lower() 77 | if choice == "y": 78 | break 79 | elif choice == "n": 80 | return False, "aborted" 81 | if self.verbose: 82 | self.log_stream.debug("getting ID token") 83 | startTime = datetime.datetime.utcnow() 84 | data = { 85 | "client_id": client_id, 86 | "grant_type": "urn:ietf:params:oauth:grant-type:device_code", 87 | "device_code": device_code, 88 | } 89 | if client_secret: 90 | data["client_secret"] = client_secret 91 | rdata = urlencode(data).encode() 92 | req = Request(token_endpoint, rdata) 93 | req.add_header("content-type", "application/x-www-form-urlencoded") 94 | while datetime.datetime.utcnow() - startTime < datetime.timedelta(seconds=expires_in): 95 | try: 96 | conn = urlopen(req) 97 | text = conn.read().decode() 98 | if self.verbose: 99 | self.log_stream.debug(text) 100 | id_token = json.loads(text)["id_token"] 101 | with open(self.get_token_path(), "w") as f: 102 | f.write(text) 103 | return True, id_token 104 | except HTTPError as e: 105 | text = e.read() 106 | try: 107 | description = json.loads(text) 108 | # pending 109 | if description["error"] == "authorization_pending": 110 | time.sleep(interval + 1) 111 | continue 112 | except Exception: 113 | pass 114 | return False, "code={0}. reason={1}. description={2}".format(e.code, e.reason, text) 115 | except Exception as e: 116 | return False, str(e) 117 | 118 | # refresh token 119 | def refresh_token(self, token_endpoint, client_id, client_secret, refresh_token_string): 120 | if self.verbose: 121 | self.log_stream.debug("refreshing token") 122 | data = {"client_id": client_id, "client_secret": client_secret, "grant_type": "refresh_token", "refresh_token": refresh_token_string} 123 | rdata = urlencode(data).encode() 124 | req = Request(token_endpoint, rdata) 125 | req.add_header("content-type", "application/x-www-form-urlencoded") 126 | try: 127 | conn = urlopen(req) 128 | text = conn.read().decode() 129 | if self.verbose: 130 | self.log_stream.debug(text) 131 | id_token = json.loads(text)["id_token"] 132 | with open(self.get_token_path(), "w") as f: 133 | f.write(text) 134 | return True, id_token 135 | except HTTPError as e: 136 | return False, "code={0}. reason={1}. description={2}".format(e.code, e.reason, e.read()) 137 | except Exception as e: 138 | return False, str(e) 139 | 140 | # fetch page 141 | def fetch_page(self, url): 142 | path = os.path.join(self.token_dir, CACHE_PREFIX + str(uuid.uuid5(uuid.NAMESPACE_URL, str(url)))) 143 | if os.path.exists(path) and datetime.datetime.now() - datetime.datetime.fromtimestamp(os.path.getmtime(path)) < datetime.timedelta(hours=1): 144 | try: 145 | with open(path) as f: 146 | return True, json.load(f) 147 | except Exception as e: 148 | self.log_stream.debug("cached {0} is corrupted: {1}".format(os.path.basename(url), str(e))) 149 | if self.verbose: 150 | self.log_stream.debug("fetching {0}".format(url)) 151 | try: 152 | context = ssl._create_unverified_context() 153 | conn = urlopen(url, context=context) 154 | text = conn.read().decode() 155 | if self.verbose: 156 | self.log_stream.debug(text) 157 | with open(path, "w") as f: 158 | f.write(text) 159 | with open(path) as f: 160 | return True, json.load(f) 161 | except HTTPError as e: 162 | return False, "code={0}. reason={1}. description={2}".format(e.code, e.reason, e.read()) 163 | except Exception as e: 164 | return False, str(e) 165 | 166 | # check token expiry 167 | def check_token(self): 168 | token_file = self.get_token_path() 169 | if os.path.exists(token_file): 170 | with open(token_file) as f: 171 | if self.verbose: 172 | self.log_stream.debug("check {0}".format(token_file)) 173 | try: 174 | # decode ID token 175 | data = json.load(f) 176 | dec = decode_id_token(data["id_token"]) 177 | exp_time = datetime.datetime.utcfromtimestamp(dec["exp"]) 178 | delta = exp_time - datetime.datetime.utcnow() 179 | if self.verbose: 180 | self.log_stream.debug("token expiration time : {0} UTC".format(exp_time.strftime("%Y-%m-%d %H:%M:%S"))) 181 | # check expiration time 182 | if delta < datetime.timedelta(minutes=5): 183 | # return refresh token 184 | if "refresh_token" in data: 185 | if self.verbose: 186 | self.log_stream.debug("to refresh token") 187 | return False, data["refresh_token"], dec 188 | else: 189 | # return valid token 190 | if self.verbose: 191 | self.log_stream.debug("valid token is available") 192 | return True, data["id_token"], dec 193 | except Exception as e: 194 | self.log_stream.error("failed to decode cached token with {0}".format(e)) 195 | if self.verbose: 196 | self.log_stream.debug("cached token unavailable") 197 | return False, None, None 198 | 199 | # run device authorization flow 200 | def run_device_authorization_flow(self): 201 | # check toke expiry 202 | s, o, dec = self.check_token() 203 | if s: 204 | # still valid 205 | return True, o 206 | refresh_token_string = o 207 | # get auth config 208 | s, o = self.fetch_page(self.auth_config_url) 209 | if not s: 210 | return False, "Failed to get Auth configuration: " + o 211 | auth_config = o 212 | # get endpoint config 213 | s, o = self.fetch_page(auth_config["oidc_config_url"]) 214 | if not s: 215 | return False, "Failed to get endpoint configuration: " + o 216 | endpoint_config = o 217 | # refresh token 218 | if refresh_token_string is not None: 219 | s, o = self.refresh_token(endpoint_config["token_endpoint"], auth_config["client_id"], auth_config["client_secret"], refresh_token_string) 220 | # refreshed 221 | if s: 222 | return True, o 223 | else: 224 | if self.verbose: 225 | self.log_stream.debug("failed to refresh token: {0}".format(o)) 226 | # get device code 227 | s, o = self.get_device_code(endpoint_config["device_authorization_endpoint"], auth_config["client_id"], auth_config["audience"]) 228 | if not s: 229 | return False, "Failed to get device code: " + o 230 | # get ID token 231 | self.log_stream.info(("Please go to {0} and sign in. " "Waiting until authentication is completed").format(o["verification_uri_complete"])) 232 | if "interval" in o: 233 | interval = o["interval"] 234 | else: 235 | interval = 5 236 | s, o = self.get_id_token( 237 | endpoint_config["token_endpoint"], auth_config["client_id"], auth_config["client_secret"], o["device_code"], interval, o["expires_in"] 238 | ) 239 | if not s: 240 | return False, "Failed to get ID token: " + o 241 | self.log_stream.info("All set") 242 | return True, o 243 | 244 | # cleanup 245 | def cleanup(self): 246 | for patt in [TOKEN_BASENAME, CACHE_PREFIX]: 247 | for f in glob.glob(os.path.join(self.token_dir, patt + "*")): 248 | os.remove(f) 249 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /pandaclient/panda_api.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import importlib 3 | import json 4 | import os 5 | import sys 6 | import tempfile 7 | 8 | from . import Client, PBookCore, PLogger 9 | 10 | 11 | class PandaAPI(object): 12 | # constructor 13 | def __init__(self): 14 | self.command_body = {} 15 | self.pbook = None 16 | self.log = PLogger.getPandaLogger() 17 | 18 | # kill a task 19 | def kill_task(self, task_id, verbose=True): 20 | """kill a task 21 | args: 22 | task_id: jediTaskID of the task to be killed 23 | verbose: True to see debug messages 24 | returns: 25 | status code 26 | 0: communication succeeded to the panda server 27 | 255: communication failure 28 | tuple of return code and diagnostic message 29 | 0: request is registered 30 | 1: server error 31 | 2: task not found 32 | 3: permission denied 33 | 4: irrelevant task status 34 | 100: non SSL connection 35 | 101: irrelevant taskID 36 | """ 37 | return Client.killTask(task_id) 38 | 39 | # finish a task 40 | def finish_task(self, task_id, wait_running=False, verbose=False): 41 | """finish a task 42 | args: 43 | task_id: jediTaskID of the task to finish 44 | wait_running: True to wait until running jobs are done 45 | verbose: True to see debug messages 46 | returns: 47 | status code 48 | 0: communication succeeded to the panda server 49 | 255: communication failure 50 | tuple of return code and diagnostic message 51 | 0: request is registered 52 | 1: server error 53 | 2: task not found 54 | 3: permission denied 55 | 4: irrelevant task status 56 | 100: non SSL connection 57 | 101: irrelevant taskID 58 | """ 59 | return Client.finishTask(task_id, wait_running, verbose) 60 | 61 | # retry a task 62 | def retry_task(self, task_id, new_parameters=None, verbose=False): 63 | """retry a task 64 | args: 65 | task_id: jediTaskID of the task to retry 66 | new_parameters: a dictionary of task parameters to overwrite 67 | verbose: True to see debug messages 68 | returns: 69 | status code 70 | 0: communication succeeded to the panda server 71 | 255: communication failure 72 | tuple of return code and diagnostic message 73 | 0: request is registered 74 | 1: server error 75 | 2: task not found 76 | 3: permission denied 77 | 4: irrelevant task status 78 | 100: non SSL connection 79 | 101: irrelevant taskID 80 | """ 81 | return Client.retryTask(task_id, verbose, True, new_parameters) 82 | 83 | # get tasks 84 | def get_tasks(self, task_ids=None, limit=1000, days=14, status=None, username=None): 85 | """get a list of task dictionaries 86 | args: 87 | task_ids: a list of task IDs, or None to get recent tasks 88 | limit: the max number of tasks to fetch from the server 89 | days: tasks for last N days to fetch 90 | status: filtering with task status 91 | username: user name of the tasks, or None to get own tasks 92 | returns: 93 | a list of task dictionaries 94 | """ 95 | if not self.pbook: 96 | self.pbook = PBookCore.PBookCore() 97 | return self.pbook.show(task_ids, limit=limit, days=days, format="json", status=status, username=username) 98 | 99 | # show tasks 100 | def show_tasks(self, task_ids=None, limit=1000, days=14, format="standard", status=None, username=None): 101 | """show tasks 102 | args: 103 | task_ids: a list of task IDs, or None to get recent tasks 104 | limit: the max number of tasks to fetch from the server 105 | days: tasks for last N days to fetch 106 | format: standard, long, or plain 107 | status: filtering with task status 108 | username: user name of the tasks, or None to get own tasks 109 | returns: 110 | None 111 | """ 112 | if not self.pbook: 113 | self.pbook = PBookCore.PBookCore() 114 | self.pbook.show(task_ids, limit=limit, days=days, format=format, status=status, username=username) 115 | 116 | # submit a task 117 | def submit_task(self, task_params, verbose=False): 118 | """submit a task using low-level API 119 | args: 120 | task_params: a dictionary of task parameters 121 | verbose: True to see debug messages 122 | returns: 123 | status code 124 | 0: communication succeeded to the panda server 125 | 255: communication failure 126 | tuple of return code, message from the server, and task ID if successful 127 | 0: request is processed 128 | 1: duplication in DEFT 129 | 2: duplication in JEDI 130 | 3: accepted for incremental execution 131 | 4: server error 132 | """ 133 | return Client.insertTaskParams(task_params, verbose=verbose, properErrorCode=True) 134 | 135 | # get metadata of all jobs in a task 136 | def get_job_metadata(self, task_id, output_json_filename): 137 | """get metadata of all jobs in a task 138 | args: 139 | task_id: task ID 140 | output_json_filename: output json filename 141 | """ 142 | if not self.pbook: 143 | self.pbook = PBookCore.PBookCore() 144 | return self.pbook.getUserJobMetadata(task_id, output_json_filename) 145 | 146 | # execute xyz 147 | def execute_xyz(self, command_name, module_name, args, console_log=True): 148 | dump_file = None 149 | stat = False 150 | ret = None 151 | err_str = None 152 | try: 153 | # convert args 154 | sys.argv = copy.copy(args) 155 | sys.argv.insert(0, command_name) 156 | # set dump file 157 | if "--dumpJson" not in sys.argv: 158 | sys.argv.append("--dumpJson") 159 | with tempfile.NamedTemporaryFile(delete=False) as f: 160 | sys.argv.append(f.name) 161 | dump_file = f.name 162 | # disable logging 163 | if not console_log: 164 | PLogger.disable_logging() 165 | # run 166 | if command_name in self.command_body: 167 | if hasattr(self.command_body[command_name], "main"): 168 | getattr(self.command_body[command_name], "main")() 169 | else: 170 | self.command_body[command_name].reload() 171 | else: 172 | self.command_body[command_name] = importlib.import_module(module_name) 173 | if hasattr(self.command_body[command_name], "main"): 174 | getattr(self.command_body[command_name], "main")() 175 | stat = True 176 | except SystemExit as e: 177 | if e.code == 0: 178 | stat = True 179 | else: 180 | err_str = "failed with code={0}".format(e.code) 181 | except Exception as e: 182 | err_str = "failed with {0}".format(str(e)) 183 | finally: 184 | # enable logging 185 | if not console_log: 186 | PLogger.enable_logging() 187 | if err_str: 188 | self.log.error(err_str) 189 | # read dump fle 190 | try: 191 | with open(sys.argv[sys.argv.index("--dumpJson") + 1]) as f: 192 | ret = json.load(f) 193 | if len(ret) == 1: 194 | ret = ret[0] 195 | except Exception: 196 | pass 197 | # delete dump file 198 | if not dump_file: 199 | os.remove(dump_file) 200 | return stat, ret 201 | 202 | # execute prun 203 | def execute_prun(self, args, console_log=True): 204 | """execute prun command 205 | 206 | args: 207 | args: The arguments used to execute prun. This is a list of strings, such as ["--outDS","user.hoge.001"] 208 | console_log: False to disable console logging 209 | 210 | returns: 211 | status: True if succeeded. Otherwise, False 212 | a dictionary: Task submission attributes including jediTaskID 213 | """ 214 | return self.execute_xyz("prun", "pandaclient.PrunScript", args, console_log) 215 | 216 | # execute pathena 217 | def execute_pathena(self, args, console_log=True): 218 | """execute pathena command 219 | 220 | args: 221 | args: The arguments used to execute pathena. This is a list of strings, such as ["--outDS","user.hoge.001"] 222 | console_log: False to disable console logging 223 | 224 | returns: 225 | status: True if succeeded. Otherwise, False 226 | a dictionary: Task submission attributes including jediTaskID 227 | """ 228 | return self.execute_xyz("pathena", "pandaclient.PathenaScript", args, console_log) 229 | 230 | # execute phpo 231 | def execute_phpo(self, args, console_log=True): 232 | """execute phpo command 233 | 234 | args: 235 | args: The arguments used to execute phpo. This is a list of strings, such as ["--outDS","user.hoge.001"] 236 | console_log: False to disable console logging 237 | 238 | returns: 239 | status: True if succeeded. Otherwise, False 240 | a dictionary: Task submission attributes including jediTaskID 241 | """ 242 | return self.execute_xyz("phpo", "pandaclient.PhpoScript", args, console_log) 243 | 244 | # execute pchain 245 | def execute_pchain(self, args, console_log=True): 246 | """execute pchain command 247 | 248 | args: 249 | args: The arguments used to execute chain. This is a list of strings, such as ["--outDS","user.hoge.001"] 250 | console_log: False to disable console logging 251 | 252 | returns: 253 | status: True if succeeded. Otherwise, False 254 | a dictionary: Task submission attributes including requestID 255 | """ 256 | return self.execute_xyz("pchain", "pandaclient.PchainScript", args, console_log) 257 | 258 | # hello 259 | def hello(self, verbose=False): 260 | """Health check with the PanDA server 261 | args: 262 | verbose: True to see verbose message 263 | returns: 264 | status code 265 | 0: communication succeeded to the panda server 266 | 255: communication failure 267 | diagnostic message 268 | """ 269 | return Client.hello(verbose) 270 | 271 | # increase attempt numbers to retry failed jobs 272 | def increase_attempt_nr(self, task_id, increase=3, verbose=False): 273 | """increase attempt numbers to retry failed jobs 274 | args: 275 | task_id: jediTaskID of the task 276 | increase: increase for attempt numbers 277 | verbose: True to see verbose message 278 | returns: 279 | status code 280 | 0: communication succeeded to the panda server 281 | 255: communication failure 282 | return code 283 | 0: succeeded 284 | 1: unknown task 285 | 2: invalid task status 286 | 3: permission denied 287 | 4: wrong parameter 288 | None: database error 289 | """ 290 | return Client.increase_attempt_nr(task_id, increase, verbose) 291 | 292 | 293 | pandaAPI = PandaAPI() 294 | del PandaAPI 295 | 296 | 297 | def get_api(): 298 | return pandaAPI 299 | -------------------------------------------------------------------------------- /pandaclient/LocalJobsetSpec.py: -------------------------------------------------------------------------------- 1 | """ 2 | local jobset specification 3 | 4 | """ 5 | 6 | import re 7 | 8 | 9 | class LocalJobsetSpec(object): 10 | # attributes 11 | _attributes = ('JobsetID','dbStatus','JobMap','PandaID','inDS','outDS', 12 | 'parentSetID','retrySetID','creationTime','jobStatus', 13 | 'jediTaskID','taskStatus') 14 | # slots 15 | __slots__ = _attributes + ('flag_showSubstatus','flag_longFormat') 16 | 17 | 18 | # constructor 19 | def __init__(self): 20 | # install attributes 21 | for attr in self._attributes: 22 | setattr(self,attr,None) 23 | self.flag_showSubstatus = '' 24 | self.flag_longFormat = False 25 | 26 | 27 | # string format 28 | def __str__(self): 29 | # set longFormat when showSubstatus 30 | if self.flag_showSubstatus != '': 31 | self.flag_longFormat = True 32 | # get JobID list 33 | jobIDs = list(self.JobMap) 34 | jobIDs.sort() 35 | # initialize 36 | firstJob = True 37 | strFormat = "%15s : %s\n" 38 | strOut1 = "" 39 | strOut2 = "" 40 | strOutJob = '' 41 | strJobID = '' 42 | for jobID in jobIDs: 43 | strJobID += '%s,' % jobID 44 | strJobID = strJobID[:-1] 45 | # loop over all jobs 46 | totalBuild = 0 47 | totalRun = 0 48 | totalMerge = 0 49 | totalJobStatus = {} 50 | usingMerge = False 51 | for jobID in jobIDs: 52 | job = self.JobMap[jobID] 53 | if firstJob: 54 | # get common values from the first jobID 55 | firstJob = False 56 | # release 57 | relStr = '' 58 | if job.releaseVar not in ['','NULL','None',None]: 59 | relStr = job.releaseVar 60 | # cache 61 | cacheStr = '' 62 | if job.cacheVar not in ['','NULL','None',None]: 63 | cacheStr = job.cacheVar 64 | # common string representation 65 | if self.isJEDI(): 66 | strOut1 += strFormat % ("jediTaskID", self.jediTaskID) 67 | strOut1 += strFormat % ("taskStatus", self.taskStatus) 68 | strOut1 += strFormat % ("JobsetID", self.JobsetID) 69 | strOut1 += strFormat % ("type", job.jobType) 70 | strOut1 += strFormat % ("release", relStr) 71 | strOut1 += strFormat % ("cache", cacheStr) 72 | #strOut2 += strFormat % ("JobID" , strJobID) 73 | #strOut2 += strFormat % ("PandaID", self.PandaID) 74 | strOut2 += strFormat % ("inDS", self.inDS) 75 | strOut2 += strFormat % ("outDS", self.outDS) 76 | if not self.isJEDI(): 77 | strOut2 += strFormat % ("parentSetID", self.parentSetID) 78 | strOut2 += strFormat % ("retrySetID", self.retrySetID) 79 | strOut2 += strFormat % ("creationTime", job.creationTime.strftime('%Y-%m-%d %H:%M:%S')) 80 | strOut2 += strFormat % ("lastUpdate", job.lastUpdate.strftime('%Y-%m-%d %H:%M:%S')) 81 | strOut2 += strFormat % ("params", job.jobParams) 82 | if not self.isJEDI(): 83 | strOut2 += strFormat % ("status", self.dbStatus) 84 | else: 85 | strOut2 += strFormat % ("inputStatus",'') 86 | # job status 87 | statusMap = {} 88 | for item in job.jobStatus.split(','): 89 | match = re.search('^(\w+)\*(\d+)$',item) 90 | if match is None: 91 | # non compact 92 | if item not in statusMap: 93 | statusMap[item] = 0 94 | statusMap[item] += 1 95 | else: 96 | # compact 97 | tmpStatus = match.group(1) 98 | tmpCount = int(match.group(2)) 99 | if tmpStatus not in statusMap: 100 | statusMap[tmpStatus] = 0 101 | statusMap[tmpStatus] += tmpCount 102 | # merge 103 | if job.mergeJobStatus not in ['NA']: 104 | usingMerge = True 105 | # get PandaIDs for each status 106 | pandaIDstatusMap = {} 107 | if not self.isJEDI(): 108 | tmpStatusList = job.jobStatus.split(',') 109 | tmpPandaIDList = job.PandaID.split(',') 110 | for tmpIndex,tmpPandaID in enumerate(tmpPandaIDList): 111 | if tmpIndex < len(tmpStatusList): 112 | tmpStatus = tmpStatusList[tmpIndex] 113 | else: 114 | # use unknown for out-range 115 | tmpStatus = 'unknown' 116 | # append for all jobs 117 | if tmpStatus not in totalJobStatus: 118 | totalJobStatus[tmpStatus] = 0 119 | totalJobStatus[tmpStatus] += 1 120 | # status of interest 121 | if tmpStatus not in self.flag_showSubstatus.split(','): 122 | continue 123 | # append for individual job 124 | if tmpStatus not in pandaIDstatusMap: 125 | pandaIDstatusMap[tmpStatus] = 'PandaID=' 126 | pandaIDstatusMap[tmpStatus] += '%s,' % tmpPandaID 127 | else: 128 | totalJobStatus = statusMap 129 | statusStr = job.dbStatus 130 | for tmpStatus in statusMap: 131 | tmpCount = statusMap[tmpStatus] 132 | statusStr += '\n%8s %10s : %s' % ('',tmpStatus,tmpCount) 133 | if self.flag_showSubstatus: 134 | if tmpStatus in pandaIDstatusMap: 135 | statusStr += '\n%8s %10s %s' % ('','',pandaIDstatusMap[tmpStatus][:-1]) 136 | # number of jobs 137 | nJobs = len(job.PandaID.split(',')) 138 | if job.buildStatus != '': 139 | # including buildJob 140 | nJobsStr = "%d + 1(build)" % (nJobs-1) 141 | totalBuild += 1 142 | if usingMerge and job.jobType in ['usermerge']: 143 | totalMerge += (nJobs-1) 144 | else: 145 | totalRun += (nJobs-1) 146 | else: 147 | nJobsStr = "%d" % nJobs 148 | if usingMerge and job.jobType in ['usermerge']: 149 | totalMerge += nJobs 150 | else: 151 | totalRun += nJobs 152 | # merging 153 | if self.isJEDI() and job.mergeJobID != '': 154 | totalMerge += len(job.mergeJobID.split(',')) 155 | # job specific string representation 156 | if self.flag_longFormat: 157 | strOutJob += '\n' 158 | strOutJob += strFormat % ("JobID", job.JobID) 159 | #strOutJob += strFormat % ("nJobs", nJobsStr) 160 | strOutJob += strFormat % ("site", job.site) 161 | strOutJob += strFormat % ("libDS", str(job.libDS)) 162 | strOutJob += strFormat % ("retryID", job.retryID) 163 | strOutJob += strFormat % ("provenanceID", job.provenanceID) 164 | strOutJob += strFormat % ("jobStatus", statusStr) 165 | # number of jobs 166 | nJobsStr = "%d" % totalRun 167 | if usingMerge: 168 | nJobsStr += " + %s(merge)" % totalMerge 169 | if totalBuild != 0: 170 | nJobsStr += " + %s(build)" % totalBuild 171 | #strOut1 += strFormat % ("nJobs", nJobsStr) 172 | strOut = strOut1 + strOut2 173 | # not long format 174 | if not self.flag_longFormat: 175 | for tmpStatus in totalJobStatus: 176 | tmpCount = totalJobStatus[tmpStatus] 177 | strOut += '%8s %10s : %s\n' % ('',tmpStatus,tmpCount) 178 | else: 179 | strOut += strOutJob 180 | # disable showSubstatus and longFormat 181 | self.flag_showSubstatus = '' 182 | self.flag_longFormat = False 183 | # return 184 | return strOut 185 | 186 | 187 | # override __getattribute__ 188 | def __getattribute__(self,name): 189 | # scan all JobIDs to get dbStatus 190 | if name == 'dbStatus': 191 | tmpJobs = object.__getattribute__(self,'JobMap') 192 | runningFlag = False 193 | for tmpJobID in tmpJobs: 194 | tmpJob = tmpJobs[tmpJobID] 195 | if tmpJob.dbStatus == 'killing': 196 | return 'killing' 197 | if tmpJob.dbStatus == 'running': 198 | runningFlag = True 199 | if runningFlag: 200 | return 'running' 201 | return 'frozen' 202 | ret = object.__getattribute__(self,name) 203 | return ret 204 | 205 | 206 | # set jobs 207 | def setJobs(self,jobs): 208 | # append 209 | retryIDs = [] 210 | parentIDs = [] 211 | for job in jobs: 212 | # set initial parameters 213 | if self.JobsetID is None: 214 | self.JobsetID = job.groupID 215 | self.JobMap = {} 216 | self.creationTime = job.creationTime 217 | self.jediTaskID = job.jediTaskID 218 | self.taskStatus = job.taskStatus 219 | self.JobMap[job.JobID] = job 220 | # get parent/retry 221 | if job.retryJobsetID not in [0,-1,'0','-1']: 222 | if job.retryJobsetID not in retryIDs: 223 | retryIDs.append(job.retryJobsetID) 224 | if job.parentJobsetID not in [0,-1,'0','-1']: 225 | if job.parentJobsetID not in parentIDs: 226 | parentIDs.append(job.parentJobsetID) 227 | # set parent/retry 228 | retryIDs.sort() 229 | parentIDs.sort() 230 | self.retrySetID = '' 231 | for tmpID in retryIDs: 232 | self.retrySetID += '%s,' % tmpID 233 | self.retrySetID = self.retrySetID[:-1] 234 | self.parentSetID = '' 235 | for tmpID in parentIDs: 236 | self.parentSetID += '%s,' % tmpID 237 | self.parentSetID = self.parentSetID[:-1] 238 | # aggregate some info 239 | pStr = '' 240 | sStatus = '' 241 | strInDS = '' 242 | strOutDS = '' 243 | tmpInDSList = [] 244 | tmpOutDSList = [] 245 | jobIDs = list(self.JobMap) 246 | jobIDs.sort() 247 | for jobID in jobIDs: 248 | job = self.JobMap[jobID] 249 | # PandaID 250 | tmpPStr = job.encodeCompact(includeMerge=True)['PandaID'] 251 | if tmpPStr != '': 252 | pStr += tmpPStr 253 | pStr += ',' 254 | # inDS and outDS 255 | try: 256 | for tmpItem in str(job.inDS).split(','): 257 | if tmpItem not in tmpInDSList: 258 | tmpInDSList.append(tmpItem) 259 | strInDS += '%s,' % tmpItem 260 | except Exception: 261 | pass 262 | try: 263 | for tmpItem in str(job.outDS).split(','): 264 | if tmpItem not in tmpOutDSList: 265 | tmpOutDSList.append(tmpItem) 266 | strOutDS += '%s,' % tmpItem 267 | except Exception: 268 | pass 269 | # set job status 270 | sStatus += job.jobStatus + ',' 271 | # set 272 | self.PandaID = pStr[:-1] 273 | self.inDS = strInDS[:-1] 274 | self.outDS = strOutDS[:-1] 275 | self.jobStatus = sStatus[:-1] 276 | 277 | 278 | # check if JEDI 279 | def isJEDI(self): 280 | if self.jediTaskID in [-1,'-1','']: 281 | return False 282 | return True 283 | -------------------------------------------------------------------------------- /pandaclient/MiscUtils.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import json 3 | import os 4 | import re 5 | import subprocess 6 | import sys 7 | import traceback 8 | import uuid 9 | 10 | try: 11 | import cPickle as pickle 12 | except ImportError: 13 | import pickle 14 | try: 15 | raw_input 16 | except NameError: 17 | raw_input = input 18 | try: 19 | unicode 20 | except Exception: 21 | unicode = str 22 | 23 | # set modules for unpickling in client-light 24 | try: 25 | from pandaserver.taskbuffer import JobSpec 26 | except ImportError: 27 | import pandaclient 28 | 29 | sys.modules["pandaserver"] = pandaclient 30 | from . import JobSpec 31 | 32 | sys.modules["pandaserver.taskbuffer.JobSpec"] = JobSpec 33 | JobSpec.JobSpec.__module__ = "pandaserver.taskbuffer.JobSpec" 34 | 35 | from . import FileSpec 36 | 37 | sys.modules["pandaserver.taskbuffer.FileSpec"] = FileSpec 38 | FileSpec.FileSpec.__module__ = "pandaserver.taskbuffer.FileSpec" 39 | 40 | 41 | # wrapper for uuidgen 42 | def wrappedUuidGen(): 43 | return str(uuid.uuid4()) 44 | 45 | 46 | # make JEDI job parameter 47 | def makeJediJobParam( 48 | lfn, 49 | dataset, 50 | paramType, 51 | padding=True, 52 | hidden=False, 53 | expand=False, 54 | include="", 55 | exclude="", 56 | nFilesPerJob=None, 57 | offset=0, 58 | destination="", 59 | token="", 60 | useNumFilesAsRatio=False, 61 | randomAtt=False, 62 | reusableAtt=False, 63 | allowNoOutput=None, 64 | outDS=None, 65 | file_list=None, 66 | ): 67 | dictItem = {} 68 | if paramType == "output": 69 | dictItem["type"] = "template" 70 | dictItem["value"] = lfn 71 | dictItem["param_type"] = paramType 72 | dictItem["dataset"] = dataset 73 | dictItem["container"] = dataset 74 | if destination != "": 75 | dictItem["destination"] = destination 76 | if token != "": 77 | dictItem["token"] = token 78 | if not padding: 79 | dictItem["padding"] = padding 80 | if allowNoOutput is not None: 81 | for tmpPatt in allowNoOutput: 82 | if tmpPatt == "": 83 | continue 84 | tmpPatt = "^.*" + tmpPatt + "$" 85 | if re.search(tmpPatt, lfn) is not None: 86 | dictItem["allowNoOutput"] = True 87 | break 88 | elif paramType == "input": 89 | dictItem["type"] = "template" 90 | dictItem["value"] = lfn 91 | dictItem["param_type"] = paramType 92 | dictItem["dataset"] = dataset 93 | if offset > 0: 94 | dictItem["offset"] = offset 95 | if include != "": 96 | dictItem["include"] = include 97 | if exclude != "": 98 | dictItem["exclude"] = exclude 99 | if expand: 100 | dictItem["expand"] = expand 101 | elif outDS: 102 | dictItem["consolidate"] = ".".join(outDS.split(".")[:2]) + "." + wrappedUuidGen() + "/" 103 | if nFilesPerJob not in [None, 0]: 104 | if useNumFilesAsRatio: 105 | dictItem["ratio"] = nFilesPerJob 106 | else: 107 | dictItem["nFilesPerJob"] = nFilesPerJob 108 | if file_list: 109 | dictItem["files"] = file_list 110 | if hidden: 111 | dictItem["hidden"] = hidden 112 | if randomAtt: 113 | dictItem["random"] = True 114 | if reusableAtt: 115 | dictItem["reusable"] = True 116 | return [dictItem] 117 | 118 | 119 | # get dataset name and num of files for a stream 120 | def getDatasetNameAndNumFiles(streamDS, nFilesPerJob, streamName): 121 | if streamDS == "": 122 | # read from stdin 123 | print("\nThis job uses %s stream" % streamName) 124 | while True: 125 | streamDS = raw_input("Enter dataset name for {0}: ".format(streamName)) 126 | streamDS = streamDS.strip() 127 | if streamDS != "": 128 | break 129 | # number of files per one signal 130 | if nFilesPerJob < 0: 131 | while True: 132 | tmpStr = raw_input("Enter the number of %s files per job : " % streamName) 133 | try: 134 | nFilesPerJob = int(tmpStr) 135 | break 136 | except Exception: 137 | pass 138 | # return 139 | return streamDS, nFilesPerJob 140 | 141 | 142 | # convert UTF-8 to ASCII in json dumps 143 | def unicodeConvert(input): 144 | if isinstance(input, dict): 145 | retMap = {} 146 | for tmpKey in input: 147 | tmpVal = input[tmpKey] 148 | retMap[unicodeConvert(tmpKey)] = unicodeConvert(tmpVal) 149 | return retMap 150 | elif isinstance(input, list): 151 | retList = [] 152 | for tmpItem in input: 153 | retList.append(unicodeConvert(tmpItem)) 154 | return retList 155 | elif isinstance(input, unicode): 156 | return input.encode("ascii", "ignore").decode() 157 | return input 158 | 159 | 160 | # decode json with ASCII 161 | def decodeJSON(input_file): 162 | with open(input_file) as f: 163 | return json.load(f, object_hook=unicodeConvert) 164 | 165 | 166 | # replacement for commands 167 | def commands_get_status_output(com): 168 | data = "" 169 | try: 170 | # for python 2.6 171 | # data = subprocess.check_output(com, shell=True, universal_newlines=True, stderr=subprocess.STDOUT) 172 | p = subprocess.Popen( 173 | com, 174 | shell=True, 175 | universal_newlines=True, 176 | stdout=subprocess.PIPE, 177 | stderr=subprocess.STDOUT, 178 | ) 179 | data, unused_err = p.communicate() 180 | retcode = p.poll() 181 | if retcode: 182 | ex = subprocess.CalledProcessError(retcode, com) 183 | raise ex 184 | status = 0 185 | except subprocess.CalledProcessError as ex: 186 | # for python 2.6 187 | # data = ex.output 188 | status = ex.returncode 189 | if data[-1:] == "\n": 190 | data = data[:-1] 191 | return status, data 192 | 193 | 194 | def commands_get_output(com): 195 | return commands_get_status_output(com)[1] 196 | 197 | 198 | def commands_fail_on_non_zero_exit_status( 199 | com, 200 | error_status_on_failure, 201 | verbose_cmd=False, 202 | verbose_output=False, 203 | logger=None, 204 | error_log_msg="", 205 | ): 206 | # print command if verbose 207 | if verbose_cmd: 208 | print(com) 209 | 210 | # execute command, get status code and message printed by the command 211 | status, data = commands_get_status_output(com) 212 | 213 | # fail for non zero exit status 214 | if status != 0: 215 | if not verbose_cmd: 216 | print(com) 217 | # print error message before failing 218 | print(data) 219 | # report error message if logger and log message have been provided 220 | if logger and error_log_msg: 221 | logger.error(error_log_msg) 222 | 223 | if type(error_status_on_failure) == int: 224 | # use error status provided to the function 225 | sys.exit(error_status_on_failure) 226 | elif error_status_on_failure == "sameAsStatus": 227 | # use error status exit code returned 228 | # by the execution of the command 229 | sys.exit(status) 230 | else: 231 | # default exit status otherwise 232 | sys.exit(1) 233 | 234 | # print command output message if verbose 235 | if verbose_output and data: 236 | print(data) 237 | 238 | return status, data 239 | 240 | 241 | # decorator to run with the original environment 242 | def run_with_original_env(func): 243 | def new_func(*args, **kwargs): 244 | if "LD_LIBRARY_PATH_ORIG" in os.environ and "LD_LIBRARY_PATH" in os.environ: 245 | os.environ["LD_LIBRARY_PATH_RESERVE"] = os.environ["LD_LIBRARY_PATH"] 246 | os.environ["LD_LIBRARY_PATH"] = os.environ["LD_LIBRARY_PATH_ORIG"] 247 | if "PYTHONPATH_ORIG" in os.environ: 248 | os.environ["PYTHONPATH_RESERVE"] = os.environ["PYTHONPATH"] 249 | os.environ["PYTHONPATH"] = os.environ["PYTHONPATH_ORIG"] 250 | if "PYTHONHOME_ORIG" in os.environ and os.environ["PYTHONHOME_ORIG"] != "": 251 | if "PYTHONHOME" in os.environ: 252 | os.environ["PYTHONHOME_RESERVE"] = os.environ["PYTHONHOME"] 253 | os.environ["PYTHONHOME"] = os.environ["PYTHONHOME_ORIG"] 254 | try: 255 | return func(*args, **kwargs) 256 | except Exception as e: 257 | print(str(e) + traceback.format_exc()) 258 | raise e 259 | finally: 260 | if "LD_LIBRARY_PATH_RESERVE" in os.environ: 261 | os.environ["LD_LIBRARY_PATH"] = os.environ["LD_LIBRARY_PATH_RESERVE"] 262 | if "PYTHONPATH_RESERVE" in os.environ: 263 | os.environ["PYTHONPATH"] = os.environ["PYTHONPATH_RESERVE"] 264 | if "PYTHONHOME_RESERVE" in os.environ: 265 | os.environ["PYTHONHOME"] = os.environ["PYTHONHOME_RESERVE"] 266 | 267 | return new_func 268 | 269 | 270 | # run commands with the original environment 271 | @run_with_original_env 272 | def commands_get_output_with_env(com): 273 | return commands_get_output(com) 274 | 275 | 276 | @run_with_original_env 277 | def commands_get_status_output_with_env(com): 278 | return commands_get_status_output(com) 279 | 280 | 281 | # unpickle python2 pickle with python3 282 | def pickle_loads(str_input): 283 | try: 284 | return pickle.loads(str_input) 285 | except Exception: 286 | try: 287 | return pickle.loads(str_input.encode("utf-8"), encoding="latin1") 288 | except Exception: 289 | raise Exception("failed to unpickle") 290 | 291 | 292 | # parse secondary dataset option 293 | def parse_secondary_datasets_opt(secondaryDSs): 294 | if secondaryDSs != "": 295 | # parse 296 | tmpMap = {} 297 | for tmpItem in secondaryDSs.split(","): 298 | if "#" in tmpItem: 299 | tmpItems = tmpItem.split("#") 300 | else: 301 | tmpItems = tmpItem.split(":") 302 | if 3 <= len(tmpItems) <= 6: 303 | tmpDsName = tmpItems[2] 304 | # change ^ to , 305 | tmpDsName = tmpDsName.replace("^", ",") 306 | # make map 307 | tmpMap[tmpDsName] = { 308 | "nFiles": int(tmpItems[1]), 309 | "streamName": tmpItems[0], 310 | "pattern": "", 311 | "nSkip": 0, 312 | "files": [], 313 | } 314 | # using filtering pattern 315 | if len(tmpItems) >= 4 and tmpItems[3]: 316 | tmpMap[tmpItems[2]]["pattern"] = tmpItems[3] 317 | # nSkip 318 | if len(tmpItems) >= 5 and tmpItems[4]: 319 | tmpMap[tmpItems[2]]["nSkip"] = int(tmpItems[4]) 320 | # files 321 | if len(tmpItems) >= 6 and tmpItems[5]: 322 | with open(tmpItems[5]) as f: 323 | for l in f: 324 | l = l.strip() 325 | if l: 326 | tmpMap[tmpItems[2]]["files"].append(l) 327 | else: 328 | errStr = "Wrong format %s in --secondaryDSs. Must be " "StreamName:nFiles:DatasetName[:Pattern[:nSkipFiles[:FileNameList]]]" % tmpItem 329 | return False, errStr 330 | # set 331 | secondaryDSs = tmpMap 332 | else: 333 | secondaryDSs = {} 334 | return True, secondaryDSs 335 | 336 | 337 | # convert datetime to string 338 | class NonJsonObjectEncoder(json.JSONEncoder): 339 | def default(self, obj): 340 | if isinstance(obj, datetime.datetime): 341 | return {"_datetime_object": obj.strftime("%Y-%m-%d %H:%M:%S.%f")} 342 | return json.JSONEncoder.default(self, obj) 343 | 344 | 345 | # hook for json decoder 346 | def as_python_object(dct): 347 | if "_datetime_object" in dct: 348 | return datetime.datetime.strptime(str(dct["_datetime_object"]), "%Y-%m-%d %H:%M:%S.%f") 349 | return dct 350 | 351 | 352 | # dump jobs to serialized json 353 | def dump_jobs_json(jobs): 354 | state_objects = [] 355 | for job_spec in jobs: 356 | state_objects.append(job_spec.dump_to_json_serializable()) 357 | return json.dumps(state_objects, cls=NonJsonObjectEncoder) 358 | 359 | 360 | # load serialized json to jobs 361 | def load_jobs_json(state): 362 | state_objects = json.loads(state, object_hook=as_python_object) 363 | jobs = [] 364 | for job_state in state_objects: 365 | job_spec = JobSpec.JobSpec() 366 | job_spec.load_from_json_serializable(job_state) 367 | jobs.append(job_spec) 368 | return jobs 369 | 370 | # load dictionary list into jobs 371 | def load_jobs(job_dicts): 372 | jobs = [] 373 | for job_dict in job_dicts: 374 | job_spec = JobSpec.JobSpec() 375 | try: 376 | job_spec.load_from_dict(job_dict) 377 | except Exception: 378 | job_spec = None 379 | jobs.append(job_spec) 380 | return jobs 381 | 382 | 383 | # ask a yes/no question and return answer 384 | def query_yes_no(question): 385 | prompt = "[y/n]: " 386 | valid = {"yes": True, "y": True, "ye": True, "no": False, "n": False} 387 | info_str = " (Use -y if you are confident and want to skip this question) " 388 | while True: 389 | sys.stdout.write(question + info_str + prompt) 390 | choice = raw_input().lower() 391 | if choice in valid: 392 | return valid[choice] 393 | else: 394 | sys.stdout.write("Please respond with 'y' or 'n'") 395 | -------------------------------------------------------------------------------- /pandaclient/PchainScript.py: -------------------------------------------------------------------------------- 1 | import atexit 2 | import copy 3 | import json 4 | import os 5 | import re 6 | import shlex 7 | import shutil 8 | import sys 9 | 10 | from pandaclient import ( 11 | Client, 12 | MiscUtils, 13 | PandaToolsPkgInfo, 14 | PLogger, 15 | PrunScript, 16 | PsubUtils, 17 | ) 18 | from pandaclient.Group_argparse import get_parser 19 | 20 | try: 21 | from urllib import quote 22 | except ImportError: 23 | from urllib.parse import quote 24 | 25 | try: 26 | unicode 27 | except Exception: 28 | unicode = str 29 | 30 | 31 | # main 32 | def main(): 33 | # tweak sys.argv 34 | sys.argv.pop(0) 35 | sys.argv.insert(0, "pchain") 36 | 37 | usage = """pchain [options] 38 | """ 39 | 40 | optP = get_parser(usage=usage, conflict_handler="resolve") 41 | 42 | group_output = optP.add_group("output", "output dataset/files") 43 | group_config = optP.add_group("config", "workflow configuration") 44 | group_submit = optP.add_group("submit", "job submission/site/retry") 45 | group_expert = optP.add_group("expert", "for experts/developers only") 46 | group_build = optP.add_group("build", "build/compile the package and env setup") 47 | group_check = optP.add_group("check", "check workflow description") 48 | 49 | optP.add_helpGroup() 50 | 51 | group_config.add_argument("--version", action="store_const", const=True, dest="version", default=False, help="Displays version") 52 | group_config.add_argument("-v", action="store_const", const=True, dest="verbose", default=False, help="Verbose") 53 | group_config.add_argument( 54 | "--dumpJson", 55 | action="store", 56 | dest="dumpJson", 57 | default=None, 58 | help="Dump all command-line parameters and submission result such as returnCode, returnOut, and requestID to a json file", 59 | ) 60 | group_check.add_argument("--check", action="store_const", const=True, dest="checkOnly", default=False, help="Check workflow description locally") 61 | group_check.add_argument( 62 | "--debug", action="store_const", const=True, dest="debugCheck", default=False, help="verbose mode when checking workflow description locally" 63 | ) 64 | 65 | group_output.add_argument("--cwl", action="store", dest="cwl", default=None, help="Name of the main CWL file to describe the workflow") 66 | group_output.add_argument("--yaml", action="store", dest="yaml", default=None, help="Name of the yaml file for workflow parameters") 67 | group_output.add_argument("--snakefile", action="store", dest="snakefile", default=None, help="Name of the main Snakefile to describe the workflow") 68 | group_output.add_argument( 69 | "--maxSizeInSandbox", 70 | action="store", 71 | dest="maxSizeInSandbox", 72 | default=1, 73 | type=int, 74 | help="Maximum size in MB of files in the workflow sandbox (default 1 MB)", 75 | ) 76 | 77 | group_build.add_argument( 78 | "--useAthenaPackages", 79 | action="store_const", 80 | const=True, 81 | dest="useAthenaPackages", 82 | default=False, 83 | help="One or more tasks in the workflow uses locally-built Athena packages", 84 | ) 85 | group_build.add_argument("--vo", action="store", dest="vo", default=None, help="virtual organization name") 86 | group_build.add_argument( 87 | "--extFile", 88 | action="store", 89 | dest="extFile", 90 | default="", 91 | help="root or large files under WORKDIR are not sent to WNs by default. " 92 | "If you want to send some skipped files, specify their names, " 93 | "e.g., data.root,big.tgz,*.o", 94 | ) 95 | 96 | group_output.add_argument("--outDS", action="store", dest="outDS", default=None, help="Name of the dataset for output and log files") 97 | group_output.add_argument("--official", action="store_const", const=True, dest="official", default=False, help="Produce official dataset") 98 | 99 | group_submit.add_argument("--noSubmit", action="store_const", const=True, dest="noSubmit", default=False, help="Dry-run") 100 | group_submit.add_argument("-3", action="store_true", dest="python3", default=False, help="Use python3") 101 | group_submit.add_argument( 102 | "--voms", 103 | action="store", 104 | dest="vomsRoles", 105 | default=None, 106 | type=str, 107 | help="generate proxy with paticular roles. " "e.g., atlas:/atlas/ca/Role=production,atlas:/atlas/fr/Role=pilot", 108 | ) 109 | group_submit.add_argument("--noEmail", action="store_const", const=True, dest="noEmail", default=False, help="Suppress email notification") 110 | group_submit.add_argument("--prodSourceLabel", action="store", dest="prodSourceLabel", default="", help="set prodSourceLabel") 111 | group_submit.add_argument("--workingGroup", action="store", dest="workingGroup", default=None, help="set workingGroup") 112 | group_submit.add_argument("--workflowName", action="store", dest="workflowName", default=None, help="set workflow name") 113 | 114 | group_expert.add_argument( 115 | "--intrSrv", 116 | action="store_const", 117 | const=True, 118 | dest="intrSrv", 119 | default=False, 120 | help="Please don't use this option. Only for developers to use the intr panda server", 121 | ) 122 | group_expert.add_argument( 123 | "--relayHost", action="store", dest="relayHost", default=None, help="Please don't use this option. Only for developers to use the relay host" 124 | ) 125 | 126 | # get logger 127 | tmpLog = PLogger.getPandaLogger() 128 | 129 | # show version 130 | if "--version" in sys.argv: 131 | print("Version: %s" % PandaToolsPkgInfo.release_version) 132 | sys.exit(0) 133 | 134 | # parse args 135 | options = optP.parse_args() 136 | 137 | # check 138 | if options.cwl: 139 | workflow_language = "cwl" 140 | workflow_file = options.cwl 141 | workflow_input = options.yaml 142 | args_to_check = ["yaml", "outDS"] 143 | elif options.snakefile: 144 | workflow_language = "snakemake" 145 | workflow_file = options.snakefile 146 | workflow_input = "" 147 | args_to_check = ["outDS"] 148 | else: 149 | tmpLog.error("argument --cwl or --snakefile is required") 150 | sys.exit(1) 151 | 152 | for arg_name in args_to_check: 153 | if not getattr(options, arg_name): 154 | tmpLog.error("argument --{0} is required".format(arg_name)) 155 | sys.exit(1) 156 | 157 | # check grid-proxy 158 | PsubUtils.check_proxy(options.verbose, options.vomsRoles) 159 | 160 | # check output name 161 | nickName = PsubUtils.getNickname() 162 | if not PsubUtils.checkOutDsName(options.outDS, options.official, nickName, verbose=options.verbose): 163 | tmpStr = "invalid output dataset name: %s" % options.outDS 164 | tmpLog.error(tmpStr) 165 | sys.exit(1) 166 | 167 | # create tmp dir 168 | curDir = os.getcwd() 169 | tmpDir = os.path.join(curDir, MiscUtils.wrappedUuidGen()) 170 | os.makedirs(tmpDir) 171 | 172 | # exit action 173 | def _onExit(dir, del_command): 174 | del_command("rm -rf %s" % dir) 175 | 176 | atexit.register(_onExit, tmpDir, MiscUtils.commands_get_output) 177 | 178 | # sandbox 179 | if options.verbose: 180 | tmpLog.debug("making sandbox") 181 | archiveName = "jobO.%s.tar.gz" % MiscUtils.wrappedUuidGen() 182 | archiveFullName = os.path.join(tmpDir, archiveName) 183 | find_opt = " -type f -size -{0}k".format(options.maxSizeInSandbox * 1024) 184 | tmpOut = MiscUtils.commands_get_output("find . {0} | tar cvfz {1} --files-from - ".format(find_opt, archiveFullName)) 185 | 186 | if options.verbose: 187 | print(tmpOut + "\n") 188 | tmpLog.debug("checking sandbox") 189 | tmpOut = MiscUtils.commands_get_output("tar tvfz {0}".format(archiveFullName)) 190 | print(tmpOut + "\n") 191 | 192 | if not options.noSubmit: 193 | tmpLog.info("uploading workflow sandbox") 194 | if options.vo: 195 | use_cache_srv = False 196 | else: 197 | use_cache_srv = True 198 | os.chdir(tmpDir) 199 | status, out = Client.putFile(archiveName, options.verbose, useCacheSrv=use_cache_srv, reuseSandbox=True) 200 | 201 | if out.startswith("NewFileName:"): 202 | # found the same input sandbox to reuse 203 | archiveName = out.split(":")[-1] 204 | elif out != "True": 205 | # failed 206 | print(out) 207 | tmpLog.error("Failed with %s" % status) 208 | sys.exit(1) 209 | os.chdir(curDir) 210 | try: 211 | shutil.rmtree(tmpDir) 212 | except Exception: 213 | pass 214 | 215 | # check if the workflow uses athena packages 216 | if not options.useAthenaPackages: 217 | with open(workflow_file, "r") as f: 218 | for line in f.readlines(): 219 | if re.search(r"^\s*[^#]\s*opt_useAthenaPackages", line): 220 | options.useAthenaPackages = True 221 | break 222 | 223 | matchURL = re.search("(http.*://[^/]+)/", Client.baseURLCSRVSSL) 224 | sourceURL = matchURL.group(1) 225 | 226 | params = { 227 | "taskParams": {}, 228 | "sourceURL": sourceURL, 229 | "sandbox": archiveName, 230 | "workflowSpecFile": workflow_file, 231 | "workflowInputFile": workflow_input, 232 | "language": workflow_language, 233 | "outDS": options.outDS, 234 | "base_platform": os.environ.get("ALRB_USER_PLATFORM", "centos7"), 235 | } 236 | if options.workflowName: 237 | params["workflow_name"] = options.workflowName 238 | 239 | # making task params with dummy exec 240 | task_type_args = {"container": "--containerImage __dummy_container__"} 241 | if options.useAthenaPackages: 242 | task_type_args["athena"] = "--useAthenaPackages" 243 | for task_type in task_type_args: 244 | os.chdir(curDir) 245 | prun_exec_str = "--exec __dummy_exec_str__ --outDS {0} {1}".format(options.outDS, task_type_args[task_type]) 246 | if options.noSubmit: 247 | prun_exec_str += " --noSubmit" 248 | if options.verbose: 249 | prun_exec_str += " -v" 250 | if options.vo: 251 | prun_exec_str += " --vo {0}".format(options.vo) 252 | if options.prodSourceLabel: 253 | prun_exec_str += " --prodSourceLabel {0}".format(options.prodSourceLabel) 254 | if options.workingGroup: 255 | prun_exec_str += " --workingGroup {0}".format(options.workingGroup) 256 | if options.official: 257 | prun_exec_str += " --official" 258 | if options.extFile: 259 | prun_exec_str += " --extFile {0}".format(options.extFile) 260 | arg_dict = {"get_taskparams": True, "ext_args": shlex.split(prun_exec_str)} 261 | if options.checkOnly: 262 | arg_dict["dry_mode"] = True 263 | 264 | taskParamMap = PrunScript.main(**arg_dict) 265 | del taskParamMap["noInput"] 266 | del taskParamMap["nEvents"] 267 | del taskParamMap["nEventsPerJob"] 268 | 269 | params["taskParams"][task_type] = taskParamMap 270 | 271 | if options.noSubmit: 272 | if options.noSubmit: 273 | if options.verbose: 274 | tmpLog.debug("==== taskParams ====") 275 | tmpKeys = list(taskParamMap) 276 | tmpKeys.sort() 277 | for tmpKey in tmpKeys: 278 | if tmpKey in ["taskParams"]: 279 | continue 280 | print("%s : %s" % (tmpKey, taskParamMap[tmpKey])) 281 | sys.exit(0) 282 | 283 | data = {"relay_host": options.relayHost, "verbose": options.verbose} 284 | if not options.checkOnly: 285 | action_type = "submit" 286 | else: 287 | action_type = "check" 288 | data["check"] = True 289 | 290 | # set to use INTR server just before taking action so that sandbox files go to the regular place 291 | if options.intrSrv: 292 | Client.useIntrServer() 293 | 294 | # action 295 | tmpLog.info("{0} workflow {1}".format(action_type, options.outDS)) 296 | tmpStat, tmpOut = Client.send_workflow_request(params, **data) 297 | 298 | # result 299 | exit_code = 0 300 | request_id = None 301 | tmp_str = "" 302 | if tmpStat != 0: 303 | tmp_str = "workflow {0} failed with {1}".format(action_type, tmpStat) 304 | tmpLog.error(tmp_str) 305 | exit_code = 1 306 | else: 307 | if tmpOut[0]: 308 | stat_code = tmpOut[1]["status"] 309 | check_log = "messages from the server\n\n" + tmpOut[1]["log"] 310 | if options.checkOnly: 311 | tmpLog.info(check_log) 312 | if stat_code: 313 | tmpLog.info("successfully verified workflow description") 314 | else: 315 | tmpLog.error("workflow description is corrupted") 316 | else: 317 | if stat_code: 318 | request_id = tmpOut[1]["request_id"] 319 | tmp_str = "successfully submitted with request_id={0}".format(request_id) 320 | tmpLog.info(tmp_str) 321 | else: 322 | tmpLog.info(check_log) 323 | tmp_str = "workflow submission failed with {0}".format(stat_code) 324 | tmpLog.error(tmp_str) 325 | exit_code = stat_code 326 | else: 327 | tmp_str = "workflow {0} failed. {1}".format(action_type, tmpOut[1]) 328 | tmpLog.error(tmp_str) 329 | exit_code = 1 330 | 331 | # dump json 332 | if options.dumpJson: 333 | dump_item = copy.deepcopy(vars(options)) 334 | dump_item["returnCode"] = exit_code 335 | dump_item["returnOut"] = tmp_str 336 | dump_item["requestID"] = request_id 337 | with open(options.dumpJson, "w") as f: 338 | json.dump(dump_item, f) 339 | 340 | return exit_code 341 | -------------------------------------------------------------------------------- /pandaclient/LocalJobSpec.py: -------------------------------------------------------------------------------- 1 | """ 2 | local job specification 3 | 4 | """ 5 | 6 | import re 7 | try: 8 | from urllib import quote, unquote 9 | except ImportError: 10 | from urllib.parse import quote, unquote 11 | import datetime 12 | try: 13 | long() 14 | except Exception: 15 | long = int 16 | try: 17 | unicode() 18 | except Exception: 19 | unicode = str 20 | 21 | 22 | class LocalJobSpec(object): 23 | # attributes 24 | _attributes = ('id','JobID','PandaID','jobStatus','site','cloud','jobType', 25 | 'jobName','inDS','outDS','libDS','provenanceID','creationTime', 26 | 'lastUpdate','jobParams','dbStatus','buildStatus','retryID', 27 | 'commandToPilot') 28 | # appended attributes 29 | appended = { 30 | 'groupID' : 'INTEGER', 31 | 'releaseVar' : 'VARCHAR(128)', 32 | 'cacheVar' : 'VARCHAR(128)', 33 | 'retryJobsetID' : 'INTEGER', 34 | 'parentJobsetID': 'INTEGER', 35 | 'mergeJobStatus': 'VARCHAR(20)', 36 | 'mergeJobID' : 'TEXT', 37 | 'nRebro' : 'INTEGER', 38 | 'jediTaskID' : 'INTEGER', 39 | 'taskStatus' : 'VARCHAR(16)', 40 | } 41 | 42 | _attributes += tuple(appended.keys()) 43 | # slots 44 | __slots__ = _attributes + ('flag_showSubstatus','flag_longFormat') 45 | 46 | 47 | # constructor 48 | def __init__(self): 49 | # install attributes 50 | for attr in self._attributes: 51 | setattr(self,attr,None) 52 | self.flag_showSubstatus = '' 53 | self.flag_longFormat = False 54 | 55 | # string format 56 | def __str__(self): 57 | # job status 58 | statusMap = {} 59 | for item in self.jobStatus.split(','): 60 | match = re.search('^(\w+)\*(\d+)$',item) 61 | if match is None: 62 | # non compact 63 | if item not in statusMap: 64 | statusMap[item] = 0 65 | statusMap[item] += 1 66 | else: 67 | # compact 68 | tmpStatus = match.group(1) 69 | tmpCount = int(match.group(2)) 70 | if tmpStatus not in statusMap: 71 | statusMap[tmpStatus] = 0 72 | statusMap[tmpStatus] += tmpCount 73 | # show PandaIDs in particular states 74 | pandaIDstatusMap = {} 75 | if self.flag_showSubstatus != '': 76 | # get PandaIDs for each status 77 | tmpStatusList = self.jobStatus.split(',') 78 | tmpPandaIDList = self.PandaID.split(',') 79 | for tmpIndex,tmpPandaID in enumerate(tmpPandaIDList): 80 | if tmpIndex < len(tmpStatusList): 81 | tmpStatus = tmpStatusList[tmpIndex] 82 | else: 83 | # use unkown for out-range 84 | tmpStatus = 'unknown' 85 | # status of interest 86 | if tmpStatus not in self.flag_showSubstatus.split(','): 87 | continue 88 | # append 89 | if tmpStatus not in pandaIDstatusMap: 90 | pandaIDstatusMap[tmpStatus] = 'PandaID=' 91 | pandaIDstatusMap[tmpStatus] += '%s,' % tmpPandaID 92 | statusStr = self.dbStatus 93 | for tmpStatus in statusMap: 94 | tmpCount = statusMap[tmpStatus] 95 | statusStr += '\n%8s %10s : %s' % ('',tmpStatus,tmpCount) 96 | if self.flag_showSubstatus: 97 | if tmpStatus in pandaIDstatusMap: 98 | statusStr += '\n%8s %10s %s' % ('','',pandaIDstatusMap[tmpStatus][:-1]) 99 | # disable showSubstatus 100 | self.flag_showSubstatus = '' 101 | # number of jobs 102 | nJobs = len(self.PandaID.split(',')) 103 | if self.buildStatus != '': 104 | # including buildJob 105 | nJobsStr = "%d + 1(build)" % (nJobs-1) 106 | else: 107 | nJobsStr = "%d" % nJobs 108 | # remove duplication in inDS and outDS 109 | strInDS = '' 110 | try: 111 | tmpInDSList = [] 112 | for tmpItem in str(self.inDS).split(','): 113 | if tmpItem not in tmpInDSList: 114 | tmpInDSList.append(tmpItem) 115 | strInDS += '%s,' % tmpItem 116 | strInDS = strInDS[:-1] 117 | except Exception: 118 | pass 119 | strOutDS = '' 120 | try: 121 | tmpOutDSList = [] 122 | for tmpItem in str(self.outDS).split(','): 123 | if tmpItem not in tmpOutDSList: 124 | tmpOutDSList.append(tmpItem) 125 | strOutDS += '%s,' % tmpItem 126 | strOutDS = strOutDS[:-1] 127 | except Exception: 128 | pass 129 | # parse 130 | relStr = '' 131 | if self.releaseVar not in ['','NULL','None',None]: 132 | relStr = self.releaseVar 133 | # cache 134 | cacheStr = '' 135 | if self.cacheVar not in ['','NULL','None',None]: 136 | cacheStr = self.cacheVar 137 | # string representation 138 | strFormat = "%15s : %s\n" 139 | strOut = "" 140 | strOut += strFormat % ("JobID", self.JobID) 141 | if self.groupID in ['','NULL',0,'0',-1,'-1']: 142 | strOut += strFormat % ("JobsetID", '') 143 | else: 144 | strOut += strFormat % ("JobsetID", self.groupID) 145 | strOut += strFormat % ("type", self.jobType) 146 | strOut += strFormat % ("release", relStr) 147 | strOut += strFormat % ("cache", cacheStr) 148 | strOut += strFormat % ("PandaID", self.encodeCompact()['PandaID']) 149 | strOut += strFormat % ("nJobs", nJobsStr) 150 | strOut += strFormat % ("site", self.site) 151 | strOut += strFormat % ("cloud", self.cloud) 152 | strOut += strFormat % ("inDS", strInDS) 153 | strOut += strFormat % ("outDS", strOutDS) 154 | strOut += strFormat % ("libDS", str(self.libDS)) 155 | strOut += strFormat % ("retryID", self.retryID) 156 | strOut += strFormat % ("provenanceID", self.provenanceID) 157 | if self.mergeJobStatus not in ['NA']: 158 | strOut += strFormat % ("mergeJobStatus", self.mergeJobStatus) 159 | strOut += strFormat % ("mergeJobID", self.mergeJobID) 160 | strOut += strFormat % ("creationTime", self.creationTime.strftime('%Y-%m-%d %H:%M:%S')) 161 | strOut += strFormat % ("lastUpdate", self.lastUpdate.strftime('%Y-%m-%d %H:%M:%S')) 162 | strOut += strFormat % ("params", self.jobParams) 163 | strOut += strFormat % ("jobStatus", statusStr) 164 | # return 165 | return strOut 166 | 167 | 168 | # override __getattribute__ for SQL 169 | def __getattribute__(self,name): 170 | ret = object.__getattribute__(self,name) 171 | if ret is None: 172 | return "NULL" 173 | return ret 174 | 175 | 176 | # pack tuple into JobSpec 177 | def pack(self,values): 178 | for i in range(len(self._attributes)): 179 | attr= self._attributes[i] 180 | val = values[i] 181 | setattr(self,attr,val) 182 | # expand compact values 183 | self.decodeCompact() 184 | 185 | 186 | # return a tuple of values 187 | def values(self,forUpdate=False): 188 | # make compact values 189 | encVal = self.encodeCompact() 190 | if forUpdate: 191 | # for UPDATE 192 | retS = "" 193 | else: 194 | # for INSERT 195 | retS = "(" 196 | # loop over all attributes 197 | for attr in self._attributes: 198 | if attr in encVal: 199 | val = encVal[attr] 200 | else: 201 | val = getattr(self,attr) 202 | # convert datetime to str 203 | if type(val) == datetime.datetime: 204 | val = val.strftime('%Y-%m-%d %H:%M:%S') 205 | # add colum name for UPDATE 206 | if forUpdate: 207 | if attr == 'id': 208 | continue 209 | retS += '%s=' % attr 210 | # value 211 | if val == 'NULL': 212 | retS += 'NULL,' 213 | else: 214 | retS += "'%s'," % str(val) 215 | retS = retS[:-1] 216 | if not forUpdate: 217 | retS += ')' 218 | return retS 219 | 220 | 221 | # expand compact values 222 | def decodeCompact(self): 223 | # PandaID 224 | pStr = '' 225 | for item in self.PandaID.split(','): 226 | match = re.search('^(\d+)-(\d+)$',item) 227 | if match is None: 228 | # non compact 229 | pStr += (item+',') 230 | else: 231 | # compact 232 | sID = long(match.group(1)) 233 | eID = long(match.group(2)) 234 | for tmpID in range(sID,eID+1): 235 | pStr += "%s," % tmpID 236 | self.PandaID = pStr[:-1] 237 | # status 238 | sStr = '' 239 | for item in self.jobStatus.split(','): 240 | match = re.search('^(\w+)\*(\d+)$',item) 241 | if match is None: 242 | # non compact 243 | sStr += (item+',') 244 | else: 245 | # compact 246 | tmpStatus = match.group(1) 247 | tmpCount = int(match.group(2)) 248 | for tmpN in range(tmpCount): 249 | sStr += "%s," % tmpStatus 250 | self.jobStatus = sStr[:-1] 251 | # job parameters 252 | self.jobParams = unquote(self.jobParams) 253 | # datetime 254 | for attr in self._attributes: 255 | val = getattr(self, attr) 256 | if not isinstance(val, (str, unicode)): 257 | continue 258 | # convert str to datetime 259 | match = re.search('^(\d+)-(\d+)-(\d+) (\d+):(\d+):(\d+)$',val) 260 | if match is not None: 261 | tmpDate = datetime.datetime(year = int(match.group(1)), 262 | month = int(match.group(2)), 263 | day = int(match.group(3)), 264 | hour = int(match.group(4)), 265 | minute = int(match.group(5)), 266 | second = int(match.group(6))) 267 | setattr(self,attr,tmpDate) 268 | # jobsetID 269 | if self.groupID in ['','NULL']: 270 | self.groupID = 0 271 | 272 | 273 | # make compact values 274 | def encodeCompact(self,includeMerge=False): 275 | ret = {} 276 | if self.isJEDI(): 277 | if self.taskStatus in ['finished','failed','done','broken','aborted']: 278 | self.dbStatus = 'frozen' 279 | else: 280 | self.dbStatus = 'running' 281 | # job parameters 282 | ret['jobParams'] = quote(self.jobParams) 283 | # PandaID 284 | pStr = '' 285 | sID = None 286 | eID = None 287 | tmpPandaIDs = self.PandaID.split(',') 288 | if includeMerge: 289 | tmpPandaIDs += self.mergeJobID.split(',') 290 | for item in tmpPandaIDs: 291 | if item in ['','None']: 292 | continue 293 | # convert to long 294 | try: 295 | tmpID = long(item) 296 | except Exception: 297 | sID = item 298 | eID = item 299 | break 300 | # set start/end ID 301 | if sID is None: 302 | sID = tmpID 303 | eID = tmpID 304 | continue 305 | # successive number 306 | if eID+1 == tmpID: 307 | eID = tmpID 308 | continue 309 | # jump 310 | if sID == eID: 311 | pStr += '%s,' % sID 312 | else: 313 | pStr += '%s-%s,' % (sID,eID) 314 | # reset 315 | sID = tmpID 316 | eID = tmpID 317 | # last bunch 318 | if sID == eID: 319 | pStr += '%s,' % sID 320 | else: 321 | pStr += '%s-%s,' % (sID,eID) 322 | ret['PandaID'] = pStr[:-1] 323 | if self.isJEDI(): 324 | return ret 325 | # job status 326 | sStr = '' 327 | sStatus = None 328 | nStatus = 0 329 | toBeFrozen = True 330 | for tmpStatus in self.jobStatus.split(','): 331 | # check if is should be frozen 332 | if toBeFrozen and tmpStatus not in ['finished','failed','partial','cancelled']: 333 | toBeFrozen = False 334 | # set start status 335 | if sStatus is None: 336 | sStatus = tmpStatus 337 | nStatus += 1 338 | continue 339 | # same status 340 | if sStatus == tmpStatus: 341 | nStatus += 1 342 | continue 343 | # jump 344 | if nStatus == 1: 345 | sStr += '%s,' % sStatus 346 | else: 347 | sStr += '%s*%s,' % (sStatus,nStatus) 348 | # reset 349 | sStatus = tmpStatus 350 | nStatus = 1 351 | # last bunch 352 | if nStatus == 1: 353 | sStr += '%s,' % sStatus 354 | else: 355 | sStr += '%s*%s,' % (sStatus,nStatus) 356 | ret['jobStatus'] = sStr[:-1] 357 | # set merge job status 358 | if '--mergeOutput' in self.jobParams and self.jobType not in ['usermerge']: 359 | if self.mergeJobStatus not in ['NA','standby','generating','generated','aborted']: 360 | self.mergeJobStatus = 'standby' 361 | else: 362 | self.mergeJobStatus = 'NA' 363 | # set dbStatus 364 | if toBeFrozen: 365 | if self.mergeJobStatus in ['standby','generating']: 366 | # intermediate state while generating merge jobs 367 | self.dbStatus = 'running' 368 | else: 369 | self.dbStatus = 'frozen' 370 | else: 371 | if self.commandToPilot=='tobekilled': 372 | self.dbStatus = 'killing' 373 | else: 374 | self.dbStatus = 'running' 375 | # return 376 | return ret 377 | 378 | 379 | # merge job generation is active 380 | def activeMergeGen(self): 381 | if '--mergeOutput' in self.jobParams and self.mergeJobStatus in ['standby','generating'] \ 382 | and self.jobType not in ['usermerge']: 383 | return True 384 | return False 385 | 386 | 387 | # return column names for INSERT or full SELECT 388 | def columnNames(cls): 389 | ret = "" 390 | for attr in cls._attributes: 391 | if ret != "": 392 | ret += ',' 393 | ret += attr 394 | return ret 395 | columnNames = classmethod(columnNames) 396 | 397 | 398 | # check if JEDI 399 | def isJEDI(self): 400 | if self.jediTaskID in [-1,'-1','']: 401 | return False 402 | return True 403 | -------------------------------------------------------------------------------- /pandaclient/MyproxyUtils.py: -------------------------------------------------------------------------------- 1 | # extraction from myproxyUtils 2 | 3 | import re 4 | import sys 5 | import getpass 6 | from . import Client 7 | from . import MiscUtils 8 | from .MiscUtils import commands_get_status_output, commands_get_output 9 | 10 | 11 | """Classes with methods to interact with a myproxy server. 12 | 13 | * Class MyProxyInterface attributes: 14 | - servername = name of the myproxy server 15 | - vomsattributes = voms attributes, needed to use glexec 16 | - userDN = DN of the user 17 | - pilotownerDN = DN of the pilot owner 18 | who will be allowed to retrieve the proxy 19 | - proxypath = location for the retrieved proxy 20 | - pilotproxypath = location of the pilot owner proxy 21 | 22 | - command = command, for testing purposes 23 | 24 | - Two dates, in seconds since Epoch, to compare with 25 | the version of the myproxy tools installed on the WN 26 | 27 | - myproxyinit_refdate = 1181620800 = 12 Jun 2007 28 | - myproxylogon_refdate = 1181620800 = 12 Jun 2007 29 | 30 | - automatic_renewal = True/False to determine if the proxy 31 | has to be periodically retrieved 32 | - time_threshold = time threshold to trigger a new retrieval 33 | 34 | 35 | * Class MyProxyInterface methods: 36 | - delegate: to delegate a valid proxy using command myproxy-init 37 | - retrieve: to recover the stored proxy using command myproxy-logon 38 | - delproxy: to delete the retrieved proxy 39 | 40 | Proxies must to have voms attributes, to be used by glexec. 41 | 42 | user's DN is used as username. This DN can be got 43 | from the job's schema, defined in JobSpec.py file, 44 | and passed by a dictionary variable named param[] 45 | 46 | Proxies are stored without password. 47 | 48 | Only the pilot onwer is allowed to retrieve the proxy. 49 | His DN is specified when the proxy is uploaded, 50 | using the -Z option. 51 | 52 | 53 | * Class MyProxyError is implemented to throw exceptions 54 | when something is going wrong. 55 | 56 | 57 | * Functions to manage the proxy retrieval and the gLExec preparation 58 | have been moved from SimplePilot.py to here for clarity 59 | 60 | 61 | Author: Jose Caballero (Brookhaven National Laboratory) 62 | E-mail: jcaballero (at) bnl.gov 63 | Last revision: 5/22/2008 64 | Version: 1.5 65 | """ 66 | 67 | class MyProxyError(Exception): 68 | """class to throw exceptions when something is going wrong 69 | during the delegation/retrieval of the proxy credentials 70 | to/from a myproxy server 71 | """ 72 | 73 | #constructor 74 | def __init__(self, index, message=''): 75 | """index is the key of a dictionary 76 | with strings explaining what happened 77 | 78 | message is the output returned from the application which failed 79 | """ 80 | self.__index = index 81 | self.__message = message 82 | 83 | self.__err={} 84 | 85 | self.__err[2100] = ' server name not specified\n' 86 | self.__err[2101] = ' voms attributes not specified\n' 87 | self.__err[2102] = ' user DN not specified\n' 88 | self.__err[2103] = ' pilot owner DN not specified\n' 89 | self.__err[2104] = ' invalid path for the delegated proxy\n' 90 | self.__err[2105] = ' invalid pilot proxy path\n' 91 | self.__err[2106] = ' no path to delegated proxy specified\n' 92 | 93 | self.__err[2200] = ' myproxy-init not available in PATH\n' 94 | self.__err[2201] = ' myproxy-logon not available in PATH\n' 95 | 96 | self.__err[2202] = ' myproxy-init version not valid\n' 97 | self.__err[2203] = ' myproxy-logon version not valid\n' 98 | 99 | self.__err[2300] = ' proxy delegation failed\n' 100 | self.__err[2301] = ' proxy retrieval failed\n' 101 | 102 | self.__err[2400] = ' security violation. Logname and DN do not match\n' 103 | 104 | # output message 105 | self.__output = '\n>>> MyProxyError %s:' % self.__index + self.__err[self.__index] 106 | 107 | if self.__message != '': 108 | self.__output += '\n>>> MyProxyError: Begin of error message\n' 109 | self.__output += self.__message 110 | self.__output += '\n>>> MyProxyError: End of error message\n' 111 | 112 | def __str__(self): 113 | return self.__output 114 | 115 | # method to read the index 116 | def getIndex(self): 117 | return self.__index 118 | index = property(getIndex) 119 | 120 | 121 | class MyProxyInterface(object): 122 | """class with basic features to delegate/retrieve 123 | a valid proxy credential to/from a myproxy server 124 | """ 125 | 126 | # constructor 127 | def __init__(self): 128 | self.__servername = '' # name of the myproxy server 129 | self.__vomsattributes = 'atlas' # voms attributes, needed to use glexec 130 | self.__userDN = '' # DN of the user 131 | self.__pilotownerDN = '' # DN of the pilot owner 132 | # who will be allowed to retrieve the proxy 133 | self.__proxypath = '' # location for the retrieved proxy 134 | self.__pilotproxypath = '' # location of the pilot owner proxy 135 | 136 | self.__command = '' # command, for testing purposes 137 | 138 | # dates, in seconds since Epoch, used as reference for myproxy tools 139 | # to compare with version installed on the WN 140 | # Reference value is 12 Jun 2007 141 | # which corresponds with 1181620800 seconds 142 | self.__myproxyinit_refdate = 1181620800 143 | self.__myproxylogon_refdate = 1181620800 144 | 145 | # variables to manage the periodically retrieval 146 | # by a separate script 147 | self.__automatic_retrieval = 0 # by default, the option is disabled. 148 | self.__time_threshold = 3600 # by default, 1 hour 149 | # src file to setup grid runtime 150 | self.srcFile = Client._getGridSrc() 151 | 152 | # methods to write/read the name of the myproxy server 153 | def setServerName(self,value): 154 | self.__servername = value 155 | def getServerName(self): 156 | return self.__servername 157 | servername = property( getServerName, setServerName ) 158 | 159 | # methods to write/read the voms attributes 160 | def setVomsAttributes(self,value): 161 | self.__vomsattributes = value 162 | def getVomsAttributes(self): 163 | return self.__vomsattributes 164 | vomsattributes = property( getVomsAttributes, setVomsAttributes ) 165 | 166 | # methods to write/read the DN of the user 167 | # This value can be got from dictionary param 168 | # implementing the job schema (file JobSpec.py) 169 | 170 | def __processDN_xtrastrings(self,value): 171 | # delegated proxies do not include strings like "/CN=proxy" or 172 | # "/CN=12345678" in the DN. Extra strings have to be removed 173 | if value.count('/CN=') > 1: 174 | first_index = value.find('/CN=') 175 | second_index = value.find('/CN=', first_index+1) 176 | value = value[0:second_index] 177 | 178 | return value 179 | 180 | def __processDN_whitespaces(self,value): 181 | # usually the last part of the DN is like 182 | # '/CN= ' 183 | # with white spaces. It must be 184 | # '/CN=\ \ ' 185 | # to scape the white spaces 186 | pattern = re.compile('[^\\\]\s') # a whitespace preceded 187 | # by anything but \ 188 | if pattern.search(value): 189 | value = value.replace(' ', '\ ') 190 | 191 | return value 192 | 193 | def __processDN_parenthesis(self,value): 194 | 195 | # the DN could contain parenthesis characteres 196 | # They have to be preceded by a backslash also 197 | pattern = re.compile( '[^\\\]\(' ) # a parenthesis "(" preceded 198 | # by anything but \ 199 | if pattern.search(value): 200 | value = value.replace( '(', '\(' ) 201 | 202 | pattern = re.compile( '[^\\\]\)' ) # a parenthesis ")" preceded 203 | # by anything but \ 204 | if pattern.search(value): 205 | value = value.replace( ')', '\)' ) 206 | 207 | return value 208 | 209 | def __processDN(self,value): 210 | 211 | value = self.__processDN_xtrastrings(value) 212 | value = self.__processDN_whitespaces(value) 213 | value = self.__processDN_parenthesis(value) 214 | 215 | return value 216 | 217 | 218 | def setUserDN(self,value): 219 | self.__userDN = self.__processDN(value) 220 | def getUserDN(self): 221 | return self.__userDN 222 | userDN = property( getUserDN, setUserDN ) 223 | 224 | # methods to write/read the DN of the pilot owner 225 | def setPilotOwnerDN(self,value): 226 | self.__pilotownerDN = value 227 | def getPilotOwnerDN(self): 228 | return self.__pilotownerDN 229 | pilotownerDN = property( getPilotOwnerDN, setPilotOwnerDN ) 230 | 231 | # methods to write/read the location of the retrieved proxy 232 | def setProxyPath(self,value): 233 | # checking the path is valid. 234 | # It has to be a file in the /tmp directory: 235 | if value.startswith('/tmp'): 236 | self.__proxypath = value 237 | else: 238 | raise MyProxyError(2104) 239 | 240 | def getProxyPath(self): 241 | # A file in the /tmp directory is created if no 242 | # other path has been set already 243 | if self.__proxypath == '': 244 | self.__proxypath = commands_get_output( 'mktemp' ) 245 | 246 | return self.__proxypath 247 | 248 | proxypath = property( getProxyPath, setProxyPath ) 249 | 250 | # methods to write/read the path to the pilot owner proxy 251 | def setPilotProxyPath(self,value): 252 | self.__pilotproxypath = value 253 | 254 | def getPilotProxyPath(self): 255 | # checking the path is valid 256 | st,out = commands_get_status_output('grid-proxy-info -exists -file %s' 257 | % self.__pilotproxypath) 258 | if st != 0: #invalid path 259 | print("\nError: not valid proxy in path %" % self.__pilotproxypath) 260 | print("\nUsing the already existing proxy to get the path") 261 | st, path = commands_get_status_output('grid-proxy-info -path') 262 | if st == 0: 263 | self.__pilotproxypath = path 264 | else: 265 | raise MyProxyError(2105) 266 | 267 | if self.__pilotproxypath == '': 268 | print("\nWarning: not valid pilot proxy path specified already") 269 | print("\nUsing the already existing proxy to get the path") 270 | st, path = commands_get_status_output('grid-proxy-info -path') 271 | if st == 0: 272 | self.__pilotproxypath = path 273 | else: 274 | raise MyProxyError(2105) 275 | 276 | return self.__pilotproxypath 277 | 278 | pilotproxypath = property( getPilotProxyPath, setPilotProxyPath ) 279 | 280 | # method to read the command (read-only variable) 281 | # for testing purposes 282 | def getCommand(self): 283 | return self.__command 284 | command = property( getCommand ) 285 | 286 | # methods to read the reference dates 287 | # (read-only variables) 288 | def getMyproxyinitRefdate(self): 289 | return self.__myproxyinit_refdate 290 | myproxyinit_refdate = property( getMyproxyinitRefdate ) 291 | 292 | def getMyproxylogonRefdate(self): 293 | return self.__myproxylogon_refdate 294 | myproxylogon_refdate = property( getMyproxylogonRefdate ) 295 | 296 | # methods to write/read the value of automatic_retrieval variable 297 | def setAutomaticRetrieval(self,value): 298 | if value==0 or value==1: 299 | self.__automatic_retrieval = value 300 | def getAutomaticRetrieval(self): 301 | return self.__automatic_retrieval 302 | automatic_retrieval = property( getAutomaticRetrieval, setAutomaticRetrieval ) 303 | 304 | 305 | # methods to write/read the number of seconds used as time threshold 306 | # to the automatic retrieval 307 | def setTimeThreshold(self,value): 308 | self.__time_threshold = value 309 | def getTimeThreshold(self): 310 | return self.__time_threshold 311 | time_threshold = property( getTimeThreshold, setTimeThreshold ) 312 | 313 | 314 | def delegate(self,gridPassPhrase='',verbose=False): 315 | """method to upload a valid proxy credential in a myproxy server. 316 | The proxy must to have voms attributes. 317 | Only the pilot owner will be allowed to retrieve the proxy, 318 | specified by -Z option. 319 | The DN of the user is used as username. 320 | """ 321 | 322 | if self.servername == '' : 323 | raise MyProxyError(2100) 324 | if self.vomsattributes == '' : 325 | raise MyProxyError(2101) 326 | if self.pilotownerDN == '' : 327 | raise MyProxyError(2103) 328 | 329 | cmd = 'myproxy-init' 330 | 331 | # credname 332 | credname = re.sub('-','',MiscUtils.wrappedUuidGen()) 333 | 334 | print("=== upload proxy for glexec") 335 | # command options 336 | cmd += ' -s %s' % self.servername # myproxy sever name 337 | cmd += " -x -Z '%s'" % self.pilotownerDN # only user with this DN 338 | # is allowed to retrieve it 339 | cmd += ' --voms %s' % self.vomsattributes # voms attributes 340 | cmd += ' -d' # uses DN as username 341 | cmd += ' --credname %s' % credname 342 | if gridPassPhrase == '': 343 | if sys.stdin.isatty(): 344 | gridPassPhrase = getpass.getpass('Enter GRID pass phrase:') 345 | else: 346 | sys.stdout.write('Enter GRID pass phrase:') 347 | sys.stdout.flush() 348 | gridPassPhrase = sys.stdin.readline().rstrip() 349 | print('') 350 | gridPassPhrase = gridPassPhrase.replace('$','\$') 351 | cmd = 'echo "%s" | %s -S' % (gridPassPhrase,cmd) 352 | cmd = self.srcFile + ' unset GT_PROXY_MODE; ' + cmd 353 | 354 | self.__command = cmd # for testing purpose 355 | 356 | if verbose: 357 | cmdStr = cmd 358 | if gridPassPhrase != '': 359 | cmdStr = re.sub(gridPassPhrase,'*****',cmd) 360 | print(cmdStr) 361 | status,out = commands_get_status_output( cmd ) 362 | if verbose: 363 | print(out) 364 | if status != 0: 365 | if out.find('Warning: your certificate and proxy will expire') == -1: 366 | if not verbose: 367 | print(out) 368 | raise MyProxyError(2300) 369 | return credname 370 | 371 | 372 | # check proxy 373 | def check(self,credname,verbose=False): 374 | # construct command to get MyProxy credentials 375 | cmd = self.srcFile + ' myproxy-info -d ' 376 | cmd += '-s %s' % self.servername 377 | if verbose: 378 | print(cmd) 379 | status,output = commands_get_status_output(cmd) 380 | if verbose: 381 | print(output) 382 | # check timeleft 383 | credSector = False 384 | for line in output.split('\n'): 385 | # look for name: 386 | match = re.search('^\s+name:\s+([a-zA-Z0-9]+)',line) 387 | if match is not None: 388 | if match.group(1) == credname: 389 | credSector = True 390 | else: 391 | credSector = False 392 | continue 393 | if not credSector: 394 | continue 395 | # look for timeleft: 396 | match = re.search('^\s+timeleft:\s+([0-9:]+)',line) 397 | if match is not None: 398 | hour = match.group(1).split(':')[0] 399 | hour = int(hour) 400 | # valid more than 3 days 401 | if hour > 24*3: 402 | return True 403 | return False 404 | # not found 405 | return False 406 | -------------------------------------------------------------------------------- /pandaclient/ParseJobXML.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import sys 3 | try: 4 | from urllib import quote 5 | except ImportError: 6 | from urllib.parse import quote 7 | import xml.dom.minidom 8 | 9 | class dom_job: 10 | """ infiles[inds]=[file1,file2...] 11 | outfiles = [file1,file2...] 12 | command - script that will be executed on the grid 13 | prepend - list of (option,value) prepended to output file name 14 | forward - list of (option,value) forwarded to the grid job 15 | """ 16 | def __init__(s,domjob=None,primaryds=None,defaultcmd=None,defaultout=[]): 17 | """ Loads from xml file. 18 | If primaryds is set, makes sure it is present in job spec """ 19 | s.infiles = {} 20 | s.outfiles = [] 21 | s.command=defaultcmd 22 | s.prepend = [] 23 | s.forward = [] 24 | if not domjob: 25 | return 26 | # script executed on the grid node for this job 27 | if len(domjob.getElementsByTagName('command'))>0: 28 | s.command = dom_parser.text(domjob.getElementsByTagName('command')[0]) 29 | # input files 30 | for inds in domjob.getElementsByTagName('inds'): 31 | name = dom_parser.text(inds.getElementsByTagName('name')[0]) 32 | files = inds.getElementsByTagName('file') 33 | if len(files)==0: 34 | continue 35 | s.infiles[name]=[] 36 | for file in files: 37 | s.infiles[name].append(dom_parser.text(file)) 38 | if primaryds and primaryds not in s.infiles.keys(): 39 | print('ERROR: primaryds=%s must be present in each job'%primaryds) 40 | sys.exit(0) 41 | # output files (also, drop duplicates within this job) 42 | outfiles = set(defaultout) 43 | [outfiles.add(dom_parser.text(v)) for v in domjob.getElementsByTagName('output')] 44 | s.outfiles = list(outfiles) 45 | # gearing options 46 | for o in domjob.getElementsByTagName('option'): 47 | name=o.attributes['name'].value 48 | value=dom_parser.text(o) 49 | prepend=dom_parser.true(o.attributes['prepend'].value) 50 | forward=dom_parser.true(o.attributes['forward'].value) 51 | if prepend: 52 | s.prepend.append((name,value)) 53 | if forward: 54 | s.forward.append((name,value)) 55 | def to_dom(s): 56 | """ Converts this job to a dom tree branch """ 57 | x = xml.dom.minidom.Document() 58 | job = x.createElement('job') 59 | for inds in s.infiles.keys(): 60 | job.appendChild(x.createElement('inds')) 61 | job.childNodes[-1].appendChild(x.createElement('name')) 62 | job.childNodes[-1].childNodes[-1].appendChild(x.createTextNode(inds)) 63 | for file in s.infiles[inds]: 64 | job.childNodes[-1].appendChild(x.createElement('file')) 65 | job.childNodes[-1].childNodes[-1].appendChild(x.createTextNode(file)) 66 | for outfile in s.outfiles: 67 | job.appendChild(x.createElement('output')) 68 | job.childNodes[-1].appendChild(x.createTextNode(outfile)) 69 | if s.command: 70 | job.appendChild(x.createElement('command')) 71 | job.childNodes[-1].appendChild(x.createTextNode(s.command)) 72 | for option in s.prepend + list(set(s.prepend+s.forward)-set(s.prepend)): 73 | job.appendChild(x.createElement('option')) 74 | job.childNodes[-1].setAttribute('name',str(option[0])) 75 | if option in s.forward: 76 | job.childNodes[-1].setAttribute('forward','true') 77 | else: 78 | job.childNodes[-1].setAttribute('forward','false') 79 | if option in s.prepend: 80 | job.childNodes[-1].setAttribute('prepend','true') 81 | else: 82 | job.childNodes[-1].setAttribute('prepend','false') 83 | job.childNodes[-1].appendChild(x.createTextNode(str(option[1]))) 84 | return job 85 | def files_in_DS(s,DS): 86 | """ Returns a list of files used in a given job in a given dataset""" 87 | if DS in s.infiles: 88 | return s.infiles[DS] 89 | else: 90 | return [] 91 | def forward_opts(s): 92 | """ passable string of forward options """ 93 | return ' '.join( ['%s=%s'%(v[0],v[1]) for v in s.forward]) 94 | def prepend_string(s): 95 | """ a tag string prepended to output files """ 96 | return '_'.join( ['%s%s'%(v[0],v[1]) for v in s.prepend]) 97 | def exec_string(s): 98 | """ exec string for prun. 99 | If user requested to run script run.sh (via run.sh), it will return 100 | opt1=value1 opt2=value2 opt3=value3 run.sh 101 | This way, all options will be set inside run.sh 102 | """ 103 | return '%s %s'%(s.forward_opts(),s.command) 104 | def exec_string_enc(s): 105 | """ exec string for prun. 106 | If user requested to run script run.sh (via run.sh), it will return 107 | opt1=value1 opt2=value2 opt3=value3 run.sh 108 | This way, all options will be set inside run.sh 109 | """ 110 | comStr = '%s %s'%(s.forward_opts(),s.command) 111 | return quote(comStr) 112 | def get_outmap_str(s,outMap): 113 | """ return mapping of original and new filenames 114 | """ 115 | newMap = {} 116 | for oldLFN in outMap: 117 | fileSpec = outMap[oldLFN] 118 | newMap[oldLFN] = str(fileSpec.lfn) 119 | return str(newMap) 120 | def outputs_list(s,prepend=False): 121 | """ python list with finalized output file names """ 122 | if prepend and s.prepend_string(): 123 | return [s.prepend_string()+'.'+o for o in s.outfiles] 124 | else: 125 | return [o for o in s.outfiles] 126 | def outputs(s,prepend=False): 127 | """ Comma-separated list of output files accepted by prun """ 128 | return ','.join(s.outputs_list(prepend)) 129 | 130 | class dom_parser: 131 | def __init__(s,fname=None,xmlStr=None): 132 | """ creates a dom object out of a text file (if provided) """ 133 | s.fname = fname 134 | s.dom = None 135 | s.title = None 136 | s.tag = None 137 | s.command = None 138 | s.outds = None 139 | s.inds = {} 140 | s.global_outfiles = [] 141 | s.jobs = [] 142 | s.primaryds = None 143 | if fname: 144 | s.dom = xml.dom.minidom.parse(fname) 145 | s.parse() 146 | s.check() 147 | if xmlStr is not None: 148 | s.dom = xml.dom.minidom.parseString(xmlStr) 149 | s.parse() 150 | s.check() 151 | 152 | @staticmethod 153 | def break_regex(v,N=100): 154 | """ breaks up a very long regex into a comma-separeted list of filters """ 155 | _REGEXLIM=2**15-1000 156 | spl=v.split('|') 157 | res=[] 158 | for i in range(0,len(spl),N): 159 | piece = '|'.join(spl[i:i+N]) 160 | assert len(piece)<_REGEXLIM,'Input dataset contains very long filenames. You must reduce parameter N in break_regex()' 161 | res.append( piece ) 162 | return ','.join(res) 163 | @staticmethod 164 | def true(v): 165 | """ define True """ 166 | return v in ('1', 'true', 'True', 'TRUE', 'yes', 'Yes', 'YES') 167 | @staticmethod 168 | def text(pnode): 169 | """ extracts the value stored in the node """ 170 | rc = [] 171 | for node in pnode.childNodes: 172 | if node.nodeType == node.TEXT_NODE: 173 | rc.append(str(node.data).strip()) 174 | return ''.join(rc) 175 | def parse(s): 176 | """ loads submission configuration from an xml file """ 177 | try: 178 | # general settings 179 | if len(s.dom.getElementsByTagName('title'))>0: 180 | s.title = dom_parser.text(s.dom.getElementsByTagName('title')[0]) 181 | else: 182 | s.title = 'Default title' 183 | if len(s.dom.getElementsByTagName('tag'))>0: 184 | s.tag = dom_parser.text(s.dom.getElementsByTagName('tag')[0]) 185 | else: 186 | s.tag = 'default_tag' 187 | s.command = None # can be overridden in subjobs 188 | for elm in s.dom.getElementsByTagName('submission')[0].childNodes: 189 | if elm.nodeName != 'command': 190 | continue 191 | s.command = dom_parser.text(elm) 192 | break 193 | s.global_outfiles = [] # subjobs can append *additional* outputs 194 | for elm in s.dom.getElementsByTagName('submission')[0].childNodes: 195 | if elm.nodeName != 'output': 196 | continue 197 | s.global_outfiles.append(dom_parser.text(elm)) 198 | s.outds = dom_parser.text(s.dom.getElementsByTagName('outds')[0]) 199 | # declaration of all input datasets 200 | primarydss = [] 201 | for elm in s.dom.getElementsByTagName('submission')[0].childNodes: 202 | if elm.nodeName != 'inds': 203 | continue 204 | if 'primary' in elm.attributes.keys() and dom_parser.true(elm.attributes['primary'].value): 205 | primary=True 206 | else: 207 | primary=False 208 | stream=dom_parser.text(elm.getElementsByTagName('stream')[0]) 209 | name=dom_parser.text(elm.getElementsByTagName('name')[0]) 210 | s.inds[name]=stream 211 | if primary: 212 | primarydss.append(name) 213 | # see if one of the input datasets was explicitly labeled as inDS 214 | if len(primarydss)==1: 215 | s.primaryds = primarydss[0] 216 | else: 217 | s.primaryds = None 218 | for job in s.dom.getElementsByTagName('job'): 219 | s.jobs.append(dom_job(job,primaryds=s.primaryds,defaultcmd=s.command,defaultout=s.global_outfiles)) 220 | except Exception: 221 | print('ERROR: failed to parse',s.fname) 222 | raise 223 | def to_dom(s): 224 | """ Converts this submission to a dom tree branch """ 225 | x = xml.dom.minidom.Document() 226 | submission = x.createElement('submission') 227 | if s.title: 228 | submission.appendChild(x.createElement('title')) 229 | submission.childNodes[-1].appendChild(x.createTextNode(s.title)) 230 | if s.tag: 231 | submission.appendChild(x.createElement('tag')) 232 | submission.childNodes[-1].appendChild(x.createTextNode(s.tag)) 233 | for name in s.inds: 234 | stream = s.inds[name] 235 | submission.appendChild(x.createElement('inds')) 236 | if name==s.primaryds: 237 | submission.childNodes[-1].setAttribute('primary','true') 238 | else: 239 | submission.childNodes[-1].setAttribute('primary','false') 240 | submission.childNodes[-1].appendChild(x.createElement('stream')) 241 | submission.childNodes[-1].childNodes[-1].appendChild(x.createTextNode(stream)) 242 | submission.childNodes[-1].appendChild(x.createElement('name')) 243 | submission.childNodes[-1].childNodes[-1].appendChild(x.createTextNode(name)) 244 | if s.command: 245 | submission.appendChild(x.createElement('command')) 246 | submission.childNodes[-1].appendChild(x.createTextNode(s.command)) 247 | for outfile in s.global_outfiles: 248 | submission.appendChild(x.createElement('output')) 249 | submission.childNodes[-1].appendChild(x.createTextNode(outfile)) 250 | submission.appendChild(x.createElement('outds')) 251 | submission.childNodes[-1].appendChild(x.createTextNode(s.outds)) 252 | for job in s.jobs: 253 | submission.appendChild(job.to_dom()) 254 | return submission 255 | def check(s): 256 | """ checks that all output files have unique qualifiers """ 257 | quals=[] 258 | for j in s.jobs: 259 | quals+=j.outputs_list(True) 260 | if len(list(set(quals))) != len(quals): 261 | print('ERROR: found non-unique output file names across the jobs') 262 | print('(you likely need to review xml options with prepend=true)') 263 | sys.exit(0) 264 | def input_datasets(s): 265 | """ returns a list of all used input datasets """ 266 | DSs = set() 267 | for j in s.jobs: 268 | for ds in j.infiles.keys(): 269 | DSs.add(ds) 270 | return list(DSs) 271 | def outDS(s): 272 | """ output dataset """ 273 | return s.outds 274 | def inDS(s): 275 | """ chooses a dataset we'll call inDS; others will become secondaryDS """ 276 | # user manually labeled one of datasets as primary, so make it inDS: 277 | if s.primaryds: 278 | return s.primaryds 279 | # OR: choose inDS dataset randomly 280 | else: 281 | return s.input_datasets()[0] 282 | def secondaryDSs(s): 283 | """ returns all secondaryDSs. This excludes inDS, unless inDS is managed by prun""" 284 | return [d for d in s.input_datasets() if d!=s.inDS() ] 285 | def secondaryDSs_config(s,filter=True): 286 | """ returns secondaryDSs string in prun format: 287 | StreamName:nFilesPerJob:DatasetName[:MatchingPattern[:nSkipFiles]] 288 | nFilesPerJob is set to zero, so that it is updated later from actual file count. 289 | MatchingPattern is an OR-separated list of actual file names. 290 | """ 291 | out = [] 292 | DSs = s.secondaryDSs() 293 | for i,DS in enumerate(DSs): 294 | if DS in s.inds: 295 | stream=s.inds[DS] 296 | else: 297 | stream='IN%d'%(i+1,) 298 | # remove scope: since it conflicts with delimiter (:) 299 | DS = DS.split(':')[-1] 300 | if filter: 301 | out.append('%s:0:%s:%s'%(stream,DS,s.files_in_DS(DS,regex=True))) 302 | else: 303 | out.append('%s:0:%s'%(stream,DS)) 304 | return ','.join(out) 305 | def writeInputToTxt(s): 306 | """ Prepares prun option --writeInputToTxt 307 | comma-separated list of STREAM:STREAM.files.dat 308 | """ 309 | out = [] 310 | DSs = s.secondaryDSs() 311 | for i,DS in enumerate(DSs): 312 | if DS in s.inds: 313 | stream=s.inds[DS] 314 | else: 315 | stream='IN%d'%(i+1,) 316 | out.append('%s:%s.files.dat'%(stream,stream)) 317 | out.append('IN:IN.files.dat') 318 | return ','.join(out) 319 | def files_in_DS(s,DS,regex=False): 320 | """ Returns a list of all files from a given dataset 321 | that will be used in at least one job in this submission 322 | If regex==True, the list is converted to a regex string 323 | """ 324 | assert DS in s.input_datasets(),'ERROR: dataset %s was not requested in the xml file'%DS 325 | files = [] 326 | for j in s.jobs: 327 | if DS in j.infiles.keys(): 328 | files+=j.infiles[DS] 329 | if regex: 330 | return '|'.join(sorted(list(set(files)))) 331 | else: 332 | return sorted(list(set(files))) 333 | def nFiles_in_DS(s,DS): 334 | return len(s.files_in_DS(DS)) 335 | def nJobs(s): 336 | return len(s.jobs) 337 | def dump(s,verbose=True): 338 | """ prints a summary of this submission """ 339 | def P(key,value=''): 340 | if value=='': 341 | print(key) 342 | else: 343 | print((key+':').ljust(14),) 344 | print(value) 345 | P('XML FILE LOADED',s.fname) 346 | P('Title',s.title) 347 | P('Command',s.command) 348 | P('InDS',s.inDS()) 349 | P('Output DS',s.outds) 350 | P('njobs',s.nJobs()) 351 | if verbose: 352 | for i,job in enumerate(s.jobs): 353 | P('===============> JOB%d'%i) 354 | P('command',job.exec_string()) 355 | P('outfiles',job.outputs()) 356 | P('INPUTS:') 357 | j=0 358 | for dsname in job.infiles: 359 | files = job.infiles[dsname] 360 | P(' Dataset%d'%j,dsname) 361 | for k,fname in enumerate(files): 362 | P(' File%d'%k,fname) 363 | j+=1 364 | 365 | if __name__=="__main__": 366 | p = dom_parser('./job.xml') 367 | p.dump() 368 | --------------------------------------------------------------------------------