├── .flake8 ├── .github ├── FUNDING.yml ├── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── release_checklist.md │ └── user-story.md ├── dependabot.yml └── workflows │ ├── ci_helm_publish.yaml │ ├── ci_helm_tests.yaml │ ├── ci_servicex.yaml │ └── deploy-config.json ├── .gitignore ├── .readthedocs.yml ├── CHANGELOG.md ├── LICENSE ├── README.md ├── code_generator_TopCPToolkit ├── .flake8 ├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── app.conf ├── boot.sh ├── poetry.lock ├── pyproject.toml ├── servicex │ ├── TopCP_code_generator │ │ ├── __init__.py │ │ ├── query_translate.py │ │ └── request_translator.py │ ├── __init__.py │ └── templates │ │ └── transform_single_file.py ├── tests │ ├── __init__.py │ └── test_src.py └── transformer_capabilities.json ├── code_generator_funcadl_uproot ├── .flake8 ├── .github │ └── workflows │ │ └── ci.yaml ├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── app.conf ├── boot.sh ├── poetry.lock ├── pyproject.toml ├── scripts │ └── from_ast_to_zip.py ├── tests │ ├── __init__.py │ └── test_ast_translator.py ├── transformer_capabilities.json └── uproot_code_generator │ ├── __init__.py │ ├── ast_translator.py │ └── templates │ └── transform_single_file.py ├── code_generator_funcadl_xAOD ├── .coveragerc ├── .flake8 ├── .github │ └── workflows │ │ └── ci.yaml ├── .gitignore ├── .vscode │ ├── launch.json │ └── settings.json ├── Codegen_func_adl_xAOD.postman_collection.json ├── Dockerfile ├── LICENSE ├── README.md ├── app.atlas.xaod.conf ├── app.cms.aod.conf ├── boot.sh ├── poetry.lock ├── pyproject.toml ├── scripts │ └── from_ast_to_zip.py ├── tests │ ├── __init__.py │ └── test_ast_translator.py ├── transformer_capabilities.json └── xaod_code_generator │ ├── __init__.py │ ├── ast_translator.py │ └── templates │ └── transform_single_file.sh ├── code_generator_python ├── .flake8 ├── .github │ └── workflows │ │ └── ci.yaml ├── .gitignore ├── Dockerfile ├── README.md ├── boot.sh ├── poetry.lock ├── pyproject.toml ├── python_code_generator │ ├── __init__.py │ ├── python_translator.py │ └── templates │ │ └── transform_single_file.py ├── scripts │ └── from_text_to_zip.py ├── tests │ ├── __init__.py │ └── test_python_translator.py └── transformer_capabilities.json ├── code_generator_raw_uproot ├── .flake8 ├── .github │ └── workflows │ │ └── ci.yaml ├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── app.conf ├── boot.sh ├── poetry.lock ├── pyproject.toml ├── servicex │ ├── __init__.py │ ├── raw_uproot_code_generator │ │ ├── __init__.py │ │ └── request_translator.py │ └── templates │ │ └── transform_single_file.py ├── tests │ ├── __init__.py │ └── test_src.py └── transformer_capabilities.json ├── did_finder_cernopendata ├── .flake8 ├── .gitattributes ├── .github │ └── workflows │ │ └── publish.yaml ├── .gitignore ├── Dockerfile ├── README.md ├── poetry.lock ├── pyproject.toml ├── samples │ └── simple_plot.ipynb ├── scripts │ └── start_celery_worker.sh ├── src │ └── did_finder_cernopendata │ │ ├── __init__.py │ │ └── celery.py └── tests │ └── did_finder_cernopendata_tests │ ├── __init__.py │ └── test_servicex_did_finder_cernopendata.py ├── did_finder_rucio ├── .coveragerc ├── .dockerignore ├── .flake8 ├── .github │ ├── ISSUE_TEMPLATE │ │ └── user-story.md │ └── workflows │ │ └── ci.yaml ├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── direct_test │ └── test1.py ├── poetry.lock ├── proxy-exporter.sh ├── pyproject.toml ├── sample │ └── xaod_query.ipynb ├── scripts │ └── start_celery_worker.sh ├── src │ └── rucio_did_finder │ │ ├── __init__.py │ │ ├── celery.py │ │ ├── lookup_request.py │ │ ├── replica_distance.py │ │ └── rucio_adapter.py └── tests │ ├── __init__.py │ └── did_finder │ ├── __init__.py │ ├── test_lookup_request.py │ └── test_replica_sorter.py ├── did_finder_xrootd ├── .flake8 ├── .gitattributes ├── .gitignore ├── Dockerfile ├── README.md ├── poetry.lock ├── pyproject.toml ├── scripts │ └── start_celery_worker.sh ├── src │ └── servicex_did_finder_xrootd │ │ ├── __init__.py │ │ └── celery.py └── tests │ ├── __init__.py │ └── test_servicex_did_finder_xrootd.py ├── docs ├── deployment │ ├── basic.md │ ├── output_options.md │ ├── production.md │ ├── reference.md │ └── user_management.md ├── development │ ├── architecture.md │ └── contributing.md ├── example_secrets.yaml ├── img │ ├── ServiceX-BlackTextOnly-Transparent.png │ ├── ServiceX-Color-ImageOnly-Transparent.png │ ├── ServiceX-Color-Transparent.png │ ├── ServiceX-WhiteText-Transparent.png │ ├── ServiceX-WhiteTextOnly-Transparent.png │ ├── ServiceX-architecture.png │ ├── ServiceX-request-lifecycle.png │ ├── Z_ee_example.jpg │ ├── develop.png │ ├── download-servicex-yaml.jpg │ ├── hep_tables_example.png │ ├── organize.png │ ├── organize2.png │ ├── sx-architecture.png │ └── sx-schema.png ├── index.md ├── introduction.md └── requirements.txt ├── examples └── ATLAS │ ├── README.md │ ├── UprootExample.ipynb │ ├── Z_ee.ipynb │ └── Z_ee.py ├── helm ├── .github │ └── workflows │ │ └── check-chart.yaml ├── .gitignore ├── demo-values.yaml ├── example_secrets.yaml ├── release-helm-chart.sh ├── scripts │ ├── generated_code_busybox.yaml │ ├── object_store_analysis.py │ ├── port-forward.sh │ ├── status_scraper.py │ └── transformer_pvc.yaml └── servicex │ ├── .helmignore │ ├── Chart.yaml │ ├── local-dev-values.yaml │ ├── requirements.lock │ ├── requirements.yaml │ ├── templates │ ├── NOTES.txt │ ├── _helpers.tpl │ ├── app │ │ ├── configmap.yaml │ │ ├── deployment.yaml │ │ ├── ingress.yaml │ │ └── service.yaml │ ├── codegen │ │ ├── deployment.yaml │ │ └── service.yaml │ ├── data-lifecycle │ │ └── cronjob.yaml │ ├── did-finder-cernopendata │ │ └── deployment.yaml │ ├── did-finder-rucio │ │ ├── deployment.yaml │ │ └── rucio-configmap.yaml │ ├── did-finder-xrootd │ │ └── deployment.yaml │ ├── rbac │ │ ├── role.yaml │ │ ├── rolebinding.yaml │ │ └── serviceaccount.yaml │ └── x509-secrets │ │ ├── deployment.yaml │ │ └── proxy-secret.yaml │ └── values.yaml ├── local_test_harness ├── .env ├── README.md ├── compose.yml ├── config.yml └── test.py ├── mkdocs.yml ├── poetry.lock ├── pyproject.toml ├── servicex_app ├── .dockerignore ├── .flake8 ├── .gitattributes ├── .github │ ├── ISSUE_TEMPLATE │ │ └── user-story.md │ └── workflows │ │ └── ci.yaml ├── .gitignore ├── .pre-commit-config.yaml ├── Dockerfile ├── LICENSE ├── README.rst ├── ServiceXTest.postman_collection.json ├── app.conf.template ├── boot.sh ├── codecov.yml ├── doc │ ├── sequence_diagram.drawio │ └── sequence_diagram.png ├── docker-dev.conf ├── gai.conf ├── migrations │ ├── README │ ├── alembic.ini │ ├── env.py │ ├── script.py.mako │ └── versions │ │ ├── 04b9fb8ffee1_v1_0_rc4_a2.py │ │ ├── 1.5.6_.py │ │ ├── 1.5.7.py │ │ ├── 1.5.7a.py │ │ ├── 208309600a27_cleanup_file_status.py │ │ ├── 5893086ec440_removing_kafka_column.py │ │ ├── 79d404108ebe_multipath_support.py │ │ ├── 99e97a63d1bd_v1_0_rc_2.py │ │ ├── a33a96f0f035_v1_0_rc4_a3.py │ │ ├── a6cbb6201d3d_v1_0_rc4_a1.py │ │ ├── b389abb05262_v1_0_rc_1.py │ │ ├── dd1f9a8a2aee_v1_0_rc_3.py │ │ ├── v1_1_3.py │ │ ├── v1_1_4.py │ │ ├── v1_3_0.py │ │ └── v1_5_5.py ├── poetry.lock ├── pyproject.toml ├── sequence_diagram.drawio ├── servicex_app │ ├── __init__.py │ ├── app.py │ ├── celery_task_router.py │ ├── cli │ │ ├── create_default_users.py │ │ └── user_commands.py │ ├── code_gen_adapter.py │ ├── dataset_manager.py │ ├── decorators.py │ ├── did_parser.py │ ├── docker_repo_adapter.py │ ├── lookup_result_processor.py │ ├── mailgun_adaptor.py │ ├── models.py │ ├── object_store_manager.py │ ├── rabbit_adaptor.py │ ├── reliable_requests.py │ ├── resources │ │ ├── __init__.py │ │ ├── datasets │ │ │ ├── __init__.py │ │ │ ├── delete_dataset.py │ │ │ ├── get_all.py │ │ │ └── get_one.py │ │ ├── info.py │ │ ├── internal │ │ │ ├── __init__.py │ │ │ ├── add_file_to_dataset.py │ │ │ ├── data_lifecycle_ops.py │ │ │ ├── fileset_complete.py │ │ │ ├── transform_status.py │ │ │ └── transformer_file_complete.py │ │ ├── servicex_resource.py │ │ ├── transformation │ │ │ ├── __init__.py │ │ │ ├── cancel.py │ │ │ ├── delete.py │ │ │ ├── deployment.py │ │ │ ├── get_all.py │ │ │ ├── get_one.py │ │ │ ├── status.py │ │ │ └── submit.py │ │ └── users │ │ │ ├── __init__.py │ │ │ ├── accept_user.py │ │ │ ├── all_users.py │ │ │ ├── delete_user.py │ │ │ ├── pending_users.py │ │ │ ├── slack_interaction.py │ │ │ └── token_refresh.py │ ├── routes.py │ ├── static │ │ ├── Spinner-1s-64px.gif │ │ ├── favicon.ico │ │ └── main.css │ ├── templates │ │ ├── about.html │ │ ├── base.html │ │ ├── emails │ │ │ └── welcome.html │ │ ├── get_started.html │ │ ├── global_dashboard.html │ │ ├── home.html │ │ ├── logs.html │ │ ├── monitor.html │ │ ├── profile.html │ │ ├── profile_form.html │ │ ├── requests_table.html │ │ ├── sort_dropdown.html │ │ ├── transformation_request.html │ │ ├── transformation_results.html │ │ └── user_dashboard.html │ ├── transformer_manager.py │ └── web │ │ ├── __init__.py │ │ ├── about.py │ │ ├── api_token.py │ │ ├── auth_callback.py │ │ ├── create_profile.py │ │ ├── dashboard.py │ │ ├── edit_profile.py │ │ ├── forms.py │ │ ├── global_dashboard.py │ │ ├── home.py │ │ ├── logs.py │ │ ├── monitor.py │ │ ├── multiple_codegen_list.py │ │ ├── servicex_file.py │ │ ├── sign_in.py │ │ ├── sign_out.py │ │ ├── slack_msg_builder.py │ │ ├── transformation_request.py │ │ ├── transformation_results.py │ │ ├── user_dashboard.py │ │ ├── utils.py │ │ └── view_profile.py └── servicex_app_test │ ├── __init__.py │ ├── busybox │ ├── __init__.py │ └── celery_busybox.py │ ├── conftest.py │ ├── resource_test_base.py │ ├── resources │ ├── __init__.py │ ├── datasets │ │ ├── __init__.py │ │ ├── test_delete.py │ │ ├── test_get_all.py │ │ └── test_get_one.py │ ├── internal │ │ ├── __init__.py │ │ ├── test_add_file_to_dataset.py │ │ ├── test_data_lifecycle_ops.py │ │ ├── test_fileset_complete.py │ │ ├── test_transform_file_complete.py │ │ └── test_transform_status.py │ ├── test_servicex_info.py │ ├── test_servicex_resource.py │ ├── test_web_app_init.py │ ├── transformation │ │ ├── __init__.py │ │ ├── test_cancel.py │ │ ├── test_delete.py │ │ ├── test_deployment.py │ │ ├── test_get_all.py │ │ ├── test_get_one.py │ │ ├── test_status.py │ │ └── test_submit.py │ └── users │ │ ├── __init__.py │ │ ├── test_accept_user.py │ │ ├── test_all_users.py │ │ ├── test_delete_user.py │ │ ├── test_pending_users.py │ │ └── test_slack_interaction.py │ ├── test.config │ ├── test_app_init.py │ ├── test_celery_task_router.py │ ├── test_code_gen_adapter.py │ ├── test_dataset_manager.py │ ├── test_decorators.py │ ├── test_did_parser.py │ ├── test_docker_repo_adapter.py │ ├── test_lookup_result_processor.py │ ├── test_mailgun_adaptor.py │ ├── test_models.py │ ├── test_object_store_manager.py │ ├── test_rabbit_adaptor.py │ ├── test_transformer_manager.py │ └── web │ ├── __init__.py │ ├── test_about.py │ ├── test_api_token.py │ ├── test_auth_callback.py │ ├── test_create_profile.py │ ├── test_edit_profile.py │ ├── test_global_dashboard.py │ ├── test_home.py │ ├── test_servicex_file.py │ ├── test_sign_in.py │ ├── test_sign_out.py │ ├── test_transformation_request.py │ ├── test_transformation_results.py │ ├── test_user_dashboard.py │ ├── test_view_profile.py │ └── web_test_base.py ├── transformer_sidecar ├── .flake8 ├── .github │ └── workflows │ │ └── buildpush.yaml ├── .gitignore ├── Dockerfile ├── README.md ├── gai.conf ├── poetry.lock ├── pyproject.toml ├── pytest.ini ├── scripts │ ├── proxy-exporter.sh │ └── watch.sh ├── src │ └── transformer_sidecar │ │ ├── __init__.py │ │ ├── object_store_manager.py │ │ ├── science_container_command.py │ │ ├── servicex_adapter.py │ │ ├── transformer.py │ │ ├── transformer_argument_parser.py │ │ ├── transformer_logging │ │ ├── __init__.py │ │ ├── logstash_formatter.py │ │ └── stream_formatter.py │ │ └── transformer_stats │ │ ├── __init__.py │ │ ├── aod_stats.py │ │ ├── raw_uproot_stats.py │ │ ├── topcp_stats.py │ │ └── uproot_stats.py ├── test_posix_vol │ ├── .gitignore │ ├── generated_code │ │ ├── generated_transformer.py │ │ ├── transform_single_file.py │ │ └── transformer_capabilities.json │ └── transformer_capabilities.json └── tests │ ├── sandbox │ └── transform_file.py │ ├── test.root │ ├── test_object_store_manager.py │ ├── test_science_container_command.py │ ├── test_servicex_adapter.py │ ├── test_transformer.py │ ├── test_transformer_argument_parser.py │ ├── transformer_capabilities.json │ └── transformer_stats │ ├── test_aod_stats.py │ ├── test_topcp_stats.py │ └── test_uproot_stats.py └── x509_secrets ├── .github └── workflows │ └── ci.yaml ├── .gitignore ├── Dockerfile ├── README.md ├── poetry.lock ├── pyproject.toml ├── tag_and_release.sh └── x509_updater.py /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 99 3 | exclude = 4 | .tox, 5 | __pycache__ 6 | validate_requests.py 7 | find_m.py 8 | messaging.py 9 | .local 10 | build 11 | migrations 12 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: 4 | patreon: # Replace with a single Patreon username 5 | open_collective: # Replace with a single Open Collective username 6 | ko_fi: # Replace with a single Ko-fi username 7 | tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel 8 | community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry 9 | liberapay: # Replace with a single Liberapay username 10 | issuehunt: # Replace with a single IssueHunt username 11 | otechie: # Replace with a single Otechie username 12 | custom: 'https://iris-hep.org' 13 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Additional context** 27 | Add any other context about the problem here. 28 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/user-story.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: User Story 3 | about: A feature from an end-user perspective 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | # Story 11 | _As a I want to so I can _ 12 | 13 | # Assumptions 14 | 1. List assumptions behind this story 15 | 16 | # Acceptance Criteria 17 | 1. How will we know when this story is complete 18 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | # Maintain dependencies for GitHub Actions 4 | - package-ecosystem: "github-actions" 5 | directory: "/" 6 | schedule: 7 | interval: "weekly" 8 | groups: 9 | actions: 10 | patterns: 11 | - "*" 12 | labels: 13 | - "dependencies" 14 | - "github-actions" 15 | 16 | # Maintain dependencies for poetry 17 | - package-ecosystem: "pip" 18 | directory: "/" 19 | schedule: 20 | interval: "weekly" 21 | groups: 22 | actions: 23 | patterns: 24 | - "*" 25 | labels: 26 | - "dependencies" 27 | - "python" 28 | -------------------------------------------------------------------------------- /.github/workflows/ci_helm_tests.yaml: -------------------------------------------------------------------------------- 1 | name: Lint and Test Helm Charts 2 | 3 | on: [pull_request, workflow_call] 4 | 5 | jobs: 6 | lint-test: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - name: Checkout 10 | uses: actions/checkout@v4.2.2 11 | with: 12 | fetch-depth: 0 13 | 14 | - name: Set up Helm 15 | uses: azure/setup-helm@v4 16 | with: 17 | version: v3.9.2 18 | 19 | - uses: actions/setup-python@v5.6.0 20 | with: 21 | python-version: "3.10" 22 | 23 | - name: Set up chart-testing 24 | uses: helm/chart-testing-action@v2.7.0 25 | 26 | - name: add bitnami repos to helm 27 | run: | 28 | helm repo add bitnami https://charts.bitnami.com/bitnami 29 | 30 | - name: Run chart-testing (list-changed) 31 | id: list-changed 32 | run: | 33 | changed=$(ct list-changed --target-branch ${{ github.event.repository.default_branch }} ) 34 | if [[ -n "$changed" ]]; then 35 | echo "changed=true" >> $GITHUB_OUTPUT 36 | fi 37 | 38 | - name: Run chart-testing (lint) 39 | run: ct lint --target-branch ${{ github.event.repository.default_branch }} --chart-dirs ${GITHUB_WORKSPACE}/helm/servicex --charts helm/servicex 40 | -------------------------------------------------------------------------------- /.github/workflows/deploy-config.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "dir_name": "servicex_app", 4 | "image_name": "servicex_app", 5 | "test_required": true 6 | }, 7 | { 8 | "dir_name": "code_generator_python", 9 | "image_name": "servicex_code_gen_python", 10 | "test_required": true 11 | }, 12 | { 13 | "dir_name": "did_finder_cernopendata", 14 | "image_name": "servicex-did-finder-cernopendata", 15 | "test_required": true 16 | }, 17 | { 18 | "dir_name": "did_finder_rucio", 19 | "image_name": "servicex-did-finder", 20 | "test_required": false 21 | }, 22 | { 23 | "dir_name": "did_finder_xrootd", 24 | "image_name": "servicex-did-finder-xrootd", 25 | "test_required": true 26 | }, 27 | { 28 | "dir_name": "code_generator_funcadl_uproot", 29 | "image_name": "servicex_code_gen_func_adl_uproot", 30 | "test_required": true 31 | }, 32 | { 33 | "dir_name": "code_generator_raw_uproot", 34 | "image_name": "servicex_code_gen_raw_uproot", 35 | "test_required": true 36 | }, 37 | { 38 | "image_name": "servicex_sidecar_transformer", 39 | "dir_name": "transformer_sidecar", 40 | "test_required": true 41 | }, 42 | { 43 | "dir_name": "x509_secrets", 44 | "image_name": "x509-secrets", 45 | "test_required": false 46 | }, 47 | { 48 | "dir_name": "code_generator_funcadl_xAOD", 49 | "image_name": "servicex_code_gen_cms_aod", 50 | "test_required": true 51 | }, 52 | { 53 | "dir_name": "code_generator_funcadl_xAOD", 54 | "image_name": "servicex_code_gen_atlas_xaod", 55 | "test_required": true 56 | }, 57 | { 58 | "dir_name": "code_generator_TopCPToolkit", 59 | "image_name": "servicex_code_gen_topcp", 60 | "test_required": true 61 | } 62 | ] 63 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .servicex 2 | .idea/ 3 | .vscode/ 4 | poetry.toml 5 | .pyc 6 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | # Read the Docs configuration file 2 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 3 | 4 | # Required 5 | version: 2 6 | 7 | # Build documentation with MkDocs 8 | mkdocs: 9 | configuration: mkdocs.yml 10 | 11 | # Optionally build your docs in additional formats such as PDF 12 | formats: 13 | - pdf 14 | 15 | # Optionally set the version of Python and requirements required to build your docs 16 | build: 17 | os: "ubuntu-22.04" 18 | tools: 19 | python: "3.10" 20 | 21 | python: 22 | install: 23 | - requirements: docs/requirements.txt 24 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2018, 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /code_generator_TopCPToolkit/.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 99 3 | exclude = 4 | .tox, 5 | __pycache__ 6 | validate_requests.py 7 | find_m.py 8 | messaging.py 9 | .local 10 | build 11 | migrations 12 | 13 | 14 | -------------------------------------------------------------------------------- /code_generator_TopCPToolkit/.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | *egg-info* 3 | *.pyc 4 | .pytest_cache/* 5 | .coverage 6 | coverage.xml 7 | generated/ 8 | .venv/* 9 | .python-version -------------------------------------------------------------------------------- /code_generator_TopCPToolkit/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.10 2 | 3 | RUN useradd -ms /bin/bash servicex 4 | RUN apt-get update && apt-get install -y netcat-traditional && rm -rf /var/lib/apt/lists/* 5 | 6 | WORKDIR /home/servicex 7 | RUN mkdir ./servicex 8 | 9 | ENV POETRY_VERSION=2.1.1 10 | RUN pip install poetry==$POETRY_VERSION 11 | 12 | COPY pyproject.toml pyproject.toml 13 | COPY poetry.lock poetry.lock 14 | 15 | RUN poetry config virtualenvs.create false && \ 16 | poetry install --no-root --no-interaction --no-ansi 17 | 18 | RUN pip install gunicorn 19 | 20 | COPY boot.sh ./ 21 | RUN chmod 755 boot.sh 22 | 23 | COPY transformer_capabilities.json ./ 24 | RUN chmod 644 transformer_capabilities.json 25 | 26 | COPY servicex/ ./servicex 27 | RUN chmod 777 -R servicex 28 | 29 | COPY app.conf . 30 | RUN chmod 755 app.conf 31 | 32 | USER servicex 33 | 34 | ENV CODEGEN_CONFIG_FILE="/home/servicex/app.conf" 35 | 36 | EXPOSE 5000 37 | ENTRYPOINT ["./boot.sh"] 38 | -------------------------------------------------------------------------------- /code_generator_TopCPToolkit/LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2025 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /code_generator_TopCPToolkit/README.md: -------------------------------------------------------------------------------- 1 | [![GitHub Actions Status](https://github.com/ssl-hep/ServiceX_Code_Generator_FuncADL_uproot/workflows/CI/CD/badge.svg)](https://github.com/ssl-hep/ServiceX_Code_Generator_FuncADL_uproot/actions) 2 | [![Code Coverage](https://codecov.io/gh/ssl-hep/ServiceX_Code_Generator_FuncADL_uproot/graph/badge.svg)](https://codecov.io/gh/ssl-hep/ServiceX_Code_Generator_FuncADL_uproot) 3 | 4 | 5 | ServiceX Code Generator 6 | ----------------------- 7 | This microservice is a REST API that will run TopCPToolkit remotely. The query to extract the data is encoded in a yaml string. 8 | 9 | Query Format 10 | ------------ 11 | ``` 12 | 13 | ``` 14 | 15 | Usage 16 | ----- 17 | This repo builds a container to be used in the `ServiceX` application. You can see the containers on docker hub. 18 | 19 | Development 20 | ----------- 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /code_generator_TopCPToolkit/app.conf: -------------------------------------------------------------------------------- 1 | TARGET_BACKEND = 'TopCP' 2 | -------------------------------------------------------------------------------- /code_generator_TopCPToolkit/boot.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Running the web server? 4 | action=${1:-web_service} 5 | if [ "$action" = "web_service" ] ; then 6 | mkdir instance 7 | exec gunicorn -b :5000 --workers=2 --threads=1 --access-logfile - --error-logfile - "servicex.TopCP_code_generator:create_app()" 8 | else 9 | echo "Unknown action '$action'" 10 | fi 11 | -------------------------------------------------------------------------------- /code_generator_TopCPToolkit/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "code-generator-funcadl-uproot-raw" 3 | version = "0.1.0" 4 | description = "" 5 | authors = ["Your Name "] 6 | readme = "README.md" 7 | packages = [{include = "code_generator_funcadl_uproot_raw"}] 8 | 9 | [tool.poetry.dependencies] 10 | python = "~3.10" 11 | servicex-code-gen-lib = "^1.2" 12 | ruamel-yaml = "^0.18.6" 13 | 14 | [tool.poetry.group.test] 15 | optional = true 16 | 17 | [tool.poetry.group.test.dependencies] 18 | pytest-flask = "^1.2.0" 19 | pytest-cov = "^4.0.0" 20 | pytest-mock = "^3.10.0" 21 | pytest = "^7.1.3" 22 | autopep8 = "^1.7.0" 23 | flake8 = "^5.0.4" 24 | coverage = "^6.5.0" 25 | 26 | [build-system] 27 | requires = ["poetry-core"] 28 | build-backend = "poetry.core.masonry.api" 29 | -------------------------------------------------------------------------------- /code_generator_TopCPToolkit/servicex/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019, IRIS-HEP 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are met: 6 | # 7 | # * Redistributions of source code must retain the above copyright notice, this 8 | # list of conditions and the following disclaimer. 9 | # 10 | # * Redistributions in binary form must reproduce the above copyright notice, 11 | # this list of conditions and the following disclaimer in the documentation 12 | # and/or other materials provided with the distribution. 13 | # 14 | # * Neither the name of the copyright holder nor the names of its 15 | # contributors may be used to endorse or promote products derived from 16 | # this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /code_generator_TopCPToolkit/servicex/templates/transform_single_file.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | from pathlib import Path 4 | import subprocess 5 | import shutil 6 | import generated_transformer 7 | 8 | instance = os.environ.get('INSTANCE_NAME', 'Unknown') 9 | 10 | 11 | def transform_single_file(file_path: str, output_path: Path, output_format: str): 12 | # create input.txt file for event loop and insert file_path as only line 13 | with open("input.txt", "w") as f: 14 | f.write(file_path) 15 | # move reco.yaml, parton.yaml and particle.yaml if they exit to CONFIG_LOC loacation 16 | if os.path.exists("/generated/reco.yaml"): 17 | shutil.copyfile("/generated/reco.yaml", 18 | os.path.join(os.environ.get("CONFIG_LOC"), "reco.yaml")) 19 | if os.path.exists("/generated/parton.yaml"): 20 | shutil.copyfile("/generated/parton.yaml", 21 | os.path.join(os.environ.get("CONFIG_LOC"), "parton.yaml")) 22 | if os.path.exists("/generated/particle.yaml"): 23 | shutil.copyfile("/generated/particle.yaml", 24 | os.path.join(os.environ.get("CONFIG_LOC"), "particle.yaml")) 25 | 26 | generated_transformer.runTop_el() 27 | subprocess.run(["mv", "output.root", output_path]) 28 | 29 | 30 | if __name__ == "__main__": 31 | transform_single_file(sys.argv[1], Path(sys.argv[2]), sys.argv[3]) 32 | -------------------------------------------------------------------------------- /code_generator_TopCPToolkit/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019, IRIS-HEP 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are met: 6 | # 7 | # * Redistributions of source code must retain the above copyright notice, this 8 | # list of conditions and the following disclaimer. 9 | # 10 | # * Redistributions in binary form must reproduce the above copyright notice, 11 | # this list of conditions and the following disclaimer in the documentation 12 | # and/or other materials provided with the distribution. 13 | # 14 | # * Neither the name of the copyright holder nor the names of its 15 | # contributors may be used to endorse or promote products derived from 16 | # this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /code_generator_TopCPToolkit/transformer_capabilities.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "TopCPToolkit transformer", 3 | "description": "Runs TopCPToolkit analysis", 4 | "limitations": "Would be good to note what isn't implemented", 5 | "file-formats": ["root-file"], 6 | "stats-parser": "TopCPStats", 7 | "language": "python", 8 | "command": "/generated/transform_single_file.py" 9 | } -------------------------------------------------------------------------------- /code_generator_funcadl_uproot/.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 99 3 | exclude = 4 | .tox, 5 | __pycache__ 6 | validate_requests.py 7 | find_m.py 8 | messaging.py 9 | .local 10 | build 11 | migrations 12 | -------------------------------------------------------------------------------- /code_generator_funcadl_uproot/.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | *egg-info* 3 | *.pyc 4 | .pytest_cache/* 5 | .coverage 6 | coverage.xml 7 | generated/ 8 | .venv/* 9 | .python-version 10 | -------------------------------------------------------------------------------- /code_generator_funcadl_uproot/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.10 2 | 3 | RUN useradd -ms /bin/bash servicex 4 | 5 | WORKDIR /home/servicex 6 | RUN mkdir ./uproot_code_generator 7 | 8 | ENV POETRY_VERSION=2.1.1 9 | RUN pip install poetry==$POETRY_VERSION 10 | 11 | COPY pyproject.toml pyproject.toml 12 | COPY poetry.lock poetry.lock 13 | 14 | RUN poetry config virtualenvs.create false && \ 15 | poetry install --no-root --no-interaction --no-ansi 16 | 17 | RUN pip install gunicorn 18 | 19 | COPY boot.sh ./ 20 | COPY transformer_capabilities.json ./ 21 | COPY uproot_code_generator/ ./uproot_code_generator/ 22 | COPY scripts/from_ast_to_zip.py . 23 | RUN chmod +x boot.sh 24 | 25 | USER servicex 26 | COPY app.conf . 27 | 28 | ENV CODEGEN_CONFIG_FILE "/home/servicex/app.conf" 29 | 30 | EXPOSE 5000 31 | ENTRYPOINT ["./boot.sh"] 32 | 33 | -------------------------------------------------------------------------------- /code_generator_funcadl_uproot/LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2018-2019, Jim Pivarski 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /code_generator_funcadl_uproot/app.conf: -------------------------------------------------------------------------------- 1 | TARGET_BACKEND = 'uproot' 2 | -------------------------------------------------------------------------------- /code_generator_funcadl_uproot/boot.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Running the web server or a utility? 4 | action=${1:-web_service} 5 | if [ "$action" = "web_service" ] ; then 6 | mkdir instance 7 | exec gunicorn -b [::]:5000 --workers=2 --threads=1 --access-logfile - --error-logfile - "uproot_code_generator:create_app()" 8 | elif [ "$action" = "translate" ]; then 9 | python from_ast_to_zip.py "${@:2}" 10 | else 11 | echo "Unknown action '$action'" 12 | fi 13 | -------------------------------------------------------------------------------- /code_generator_funcadl_uproot/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "code-generator-funcadl-uproot" 3 | version = "0.1.0" 4 | description = "" 5 | authors = ["Your Name "] 6 | readme = "README.md" 7 | packages = [{include = "code_generator_funcadl_uproot"}] 8 | 9 | [tool.poetry.dependencies] 10 | python = "~3.10" 11 | func-adl = "3.2.7" 12 | func-adl-uproot = "2.1.1" 13 | qastle = "0.17.0" 14 | servicex-code-gen-lib = "^1.2" 15 | 16 | [tool.poetry.group.test] 17 | optional = true 18 | 19 | [tool.poetry.group.test.dependencies] 20 | pytest-flask = "^1.2.0" 21 | pytest-cov = "^4.0.0" 22 | pytest-mock = "^3.10.0" 23 | pytest = "^7.1.3" 24 | autopep8 = "^1.7.0" 25 | flake8 = "^5.0.4" 26 | coverage = "^6.5.0" 27 | 28 | [build-system] 29 | requires = ["poetry-core"] 30 | build-backend = "poetry.core.masonry.api" 31 | -------------------------------------------------------------------------------- /code_generator_funcadl_uproot/scripts/from_ast_to_zip.py: -------------------------------------------------------------------------------- 1 | # A script that will take as input a text ast (on the command line) and 2 | # write out a zip file. 3 | import sys 4 | from servicex.xaod_code_generator.ast_translator import AstTranslator 5 | 6 | if __name__ == "__main__": 7 | import argparse 8 | parser = argparse.ArgumentParser() 9 | parser.add_argument("-a", "--ast", 10 | help="The text AST to be converted into zip file. STDIN if this is left off") # noqa: E501 11 | parser.add_argument("-z", "--zipfile", 12 | help="The name of the zip file to write out. STDOUT if this is left off") 13 | args = parser.parse_args() 14 | 15 | # Get the input AST 16 | ast_text = args.ast if args.ast is not None else sys.stdin.read().strip() 17 | 18 | # Output file 19 | translator = AstTranslator() 20 | zip_data = translator.translate_text_ast_to_zip(ast_text) 21 | if args.zipfile is None: 22 | sys.stdout.buffer.write(zip_data) 23 | else: 24 | with open(args.zipfile, 'wb') as w: 25 | w.write(zip_data) 26 | -------------------------------------------------------------------------------- /code_generator_funcadl_uproot/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019, IRIS-HEP 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are met: 6 | # 7 | # * Redistributions of source code must retain the above copyright notice, this 8 | # list of conditions and the following disclaimer. 9 | # 10 | # * Redistributions in binary form must reproduce the above copyright notice, 11 | # this list of conditions and the following disclaimer in the documentation 12 | # and/or other materials provided with the distribution. 13 | # 14 | # * Neither the name of the copyright holder nor the names of its 15 | # contributors may be used to endorse or promote products derived from 16 | # this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /code_generator_funcadl_uproot/transformer_capabilities.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "FuncADL based uproot transformer", 3 | "description": "Extracts data from flat ntuple style root files.", 4 | "limitations": "Would be good to note what isn't implemented", 5 | "file-formats": ["parquet", "root"], 6 | "stats-parser": "UprootStats", 7 | "language": "python", 8 | "command": "/generated/transform_single_file.py" 9 | } -------------------------------------------------------------------------------- /code_generator_funcadl_xAOD/.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = True 3 | source = servicex 4 | 5 | [report] 6 | exclude_lines = 7 | if self.debug: 8 | pragma: no cover 9 | raise NotImplementedError 10 | if __name__ == .__main__.: 11 | ignore_errors = True 12 | omit = 13 | tests/* 14 | -------------------------------------------------------------------------------- /code_generator_funcadl_xAOD/.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 150 3 | exclude = 4 | .tox, 5 | __pycache__ 6 | validate_requests.py 7 | find_m.py 8 | messaging.py 9 | .local 10 | build 11 | migrations 12 | 13 | 14 | -------------------------------------------------------------------------------- /code_generator_funcadl_xAOD/.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | *egg-info* 3 | *.pyc 4 | .pytest_cache/* 5 | .coverage 6 | coverage.xml 7 | generated/ 8 | .venv/* 9 | .idea/ 10 | .python-version -------------------------------------------------------------------------------- /code_generator_funcadl_xAOD/.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": "Python: Flask", 9 | "type": "python", 10 | "request": "launch", 11 | "module": "flask", 12 | "env": { 13 | "FLASK_APP": "servicex/code_generator_service", 14 | "FLASK_ENV": "development", 15 | "FLASK_DEBUG": "0", 16 | "APP_CONFIG_FILE": "../app.conf" 17 | }, 18 | "args": [ 19 | "run", 20 | "--no-debugger", 21 | "--no-reload" 22 | ], 23 | "jinja": true 24 | } 25 | ] 26 | } -------------------------------------------------------------------------------- /code_generator_funcadl_xAOD/.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "cSpell.words": [ 3 | "Codecov", 4 | "Dlib", 5 | "LINQ", 6 | "envvar", 7 | "gwatts", 8 | "jsonify", 9 | "localds", 10 | "microservice", 11 | "noqa", 12 | "pytest", 13 | "qastle", 14 | "servicex", 15 | "setuptools", 16 | "xaod" 17 | ], 18 | "pyright.openFilesOnly": false, 19 | "python.linting.flake8Enabled": true, 20 | "python.pythonPath": ".venv\\Scripts\\python.exe", 21 | "python.linting.flake8Args": [ 22 | "--ignore=E501" 23 | ], 24 | "restructuredtext.confPath": "", 25 | } -------------------------------------------------------------------------------- /code_generator_funcadl_xAOD/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.10 2 | 3 | ARG APP_CONFIG_FILE="app.atlas.xaod.conf" 4 | 5 | ENV POETRY_VERSION=2.1.1 6 | RUN pip install poetry==$POETRY_VERSION 7 | 8 | RUN useradd -ms /bin/bash servicex 9 | 10 | WORKDIR /home/servicex 11 | RUN mkdir ./xaod_code_generator 12 | 13 | COPY pyproject.toml pyproject.toml 14 | COPY poetry.lock poetry.lock 15 | 16 | RUN poetry config virtualenvs.create false && \ 17 | poetry install --no-root --no-interaction --no-ansi 18 | RUN pip install gunicorn 19 | RUN pip list 20 | 21 | COPY boot.sh ./ 22 | COPY transformer_capabilities.json ./ 23 | COPY xaod_code_generator/ ./xaod_code_generator 24 | COPY scripts/from_ast_to_zip.py . 25 | RUN chmod +x boot.sh 26 | 27 | USER servicex 28 | COPY ${APP_CONFIG_FILE} app.conf 29 | ENV CODEGEN_CONFIG_FILE /home/servicex/app.conf 30 | 31 | EXPOSE 5000 32 | ENTRYPOINT ["/home/servicex/boot.sh"] 33 | -------------------------------------------------------------------------------- /code_generator_funcadl_xAOD/LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2018-2019, Jim Pivarski 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /code_generator_funcadl_xAOD/app.atlas.xaod.conf: -------------------------------------------------------------------------------- 1 | TARGET_BACKEND = 'ATLAS xAOD' 2 | -------------------------------------------------------------------------------- /code_generator_funcadl_xAOD/app.cms.aod.conf: -------------------------------------------------------------------------------- 1 | TARGET_BACKEND = 'CMS AOD' 2 | -------------------------------------------------------------------------------- /code_generator_funcadl_xAOD/boot.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Running the web server or a utility? 4 | action=${1:-web_service} 5 | if [ "$action" = "web_service" ] ; then 6 | mkdir instance 7 | exec gunicorn -b [::]:5000 --workers=2 --threads=1 --log-level=info --access-logfile /tmp/access --error-logfile /tmp/error "xaod_code_generator:create_app()" 8 | elif [ "$action" = "translate" ]; then 9 | python from_ast_to_zip.py "${@:2}" 10 | else 11 | echo "Unknown action '$action'" 12 | fi 13 | -------------------------------------------------------------------------------- /code_generator_funcadl_xAOD/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "code-generator-funcadl-xaod" 3 | version = "0.1.0" 4 | description = "" 5 | authors = ["Your Name "] 6 | readme = "README.md" 7 | packages = [{ include = "code_generator_funcadl_xaod" }] 8 | 9 | [tool.poetry.dependencies] 10 | python = "~3.10" 11 | func-adl-xAOD = "^2.2.0" 12 | servicex-code-gen-lib = "^1.2" 13 | 14 | [tool.poetry.group.test] 15 | optional = true 16 | 17 | [tool.poetry.group.test.dependencies] 18 | pytest-flask = "^1.2.0" 19 | pytest-cov = "^4.0.0" 20 | pytest-mock = "^3.10.0" 21 | pytest = "^7.1.3" 22 | autopep8 = "^1.7.0" 23 | flake8 = "^5.0.4" 24 | coverage = "^6.5.0" 25 | 26 | 27 | [build-system] 28 | requires = ["poetry-core"] 29 | build-backend = "poetry.core.masonry.api" 30 | -------------------------------------------------------------------------------- /code_generator_funcadl_xAOD/scripts/from_ast_to_zip.py: -------------------------------------------------------------------------------- 1 | # A script that will take as input a text ast (on the command line) and 2 | # write out a zip file. 3 | import sys 4 | from servicex.xaod_code_generator.ast_translator import AstTranslator 5 | 6 | if __name__ == "__main__": 7 | import argparse 8 | parser = argparse.ArgumentParser() 9 | parser.add_argument("-a", "--ast", 10 | help="The text AST to be converted into zip file. STDIN if this is left off") # noqa: E501 11 | parser.add_argument("-z", "--zipfile", 12 | help="The name of the zip file to write out. STDOUT if this is left off") 13 | parser.add_argument("--cms-aod", default=False, required=False, action='store_true', 14 | help="Use CMS AOD executor to translate the ast") 15 | parser.add_argument("--atlas-xaod", default=False, required=False, action='store_true', 16 | help="Use ATLAS xAOD executor to translate the ast") 17 | 18 | args = parser.parse_args() 19 | 20 | backend = 'ATLAS xAOD' 21 | if args.cms_aod: 22 | backend = 'CMS AOD' 23 | 24 | # Get the input AST 25 | ast_text = args.ast if args.ast is not None else sys.stdin.read().strip() 26 | 27 | # Output file 28 | translator = AstTranslator(executor=backend) 29 | zip_data = translator.translate_text_ast_to_zip(ast_text) 30 | if args.zipfile is None: 31 | sys.stdout.buffer.write(zip_data) 32 | else: 33 | with open(args.zipfile, 'wb') as w: 34 | w.write(zip_data) 35 | -------------------------------------------------------------------------------- /code_generator_funcadl_xAOD/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019, IRIS-HEP 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are met: 6 | # 7 | # * Redistributions of source code must retain the above copyright notice, this 8 | # list of conditions and the following disclaimer. 9 | # 10 | # * Redistributions in binary form must reproduce the above copyright notice, 11 | # this list of conditions and the following disclaimer in the documentation 12 | # and/or other materials provided with the distribution. 13 | # 14 | # * Neither the name of the copyright holder nor the names of its 15 | # contributors may be used to endorse or promote products derived from 16 | # this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /code_generator_funcadl_xAOD/tests/test_ast_translator.py: -------------------------------------------------------------------------------- 1 | import os 2 | import tempfile 3 | from pathlib import PosixPath 4 | 5 | import pytest 6 | from func_adl_xAOD.atlas.xaod.executor import atlas_xaod_executor 7 | from xaod_code_generator.ast_translator import AstAODTranslator 8 | 9 | from servicex_codegen.code_generator import GenerateCodeException 10 | 11 | 12 | def test_ctor(): 13 | 'Make sure default ctor works' 14 | a = AstAODTranslator("ATLAS xAOD") 15 | assert isinstance(a.executor, atlas_xaod_executor) 16 | 17 | 18 | def test_translate_good(mocker): 19 | with tempfile.TemporaryDirectory() as tmpdirname: 20 | exe = mocker.MagicMock() 21 | 22 | os.environ['TEMPLATE_PATH'] = "xaod_code_generator/templates/transform_single_file.sh" 23 | os.environ['CAPABILITIES_PATH'] = "transformer_capabilities.json" 24 | query = "(call ResultTTree (call Select (call SelectMany (call EventDataset (list 'localds://did_01')))))" # NOQA E501 25 | translator = AstAODTranslator(exe=exe) 26 | generated = translator.generate_code( 27 | query, 28 | cache_path=tmpdirname) 29 | 30 | assert generated.output_dir == tmpdirname 31 | exe.apply_ast_transformations.assert_called() 32 | assert exe.write_cpp_files.call_args[0][1] == PosixPath(tmpdirname) 33 | 34 | 35 | def test_translate_no_code(mocker): 36 | with tempfile.TemporaryDirectory() as tmpdirname: 37 | exe = mocker.MagicMock() 38 | 39 | translator = AstAODTranslator(exe=exe) 40 | with pytest.raises(GenerateCodeException): 41 | translator.generate_code("", cache_path=tmpdirname) 42 | -------------------------------------------------------------------------------- /code_generator_funcadl_xAOD/transformer_capabilities.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "FuncADL based C++ transformer", 3 | "description": "Two different transformers. One for ATLAS reads xAOD files. A second transformer reads CMS Run 1 AOD files", 4 | "limitations": "Would be good to note what isn't implemented", 5 | "file-formats": ["root"], 6 | "stats-parser": "AODStats", 7 | "language": "bash", 8 | "command": "/generated/transform_single_file.sh" 9 | 10 | } -------------------------------------------------------------------------------- /code_generator_funcadl_xAOD/xaod_code_generator/templates/transform_single_file.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set +e 3 | 4 | # If transformer has already run then we don't need to compile 5 | if [ ! -d /home/atlas/rel ]; then 6 | echo "Compile" 7 | bash /generated/runner.sh -c 8 | if [ $? != 0 ]; then 9 | echo "Compile step failed" 10 | exit 1 11 | fi 12 | fi 13 | 14 | echo "Transform a file $1 -> $2" 15 | bash /generated/runner.sh -r -d "$1" -o "$2" 16 | if [ $? != 0 ]; then 17 | echo "Transform step failed" 18 | exit 1 19 | fi 20 | -------------------------------------------------------------------------------- /code_generator_python/.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length=99 3 | -------------------------------------------------------------------------------- /code_generator_python/.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | name: CI/CD 2 | 3 | on: 4 | push: 5 | branches: 6 | - "*" 7 | tags: 8 | - "*" 9 | 10 | pull_request: 11 | 12 | jobs: 13 | test: 14 | strategy: 15 | matrix: 16 | python-version: [3.7] 17 | runs-on: ubuntu-latest 18 | 19 | steps: 20 | - uses: actions/checkout@master 21 | - name: Set up Python ${{ matrix.python-version }} 22 | uses: actions/setup-python@v4 23 | with: 24 | python-version: ${{ matrix.python-version }} 25 | - name: Install dependencies 26 | run: | 27 | python -m pip install --upgrade pip setuptools wheel 28 | pip install --no-cache-dir -r requirements.txt 29 | pip install --no-cache-dir -r requirements_dev.txt 30 | pip list 31 | - name: Lint with Flake8 32 | run: | 33 | flake8 --exclude=tests/* --ignore=E501 34 | publish: 35 | runs-on: ubuntu-latest 36 | needs: test 37 | steps: 38 | - uses: actions/checkout@master 39 | 40 | - name: Extract tag name 41 | shell: bash 42 | run: echo "##[set-output name=imagetag;]$(echo ${GITHUB_REF##*/})" 43 | id: extract_tag_name 44 | 45 | - name: Build Uproot Image 46 | uses: elgohr/Publish-Docker-Github-Action@master 47 | with: 48 | name: sslhep/servicex_code_gen_python:${{ steps.extract_tag_name.outputs.imagetag }} 49 | username: ${{ secrets.DOCKER_USERNAME }} 50 | password: ${{ secrets.DOCKER_PASSWORD }} 51 | -------------------------------------------------------------------------------- /code_generator_python/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.10 2 | 3 | RUN useradd -ms /bin/bash servicex 4 | 5 | WORKDIR /home/servicex 6 | RUN mkdir ./python_code_generator 7 | 8 | ENV POETRY_VERSION=2.1.1 9 | RUN pip install poetry==$POETRY_VERSION 10 | 11 | COPY pyproject.toml pyproject.toml 12 | COPY poetry.lock poetry.lock 13 | 14 | RUN poetry config virtualenvs.create false && \ 15 | poetry install --no-root --no-interaction --no-ansi 16 | 17 | RUN pip install gunicorn 18 | 19 | COPY boot.sh ./ 20 | COPY transformer_capabilities.json ./ 21 | COPY python_code_generator/ ./python_code_generator 22 | COPY scripts/from_text_to_zip.py . 23 | RUN chmod +x boot.sh 24 | 25 | USER servicex 26 | 27 | EXPOSE 5000 28 | ENTRYPOINT ["./boot.sh"] 29 | 30 | -------------------------------------------------------------------------------- /code_generator_python/README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # ServiceX_Code_Generator_Python 4 | 5 | # Dummy test 6 | 7 | This code generator takes python code/function and passes it unmodified for use by the transformer. 8 | 9 | For instance, the following python function was saved in the file selection.py: 10 | 11 | ``` 12 | def transform_yt(ds): 13 | slc = ds.r[ds.domain_center[0], :, :].plot(("gas", "density")) 14 | sac = slc.frb[("gas", "density")].d 15 | return sac 16 | ``` 17 | 18 | The json passed to the transformer will need this function base64 encoded. To encode your python code: 19 | 20 | ``` 21 | cat selection.py | base64 22 | ``` 23 | 24 | Then in the json the "selection" key will take the base64-encoded string: 25 | 26 | ``` 27 | selection: "ZGVmIHRyYW5zZm9ybV95dChkcyk6CiAgICBzbGMgPSBkcy5yW2RzLmRvbWFpbl9jZW50ZXJbMF0sIDosIDpdLnBsb3QoKCJnYXMiLCAiZGVuc2l0eSIpKQogICAgc2FjID0gc2xjLmZyYlsoImdhcyIsICJkZW5zaXR5IildLmQKICAgIHJldHVybiBzYWMK" 28 | ``` 29 | 30 | Here is an example using the [ServiceX frontend](https://github.com/ssl-hep/ServiceX_frontend): 31 | 32 | ``` 33 | from servicex import ServiceXDataset 34 | from servicex.servicex_python_function import ServiceXPythonFunction 35 | 36 | def transform_yt(ds): 37 | slc = ds.r[ds.domain_center[0], :, :].plot(("gas", "density")) 38 | sac = slc.frb[("gas", "density")].d 39 | return sac 40 | 41 | if __name__ == "__main__": 42 | dataset = "girder://579fb0aa7b6f0800011ea3b6#item" 43 | 44 | ds = ServiceXDataset(dataset, 45 | backend_name = "python" 46 | ) 47 | selection = ServiceXPythonFunction(ds) 48 | encoded_selection = selection._encode_function(transform_yt) 49 | r = ds.get_data_pandas_df(encoded_selection) 50 | print(r) 51 | ``` 52 | -------------------------------------------------------------------------------- /code_generator_python/boot.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Running the web server or a utility? 4 | action=${1:-web_service} 5 | if [ "$action" = "web_service" ] ; then 6 | mkdir instance 7 | exec gunicorn -b [::]:5000 --workers=2 -t 10000 --threads=1 --access-logfile - --error-logfile - "python_code_generator:create_app()" 8 | elif [ "$action" = "translate" ]; then 9 | python from_text_to_zip.py "${@:2}" 10 | else 11 | echo "Unknown action '$action'" 12 | fi 13 | -------------------------------------------------------------------------------- /code_generator_python/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "code-generator-python" 3 | version = "0.1.0" 4 | description = "" 5 | authors = [] 6 | readme = "README.md" 7 | packages = [{include = "code_generator_python"}] 8 | 9 | [tool.poetry.dependencies] 10 | python = "~3.10" 11 | servicex-code-gen-lib = "^1.2" 12 | 13 | [tool.poetry.group.test] 14 | optional = true 15 | 16 | [tool.poetry.group.test.dependencies] 17 | pytest-flask = "^1.2.0" 18 | pytest-cov = "^4.0.0" 19 | pytest-mock = "^3.10.0" 20 | pytest = "^7.1.3" 21 | autopep8 = "^1.7.0" 22 | flake8 = "^5.0.4" 23 | coverage = "^6.5.0" 24 | 25 | [build-system] 26 | requires = ["poetry-core"] 27 | build-backend = "poetry.core.masonry.api" 28 | -------------------------------------------------------------------------------- /code_generator_python/scripts/from_text_to_zip.py: -------------------------------------------------------------------------------- 1 | # A script that will take as input a text (on the command line) and 2 | # write out a zip file. 3 | import sys 4 | from servicex.xaod_code_generator.python_translator import PythonTranslator 5 | 6 | if __name__ == "__main__": 7 | import argparse 8 | parser = argparse.ArgumentParser() 9 | parser.add_argument("-t", "--text", 10 | help="The text to be converted into zip file. STDIN if this is left off") 11 | parser.add_argument("-z", "--zipfile", 12 | help="The name of the zip file to write out. STDOUT if this is left off") 13 | args = parser.parse_args() 14 | 15 | # Get the input text 16 | text = args.text if args.text is not None else sys.stdin.read().strip() 17 | 18 | # Output file 19 | translator = PythonTranslator() 20 | zip_data = translator.translate_text_python_to_zip(text) 21 | if args.zipfile is None: 22 | sys.stdout.buffer.write(zip_data) 23 | else: 24 | with open(args.zipfile, 'wb') as w: 25 | w.write(zip_data) 26 | -------------------------------------------------------------------------------- /code_generator_python/transformer_capabilities.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Python Uproot Transformer", 3 | "description": "Extracts data from flat ntuple style root files.", 4 | "limitations": "Would be good to note what isn't implemented", 5 | "file-formats": ["parquet", "root"], 6 | "stats-parser": "UprootStats", 7 | "language": "python", 8 | "command": "/generated/transform_single_file.py" 9 | } -------------------------------------------------------------------------------- /code_generator_raw_uproot/.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 99 3 | exclude = 4 | .tox, 5 | __pycache__ 6 | validate_requests.py 7 | find_m.py 8 | messaging.py 9 | .local 10 | build 11 | migrations 12 | 13 | 14 | -------------------------------------------------------------------------------- /code_generator_raw_uproot/.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | *egg-info* 3 | *.pyc 4 | .pytest_cache/* 5 | .coverage 6 | coverage.xml 7 | generated/ 8 | .venv/* 9 | .python-version -------------------------------------------------------------------------------- /code_generator_raw_uproot/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.10 2 | 3 | RUN useradd -ms /bin/bash servicex 4 | RUN apt-get update && apt-get install -y netcat-traditional && rm -rf /var/lib/apt/lists/* 5 | 6 | WORKDIR /home/servicex 7 | RUN mkdir ./servicex 8 | 9 | ENV POETRY_VERSION=2.1.1 10 | RUN pip install poetry==$POETRY_VERSION 11 | 12 | COPY pyproject.toml pyproject.toml 13 | COPY poetry.lock poetry.lock 14 | 15 | RUN poetry config virtualenvs.create false && \ 16 | poetry install --no-root --no-interaction --no-ansi 17 | 18 | RUN pip install gunicorn 19 | 20 | COPY boot.sh ./ 21 | COPY transformer_capabilities.json ./ 22 | COPY servicex/ ./servicex 23 | RUN chmod +x boot.sh 24 | 25 | USER servicex 26 | COPY app.conf . 27 | 28 | ENV CODEGEN_CONFIG_FILE "/home/servicex/app.conf" 29 | 30 | EXPOSE 5000 31 | ENTRYPOINT ["./boot.sh"] 32 | 33 | -------------------------------------------------------------------------------- /code_generator_raw_uproot/LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2018-2019, Jim Pivarski 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /code_generator_raw_uproot/app.conf: -------------------------------------------------------------------------------- 1 | TARGET_BACKEND = 'uproot-raw' 2 | -------------------------------------------------------------------------------- /code_generator_raw_uproot/boot.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Running the web server? 4 | action=${1:-web_service} 5 | if [ "$action" = "web_service" ] ; then 6 | mkdir instance 7 | exec gunicorn -b [::]:5000 --workers=2 --threads=1 --access-logfile - --error-logfile - "servicex.raw_uproot_code_generator:create_app()" 8 | else 9 | echo "Unknown action '$action'" 10 | fi 11 | -------------------------------------------------------------------------------- /code_generator_raw_uproot/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "code-generator-funcadl-uproot-raw" 3 | version = "0.1.0" 4 | description = "" 5 | authors = ["Your Name "] 6 | readme = "README.md" 7 | packages = [{include = "code_generator_funcadl_uproot_raw"}] 8 | 9 | [tool.poetry.dependencies] 10 | python = "~3.10" 11 | servicex-code-gen-lib = "^1.2" 12 | 13 | [tool.poetry.group.test] 14 | optional = true 15 | 16 | [tool.poetry.group.test.dependencies] 17 | pytest-flask = "^1.2.0" 18 | pytest-cov = "^4.0.0" 19 | pytest-mock = "^3.10.0" 20 | pytest = "^7.1.3" 21 | autopep8 = "^1.7.0" 22 | flake8 = "^5.0.4" 23 | coverage = "^6.5.0" 24 | 25 | [build-system] 26 | requires = ["poetry-core"] 27 | build-backend = "poetry.core.masonry.api" 28 | -------------------------------------------------------------------------------- /code_generator_raw_uproot/servicex/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019, IRIS-HEP 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are met: 6 | # 7 | # * Redistributions of source code must retain the above copyright notice, this 8 | # list of conditions and the following disclaimer. 9 | # 10 | # * Redistributions in binary form must reproduce the above copyright notice, 11 | # this list of conditions and the following disclaimer in the documentation 12 | # and/or other materials provided with the distribution. 13 | # 14 | # * Neither the name of the copyright holder nor the names of its 15 | # contributors may be used to endorse or promote products derived from 16 | # this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /code_generator_raw_uproot/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019, IRIS-HEP 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are met: 6 | # 7 | # * Redistributions of source code must retain the above copyright notice, this 8 | # list of conditions and the following disclaimer. 9 | # 10 | # * Redistributions in binary form must reproduce the above copyright notice, 11 | # this list of conditions and the following disclaimer in the documentation 12 | # and/or other materials provided with the distribution. 13 | # 14 | # * Neither the name of the copyright holder nor the names of its 15 | # contributors may be used to endorse or promote products derived from 16 | # this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /code_generator_raw_uproot/transformer_capabilities.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Uproot transformer using native uproot arguments", 3 | "description": "Extracts data from flat ntuple style root files.", 4 | "limitations": "Would be good to note what isn't implemented", 5 | "file-formats": ["parquet", "root-file", "root-rntuple"], 6 | "stats-parser": "RawUprootStats", 7 | "language": "python", 8 | "command": "/generated/transform_single_file.py" 9 | } -------------------------------------------------------------------------------- /did_finder_cernopendata/.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length=99 3 | -------------------------------------------------------------------------------- /did_finder_cernopendata/.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /did_finder_cernopendata/.github/workflows/publish.yaml: -------------------------------------------------------------------------------- 1 | name: CI/CD 2 | 3 | on: 4 | push: 5 | branches: 6 | - "*" 7 | tags: 8 | - "*" 9 | pull_request: 10 | 11 | jobs: 12 | test: 13 | strategy: 14 | matrix: 15 | python-version: [3.6, 3.7, 3.8, 3.9] 16 | runs-on: ubuntu-latest 17 | 18 | steps: 19 | - uses: actions/checkout@master 20 | 21 | - name: Set up Python ${{ matrix.python-version }} 22 | uses: actions/setup-python@v4 23 | with: 24 | python-version: ${{ matrix.python-version }} 25 | 26 | - name: Install dependencies 27 | run: | 28 | python -m pip install --upgrade pip setuptools wheel 29 | python -m pip install -r requirements.txt 30 | python -m pip install -r requirements_test.txt 31 | pip list 32 | 33 | - name: Lint with Flake8 34 | run: | 35 | flake8 36 | - name: Test with pytest 37 | run: | 38 | coverage run -m --source=src pytest tests 39 | coverage xml 40 | 41 | publish: 42 | runs-on: ubuntu-latest 43 | needs: test 44 | 45 | steps: 46 | - uses: actions/checkout@master 47 | 48 | - name: Extract tag name 49 | shell: bash 50 | run: echo "##[set-output name=imagetag;]$(echo ${GITHUB_REF##*/})" 51 | id: extract_tag_name 52 | 53 | - name: Build DID-Finder Image 54 | uses: elgohr/Publish-Docker-Github-Action@master 55 | with: 56 | name: sslhep/servicex-did-finder-cernopendata:${{ steps.extract_tag_name.outputs.imagetag }} 57 | username: ${{ secrets.DOCKER_USERNAME }} 58 | password: ${{ secrets.DOCKER_PASSWORD }} 59 | tag: "${GITHUB_REF##*/}" 60 | -------------------------------------------------------------------------------- /did_finder_cernopendata/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM cernopendata/cernopendata-client:0.3.0 2 | 3 | LABEL maintainer="Gordon Watts " 4 | 5 | USER root 6 | 7 | # Create app directory 8 | WORKDIR /opt/servicex 9 | 10 | # Create celery user. Assign them to group zero since that is the group OpenShift will run the container as 11 | RUN useradd -g 0 -ms /bin/bash celery 12 | 13 | ENV POETRY_VERSION=2.1.1 14 | 15 | # There is a bug in poetry with no virtual env where it can delete this package 16 | # during install. If we delete it before installing poetry it will 17 | # recognize it as its own and handle it correctly 18 | # See: https://github.com/python-poetry/poetry/issues/5977 19 | RUN pip uninstall -y certifi 20 | 21 | RUN pip install poetry==$POETRY_VERSION 22 | 23 | COPY pyproject.toml pyproject.toml 24 | COPY poetry.lock poetry.lock 25 | 26 | # Bring over the main scripts 27 | COPY . . 28 | 29 | ENV XDG_CONFIG_HOME=/opt/servicex 30 | RUN poetry config virtualenvs.in-project true 31 | RUN poetry install --no-root --no-interaction --no-ansi 32 | 33 | # Change ownership of the app directory to celery user. Also set the group to zero since 34 | # that is the group OpenShift will run the container as 35 | RUN chown -R celery:0 /opt/servicex 36 | RUN chmod -R g=u /opt/servicex 37 | 38 | # Switch to celery user 39 | USER celery 40 | 41 | # Make sure python isn't buffered 42 | ENV PYTHONUNBUFFERED=1 43 | 44 | ENV PYTHONPATH=/opt/servicex/src 45 | ENV BROKER_URL="amqp://guest:guest@localhost:5672//" 46 | 47 | ENTRYPOINT [ "scripts/start_celery_worker.sh"] 48 | -------------------------------------------------------------------------------- /did_finder_cernopendata/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "did-finder-cernopendata" 3 | version = "0.1.0" 4 | description = "" 5 | authors = ["Your Name "] 6 | readme = "README.md" 7 | packages = [{include = "did_finder_cernopendata", from = "src"}] 8 | 9 | [tool.poetry.dependencies] 10 | python = "~3.10" 11 | servicex-did-finder-lib = "^3.0.0" 12 | 13 | [tool.poetry.group.test] 14 | optional = true 15 | 16 | [tool.poetry.group.test.dependencies] 17 | pytest = "^7.1.3" 18 | flake8 = "^5.0.4" 19 | pytest-asyncio = "^0.19.0" 20 | coverage = "^6.5.0" 21 | 22 | [build-system] 23 | requires = ["poetry-core"] 24 | build-backend = "poetry.core.masonry.api" 25 | -------------------------------------------------------------------------------- /did_finder_cernopendata/scripts/start_celery_worker.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | poetry run celery --broker="$BROKER_URL" -A did_finder_cernopendata worker \ 3 | --loglevel=info -Q did_finder_cernopendata \ 4 | --concurrency=1 --hostname=did_finder_cernopendata@%h 5 | -------------------------------------------------------------------------------- /did_finder_cernopendata/src/did_finder_cernopendata/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssl-hep/ServiceX/74b29d758dfe8c78e1e0109a706df9ddf1fc56cf/did_finder_cernopendata/src/did_finder_cernopendata/__init__.py -------------------------------------------------------------------------------- /did_finder_cernopendata/tests/did_finder_cernopendata_tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2024, IRIS-HEP 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are met: 6 | # 7 | # * Redistributions of source code must retain the above copyright notice, this 8 | # list of conditions and the following disclaimer. 9 | # 10 | # * Redistributions in binary form must reproduce the above copyright notice, 11 | # this list of conditions and the following disclaimer in the documentation 12 | # and/or other materials provided with the distribution. 13 | # 14 | # * Neither the name of the copyright holder nor the names of its 15 | # contributors may be used to endorse or promote products derived from 16 | # this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /did_finder_rucio/.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | branch = True 3 | source = 4 | src/ 5 | -------------------------------------------------------------------------------- /did_finder_rucio/.dockerignore: -------------------------------------------------------------------------------- 1 | kube 2 | .gitignore 3 | config 4 | -------------------------------------------------------------------------------- /did_finder_rucio/.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 99 3 | exclude = 4 | .tox, 5 | __pycache__ 6 | validate_requests.py 7 | find_m.py 8 | messaging.py 9 | .local 10 | build 11 | migrations 12 | -------------------------------------------------------------------------------- /did_finder_rucio/.github/ISSUE_TEMPLATE/user-story.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: User Story 3 | about: A feature from an end-user perspective 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | # Story 11 | _As a I want to so I can _ 12 | 13 | # Acceptance Criteria 14 | 1. How will we know when this story is complete 15 | 16 | # Assumptions 17 | 1. List assumptions behind this story 18 | -------------------------------------------------------------------------------- /did_finder_rucio/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM sslhep/rucio-client:main 2 | 3 | LABEL maintainer="Ilija Vukotic " 4 | 5 | # Create app directory 6 | WORKDIR /opt/servicex 7 | 8 | # Create celery user. Assign them to group zero since that is the group OpenShift will run the container as 9 | RUN useradd -g 0 -ms /bin/bash celery 10 | 11 | # for CA certificates 12 | RUN mkdir -p /etc/grid-security/certificates /etc/grid-security/vomsdir 13 | 14 | RUN yum clean all 15 | RUN yum -y update 16 | 17 | # Okay, change our shell to specifically use our software collections. 18 | # (default was SHELL [ "/bin/sh", "-c" ]) 19 | # https://docs.docker.com/engine/reference/builder/#shell 20 | 21 | ENV POETRY_VERSION=2.1.1 22 | RUN python3 -m pip install --upgrade pip 23 | 24 | RUN pip install poetry==$POETRY_VERSION 25 | RUN mkdir -p /opt/servicex/pypoetry 26 | COPY pyproject.toml pyproject.toml 27 | COPY poetry.lock poetry.lock 28 | 29 | ENV XDG_CONFIG_HOME=/opt/servicex 30 | RUN poetry config virtualenvs.in-project true 31 | RUN poetry install --no-root --no-interaction --no-ansi 32 | 33 | COPY . . 34 | 35 | # Change ownership of the app directory to celery user. Also set the group to zero since 36 | # that is the group OpenShift will run the container as 37 | RUN chown -R celery:0 /opt/servicex 38 | RUN chmod -R g=u /opt/servicex 39 | 40 | ENV X509_USER_PROXY=/tmp/grid-security/x509up 41 | ENV X509_CERT_DIR=/etc/grid-security/certificates 42 | ENV PYTHONPATH=/opt/servicex/src 43 | 44 | ENV BROKER_URL="amqp://guest:guest@localhost:5672//" 45 | 46 | # Switch to celery user 47 | USER celery 48 | 49 | ENTRYPOINT [ "scripts/start_celery_worker.sh"] 50 | -------------------------------------------------------------------------------- /did_finder_rucio/proxy-exporter.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | proxydir=$(dirname ${X509_USER_PROXY}) 4 | 5 | if [[ ! -d $proxydir ]] 6 | then 7 | mkdir -p $proxydir 8 | fi 9 | 10 | while true; do 11 | 12 | while true; do 13 | cp /etc/grid-security-ro/x509up ${X509_USER_PROXY} 14 | RESULT=$? 15 | if [ $RESULT -eq 0 ]; then 16 | echo "INFO $INSTANCE_NAME did-finder none Got proxy." 17 | chmod 600 ${X509_USER_PROXY} 18 | break 19 | else 20 | echo "WARNING $INSTANCE_NAME did-finder none An issue encountered when getting proxy." 21 | sleep 5 22 | fi 23 | done 24 | 25 | # Refresh every hour 26 | sleep 3600 27 | 28 | done 29 | -------------------------------------------------------------------------------- /did_finder_rucio/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "did-finder-rucio" 3 | version = "0.1.0" 4 | description = "" 5 | authors = ["Ilija Vukotic "] 6 | readme = "README.md" 7 | packages = [{include = "rucio_did_finder", from = "src"}] 8 | 9 | [tool.poetry.dependencies] 10 | python = "^3.9" 11 | rucio-clients = "^34.2.0" 12 | xmltodict = "^0.13.0" 13 | servicex-did-finder-lib = "^3.0.1" 14 | geoip2 = "^4.7.0" 15 | requests = ">=2.25.0,<3.0.0" 16 | 17 | [tool.poetry.group.test] 18 | optional = true 19 | 20 | [tool.poetry.group.test.dependencies] 21 | pytest = "^7.2.0" 22 | coverage = "^6.5.0" 23 | pytest-mock = "^3.10.0" 24 | flake8 = "^5.0.4" 25 | 26 | 27 | [build-system] 28 | requires = ["poetry-core"] 29 | build-backend = "poetry.core.masonry.api" 30 | -------------------------------------------------------------------------------- /did_finder_rucio/scripts/start_celery_worker.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | /opt/servicex/proxy-exporter.sh & 3 | 4 | while true; do 5 | date 6 | ls ${X509_USER_PROXY} 7 | RESULT=$? 8 | if [ $RESULT -eq 0 ]; then 9 | break 10 | fi 11 | echo "INFO $INSTANCE_NAME did-finder none Waiting for the proxy." 12 | sleep 5 13 | done 14 | 15 | poetry run celery --broker="$BROKER_URL" -A rucio_did_finder worker \ 16 | --loglevel=info -Q did_finder_rucio \ 17 | --concurrency=5 --hostname=did_finder_rucio@%h 18 | -------------------------------------------------------------------------------- /did_finder_rucio/src/rucio_did_finder/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019, IRIS-HEP 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are met: 6 | # 7 | # * Redistributions of source code must retain the above copyright notice, this 8 | # list of conditions and the following disclaimer. 9 | # 10 | # * Redistributions in binary form must reproduce the above copyright notice, 11 | # this list of conditions and the following disclaimer in the documentation 12 | # and/or other materials provided with the distribution. 13 | # 14 | # * Neither the name of the copyright holder nor the names of its 15 | # contributors may be used to endorse or promote products derived from 16 | # this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /did_finder_rucio/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019, IRIS-HEP 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are met: 6 | # 7 | # * Redistributions of source code must retain the above copyright notice, this 8 | # list of conditions and the following disclaimer. 9 | # 10 | # * Redistributions in binary form must reproduce the above copyright notice, 11 | # this list of conditions and the following disclaimer in the documentation 12 | # and/or other materials provided with the distribution. 13 | # 14 | # * Neither the name of the copyright holder nor the names of its 15 | # contributors may be used to endorse or promote products derived from 16 | # this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /did_finder_rucio/tests/did_finder/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019, IRIS-HEP 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are met: 6 | # 7 | # * Redistributions of source code must retain the above copyright notice, this 8 | # list of conditions and the following disclaimer. 9 | # 10 | # * Redistributions in binary form must reproduce the above copyright notice, 11 | # this list of conditions and the following disclaimer in the documentation 12 | # and/or other materials provided with the distribution. 13 | # 14 | # * Neither the name of the copyright holder nor the names of its 15 | # contributors may be used to endorse or promote products derived from 16 | # this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /did_finder_xrootd/.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length=99 3 | -------------------------------------------------------------------------------- /did_finder_xrootd/.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /did_finder_xrootd/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.10 2 | 3 | LABEL maintainer="Peter Onyisi " 4 | 5 | # Create app directory 6 | WORKDIR /opt/servicex 7 | 8 | # Create celery user. Assign them to group zero since that is the group OpenShift will run the container as 9 | RUN useradd -g 0 -ms /bin/bash celery 10 | 11 | ENV POETRY_VERSION=2.1.1 12 | 13 | RUN apt-get update && \ 14 | apt-get install cmake --no-install-recommends --assume-yes 15 | 16 | RUN pip install poetry==$POETRY_VERSION 17 | RUN mkdir -p /opt/servicex/pypoetry 18 | COPY pyproject.toml pyproject.toml 19 | COPY poetry.lock poetry.lock 20 | 21 | ENV XDG_CONFIG_HOME=/opt/servicex 22 | RUN poetry config virtualenvs.in-project true 23 | RUN poetry install --no-root --no-interaction --no-ansi 24 | 25 | # Bring over the main scripts 26 | COPY . . 27 | 28 | # Change ownership of the app directory to celery user. Also set the group to zero since 29 | # that is the group OpenShift will run the container as 30 | RUN chown -R celery:0 /opt/servicex 31 | RUN chmod -R g=u /opt/servicex 32 | 33 | # Switch to celery user 34 | USER celery 35 | 36 | # Make sure python isn't buffered 37 | ENV PYTHONUNBUFFERED=1 38 | 39 | ENV PYTHONPATH=/opt/servicex/src 40 | 41 | ENV BROKER_URL="amqp://guest:guest@localhost:5672//" 42 | 43 | 44 | ENTRYPOINT [ "scripts/start_celery_worker.sh"] 45 | -------------------------------------------------------------------------------- /did_finder_xrootd/README.md: -------------------------------------------------------------------------------- 1 | # ServiceX_DID_finder_XRootD 2 | Find datasets by performing glob wildcarding via xrootd. 3 | 4 | ## Finding datasets 5 | 6 | XRootD is a standard for remotely accessing HEP files, supported by many storage backends (EOS, dCache, etc.) This "DID finder" allows you to specify a glob wildcard pattern, such as 7 | ``` 8 | root://eospublic.cern.ch//eos/opendata/atlas/OutreachDatasets/2020-01-22/4lep/MC/* 9 | ``` 10 | to look up all the files matching this pattern and use those as input for a ServiceX transformation. In particular, files from CERN EOS are available via the gateways `eospublic.cern.ch`, `eosatlas.cern.ch`, `eoscms.cern.ch`, etc. 11 | 12 | When accessing files from EOS, note that you must have the double slash `//` between the server name and the first element of the path! 13 | -------------------------------------------------------------------------------- /did_finder_xrootd/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "did-finder-xrootd" 3 | version = "0.1.0" 4 | description = "" 5 | authors = ["Your Name "] 6 | readme = "README.md" 7 | packages = [{include = "src/servicex_did_finder_xrootd"}] 8 | 9 | [tool.poetry.dependencies] 10 | python = ">=3.10,<4.0" 11 | servicex-did-finder-lib = "^3.0.0" 12 | xrootd = ">=5.6.9" 13 | 14 | [tool.poetry.group.test] 15 | optional = true 16 | 17 | [tool.poetry.group.test.dependencies] 18 | pytest = ">=7.1.3" 19 | flake8 = ">=5.0.4" 20 | pytest-asyncio = ">=0.19.0" 21 | coverage = ">=6.5.0" 22 | 23 | [build-system] 24 | requires = ["poetry-core"] 25 | build-backend = "poetry.core.masonry.api" 26 | -------------------------------------------------------------------------------- /did_finder_xrootd/scripts/start_celery_worker.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | poetry run celery --broker="$BROKER_URL" -A servicex_did_finder_xrootd worker \ 3 | --loglevel=info -Q did_finder_xrootd \ 4 | --concurrency=1 --hostname=did_finder_xrootd@%h 5 | -------------------------------------------------------------------------------- /did_finder_xrootd/src/servicex_did_finder_xrootd/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssl-hep/ServiceX/74b29d758dfe8c78e1e0109a706df9ddf1fc56cf/did_finder_xrootd/src/servicex_did_finder_xrootd/__init__.py -------------------------------------------------------------------------------- /did_finder_xrootd/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssl-hep/ServiceX/74b29d758dfe8c78e1e0109a706df9ddf1fc56cf/did_finder_xrootd/tests/__init__.py -------------------------------------------------------------------------------- /did_finder_xrootd/tests/test_servicex_did_finder_xrootd.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from servicex_did_finder_xrootd.celery import find_files 3 | 4 | 5 | def test_working_call(): 6 | iter = find_files( 7 | ( 8 | "root://eospublic.cern.ch//eos/opendata/atlas/" 9 | "OutreachDatasets/2020-01-22/4lep/MC/*" 10 | ), 11 | {"dataset-id": "112233"}, 12 | ) 13 | files = [f for f in iter] 14 | 15 | assert len(files) == 106 16 | assert isinstance(files[0], dict) 17 | sorted_files = sorted(files, key=lambda x: x["paths"][0]) 18 | assert sorted_files[0]["paths"][0] == ( 19 | "root://eospublic.cern.ch//eos/opendata/atlas/" 20 | "OutreachDatasets/2020-01-22/4lep/MC/mc_301215.ZPrime2000_ee.4lep.root" 21 | ) 22 | 23 | 24 | def test_exception_no_files(): 25 | iter = find_files( 26 | ( 27 | "root://eospublic.cern.ch//eos/opendata/atlas/" 28 | "OutreachDatasets/2020-01-22/4lep/MC/dummy*" 29 | ), 30 | {"dataset-id": "112233"}, 31 | ) 32 | with pytest.raises(RuntimeError): 33 | [f for f in iter] 34 | -------------------------------------------------------------------------------- /docs/deployment/output_options.md: -------------------------------------------------------------------------------- 1 | # Columnar Output Options 2 | The transformers can write the columnar data either to a deployed [Minio Object 3 | store](https://docs.min.io/docs/minio-quickstart-guide.html) or to mounted 4 | POSIX volumes. 5 | 6 | ## Minio 7 | The helm chart has an option to deploy the legacy Minio helm chart as a 8 | dependent chart. This has options to create an ingress so you can access your 9 | objects remotely. 10 | 11 | Minio is enabled by default. In transforms, set the `result-destination` 12 | property to `object-store`. 13 | 14 | ## POSIX Mounted Volumes 15 | In some environments there is an existing filesystem, and it makes sense to 16 | write the extracted files there for further processing. 17 | 18 | You can retain the Minio deployment if desired, otherwise deactivate it and 19 | prevent helm from deploying Minio by setting: 20 | ```yaml 21 | objectStore: 22 | enabled: false 23 | ``` 24 | 25 | The mounted POSIX volumes assumes that a kubernetes read-write-many persistent 26 | volume claim exists in the deployed namespace. If you need an example on 27 | creating a PVC, take a look in this repo's `scripts/transformer_pvc.yaml` file. 28 | 29 | In your helm values file you can provide this PVC name as well as a subdirectory 30 | into the claim where the files will be written. Note that this subdir path 31 | must have a trailing / to be treated as a directory: 32 | 33 | ```yaml 34 | transformer: 35 | persistence: 36 | existingClaim: transformer-pv-claim 37 | subdir: foo/bar/ 38 | ``` 39 | -------------------------------------------------------------------------------- /docs/example_secrets.yaml: -------------------------------------------------------------------------------- 1 | { 2 | "kind": "SealedSecret", 3 | "apiVersion": "bitnami.com/v1alpha1", 4 | "metadata": { 5 | "name": "servicex-secrets", 6 | "namespace": "servicex-namespace", 7 | "creationTimestamp": null, 8 | "annotations": { 9 | "sealedsecrets.bitnami.com/namespace-wide": "true" 10 | } 11 | }, 12 | "spec": { 13 | "template": { 14 | "metadata": { 15 | "name": "servicex-secrets", 16 | "namespace": "servicex-namespace", 17 | "creationTimestamp": null, 18 | "annotations": { 19 | "sealedsecrets.bitnami.com/managed": "true", 20 | "sealedsecrets.bitnami.com/namespace-wide": "true" 21 | } 22 | }, 23 | "type": "Opaque", 24 | "data": null 25 | }, 26 | "encryptedData": { 27 | "accesskey": "aaa", 28 | "flaskSecretKey": "aaa", 29 | "globusClientID": "aaa", 30 | "globusClientSecret": "aaa", 31 | "jwtSecretKey": "aaa", 32 | "mailgunAPIKey": "aaa", 33 | "postgresql-password": "aaa", 34 | "rabbitmq-erlang-cookie": "aaa", 35 | "rabbitmq-password": "aaa", 36 | "root-password": "aaa", 37 | "root-user": "aaa", 38 | "secretkey": "aaa", 39 | "slackSigningSecret": "aaa", 40 | "slackSignupWebhook": "aaa" 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /docs/img/ServiceX-BlackTextOnly-Transparent.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssl-hep/ServiceX/74b29d758dfe8c78e1e0109a706df9ddf1fc56cf/docs/img/ServiceX-BlackTextOnly-Transparent.png -------------------------------------------------------------------------------- /docs/img/ServiceX-Color-ImageOnly-Transparent.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssl-hep/ServiceX/74b29d758dfe8c78e1e0109a706df9ddf1fc56cf/docs/img/ServiceX-Color-ImageOnly-Transparent.png -------------------------------------------------------------------------------- /docs/img/ServiceX-Color-Transparent.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssl-hep/ServiceX/74b29d758dfe8c78e1e0109a706df9ddf1fc56cf/docs/img/ServiceX-Color-Transparent.png -------------------------------------------------------------------------------- /docs/img/ServiceX-WhiteText-Transparent.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssl-hep/ServiceX/74b29d758dfe8c78e1e0109a706df9ddf1fc56cf/docs/img/ServiceX-WhiteText-Transparent.png -------------------------------------------------------------------------------- /docs/img/ServiceX-WhiteTextOnly-Transparent.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssl-hep/ServiceX/74b29d758dfe8c78e1e0109a706df9ddf1fc56cf/docs/img/ServiceX-WhiteTextOnly-Transparent.png -------------------------------------------------------------------------------- /docs/img/ServiceX-architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssl-hep/ServiceX/74b29d758dfe8c78e1e0109a706df9ddf1fc56cf/docs/img/ServiceX-architecture.png -------------------------------------------------------------------------------- /docs/img/ServiceX-request-lifecycle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssl-hep/ServiceX/74b29d758dfe8c78e1e0109a706df9ddf1fc56cf/docs/img/ServiceX-request-lifecycle.png -------------------------------------------------------------------------------- /docs/img/Z_ee_example.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssl-hep/ServiceX/74b29d758dfe8c78e1e0109a706df9ddf1fc56cf/docs/img/Z_ee_example.jpg -------------------------------------------------------------------------------- /docs/img/develop.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssl-hep/ServiceX/74b29d758dfe8c78e1e0109a706df9ddf1fc56cf/docs/img/develop.png -------------------------------------------------------------------------------- /docs/img/download-servicex-yaml.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssl-hep/ServiceX/74b29d758dfe8c78e1e0109a706df9ddf1fc56cf/docs/img/download-servicex-yaml.jpg -------------------------------------------------------------------------------- /docs/img/hep_tables_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssl-hep/ServiceX/74b29d758dfe8c78e1e0109a706df9ddf1fc56cf/docs/img/hep_tables_example.png -------------------------------------------------------------------------------- /docs/img/organize.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssl-hep/ServiceX/74b29d758dfe8c78e1e0109a706df9ddf1fc56cf/docs/img/organize.png -------------------------------------------------------------------------------- /docs/img/organize2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssl-hep/ServiceX/74b29d758dfe8c78e1e0109a706df9ddf1fc56cf/docs/img/organize2.png -------------------------------------------------------------------------------- /docs/img/sx-architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssl-hep/ServiceX/74b29d758dfe8c78e1e0109a706df9ddf1fc56cf/docs/img/sx-architecture.png -------------------------------------------------------------------------------- /docs/img/sx-schema.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssl-hep/ServiceX/74b29d758dfe8c78e1e0109a706df9ddf1fc56cf/docs/img/sx-schema.png -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | hide: 3 | - navigation 4 | - toc 5 | --- 6 | # 7 | 8 | ServiceX Logo 9 | 10 | [ServiceX](https://github.com/ssl-hep/ServiceX), a component of the 11 | [IRIS-HEP](https://iris-hep.org/) Intelligent Data Delivery Service, is an experiment-agnostic 12 | service to enable on-demand columnar data delivery tailored for nearly interactive, high 13 | performance, array-based Pythonic analyses. It provides a uniform backend interface to data storage 14 | services and an intuitive frontend for users to enable columnar transformations from multiple 15 | different data formats and organizational structures. 16 | 17 | For documentation on the ServiceX client software, please go to the [ServiceX Frontend](https://servicex-frontend.readthedocs.io) page. This site contains documentation concerning deployment of the ServiceX backend server and how to contribute code. 18 | -------------------------------------------------------------------------------- /docs/introduction.md: -------------------------------------------------------------------------------- 1 | # Introduction 2 | 3 | The High Luminosity Large Hadron Collider (HL-LHC) faces enormous computational challenges in the 4 | 2020s. The HL-LHC will produce exabytes of data each year, with increasingly complex event 5 | structure due to high pileup conditions. The ATLAS and CMS experiments will record ~ 10 times as 6 | much data from ~ 100 times as many collisions as were used to discover the Higgs boson. 7 | 8 | ## Columnar data delivery 9 | 10 | ServiceX seeks to enable on-demand data delivery of columnar data in a variety of formats for 11 | physics analyses. It provides a uniform backend to data storage services, ensuring the user doesn't 12 | have to know how or where the data is stored, and is capable of on-the-fly data transformations 13 | into a variety of formats (ROOT files, Arrow arrays, Parquet files, ...) The service offers 14 | preprocessing functionality via an analysis description language called 15 | [func-adl](https://pypi.org/project/func-adl/) that allows users to filter events, request columns, 16 | and even compute new variables. This enables the user to start from any format and extract only the 17 | data needed for an analysis. 18 | 19 | ![Organization](img/organize2.png) 20 | 21 | ServiceX is designed to feed columns to a user running an analysis (e.g. via 22 | [Awkward](https://github.com/scikit-hep/awkward-array) or 23 | [Coffea](https://github.com/CoffeaTeam/coffea) tools) based on the results of a query designed by 24 | the user. 25 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | mkdocs==1.6.1 2 | mkdocs-jupyter==0.25.1 3 | mkdocs-material==9.6.14 4 | -------------------------------------------------------------------------------- /examples/ATLAS/README.md: -------------------------------------------------------------------------------- 1 | # ServiceX Example for ATLAS xAOD Files 2 | 3 | These examples show how to use ServiceX to analyze events from ATLAS xAOD files. 4 | 5 | ## Prerequisites 6 | 7 | These examples use a _coffea_ local processor to create histograms. 8 | This processor knows how to talk to a ServiceX instance and submit a request 9 | in the func_adl analysis language. 10 | 11 | To install: 12 | 13 | ```bash 14 | pip install coffea[servicex] 15 | ``` 16 | 17 | If you want to host your own notebook you'll need JupyterLab 18 | 19 | ```bash 20 | pip install jupyterlab 21 | ``` 22 | 23 | The dynamic plots require the JupyterLab widgets 24 | 25 | ```bash 26 | pip install jupyterlab-widgets 27 | ``` 28 | 29 | ## Obtain Credentials 30 | 31 | Visit SSL's [xAOD ServiceX](https://xaod.servicex.af.uchicago.edu) instance. Click 32 | on the _Sign-in_ button in the upper right hand corner. You will be asked to 33 | authenticate via the ATLAS SSO and complete a registration form. Once this form is 34 | complete, your account will be created. 35 | 36 | At this time you may return to the ServiceX page. Click on your name in the 37 | upper right hand corner and then select _Profile_ tab. Click on the download 38 | button to have a servicex.yaml file generated with your access token and 39 | downloaded to your computer. 40 | 41 | ![Download button](/docs/img/download-servicex-yaml.jpg) 42 | 43 | You may place this in your home directory or within 44 | the [servicex_frontend search path](https://github.com/ssl-hep/ServiceX_frontend#configuration). 45 | 46 | ## Examples 47 | 48 | 49 | 50 | 51 | 52 | 53 |
Z_ee Example NotebookNotebook to compute Mass of Z Boson from Z_ee
54 | -------------------------------------------------------------------------------- /examples/ATLAS/Z_ee.py: -------------------------------------------------------------------------------- 1 | from func_adl_servicex import ServiceXSourceXAOD 2 | 3 | query = "(call ResultTTree (call Select (call SelectMany (call EventDataset (list 'localds:bogus')) (lambda (list e) (call (attr e 'Jets') 'AntiKt4EMTopoJets'))) (lambda (list j) (/ (call (attr j 'pt')) 1000.0))) (list 'JetPt') 'analysis' 'junk.root')" # noqa: E501 4 | big_dataset = "rucio://mc15_13TeV:mc15_13TeV.361106.PowhegPythia8EvtGen_AZNLOCTEQ6L1_Zee.merge.DAOD_STDM3.e3601_s2576_s2132_r6630_r6264_p2363_tid05630052_00?files=3" # noqa: E501 5 | dataset = "mc16_13TeV:DAOD_TOPQ1.25521529._000009.pool.root.1" 6 | ds = ServiceXSourceXAOD(dataset, backend="local") 7 | leptons_per_event_query = ( 8 | ds.Select(lambda e: e.Electrons("Electrons")) 9 | .Select(lambda eles: eles.Where(lambda e: e.pt() / 1000.0 > 30.0)) 10 | .Select(lambda eles: eles.Where(lambda e: abs(e.eta()) < 2.5)) 11 | .Where(lambda eles: len(eles) == 2) 12 | .Select( 13 | lambda ls: ( 14 | ls.Select(lambda e: e.pt() / 1000.0), 15 | ls.Select(lambda e: e.eta()), 16 | ls.Select(lambda e: e.phi()), 17 | ls.Select(lambda e: e.m() / 1000.0), 18 | ls.Select(lambda e: e.charge()), 19 | ) 20 | ) 21 | ) 22 | 23 | print( 24 | leptons_per_event_query.AsPandasDF( 25 | [ 26 | "electrons_pt", 27 | "electrons_eta", 28 | "electrons_phi", 29 | "electrons_mass", 30 | "electrons_charge", 31 | ] 32 | ).value() 33 | ) 34 | -------------------------------------------------------------------------------- /helm/.github/workflows/check-chart.yaml: -------------------------------------------------------------------------------- 1 | name: Lint and Test Charts 2 | 3 | on: [pull_request, workflow_dispatch] 4 | 5 | jobs: 6 | lint-test: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - name: Checkout 10 | uses: actions/checkout@v4.1.1 11 | with: 12 | fetch-depth: 0 13 | 14 | - name: Set up Helm 15 | uses: azure/setup-helm@v4 16 | with: 17 | version: v3.9.2 18 | 19 | - uses: actions/setup-python@v5.1.0 20 | with: 21 | python-version: 3.7 22 | 23 | - name: Set up chart-testing 24 | uses: helm/chart-testing-action@v2.6.1 25 | 26 | - name: add bitnami repos to helm 27 | run: | 28 | helm repo add bitnami https://charts.bitnami.com/bitnami 29 | 30 | - name: Run chart-testing (list-changed) 31 | id: list-changed 32 | run: | 33 | changed=$(ct list-changed --target-branch ${{ github.event.repository.default_branch }} ) 34 | if [[ -n "$changed" ]]; then 35 | echo "{changed}={true}" >> $GITHUB_OUTPUT 36 | fi 37 | 38 | - name: Run chart-testing (lint) 39 | run: 40 | ct lint --target-branch ${{ github.event.repository.default_branch }} --chart-dirs ${GITHUB_WORKSPACE}/servicex --charts servicex 41 | 42 | # - name: Create kind cluster 43 | # uses: helm/kind-action@v1.3.0 44 | 45 | # - name: Run chart-testing (install) 46 | # run: ct install --chart-dirs ${GITHUB_WORKSPACE}/servicex --charts servicex 47 | -------------------------------------------------------------------------------- /helm/demo-values.yaml: -------------------------------------------------------------------------------- 1 | didFinder: 2 | # For easy testing, we set this to be a publicly accessable xAOD File 3 | # The DID Finder will always just return this file instead of connecting 4 | # To rucio to actually resolve the dataset 5 | # Set this to empty to actually use the DID Finder 6 | staticFile: 'https://recastwww.web.cern.ch/recastwww/data/reana-recast-demo/mc15_13TeV.123456.cap_recast_demo_signal_one.root' 7 | -------------------------------------------------------------------------------- /helm/example_secrets.yaml: -------------------------------------------------------------------------------- 1 | # These are the secrets that are needed if you want to override values 2 | # in a secure way 3 | apiVersion: v1 4 | kind: Secret 5 | metadata: 6 | creationTimestamp: null 7 | name: servicex_secrets 8 | data: 9 | globusClientID: <> 10 | globusClientSecret: <> 11 | flaskSecretKey: <> 12 | jwtSecretKey: <> 13 | slackSigningSecret: <> 14 | slackSignupWebhook: <> 15 | mailgunAPIKey: <> 16 | accesskey: <> 17 | secretkey: <> 18 | rabbitmq-password: << rabbitMQ password >> 19 | rabbitmq-erlang-cookie: << rabbitMQ erlang cookie >> 20 | postgresql-password: << postgresql password for postgres user >> 21 | geoip-license-key: << MaxMind license key for GeoIP database download >> 22 | -------------------------------------------------------------------------------- /helm/release-helm-chart.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | git checkout develop 3 | git pull origin develop 4 | 5 | helm dependencies update servicex 6 | 7 | # Update the app version and chart version in the chart 8 | sed -e "s/appVersion: .*$/appVersion: $1/" -e "s/version: .*$/version: $1/" servicex/Chart.yaml > servicex/Chart.new.yaml 9 | mv servicex/Chart.new.yaml servicex/Chart.yaml 10 | 11 | # Point all images in values.yaml to the new deployment 12 | sed -E -e "s/ tag:\s*[[:digit:]]{8}-[[:digit:]]{4}-stable.*$/ tag: $1/" -e "s/ defaultTransformerTag:\s*.+$/ defaultTransformerTag: $1/" servicex/values.yaml > servicex/values.new.yaml 13 | mv servicex/values.new.yaml servicex/values.yaml 14 | helm package servicex 15 | -------------------------------------------------------------------------------- /helm/scripts/generated_code_busybox.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Pod 3 | metadata: 4 | name: transformer-busybox 5 | spec: 6 | containers: 7 | - args: 8 | - /home/atlas/proxy-exporter.sh & sleep 5 && sleep 20000 9 | command: 10 | - bash 11 | - -c 12 | env: 13 | - name: BASH_ENV 14 | value: /home/atlas/.bashrc 15 | image: sslhep/servicex_func_adl_uproot_transformer:develop 16 | imagePullPolicy: Always 17 | name: generated-code-busybox 18 | resources: {} 19 | terminationMessagePath: /dev/termination-log 20 | terminationMessagePolicy: File 21 | volumeMounts: 22 | - mountPath: /etc/grid-security-ro 23 | name: x509-secret 24 | - mountPath: /generated 25 | name: generated-code 26 | restartPolicy: Never 27 | volumes: 28 | - name: x509-secret 29 | secret: 30 | defaultMode: 420 31 | secretName: uproot-x509-proxy 32 | - configMap: 33 | defaultMode: 420 34 | name: 0294c057-896a-4a89-a59b-80d1930f2600-generated-source 35 | name: generated-code 36 | -------------------------------------------------------------------------------- /helm/scripts/port-forward.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | 4 | # Function to start port forwarding 5 | start_port_forward() { 6 | local service=$1 7 | local ports=$2 8 | echo "Starting port forward for $service on ports $ports" 9 | kubectl port-forward service/$service $ports & 10 | } 11 | 12 | # Function to stop port forwarding 13 | stop_port_forward() { 14 | echo "Stopping all port forwards" 15 | pkill -f "kubectl port-forward" 16 | } 17 | 18 | # Start port forwarding for all services 19 | start_all() { 20 | start_port_forward "servicex-ben-postgresql" "5432:5432" 21 | start_port_forward "servicex-ben-code-gen-uproot-raw" "5001:8000" 22 | start_port_forward "servicex-ben-minio" "9000:9000" 23 | start_port_forward "servicex-ben-rabbitmq" "5672:5672" 24 | start_port_forward "servicex-ben-servicex-app" "8000:8000" 25 | 26 | echo "All port forwards started. Press Ctrl+C to stop." 27 | } 28 | 29 | # Main script 30 | case "$1" in 31 | start) 32 | start_all 33 | # Wait for user interrupt 34 | trap stop_port_forward EXIT 35 | while true; do sleep 1; done 36 | ;; 37 | stop) 38 | stop_port_forward 39 | ;; 40 | *) 41 | echo "Usage: $0 {start|stop}" 42 | exit 1 43 | ;; 44 | esac 45 | -------------------------------------------------------------------------------- /helm/scripts/transformer_pvc.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: PersistentVolume 3 | metadata: 4 | name: transformer-pv-volume 5 | labels: 6 | type: local 7 | spec: 8 | storageClassName: manual 9 | capacity: 10 | storage: 5Gi 11 | accessModes: 12 | - ReadWriteMany 13 | hostPath: 14 | path: "<<< Path to your local directory >>>" 15 | --- 16 | apiVersion: v1 17 | kind: PersistentVolumeClaim 18 | metadata: 19 | name: transformer-pv-claim 20 | spec: 21 | storageClassName: manual 22 | accessModes: 23 | - ReadWriteMany 24 | resources: 25 | requests: 26 | storage: 3Gi 27 | -------------------------------------------------------------------------------- /helm/servicex/.helmignore: -------------------------------------------------------------------------------- 1 | # Patterns to ignore when building packages. 2 | # This supports shell glob matching, relative path matching, and 3 | # negation (prefixed with !). Only one pattern per line. 4 | .DS_Store 5 | # Common VCS dirs 6 | .git/ 7 | .gitignore 8 | .bzr/ 9 | .bzrignore 10 | .hg/ 11 | .hgignore 12 | .svn/ 13 | # Common backup files 14 | *.swp 15 | *.bak 16 | *.tmp 17 | *~ 18 | # Various IDEs 19 | .project 20 | .idea/ 21 | *.tmproj 22 | .vscode/ 23 | -------------------------------------------------------------------------------- /helm/servicex/Chart.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | appVersion: 20220928-0107-stable 3 | description: Install ServiceX deployment - HEP Columnar Data Delivery Service 4 | maintainers: 5 | - name: BenGalewsky 6 | url: https://github.com/BenGalewsky 7 | - name: sthapa 8 | url: https://github.com/sthapa 9 | - name: ivukotic 10 | url: https://github.com/ivukotic 11 | name: servicex 12 | version: 1.5.7-a1 13 | -------------------------------------------------------------------------------- /helm/servicex/local-dev-values.yaml: -------------------------------------------------------------------------------- 1 | noCerts: false 2 | didFinder: 3 | rucio: 4 | enabled: true 5 | tag: sidecar-monorepo 6 | pullPolicy: Always 7 | 8 | CERNOpenData: 9 | enabled: false 10 | 11 | YT: 12 | enabled: false 13 | app: 14 | tag: sidecar-monorepo 15 | pullPolicy: IfNotPresent 16 | validateTransformerImage: false 17 | 18 | transformer: 19 | autoscalerEnabled: false 20 | sidecarTag: sidecar-monorepo 21 | sidecarPullPolicy: IfNotPresent 22 | 23 | # language: python 24 | # exec: /generated/transform_single_file.py 25 | # defaultScienceContainerImage: sslhep/servicex_func_adl_uproot_transformer 26 | 27 | language: bash 28 | exec: /generated/transform_single_file.sh 29 | scienceContainerPullPolicy: IfNotPresent 30 | 31 | codeGen: 32 | #image: sslhep/servicex_code_gen_func_adl_uproot 33 | image: sslhep/servicex_code_gen_atlas_xaod 34 | tag: sidecar-monorepo 35 | pullPolicy: IfNotPresent 36 | defaultScienceContainerImage: sslhep/servicex_func_adl_xaod_transformer 37 | defaultScienceContainerTag: sidecar-monorepo 38 | 39 | objectStore: 40 | enabled: true 41 | publicURL: localhost:9000 42 | publicClientUseTLS: false 43 | 44 | 45 | rabbitmq: 46 | auth: 47 | erlangCookie: mmmmmmmcoookie 48 | persistence: 49 | enabled: false 50 | service: 51 | type: NodePort 52 | nodePort: 30672 53 | 54 | 55 | postgres: 56 | enabled: true 57 | 58 | gridAccount: bgalewsk 59 | -------------------------------------------------------------------------------- /helm/servicex/requirements.lock: -------------------------------------------------------------------------------- 1 | dependencies: 2 | - name: rabbitmq 3 | repository: https://charts.bitnami.com/bitnami/ 4 | version: 11.4.0 5 | - name: minio 6 | repository: https://charts.bitnami.com/bitnami/ 7 | version: 11.10.26 8 | - name: postgresql 9 | repository: https://charts.bitnami.com/bitnami/ 10 | version: 11.6.26 11 | digest: sha256:1538e3fd27ac2dcfe5df320de851a8c501f8b5cc4056b13b208a92d5e70340c5 12 | generated: "2023-01-25T16:29:05.999958-06:00" 13 | -------------------------------------------------------------------------------- /helm/servicex/requirements.yaml: -------------------------------------------------------------------------------- 1 | dependencies: 2 | - name: rabbitmq 3 | version: 11.4.* 4 | repository: https://charts.bitnami.com/bitnami/ 5 | - name: minio 6 | version: 11.10.* 7 | repository: https://charts.bitnami.com/bitnami/ 8 | condition: objectStore.internal, objectStore.enabled 9 | - name: postgresql 10 | version: 11.6.* 11 | repository: https://charts.bitnami.com/bitnami/ 12 | condition: postgres.enabled 13 | -------------------------------------------------------------------------------- /helm/servicex/templates/_helpers.tpl: -------------------------------------------------------------------------------- 1 | {{/* vim: set filetype=mustache: */}} 2 | {{/* 3 | Expand the name of the chart. 4 | */}} 5 | {{- define "servicex.name" -}} 6 | {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} 7 | {{- end -}} 8 | 9 | {{/* 10 | Create a default fully qualified app name. 11 | We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). 12 | If release name contains chart name it will be used as a full name. 13 | */}} 14 | {{- define "servicex.fullname" -}} 15 | {{- if .Values.fullnameOverride -}} 16 | {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} 17 | {{- else -}} 18 | {{- $name := default .Chart.Name .Values.nameOverride -}} 19 | {{- if contains $name .Release.Name -}} 20 | {{- .Release.Name | trunc 63 | trimSuffix "-" -}} 21 | {{- else -}} 22 | {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} 23 | {{- end -}} 24 | {{- end -}} 25 | {{- end -}} 26 | 27 | {{/* 28 | Create chart name and version as used by the chart label. 29 | */}} 30 | {{- define "servicex.chart" -}} 31 | {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} 32 | {{- end -}} 33 | 34 | {{/* 35 | Common labels 36 | */}} 37 | {{- define "servicex.labels" -}} 38 | app.kubernetes.io/name: {{ include "servicex.name" . }} 39 | helm.sh/chart: {{ include "servicex.chart" . }} 40 | app.kubernetes.io/instance: {{ .Release.Name }} 41 | {{- if .Chart.AppVersion }} 42 | app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} 43 | {{- end }} 44 | app.kubernetes.io/managed-by: {{ .Release.Service }} 45 | {{- end -}} 46 | -------------------------------------------------------------------------------- /helm/servicex/templates/app/ingress.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.app.ingress.enabled -}} 2 | apiVersion: networking.k8s.io/v1 3 | kind: Ingress 4 | metadata: 5 | annotations: 6 | kubernetes.io/ingress.class: {{ .Values.app.ingress.class }} 7 | {{- if .Values.app.ingress.tls.clusterIssuer }} 8 | cert-manager.io/cluster-issuer: {{ .Values.app.ingress.tls.clusterIssuer }} 9 | acme.cert-manager.io/http01-edit-in-place: "true" 10 | {{- end }} 11 | labels: 12 | app: {{ .Release.Name }}-servicex 13 | name: {{ .Release.Name }}-servicex 14 | spec: 15 | {{- if .Values.app.ingress.tls.enabled }} 16 | tls: 17 | - hosts: 18 | - {{ .Release.Name }}.{{ .Values.app.ingress.host }} 19 | secretName: {{ tpl .Values.app.ingress.tls.secretName . }} 20 | {{- end }} 21 | rules: 22 | - host: {{ .Release.Name }}.{{ .Values.app.ingress.host }} 23 | http: 24 | paths: 25 | - path: / 26 | pathType: Prefix 27 | backend: 28 | service: 29 | name: {{ .Release.Name }}-servicex-app 30 | port: 31 | number: 8000 32 | {{- end }} 33 | -------------------------------------------------------------------------------- /helm/servicex/templates/app/service.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | name: {{ .Release.Name }}-servicex-app 6 | spec: 7 | ports: 8 | - port: 8000 9 | targetPort: 5000 10 | name: "tcp" 11 | protocol: TCP 12 | selector: 13 | app: {{ .Release.Name }}-servicex-app 14 | type: ClusterIP 15 | -------------------------------------------------------------------------------- /helm/servicex/templates/codegen/deployment.yaml: -------------------------------------------------------------------------------- 1 | {{- range $codeGenName, $v := .Values.codeGen }} 2 | --- 3 | {{- if .enabled }} 4 | apiVersion: apps/v1 5 | kind: Deployment 6 | metadata: 7 | name: {{ $.Release.Name }}-code-gen-{{ $codeGenName }} 8 | spec: 9 | replicas: 1 10 | selector: 11 | matchLabels: 12 | app: {{ $.Release.Name }}-code-gen-{{ $codeGenName }} 13 | template: 14 | metadata: 15 | labels: 16 | app: {{ $.Release.Name }}-code-gen-{{ $codeGenName }} 17 | spec: 18 | serviceAccountName: {{ template "servicex.fullname" $ }} 19 | containers: 20 | - name: {{ $.Release.Name }}-code-gen-{{ $codeGenName }} 21 | image: {{ .image }}:{{ .tag }} 22 | env: 23 | - name: INSTANCE_NAME 24 | value: {{ $.Release.Name }} 25 | - name: TRANSFORMER_SCIENCE_IMAGE 26 | value: {{ .defaultScienceContainerImage }}:{{ .defaultScienceContainerTag }} 27 | tty: true 28 | stdin: true 29 | imagePullPolicy: {{ .pullPolicy }} 30 | ports: 31 | - containerPort: 5000 32 | {{- end }} 33 | {{- end }} 34 | -------------------------------------------------------------------------------- /helm/servicex/templates/codegen/service.yaml: -------------------------------------------------------------------------------- 1 | {{- range $codeGenName, $v := .Values.codeGen }} 2 | --- 3 | {{- if .enabled }} 4 | --- 5 | apiVersion: v1 6 | kind: Service 7 | metadata: 8 | name: {{ $.Release.Name }}-code-gen-{{ $codeGenName }} 9 | spec: 10 | ports: 11 | - port: 8000 12 | targetPort: 5000 13 | name: "tcp" 14 | protocol: TCP 15 | selector: 16 | app: {{ $.Release.Name }}-code-gen-{{ $codeGenName }} 17 | type: ClusterIP 18 | {{- end }} 19 | {{- end }} 20 | -------------------------------------------------------------------------------- /helm/servicex/templates/data-lifecycle/cronjob.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.dataLifecycle.enabled -}} 2 | apiVersion: batch/v1 3 | kind: CronJob 4 | metadata: 5 | name: {{ .Release.Name }}-data-lifecycle-job 6 | spec: 7 | schedule: {{ .Values.dataLifecycle.schedule | default "* */8 * * *" | quote }} 8 | concurrencyPolicy: "Forbid" 9 | jobTemplate: 10 | spec: 11 | template: 12 | metadata: 13 | labels: 14 | app: {{ .Release.Name }}-data-lifecycle-job 15 | spec: 16 | containers: 17 | - name: {{ .Release.Name }}-data-lifecycle-job 18 | image: {{ .Values.dataLifecycle.image }}:{{ .Values.dataLifecycle.tag }} 19 | imagePullPolicy: {{ .Values.dataLifecycle.pullPolicy }} 20 | env: 21 | - name: RETENTION_PERIOD 22 | value: {{ .Values.dataLifecycle.retention }} 23 | 24 | command: 25 | - /bin/sh 26 | - -c 27 | - | 28 | CUTOFF_TIMESTAMP=$(date -d "$RETENTION_PERIOD" +%Y-%m-%dT%H:%M:%S) && 29 | echo "Request data lifecycle with cutoff timestamp: $CUTOFF_TIMESTAMP" && 30 | curl --request POST "http://{{ .Release.Name }}-servicex-app:8000/servicex/internal/data-lifecycle?cutoff_timestamp=$CUTOFF_TIMESTAMP" 31 | restartPolicy: OnFailure 32 | {{- end }} 33 | -------------------------------------------------------------------------------- /helm/servicex/templates/did-finder-cernopendata/deployment.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | {{ if .Values.didFinder.CERNOpenData.enabled}} 3 | apiVersion: apps/v1 4 | kind: Deployment 5 | metadata: 6 | name: {{ .Release.Name }}-did-finder-cernopendata 7 | spec: 8 | replicas: 1 9 | selector: 10 | matchLabels: 11 | app: {{ .Release.Name }}-did-finder-cernopendata 12 | template: 13 | metadata: 14 | labels: 15 | app: {{ .Release.Name }}-did-finder-cernopendata 16 | spec: 17 | initContainers: 18 | - name: check-url 19 | image: {{ .Values.app.checksImage }} 20 | env: 21 | - name: URL 22 | value: "http://{{ .Release.Name }}-servicex-app:8000" 23 | containers: 24 | - name: {{ .Release.Name }}-did-finder-cernopendata 25 | image: {{ .Values.didFinder.CERNOpenData.image }}:{{ .Values.didFinder.CERNOpenData.tag }} 26 | imagePullPolicy: {{ .Values.didFinder.CERNOpenData.pullPolicy }} 27 | env: 28 | - name: INSTANCE_NAME 29 | value: {{ .Release.Name }} 30 | {{- if .Values.secrets }} 31 | - name: RMQ_PASS 32 | valueFrom: 33 | secretKeyRef: 34 | name: {{ .Values.secrets }} 35 | key: rabbitmq-password 36 | - name: BROKER_URL 37 | value: amqp://user:$(RMQ_PASS)@{{ .Release.Name }}-rabbitmq:5672/?heartbeat=9000 38 | {{- else }} 39 | - name: BROKER_URL 40 | value: amqp://user:{{ .Values.rabbitmq.auth.password }}@{{ .Release.Name }}-rabbitmq:5672/%2F 41 | {{- end }} 42 | {{ end }} 43 | -------------------------------------------------------------------------------- /helm/servicex/templates/did-finder-rucio/rucio-configmap.yaml: -------------------------------------------------------------------------------- 1 | {{ if .Values.didFinder.rucio.enabled}} 2 | apiVersion: v1 3 | kind: ConfigMap 4 | metadata: 5 | name: {{ .Release.Name }}-rucio-config 6 | labels: 7 | heritage: {{ .Release.Service }} 8 | release: {{ .Release.Name }} 9 | chart: {{ .Chart.Name }}-{{ .Chart.Version }} 10 | app: {{ .Release.Name }} 11 | data: 12 | rucio.cfg: | 13 | [client] 14 | rucio_host = {{ .Values.didFinder.rucio.rucio_host }} 15 | auth_host = {{ .Values.didFinder.rucio.auth_host }} 16 | auth_type = x509_proxy 17 | ca_cert = /etc/pki/tls/certs/ca-bundle.crt 18 | account = {{ .Values.gridAccount }} 19 | client_x509_proxy = $X509_USER_PROXY 20 | request_retries = 3 21 | 22 | [policy] 23 | permission = {{ .Values.x509Secrets.vomsOrg }} 24 | schema = {{ .Values.x509Secrets.vomsOrg }} 25 | lfn2pfn_algorithm_default = hash 26 | {{ end }} 27 | -------------------------------------------------------------------------------- /helm/servicex/templates/did-finder-xrootd/deployment.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | {{ if .Values.didFinder.xrootd.enabled}} 3 | apiVersion: apps/v1 4 | kind: Deployment 5 | metadata: 6 | name: {{ .Release.Name }}-did-finder-xrootd 7 | spec: 8 | replicas: 1 9 | selector: 10 | matchLabels: 11 | app: {{ .Release.Name }}-did-finder-xrootd 12 | template: 13 | metadata: 14 | labels: 15 | app: {{ .Release.Name }}-did-finder-xrootd 16 | spec: 17 | initContainers: 18 | - name: check-url 19 | image: {{ .Values.app.checksImage }} 20 | env: 21 | - name: URL 22 | value: "http://{{ .Release.Name }}-servicex-app:8000" 23 | containers: 24 | - name: {{ .Release.Name }}-did-finder-xrootd 25 | image: {{ .Values.didFinder.xrootd.image }}:{{ .Values.didFinder.xrootd.tag }} 26 | imagePullPolicy: {{ .Values.didFinder.CERNOpenData.pullPolicy }} 27 | env: 28 | - name: INSTANCE_NAME 29 | value: {{ .Release.Name }} 30 | {{- if .Values.secrets }} 31 | - name: RMQ_PASS 32 | valueFrom: 33 | secretKeyRef: 34 | name: {{ .Values.secrets }} 35 | key: rabbitmq-password 36 | - name: BROKER_URL 37 | value: amqp://user:$(RMQ_PASS)@{{ .Release.Name }}-rabbitmq:5672/?heartbeat=9000 38 | {{- else }} 39 | - name: BROKER_URL 40 | value: amqp://user:{{ .Values.rabbitmq.auth.password }}@{{ .Release.Name }}-rabbitmq:5672/%2F 41 | {{- end }} 42 | {{ end }} 43 | -------------------------------------------------------------------------------- /helm/servicex/templates/rbac/serviceaccount.yaml: -------------------------------------------------------------------------------- 1 | {{- if .Values.rbacEnabled }} 2 | apiVersion: v1 3 | kind: ServiceAccount 4 | metadata: 5 | name: {{ template "servicex.fullname" . }} 6 | labels: 7 | app: {{ template "servicex.name" . }} 8 | chart: {{ template "servicex.chart" . }} 9 | release: "{{ .Release.Name }}" 10 | heritage: "{{ .Release.Service }}" 11 | {{- end }} 12 | -------------------------------------------------------------------------------- /helm/servicex/templates/x509-secrets/proxy-secret.yaml: -------------------------------------------------------------------------------- 1 | {{- if not .Values.noCerts }} 2 | # This is just a skeletal secret. The details will be filled in by the X509-Secret 3 | # service. We are deploying this as part of the helm chart so the X509 proxy secret 4 | # will also be deleted when the chart is deleted. 5 | apiVersion: v1 6 | kind: Secret 7 | metadata: 8 | name: x509-proxy 9 | labels: 10 | heritage: {{ .Release.Service }} 11 | release: {{ .Release.Name }} 12 | chart: {{ .Chart.Name }}-{{ .Chart.Version }} 13 | app: app 14 | type: Opaque 15 | {{- end }} 16 | -------------------------------------------------------------------------------- /local_test_harness/.env: -------------------------------------------------------------------------------- 1 | BASH_ENV='/servicex/.bashrc' 2 | LOG_LEVEL=DEBUG 3 | SCIENCE=sslhep/servicex_func_adl_uproot_transformer:uproot5 4 | CODEGEN=sslhep/servicex_code_gen_raw_uproot:develop 5 | LOCAL_FOLDER=./temp1 6 | PORT=8005 7 | -------------------------------------------------------------------------------- /local_test_harness/README.md: -------------------------------------------------------------------------------- 1 | Local Test Harness for running the codegen and science image for the root file 2 | 3 | ### Step1. Configure the config.yml file 4 | ```yaml 5 | TestName: local test 6 | codegen: 7 | port: 8005 8 | query: '[{"treename": {"nominal": "modified"}, "filter_name": ["lbn"]}]' #provide the query as string 9 | 10 | science: 11 | rootfile: root://fax.mwt2.org:1094//pnfs/uchicago.edu/atlaslocalgroupdisk/rucio/user/mgeyik/e7/ee/user.mgeyik.30182995._000093.out.root 12 | outputfile: /generated/out.parquet 13 | outputformat: root-file 14 | ``` 15 | 16 | ### Step2. Configure the .env for the compose file 17 | ```yaml 18 | BASH_ENV='/servicex/.bashrc' 19 | LOG_LEVEL=DEBUG 20 | SCIENCE=sslhep/servicex_func_adl_uproot_transformer:uproot5 #science image 21 | CODEGEN=sslhep/servicex_code_gen_raw_uproot:develop #codegen image 22 | LOCAL_FOLDER=./temp1 #local folder where you want output files 23 | PORT=8005 # port number for your codegen 24 | ``` 25 | 26 | ### Step3. Setup the proxy and run 27 | ```shell 28 | python test.py --proxy_image 29 | ``` 30 | 31 | OR if you already have the proxy setup in your `/tmp` 32 | 33 | ```shell 34 | python test.py 35 | ``` 36 | The default proxy image: `sslhep/x509-secrets:develop` 37 | -------------------------------------------------------------------------------- /local_test_harness/compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | 3 | codegen: 4 | image: $CODEGEN 5 | env_file: .env 6 | pull_policy: missing 7 | ports: 8 | - $PORT:5000 9 | 10 | science: 11 | image: $SCIENCE 12 | pull_policy: missing 13 | env_file: .env 14 | volumes: 15 | - sidecar-volume:/servicex/output 16 | - /tmp:/tmp/grid-security/ 17 | - $LOCAL_FOLDER:/generated 18 | tty: true 19 | depends_on: 20 | - codegen 21 | 22 | volumes: 23 | sidecar-volume: 24 | -------------------------------------------------------------------------------- /local_test_harness/config.yml: -------------------------------------------------------------------------------- 1 | TestName: local test 2 | codegen: 3 | port: 8005 4 | query: '[{"treename": {"nominal": "modified"}, "filter_name": ["lbn"]}]' 5 | 6 | science: 7 | rootfile: root://fax.mwt2.org:1094//pnfs/uchicago.edu/atlaslocalgroupdisk/rucio/user/mgeyik/e7/ee/user.mgeyik.30182995._000093.out.root 8 | outputfile: /generated/out.parquet 9 | outputformat: root-file 10 | -------------------------------------------------------------------------------- /mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: ServiceX 2 | site_url: https://servicex.readthedocs.io/ 3 | nav: 4 | - Home: index.md 5 | - Introduction: introduction.md 6 | - 'User Guide': 'https://servicex-frontend.readthedocs.io/' 7 | - 'Deployment Guide': 8 | - Basic deployments: deployment/basic.md 9 | - Production deployments: deployment/production.md 10 | - Columnar Data Output Options: deployment/output_options.md 11 | - Reference: deployment/reference.md 12 | - User Management: deployment/user_management.md 13 | - 'Developer Guide': 14 | - Architecture: development/architecture.md 15 | - Contributing: development/contributing.md 16 | plugins: 17 | - mkdocs-jupyter 18 | theme: 19 | name: material 20 | features: 21 | - navigation.tabs 22 | logo: img/ServiceX-Color-Transparent.png 23 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "servicex" 3 | version = "0.1.0" 4 | description = "Developmen virtual environment for ServiceX" 5 | authors = ["Ben Galewsky "] 6 | readme = "README.md" 7 | 8 | [tool.poetry.dependencies] 9 | python = "~3.10" 10 | servicex-app = {path = "servicex_app", develop = true} 11 | did-finder-cernopendata = {path = "did_finder_cernopendata", develop = true} 12 | did-finder-xrootd = {path = "did_finder_xrootd", develop = true} 13 | did-finder-rucio = {path = "did_finder_rucio", develop = true} 14 | 15 | [tool.poetry.group.test.dependencies] 16 | pytest = "^8.3.5" 17 | flake8 = "^7.2" 18 | pytest-mock = "^3.14.1" 19 | coverage = "^7.8.2" 20 | responses = "^0.25.7" 21 | pytest-asyncio = "^0.26.0" 22 | pytest-flask = "^1.3.0" 23 | 24 | [build-system] 25 | requires = ["poetry-core"] 26 | build-backend = "poetry.core.masonry.api" 27 | -------------------------------------------------------------------------------- /servicex_app/.dockerignore: -------------------------------------------------------------------------------- 1 | sqllite 2 | -------------------------------------------------------------------------------- /servicex_app/.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 150 3 | exclude = 4 | .tox, 5 | __pycache__ 6 | validate_requests.py 7 | find_m.py 8 | messaging.py 9 | .local 10 | build 11 | migrations 12 | 13 | 14 | -------------------------------------------------------------------------------- /servicex_app/.gitattributes: -------------------------------------------------------------------------------- 1 | boot.sh eol=lf 2 | -------------------------------------------------------------------------------- /servicex_app/.github/ISSUE_TEMPLATE/user-story.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: User Story 3 | about: A feature from an end-user perspective 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | # Story 11 | _As a I want to so I can _ 12 | 13 | # Acceptance Criteria 14 | 1. How will we know when this story is complete 15 | 16 | # Assumptions 17 | 1. List assumptions behind this story 18 | 19 | -------------------------------------------------------------------------------- /servicex_app/.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | name: CI/CD 2 | 3 | on: 4 | push: 5 | branches: 6 | - "*" 7 | tags: 8 | - "*" 9 | pull_request: 10 | 11 | jobs: 12 | test: 13 | strategy: 14 | matrix: 15 | python-version: ["3.7", "3.10"] 16 | runs-on: ubuntu-latest 17 | 18 | steps: 19 | - uses: actions/checkout@master 20 | - name: Set up Python ${{ matrix.python-version }} 21 | uses: actions/setup-python@v4 22 | with: 23 | python-version: ${{ matrix.python-version }} 24 | - name: Install dependencies 25 | run: | 26 | python -m pip install --upgrade pip setuptools wheel 27 | pip install --no-cache-dir -e ".[test]" 28 | pip list 29 | - name: Lint with Flake8 30 | run: | 31 | flake8 32 | - name: Test with pytest 33 | run: | 34 | python -m coverage run -m pytest -r sx 35 | - name: Report coverage with Codecov 36 | uses: codecov/codecov-action@v3 37 | with: 38 | token: ${{ secrets.CODECOV_TOKEN }} 39 | publish: 40 | runs-on: ubuntu-latest 41 | needs: test 42 | steps: 43 | - uses: actions/checkout@master 44 | 45 | - name: Extract tag name 46 | shell: bash 47 | run: echo "##[set-output name=imagetag;]$(echo ${GITHUB_REF##*/})" 48 | id: extract_tag_name 49 | 50 | - name: Build Docker Image 51 | uses: elgohr/Publish-Docker-Github-Action@master 52 | with: 53 | name: sslhep/servicex_app:${{ steps.extract_tag_name.outputs.imagetag }} 54 | username: ${{ secrets.DOCKER_USERNAME }} 55 | password: ${{ secrets.DOCKER_PASSWORD }} 56 | tag: "${GITHUB_REF##*/}" 57 | -------------------------------------------------------------------------------- /servicex_app/.gitignore: -------------------------------------------------------------------------------- 1 | sqlite 2 | app.conf 3 | .idea 4 | __pycache__/ 5 | *.pyc 6 | servicex.egg-info/ 7 | .coverage 8 | htmlcov/ 9 | coverage.xml 10 | build/ 11 | dist/ 12 | .cache 13 | .pytest_cache/ 14 | .vscode/settings.json 15 | venv/ 16 | .venv/ 17 | .python-version -------------------------------------------------------------------------------- /servicex_app/.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://gitlab.com/pycqa/flake8 3 | rev: 3.8.4 4 | hooks: 5 | - id: flake8 6 | -------------------------------------------------------------------------------- /servicex_app/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.10 2 | 3 | RUN useradd -ms /bin/bash servicex 4 | 5 | WORKDIR /home/servicex 6 | RUN mkdir ./servicex 7 | 8 | ENV POETRY_VERSION=2.1.1 9 | RUN pip install poetry==$POETRY_VERSION 10 | 11 | COPY README.rst README.rst 12 | COPY pyproject.toml pyproject.toml 13 | COPY poetry.lock poetry.lock 14 | RUN poetry config virtualenvs.create false && \ 15 | poetry install --no-root --no-interaction --no-ansi 16 | 17 | COPY *.py docker-dev.conf boot.sh ./ 18 | COPY servicex_app/ ./servicex_app 19 | COPY migrations migrations 20 | ADD gai.conf /etc/gai.conf 21 | 22 | RUN chmod +x boot.sh 23 | 24 | #ENV FLASK_APP servicex 25 | 26 | USER servicex 27 | 28 | EXPOSE 5000 29 | ENTRYPOINT ["./boot.sh"] 30 | -------------------------------------------------------------------------------- /servicex_app/LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2018-2019, Jim Pivarski 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /servicex_app/boot.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | mkdir instance 3 | # SQLite doesn't handle migrations, so rely on SQLAlchmy table creation 4 | if grep "sqlite://" $APP_CONFIG_FILE; then 5 | echo "SQLLite DB, so skipping db migrations"; 6 | else 7 | FLASK_APP=servicex_app/app.py flask db upgrade; 8 | fi 9 | [ -d "/default_users" ] && python3 servicex/cli/create_default_users.py 10 | exec gunicorn -b [::]:5000 --workers=5 --threads=1 --timeout 120 --log-level=warning --access-logfile /tmp/gunicorn.log --error-logfile - "servicex_app:create_app()" 11 | # to log requests to stdout --access-logfile - -------------------------------------------------------------------------------- /servicex_app/codecov.yml: -------------------------------------------------------------------------------- 1 | ignore: 2 | - "servicex/__init__.py" # Most of this is bypassed during test setup -------------------------------------------------------------------------------- /servicex_app/doc/sequence_diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssl-hep/ServiceX/74b29d758dfe8c78e1e0109a706df9ddf1fc56cf/servicex_app/doc/sequence_diagram.png -------------------------------------------------------------------------------- /servicex_app/gai.conf: -------------------------------------------------------------------------------- 1 | label ::1/128 0 2 | label ::/0 1 3 | label 2002::/16 2 4 | label ::/96 3 5 | label ::ffff:0:0/96 4 -------------------------------------------------------------------------------- /servicex_app/migrations/README: -------------------------------------------------------------------------------- 1 | Single-database configuration for Flask. 2 | -------------------------------------------------------------------------------- /servicex_app/migrations/alembic.ini: -------------------------------------------------------------------------------- 1 | # A generic, single database configuration. 2 | 3 | [alembic] 4 | 5 | script_location = migrations 6 | # template used to generate migration files 7 | # file_template = %%(rev)s_%%(slug)s 8 | 9 | # set to 'true' to run the environment during 10 | # the 'revision' command, regardless of autogenerate 11 | # revision_environment = false 12 | 13 | 14 | # Logging configuration 15 | [loggers] 16 | keys = root,sqlalchemy,alembic,flask_migrate 17 | 18 | [handlers] 19 | keys = console 20 | 21 | [formatters] 22 | keys = generic 23 | 24 | [logger_root] 25 | level = WARN 26 | handlers = console 27 | qualname = 28 | 29 | [logger_sqlalchemy] 30 | level = WARN 31 | handlers = 32 | qualname = sqlalchemy.engine 33 | 34 | [logger_alembic] 35 | level = INFO 36 | handlers = 37 | qualname = alembic 38 | 39 | [logger_flask_migrate] 40 | level = INFO 41 | handlers = 42 | qualname = flask_migrate 43 | 44 | [handler_console] 45 | class = StreamHandler 46 | args = (sys.stderr,) 47 | level = NOTSET 48 | formatter = generic 49 | 50 | [formatter_generic] 51 | format = %(levelname)-5.5s [%(name)s] %(message)s 52 | datefmt = %H:%M:%S 53 | -------------------------------------------------------------------------------- /servicex_app/migrations/script.py.mako: -------------------------------------------------------------------------------- 1 | """${message} 2 | 3 | Revision ID: ${up_revision} 4 | Revises: ${down_revision | comma,n} 5 | Create Date: ${create_date} 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | ${imports if imports else ""} 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = ${repr(up_revision)} 14 | down_revision = ${repr(down_revision)} 15 | branch_labels = ${repr(branch_labels)} 16 | depends_on = ${repr(depends_on)} 17 | 18 | 19 | def upgrade(): 20 | ${upgrades if upgrades else "pass"} 21 | 22 | 23 | def downgrade(): 24 | ${downgrades if downgrades else "pass"} 25 | -------------------------------------------------------------------------------- /servicex_app/migrations/versions/1.5.6_.py: -------------------------------------------------------------------------------- 1 | """empty message 2 | 3 | Revision ID: 1.5.6 4 | Revises: v1_5_5 5 | Create Date: 2024-11-23 17:30:18.079736 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | # revision identifiers, used by Alembic. 12 | revision = 'v1_5_6' 13 | down_revision = 'v1_5_5' 14 | branch_labels = None 15 | depends_on = None 16 | 17 | """ 18 | Clean up indexes and constraints in the database 19 | """ 20 | def upgrade(): 21 | op.drop_index('ix_dataset_id', table_name='files') 22 | op.drop_constraint('files_dataset_id_fkey', 'files', type_='foreignkey') 23 | op.create_foreign_key(None, 'files', 'datasets', ['dataset_id'], ['id']) 24 | op.alter_column('requests', 'files', 25 | existing_type=sa.INTEGER(), 26 | nullable=False) 27 | op.drop_index('ix_transform_result_request_id', table_name='transform_result') 28 | op.drop_index('ix_transform_result_transform_status', table_name='transform_result') 29 | op.create_index(op.f('ix_users_sub'), 'users', ['sub'], unique=True) 30 | 31 | 32 | def downgrade(): 33 | op.drop_index(op.f('ix_users_sub'), table_name='users') 34 | op.create_index('ix_transform_result_transform_status', 'transform_result', ['transform_status'], unique=False) 35 | op.create_index('ix_transform_result_request_id', 'transform_result', ['request_id'], unique=False) 36 | op.alter_column('requests', 'files', 37 | existing_type=sa.INTEGER(), 38 | nullable=True) 39 | op.drop_constraint(None, 'files', type_='foreignkey') 40 | op.create_foreign_key('files_dataset_id_fkey', 'files', 'datasets', ['dataset_id'], ['id'], ondelete='CASCADE') 41 | op.create_index('ix_dataset_id', 'files', ['dataset_id'], unique=False) 42 | -------------------------------------------------------------------------------- /servicex_app/migrations/versions/1.5.7.py: -------------------------------------------------------------------------------- 1 | """Add unique constraint to TransformResult on request_id and file_id 2 | 3 | Revision ID: 1.5.7 4 | Revises: v1_5_6 5 | Create Date: 2025-01-13 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | # revision identifiers, used by Alembic. 12 | revision = 'v1_5_7' 13 | down_revision = 'v1_5_6' 14 | branch_labels = None 15 | depends_on = None 16 | 17 | def upgrade(): 18 | # Add unique constraint 19 | op.create_unique_constraint('uix_file_request', 'transform_result', ['file_id', 'request_id']) 20 | 21 | 22 | def downgrade(): 23 | # Remove unique constraint 24 | op.drop_constraint('uix_file_request', 'transform_result', type_='unique') 25 | -------------------------------------------------------------------------------- /servicex_app/migrations/versions/1.5.7a.py: -------------------------------------------------------------------------------- 1 | """Switch to index on email not sub 2 | 3 | Revision ID: v1_5_7a 4 | Revises: v1_5_7 5 | Create Date: 2025-01-13 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | # revision identifiers, used by Alembic. 12 | revision = 'v1_5_7a' 13 | down_revision = 'v1_5_7' 14 | branch_labels = None 15 | depends_on = None 16 | 17 | def upgrade(): 18 | # Remove unique constraint 19 | op.drop_index('ix_users_sub', 'users') 20 | op.create_index('ix_users_email', 'users', [sa.text('lower(email)')], unique=True) 21 | 22 | def downgrade(): 23 | # Remove unique constraint 24 | op.drop_index('ix_users_email', 'users') 25 | op.create_index('ix_users_sub', 'users', ['sub'], unique=True) 26 | -------------------------------------------------------------------------------- /servicex_app/migrations/versions/5893086ec440_removing_kafka_column.py: -------------------------------------------------------------------------------- 1 | """removing Kafka column 2 | 3 | Revision ID: 5893086ec440 4 | Revises: a33a96f0f035 5 | Create Date: 2021-11-15 11:33:20.740149 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '5893086ec440' 14 | down_revision = 'a33a96f0f035' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | op.drop_column('requests', 'kafka_broker') 21 | op.drop_column('requests', 'chunk_size') 22 | 23 | 24 | def downgrade(): 25 | op.add_column('requests', sa.Column('kafka_broker', sa.String(length=128), nullable=True)) 26 | op.add_column('requests', sa.Column('chunk_size', sa.Integer(), nullable=True)) 27 | -------------------------------------------------------------------------------- /servicex_app/migrations/versions/79d404108ebe_multipath_support.py: -------------------------------------------------------------------------------- 1 | """multipath support 2 | 3 | Revision ID: 79d404108ebe 4 | Revises: 5893086ec440 5 | Create Date: 2022-01-25 10:12:23.947313 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | from sqlalchemy.sql import table 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '79d404108ebe' 14 | down_revision = '5893086ec440' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | op.drop_column('files', 'file_path') 21 | op.add_column('files', sa.Column('paths', sa.Text())) 22 | files_table = table( 23 | 'files', 24 | sa.Column('file_path', sa.String(length=512)), 25 | sa.Column('paths', sa.Text()) 26 | # Other columns not needed for the data migration 27 | ) 28 | op.execute( 29 | files_table 30 | .update() 31 | .values({'paths': 'old'}) 32 | ) 33 | op.alter_column('files', 'paths', nullable=False) 34 | 35 | 36 | def downgrade(): 37 | op.drop_column('files', 'paths') 38 | op.add_column('files', sa.Column('file_path', sa.String(length=512), nullable=False)) 39 | -------------------------------------------------------------------------------- /servicex_app/migrations/versions/99e97a63d1bd_v1_0_rc_2.py: -------------------------------------------------------------------------------- 1 | """V1.0-RC.2 2 | 3 | Revision ID: 99e97a63d1bd 4 | Revises: b389abb05262 5 | Create Date: 2020-07-25 16:49:02.090979 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '99e97a63d1bd' 14 | down_revision = 'b389abb05262' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('requests', sa.Column('failure_description', sa.String(length=10485760), nullable=True)) 22 | op.add_column('requests', sa.Column('status', sa.String(length=128), nullable=True)) 23 | # ### end Alembic commands ### 24 | 25 | 26 | def downgrade(): 27 | # ### commands auto generated by Alembic - please adjust! ### 28 | op.drop_column('requests', 'status') 29 | op.drop_column('requests', 'failure_description') 30 | # ### end Alembic commands ### 31 | -------------------------------------------------------------------------------- /servicex_app/migrations/versions/a33a96f0f035_v1_0_rc4_a3.py: -------------------------------------------------------------------------------- 1 | """rc4a2 2 | 3 | Revision ID: a33a96f0f035 4 | Revises: 04b9fb8ffee1 5 | Create Date: 2021-06-10 13:43:28.447134 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'a33a96f0f035' 14 | down_revision = '04b9fb8ffee1' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('requests', sa.Column('finish_time', sa.DateTime(), nullable=True)) 22 | # ### end Alembic commands ### 23 | 24 | 25 | def downgrade(): 26 | # ### commands auto generated by Alembic - please adjust! ### 27 | op.drop_column('requests', 'finish_time') 28 | # ### end Alembic commands ### 29 | -------------------------------------------------------------------------------- /servicex_app/migrations/versions/a6cbb6201d3d_v1_0_rc4_a1.py: -------------------------------------------------------------------------------- 1 | """v1.0-rc4-a1 2 | 3 | Revision ID: a6cbb6201d3d 4 | Revises: dd1f9a8a2aee 5 | Create Date: 2021-05-01 15:19:59.936722 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'a6cbb6201d3d' 14 | down_revision = 'dd1f9a8a2aee' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.add_column('requests', sa.Column('title', sa.String(length=128), nullable=True)) 22 | # ### end Alembic commands ### 23 | 24 | 25 | def downgrade(): 26 | # ### commands auto generated by Alembic - please adjust! ### 27 | op.drop_column('requests', 'title') 28 | # ### end Alembic commands ### 29 | -------------------------------------------------------------------------------- /servicex_app/migrations/versions/v1_1_3.py: -------------------------------------------------------------------------------- 1 | """Allow user token to be null. 2 | 3 | Revision ID: v1_1_3 4 | Revises: 208309600a27 5 | Create Date: 2022-12-23 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'v1_1_3' 14 | down_revision = '208309600a27' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | with op.batch_alter_table('users') as batch_op: 21 | batch_op.alter_column('refresh_token', 22 | existing_type=sa.Text, 23 | nullable=True) 24 | 25 | 26 | def downgrade(): 27 | with op.batch_alter_table('users') as batch_op: 28 | batch_op.alter_column('refresh_token', 29 | existing_type=sa.Text, 30 | nullable=False) 31 | -------------------------------------------------------------------------------- /servicex_app/migrations/versions/v1_1_4.py: -------------------------------------------------------------------------------- 1 | """Support for multiple code generators. 2 | 3 | Revision ID: v1_1_4 4 | Revises: v1_1_3 5 | Create Date: 2023-02-15 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'v1_1_4' 14 | down_revision = 'v1_1_3' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | op.add_column('requests', sa.Column("transformer_language", sa.String(256), nullable=True)) 21 | op.add_column('requests', sa.Column("transformer_command", sa.String(256), nullable=True)) 22 | 23 | 24 | def downgrade(): 25 | op.drop_column('requests', 'transformer_language') 26 | op.drop_column('requests', 'transformer_command') 27 | 28 | -------------------------------------------------------------------------------- /servicex_app/migrations/versions/v1_5_5.py: -------------------------------------------------------------------------------- 1 | """ 2 | Mark dataset rows as stale 3 | 4 | Revision ID: v1_5_5 5 | Revises: v1_3_0 6 | Create Date: 2024-11-11 16:06:00.000000 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | from sqlalchemy.dialects import postgresql 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = 'v1_5_5' 14 | down_revision = 'v1_3_0' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade(): 20 | op.add_column('datasets', sa.Column('stale', 21 | sa.Boolean(), 22 | nullable=False, 23 | server_default='false')) 24 | 25 | # Name is no longer unique, so we need to drop the unique index 26 | op.drop_constraint('datasets_name_key', 'datasets') 27 | # op.drop_index('datasets_name_key', table_name='datasets') 28 | 29 | # This was never used, so a fine time to delete it 30 | op.drop_column('transform_result', 'did') 31 | 32 | 33 | def downgrade(): 34 | op.create_unique_constraint('datasets_name_key', 'datasets', ['name']) 35 | op.drop_column('datasets', 'stale') 36 | 37 | op.add_column('transform_result', sa.Column('did', sa.String(), nullable=True)) 38 | -------------------------------------------------------------------------------- /servicex_app/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "servicex-app" 3 | version = "1.6.0-a1" 4 | homepage = "https://iris-hep.org" 5 | license = "BSD" 6 | maintainers = ["ServiceX Team "] 7 | authors = [] 8 | description = "REST Frontend to ServiceX" 9 | readme = "README.rst" 10 | 11 | packages = [{include = "servicex_app"}] 12 | 13 | [tool.poetry.dependencies] 14 | python = "~3.10" 15 | markupsafe = "^2.1.1" 16 | jinja2 = "^3.1.5" 17 | flask = "^2.3.3" 18 | werkzeug = "^3.0.6" 19 | flask-wtf = "^1.0.1" 20 | flask-cors = "^6.0.0" 21 | wtforms = "^3.0.1" 22 | email-validator = "^1.3.0" 23 | Celery = "^5.2.0" 24 | tenacity = "^9.0.0" 25 | pika = "^1.3.1" 26 | flask-restful = "^0.3.9" 27 | flask-jwt-extended = "^4.4.4" 28 | passlib = "^1.7.4" 29 | flask-sqlalchemy = "^3.0.2" 30 | kubernetes = "^25.3.0" 31 | cryptography = "^44.0.1" 32 | bootstrap-flask = "^2.1.0" 33 | blinker = "^1.5" 34 | pre-commit = "^2.20.0" 35 | minio = "^7.1.12" 36 | flask-migrate = "^3.1.0" 37 | psycopg2 = "^2.9.5" 38 | python-logstash = "^0.4.8" 39 | humanize = "^4.4.0" 40 | gunicorn = "^23.0.0" 41 | requests-toolbelt = "^0.10.1" 42 | urllib3 = "1.26.19" 43 | requests = "^2.31" 44 | authlib = "^1.4.1" 45 | flask-moment = "^1.0.6" 46 | 47 | [tool.poetry.group.test] 48 | optional = true 49 | 50 | [tool.poetry.group.test.dependencies] 51 | flake8 = "^5.0.4" 52 | pytest-mock = "^3.10.0" 53 | pytest = "^7.2.0" 54 | pytest-flask = "^1.2.0" 55 | coverage = "^6.5.0" 56 | 57 | 58 | [build-system] 59 | requires = ["poetry-core"] 60 | build-backend = "poetry.core.masonry.api" 61 | 62 | [tool.coverage.run] 63 | branch = true 64 | source = ["servicex_app"] 65 | -------------------------------------------------------------------------------- /servicex_app/servicex_app/app.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019, IRIS-HEP 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are met: 6 | # 7 | # * Redistributions of source code must retain the above copyright notice, this 8 | # list of conditions and the following disclaimer. 9 | # 10 | # * Redistributions in binary form must reproduce the above copyright notice, 11 | # this list of conditions and the following disclaimer in the documentation 12 | # and/or other materials provided with the distribution. 13 | # 14 | # * Neither the name of the copyright holder nor the names of its 15 | # contributors may be used to endorse or promote products derived from 16 | # this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | 30 | from . import create_app 31 | 32 | app = create_app() 33 | -------------------------------------------------------------------------------- /servicex_app/servicex_app/cli/create_default_users.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | 4 | with open('/default_users/users.json', 'r') as f: 5 | users = json.load(f) 6 | 7 | 8 | for user in users: 9 | print(f"Creating user record for {user['name']}") 10 | command = "poetry run flask user create " \ 11 | " '{}' " \ 12 | " '{}' " \ 13 | " '{}' " \ 14 | " '{}' " \ 15 | " '{}' ".format(user["sub"], 16 | user["email"], 17 | user["name"], 18 | user["institution"], 19 | user["refresh_token"]) 20 | os.system(command) 21 | -------------------------------------------------------------------------------- /servicex_app/servicex_app/cli/user_commands.py: -------------------------------------------------------------------------------- 1 | from flask import current_app 2 | from servicex_app.models import UserModel 3 | 4 | 5 | def check_user_exists(sub): 6 | return UserModel.find_by_sub(sub) 7 | 8 | 9 | def add_user(sub, email, name, institution, refresh_token): 10 | new_user = UserModel( 11 | sub=sub, 12 | email=email, 13 | name=name, 14 | institution=institution, 15 | refresh_token=refresh_token) 16 | 17 | if new_user.email == current_app.config.get('JWT_ADMIN'): 18 | new_user.admin = True 19 | if refresh_token: 20 | new_user.pending = False 21 | else: 22 | new_user.pending = True 23 | try: 24 | if not check_user_exists(new_user.sub): 25 | new_user.save_to_db() 26 | except Exception as ex: 27 | print(str(ex)) 28 | 29 | 30 | def list_users() -> None: 31 | users = UserModel.query.all() 32 | print("Sub, Email, Name, Institution, Pending?") 33 | for user in users: 34 | print(", ".join([user.sub, user.email, user.name, user.institution, 35 | "Pending" if user.pending else "Approved"])) 36 | 37 | 38 | def approve_user(sub: str) -> None: 39 | user = UserModel.find_by_sub(sub) 40 | if user and user.pending: 41 | user.pending = False 42 | user.save_to_db() 43 | print(f"User {sub} approved") 44 | elif user and not user.pending: 45 | print(f"User {sub} already approved") 46 | else: 47 | print(f"User {sub} not found") 48 | -------------------------------------------------------------------------------- /servicex_app/servicex_app/mailgun_adaptor.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from flask import current_app, render_template 3 | 4 | from servicex_app.reliable_requests import servicex_retry, REQUEST_TIMEOUT 5 | 6 | 7 | class MailgunAdaptor: 8 | def __init__(self): 9 | self.api_key = current_app.config.get('MAILGUN_API_KEY') 10 | self.domain = current_app.config.get('MAILGUN_DOMAIN') 11 | self.endpoint = f"https://api.mailgun.net/v3/{self.domain}/messages" 12 | 13 | @servicex_retry() 14 | def post_mailgun(self, data) -> requests.Response: 15 | res = requests.post(self.endpoint, data, 16 | auth=("api", self.api_key), 17 | timeout=REQUEST_TIMEOUT) 18 | return res 19 | 20 | def send(self, email: str, template_name: str): 21 | """ 22 | Sends an email to the given address using the given template. 23 | :param email: Email address of recipient. 24 | :param template_name: Name of HTML template file. 25 | """ 26 | if not self.api_key or not self.domain: 27 | return 28 | data = { 29 | "from": f"ServiceX ", 30 | "to": [email], 31 | "subject": "Welcome to ServiceX!", 32 | "html": render_template(f"emails/{template_name}") 33 | } 34 | 35 | res = self.post_mailgun(data) 36 | res.raise_for_status() 37 | -------------------------------------------------------------------------------- /servicex_app/servicex_app/resources/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019, IRIS-HEP 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are met: 6 | # 7 | # * Redistributions of source code must retain the above copyright notice, this 8 | # list of conditions and the following disclaimer. 9 | # 10 | # * Redistributions in binary form must reproduce the above copyright notice, 11 | # this list of conditions and the following disclaimer in the documentation 12 | # and/or other materials provided with the distribution. 13 | # 14 | # * Neither the name of the copyright holder nor the names of its 15 | # contributors may be used to endorse or promote products derived from 16 | # this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /servicex_app/servicex_app/resources/datasets/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2024, IRIS-HEP 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are met: 6 | # 7 | # * Redistributions of source code must retain the above copyright notice, this 8 | # list of conditions and the following disclaimer. 9 | # 10 | # * Redistributions in binary form must reproduce the above copyright notice, 11 | # this list of conditions and the following disclaimer in the documentation 12 | # and/or other materials provided with the distribution. 13 | # 14 | # * Neither the name of the copyright holder nor the names of its 15 | # contributors may be used to endorse or promote products derived from 16 | # this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /servicex_app/servicex_app/resources/internal/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssl-hep/ServiceX/74b29d758dfe8c78e1e0109a706df9ddf1fc56cf/servicex_app/servicex_app/resources/internal/__init__.py -------------------------------------------------------------------------------- /servicex_app/servicex_app/resources/internal/transform_status.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, timezone 2 | 3 | from flask import current_app, request 4 | from servicex_app.models import TransformRequest, db 5 | from servicex_app.resources.servicex_resource import ServiceXResource 6 | 7 | 8 | class TransformationStatusInternal(ServiceXResource): 9 | def post(self, request_id): 10 | current_app.logger.info("--- Transformation Status Update Received ---") 11 | 12 | status = request.get_json() 13 | if 'severity' not in status: 14 | status['severity'] = 'info' 15 | if 'source' not in status or 'info' not in status: 16 | return 'bad status', 400 17 | 18 | current_app.logger.info(f"--{status['info']}--") 19 | if status['severity'] == "fatal": 20 | current_app.logger.error(f"Fatal error reported from " 21 | f"{status['source']}: {status['info']}", 22 | extra={'requestId': request_id}) 23 | 24 | submitted_request = TransformRequest.lookup(request_id) 25 | submitted_request.status = 'Fatal' 26 | submitted_request.finish_time = datetime.now(tz=timezone.utc) 27 | submitted_request.failure_description = status["info"] 28 | submitted_request.save_to_db() 29 | db.session.commit() 30 | else: 31 | current_app.logger.info("Transformation Status Update", extra={ 32 | 'requestId': request_id, 'metric': status}) 33 | -------------------------------------------------------------------------------- /servicex_app/servicex_app/resources/transformation/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssl-hep/ServiceX/74b29d758dfe8c78e1e0109a706df9ddf1fc56cf/servicex_app/servicex_app/resources/transformation/__init__.py -------------------------------------------------------------------------------- /servicex_app/servicex_app/resources/transformation/deployment.py: -------------------------------------------------------------------------------- 1 | from flask import jsonify, current_app 2 | 3 | from servicex_app.decorators import auth_required 4 | from servicex_app.resources.servicex_resource import ServiceXResource 5 | from servicex_app.transformer_manager import TransformerManager 6 | 7 | 8 | class DeploymentStatus(ServiceXResource): 9 | @classmethod 10 | def make_api(cls, transformer_manager: TransformerManager): 11 | cls.transformer_manager = transformer_manager 12 | 13 | @auth_required 14 | def get(self, request_id): 15 | """ 16 | Returns information about the transformer deployment for a given request. 17 | :param request_id: UUID of transformation request. 18 | """ 19 | status = self.transformer_manager.get_deployment_status(request_id) 20 | if status is None: 21 | msg = f"Deployment not found: '{request_id}'" 22 | current_app.logger.error(msg, extra={'requestId': request_id}) 23 | return {'message': msg}, 404 24 | current_app.logger.debug(f"Transformation deployment request: {status.to_dict()}") 25 | return jsonify(status.to_dict()) 26 | -------------------------------------------------------------------------------- /servicex_app/servicex_app/resources/users/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019, IRIS-HEP 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are met: 6 | # 7 | # * Redistributions of source code must retain the above copyright notice, this 8 | # list of conditions and the following disclaimer. 9 | # 10 | # * Redistributions in binary form must reproduce the above copyright notice, 11 | # this list of conditions and the following disclaimer in the documentation 12 | # and/or other materials provided with the distribution. 13 | # 14 | # * Neither the name of the copyright holder nor the names of its 15 | # contributors may be used to endorse or promote products derived from 16 | # this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /servicex_app/servicex_app/resources/users/accept_user.py: -------------------------------------------------------------------------------- 1 | from flask_restful import reqparse 2 | from sqlalchemy.orm.exc import NoResultFound 3 | 4 | from servicex_app.resources.servicex_resource import ServiceXResource 5 | from servicex_app.decorators import admin_required 6 | from servicex_app.models import UserModel 7 | 8 | parser = reqparse.RequestParser() 9 | parser.add_argument('email', help='This field cannot be blank', required=True) 10 | 11 | 12 | class AcceptUser(ServiceXResource): 13 | @admin_required 14 | def post(self): 15 | data = parser.parse_args() 16 | email = data['email'] 17 | try: 18 | UserModel.accept(email) 19 | return {'message': 'user {} now ready for access'.format(email)} 20 | except NoResultFound as err: 21 | return {'message': str(err)}, 404 22 | -------------------------------------------------------------------------------- /servicex_app/servicex_app/resources/users/delete_user.py: -------------------------------------------------------------------------------- 1 | from servicex_app.resources.servicex_resource import ServiceXResource 2 | from servicex_app.decorators import admin_required 3 | from servicex_app.models import UserModel 4 | 5 | 6 | class DeleteUser(ServiceXResource): 7 | @admin_required 8 | def delete(self, user_id): 9 | user: UserModel = UserModel.find_by_id(user_id) 10 | if not user: 11 | return {'message': 'user {} not found'.format(user_id)}, 404 12 | user.delete_from_db() 13 | return {'message': 'user {} has been deleted'.format(user_id)}, 200 14 | -------------------------------------------------------------------------------- /servicex_app/servicex_app/static/Spinner-1s-64px.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssl-hep/ServiceX/74b29d758dfe8c78e1e0109a706df9ddf1fc56cf/servicex_app/servicex_app/static/Spinner-1s-64px.gif -------------------------------------------------------------------------------- /servicex_app/servicex_app/static/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssl-hep/ServiceX/74b29d758dfe8c78e1e0109a706df9ddf1fc56cf/servicex_app/servicex_app/static/favicon.ico -------------------------------------------------------------------------------- /servicex_app/servicex_app/static/main.css: -------------------------------------------------------------------------------- 1 | body { 2 | background: #fafafa; 3 | color: #333333; 4 | margin-top: 56px; 5 | padding: 20px 0; 6 | } 7 | 8 | .site-header .navbar-nav .nav-link { 9 | color: #cbd5db; 10 | } 11 | 12 | .site-header .navbar-nav .nav-link:hover { 13 | color: #ffffff; 14 | } 15 | 16 | .site-header .navbar-nav .nav-link.active { 17 | font-weight: 500; 18 | } 19 | 20 | .content-section { 21 | background: #ffffff; 22 | padding: 20px; 23 | border: 1px solid #dddddd; 24 | border-radius: 3px; 25 | margin-bottom: 20px; 26 | } 27 | 28 | img.spinner { 29 | /* scale: 50%; */ 30 | visibility: hidden; 31 | } 32 | 33 | iframe { 34 | border: 0; 35 | width: 1200px; 36 | height: 1400px; 37 | } -------------------------------------------------------------------------------- /servicex_app/servicex_app/templates/about.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | {% extends "base.html" %} {% block content %} 4 | 5 |
6 |

ServiceX version: {{ config['CHART']}}

7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | {% for key, value in config['CODE_GEN_IMAGES'].items() %} 33 | 34 | 35 | 36 | 37 | {% endfor %} 38 | 39 |
ComponentVersion
App{{ config['APP_IMAGE_TAG']}}
DID Finder Rucio{{ config['DID_RUCIO_FINDER_TAG']}}
DID Finder CERN Open Data{{ config['DID_CERNOPENDATA_FINDER_TAG']}}
DID Finder XRootD{{ config['DID_XROOTD_FINDER_TAG']}}
{{ key }}{{ value }}
40 |
41 | 42 | {% endblock %} 43 | -------------------------------------------------------------------------------- /servicex_app/servicex_app/templates/emails/welcome.html: -------------------------------------------------------------------------------- 1 | 2 |

3 | Your ServiceX account has been approved! 4 |

5 |
6 | Check out the ServiceX 7 | docs 8 | to get started! 9 |
10 | 11 | -------------------------------------------------------------------------------- /servicex_app/servicex_app/templates/get_started.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |
Get Started
4 |
5 |

6 | Check out our guide to making your first transformation request. 7 | Recent requests will show up here. 8 |

9 |
10 | 11 | Go to Docs 12 | 13 |
14 |
15 | -------------------------------------------------------------------------------- /servicex_app/servicex_app/templates/global_dashboard.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | 3 | {% from 'requests_table.html' import requests_table, requests_table_update_script with context %} 4 | {% from 'sort_dropdown.html' import sort_dropdown %} 5 | 6 | {% block content %} 7 | 8 |
9 |
10 |
11 |

Transformation Requests

12 | {{ sort_dropdown(dropdown_options, active_sort, active_order) }} 13 |
14 | {{ requests_table(pagination, humanize) }} 15 |
16 |
17 | 18 | {% endblock %} 19 | 20 | {% block scripts %} 21 | {{ super() }} 22 | {{ requests_table_update_script(pagination) }} 23 | {% endblock %} 24 | -------------------------------------------------------------------------------- /servicex_app/servicex_app/templates/home.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | {% block content %} 3 |

Welcome to ServiceX!

4 | {% endblock %} -------------------------------------------------------------------------------- /servicex_app/servicex_app/templates/logs.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | 3 | {% block content %} 4 | 5 | {% endblock %} 6 | 7 | 8 | {% block scripts %} 9 | {{ super() }} 10 | 11 | 22 | 23 | {% endblock %} -------------------------------------------------------------------------------- /servicex_app/servicex_app/templates/monitor.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | 3 | {% block content %} 4 | 5 | {% endblock %} 6 | 7 | {% block scripts %} 8 | {{ super() }} 9 | 10 | 21 | 22 | {% endblock %} -------------------------------------------------------------------------------- /servicex_app/servicex_app/templates/profile_form.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | {% extends "base.html" %} {% from 'bootstrap4/form.html' import render_form %} 4 | {% block content %} 5 |
6 |

{{ action }}

7 | {{ render_form(form, button_style='outline-primary', extra_classes='d-flex 8 | flex-column') }} 9 |
10 | {% endblock %} 11 | -------------------------------------------------------------------------------- /servicex_app/servicex_app/templates/sort_dropdown.html: -------------------------------------------------------------------------------- 1 | {% macro sort_dropdown(options, active_sort, active_order) %} 2 |
3 | 7 |
8 | {% for sort, order in options %} 9 | {{ sort.title() }} ({{ order }}) 10 | {% endfor %} 11 |
12 |
13 | {% endmacro %} 14 | -------------------------------------------------------------------------------- /servicex_app/servicex_app/templates/transformation_results.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | 3 | {% from 'bootstrap5/table.html' import render_table %} 4 | {% from 'bootstrap5/pagination.html' import render_pager %} 5 | {% from 'bootstrap5/pagination.html' import render_pagination %} 6 | 7 | {% block content %} 8 |
9 |
10 |

11 | Results for Transformation 12 | {{ treq.request_id }} 13 |

14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | {% for res in pagination.items %} 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | {% endfor %} 38 | 39 |
IDFile PathStatusTimeEventsBytesRate
{{ res.file_id }}{{ res.file_path }}{{ res.transform_status }}{{ res.transform_time }}{{ res.total_events }}{{ res.total_bytes }}{{ res.avg_rate }}
40 | {% if pagination.items %} 41 | {{ render_pagination(pagination, align='center') }} 42 | {% else %} 43 |
No results found!
44 | {% endif %} 45 |
46 |
47 | {% endblock %} -------------------------------------------------------------------------------- /servicex_app/servicex_app/templates/user_dashboard.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | 3 | {% from 'requests_table.html' import requests_table, requests_table_update_script with context %} 4 | {% from 'sort_dropdown.html' import sort_dropdown %} 5 | 6 | {% block content %} 7 | 8 |
9 |
10 |
11 |

Transformation Requests

12 | {{ sort_dropdown(dropdown_options, active_sort, active_order) }} 13 |
14 | {{ requests_table(pagination, humanize) }} 15 |
16 |
17 | 18 | {% endblock %} 19 | 20 | {% block scripts %} 21 | {{ super() }} 22 | {{ requests_table_update_script(pagination) }} 23 | {% endblock %} -------------------------------------------------------------------------------- /servicex_app/servicex_app/web/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssl-hep/ServiceX/74b29d758dfe8c78e1e0109a706df9ddf1fc56cf/servicex_app/servicex_app/web/__init__.py -------------------------------------------------------------------------------- /servicex_app/servicex_app/web/about.py: -------------------------------------------------------------------------------- 1 | from flask import render_template 2 | 3 | 4 | def about(): 5 | return render_template('about.html') 6 | -------------------------------------------------------------------------------- /servicex_app/servicex_app/web/api_token.py: -------------------------------------------------------------------------------- 1 | from flask import redirect, url_for, session, flash, current_app 2 | from flask_jwt_extended import create_refresh_token 3 | 4 | from servicex_app.models import db, UserModel 5 | from servicex_app.decorators import oauth_required 6 | 7 | 8 | @oauth_required 9 | def api_token(): 10 | """Generate a new ServiceX refresh token.""" 11 | email = session.get('email') 12 | user: UserModel = UserModel.find_by_email(email) 13 | user.refresh_token = create_refresh_token(email) 14 | db.session.commit() 15 | current_app.logger.info(f"Generated new API token for {email}") 16 | flash("Your new API token has been generated!", 'success') 17 | return redirect(url_for('profile')) 18 | -------------------------------------------------------------------------------- /servicex_app/servicex_app/web/dashboard.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | 3 | from flask import render_template, session 4 | from flask_restful import reqparse 5 | from servicex_app.models import TransformRequest 6 | 7 | model_attributes = { 8 | "start": TransformRequest.submit_time, 9 | "finish": TransformRequest.finish_time, 10 | "status": TransformRequest.status 11 | } 12 | parser = reqparse.RequestParser() 13 | parser.add_argument("page", default=1, type=int, location='args') 14 | sort_choices = tuple(model_attributes.keys()) 15 | parser.add_argument( 16 | "sort", 17 | choices=sort_choices, 18 | default="start", 19 | location='args', 20 | help=f"Sort must be one of: {', '.join(map(repr, sort_choices))}." 21 | ) 22 | order_choices = ("asc", "desc") 23 | parser.add_argument( 24 | "order", 25 | choices=order_choices, 26 | default="desc", 27 | location='args', 28 | help="Order must be 'asc' or 'desc'." 29 | ) 30 | 31 | 32 | def dashboard(template_name: str, user_specific=False): 33 | args = parser.parse_args() 34 | sort, order = args["sort"], args["order"] 35 | query = TransformRequest.query.filter_by() 36 | 37 | if user_specific: 38 | query = query.filter_by(submitted_by=session["user_id"]) 39 | 40 | sort_column = model_attributes[sort] 41 | sort_order = sort_column.asc() if order == "asc" else sort_column.desc() 42 | pagination = query \ 43 | .order_by(sort_order) \ 44 | .paginate(page=args["page"], per_page=15, error_out=False) 45 | return render_template( 46 | template_name, 47 | pagination=pagination, 48 | dropdown_options=list(itertools.product(sort_choices, order_choices)), 49 | active_sort=sort, 50 | active_order=order, 51 | ) 52 | -------------------------------------------------------------------------------- /servicex_app/servicex_app/web/edit_profile.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | 3 | from flask import session, render_template, redirect, url_for, \ 4 | request, flash, current_app 5 | 6 | from servicex_app.models import db, UserModel 7 | from servicex_app.decorators import oauth_required 8 | from .forms import ProfileForm 9 | 10 | 11 | @oauth_required 12 | def edit_profile(): 13 | email = session.get('email') 14 | user: UserModel = UserModel.find_by_email(email) 15 | form = ProfileForm() 16 | if request.method == 'GET': 17 | form = ProfileForm(user) 18 | elif request.method == 'POST': 19 | if form.validate_on_submit(): 20 | user.name = form.name.data 21 | user.email = form.email.data 22 | user.institution = form.institution.data 23 | user.experiment = form.experiment.data 24 | user.updated_at = datetime.utcnow() 25 | db.session.commit() 26 | flash("Your profile has been saved!", 'success') 27 | current_app.logger.info(f"Updated profile for {user.name}") 28 | return redirect(url_for('profile')) 29 | else: 30 | current_app.logger.error(f"Edit Profile Form errors {form.errors}") 31 | flash("Profile could not be saved. Please fix invalid fields below.", 'danger') 32 | return render_template("profile_form.html", form=form, action="Edit Profile") 33 | -------------------------------------------------------------------------------- /servicex_app/servicex_app/web/forms.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | 3 | from flask_wtf import FlaskForm 4 | from wtforms import StringField, SelectField, SubmitField 5 | from wtforms.validators import DataRequired, Length, Email 6 | 7 | from servicex_app.models import UserModel 8 | 9 | 10 | class ProfileForm(FlaskForm): 11 | name = StringField('Full Name', validators=[DataRequired(), Length(0, 120)]) 12 | email = StringField('Email', validators=[DataRequired(), Email()]) 13 | institution = StringField('Institution', validators=[DataRequired()]) 14 | experiment = SelectField('Experiment', validators=[DataRequired()], 15 | choices=[("ATLAS", "ATLAS"), ("CMS", "CMS")], 16 | default="ATLAS") 17 | submit = SubmitField('Save Profile') 18 | 19 | def __init__(self, user: Optional[UserModel] = None): 20 | super().__init__() 21 | if user: 22 | self.name.data = user.name 23 | self.email.data = user.email 24 | self.institution.data = user.institution 25 | self.experiment.data = user.experiment 26 | -------------------------------------------------------------------------------- /servicex_app/servicex_app/web/global_dashboard.py: -------------------------------------------------------------------------------- 1 | from servicex_app.decorators import admin_required 2 | from servicex_app.web.dashboard import dashboard 3 | 4 | 5 | @admin_required 6 | def global_dashboard(): 7 | return dashboard("global_dashboard.html", user_specific=False) 8 | -------------------------------------------------------------------------------- /servicex_app/servicex_app/web/home.py: -------------------------------------------------------------------------------- 1 | from flask import render_template 2 | 3 | 4 | def home(): 5 | return render_template('home.html') 6 | -------------------------------------------------------------------------------- /servicex_app/servicex_app/web/logs.py: -------------------------------------------------------------------------------- 1 | from flask import render_template 2 | 3 | 4 | def logs(): 5 | return render_template('logs.html') 6 | -------------------------------------------------------------------------------- /servicex_app/servicex_app/web/monitor.py: -------------------------------------------------------------------------------- 1 | from flask import render_template 2 | 3 | 4 | def monitor(): 5 | return render_template('monitor.html') 6 | -------------------------------------------------------------------------------- /servicex_app/servicex_app/web/multiple_codegen_list.py: -------------------------------------------------------------------------------- 1 | from flask import current_app, jsonify 2 | 3 | 4 | def multiple_codegen_list(): 5 | return jsonify(current_app.config.get('CODE_GEN_SERVICE_URLS')) 6 | -------------------------------------------------------------------------------- /servicex_app/servicex_app/web/sign_in.py: -------------------------------------------------------------------------------- 1 | from flask import redirect, url_for 2 | 3 | 4 | def sign_in(): 5 | """Send the user to OIDC Auth.""" 6 | return redirect(url_for('auth_callback')) 7 | -------------------------------------------------------------------------------- /servicex_app/servicex_app/web/sign_out.py: -------------------------------------------------------------------------------- 1 | from flask import redirect, url_for, current_app, session 2 | 3 | from servicex_app.decorators import oauth_required 4 | from .utils import load_oauth_client 5 | 6 | 7 | @oauth_required 8 | def sign_out(): 9 | """Revoke tokens with OIDC and destroy session state.""" 10 | from authlib.integrations.requests_client import OAuth2Session 11 | oauth = load_oauth_client() 12 | client = OAuth2Session(oauth.oauth.client_id, oauth.oauth.client_secret, 13 | scope=oauth.oauth.client_kwargs['scope']) 14 | oauth.oauth.load_server_metadata() 15 | for ty in ('access_token', 'refresh_token'): 16 | if ty in session['tokens']: 17 | client.revoke_token(oauth.oauth.server_metadata['revocation_endpoint'], 18 | token=session['tokens'][ty]) 19 | 20 | session.clear() 21 | 22 | redirect_uri = url_for('home', _external=True) 23 | if 'end_session_endpoint' in oauth.oauth.server_metadata: 24 | ga_logout_url = ''.join([ 25 | oauth.oauth.server_metadata['end_session_endpoint'], 26 | f"?client={current_app.config['OAUTH_CLIENT_ID']}", 27 | f"&redirect_uri={redirect_uri}", 28 | "&redirect_name=ServiceX Portal" 29 | ]) 30 | return redirect(ga_logout_url) 31 | else: 32 | # if metadata doesn't give us the relevant endpoint (e.g. Globus) 33 | return redirect(redirect_uri) 34 | -------------------------------------------------------------------------------- /servicex_app/servicex_app/web/transformation_request.py: -------------------------------------------------------------------------------- 1 | from flask import render_template, abort, current_app 2 | 3 | from servicex_app.decorators import oauth_required 4 | from servicex_app.models import TransformRequest 5 | 6 | 7 | @oauth_required 8 | def transformation_request(id_: str): 9 | current_app.logger.debug(f"Got transformation request: {id_}") 10 | req = TransformRequest.lookup(id_) 11 | if not req: 12 | abort(404) 13 | return render_template('transformation_request.html', req=req) 14 | -------------------------------------------------------------------------------- /servicex_app/servicex_app/web/transformation_results.py: -------------------------------------------------------------------------------- 1 | from flask import abort, render_template, request 2 | from flask_sqlalchemy.pagination import Pagination 3 | from servicex_app.decorators import oauth_required 4 | from servicex_app.models import TransformationResult, TransformRequest 5 | 6 | 7 | @oauth_required 8 | def transformation_results(id_: str): 9 | treq = TransformRequest.lookup(id_) 10 | if not treq: 11 | abort(404) 12 | page = request.args.get('page', 1, type=int) 13 | filter_by_values = {} 14 | status = request.args.get("status") 15 | if status: 16 | filter_by_values["transform_status"] = status 17 | pagination: Pagination = TransformationResult.query\ 18 | .filter_by(request_id=treq.request_id, **filter_by_values)\ 19 | .order_by(TransformationResult.file_id.asc())\ 20 | .paginate(page=page, per_page=100, error_out=False) 21 | return render_template("transformation_results.html", treq=treq, pagination=pagination) 22 | -------------------------------------------------------------------------------- /servicex_app/servicex_app/web/user_dashboard.py: -------------------------------------------------------------------------------- 1 | from servicex_app.decorators import oauth_required 2 | from servicex_app.web.dashboard import dashboard 3 | 4 | 5 | @oauth_required 6 | def user_dashboard(): 7 | return dashboard("user_dashboard.html", user_specific=True) 8 | -------------------------------------------------------------------------------- /servicex_app/servicex_app/web/utils.py: -------------------------------------------------------------------------------- 1 | from flask import current_app 2 | from authlib.integrations.flask_client import OAuth 3 | 4 | oauth = None 5 | 6 | 7 | def load_oauth_client(): 8 | global oauth 9 | if oauth is not None: 10 | return oauth 11 | oauth = OAuth() 12 | oauth.register( 13 | name='oauth', 14 | server_metadata_url=current_app.config['OAUTH_METADATA_URL'], 15 | client_id=current_app.config['OAUTH_CLIENT_ID'], 16 | client_secret=current_app.config['OAUTH_CLIENT_SECRET'], 17 | client_kwargs={'scope': 'openid profile email'} 18 | ) 19 | oauth.init_app(current_app) 20 | return oauth 21 | -------------------------------------------------------------------------------- /servicex_app/servicex_app/web/view_profile.py: -------------------------------------------------------------------------------- 1 | from flask import session, render_template, redirect, url_for 2 | 3 | from servicex_app.models import UserModel 4 | from servicex_app.decorators import oauth_required 5 | 6 | 7 | @oauth_required 8 | def view_profile(): 9 | email = session.get('email') 10 | user = UserModel.find_by_email(email) 11 | if not user: 12 | return redirect(url_for('create_profile')) 13 | return render_template('profile.html', user=user) 14 | -------------------------------------------------------------------------------- /servicex_app/servicex_app_test/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019, IRIS-HEP 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are met: 6 | # 7 | # * Redistributions of source code must retain the above copyright notice, this 8 | # list of conditions and the following disclaimer. 9 | # 10 | # * Redistributions in binary form must reproduce the above copyright notice, 11 | # this list of conditions and the following disclaimer in the documentation 12 | # and/or other materials provided with the distribution. 13 | # 14 | # * Neither the name of the copyright holder nor the names of its 15 | # contributors may be used to endorse or promote products derived from 16 | # this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /servicex_app/servicex_app_test/busybox/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2024, IRIS-HEP 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are met: 6 | # 7 | # * Redistributions of source code must retain the above copyright notice, this 8 | # list of conditions and the following disclaimer. 9 | # 10 | # * Redistributions in binary form must reproduce the above copyright notice, 11 | # this list of conditions and the following disclaimer in the documentation 12 | # and/or other materials provided with the distribution. 13 | # 14 | # * Neither the name of the copyright holder nor the names of its 15 | # contributors may be used to endorse or promote products derived from 16 | # this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /servicex_app/servicex_app_test/conftest.py: -------------------------------------------------------------------------------- 1 | from pytest import fixture 2 | 3 | 4 | @fixture 5 | def mock_jwt_extended(mocker): 6 | """ 7 | During unit tests, functions from Flask-JWT-extended are mocked to do nothing. 8 | """ 9 | mocker.patch('servicex_app.decorators.verify_jwt_in_request') 10 | -------------------------------------------------------------------------------- /servicex_app/servicex_app_test/resources/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019, IRIS-HEP 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are met: 6 | # 7 | # * Redistributions of source code must retain the above copyright notice, this 8 | # list of conditions and the following disclaimer. 9 | # 10 | # * Redistributions in binary form must reproduce the above copyright notice, 11 | # this list of conditions and the following disclaimer in the documentation 12 | # and/or other materials provided with the distribution. 13 | # 14 | # * Neither the name of the copyright holder nor the names of its 15 | # contributors may be used to endorse or promote products derived from 16 | # this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /servicex_app/servicex_app_test/resources/datasets/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2024, IRIS-HEP 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are met: 6 | # 7 | # * Redistributions of source code must retain the above copyright notice, this 8 | # list of conditions and the following disclaimer. 9 | # 10 | # * Redistributions in binary form must reproduce the above copyright notice, 11 | # this list of conditions and the following disclaimer in the documentation 12 | # and/or other materials provided with the distribution. 13 | # 14 | # * Neither the name of the copyright holder nor the names of its 15 | # contributors may be used to endorse or promote products derived from 16 | # this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /servicex_app/servicex_app_test/resources/internal/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssl-hep/ServiceX/74b29d758dfe8c78e1e0109a706df9ddf1fc56cf/servicex_app/servicex_app_test/resources/internal/__init__.py -------------------------------------------------------------------------------- /servicex_app/servicex_app_test/resources/transformation/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssl-hep/ServiceX/74b29d758dfe8c78e1e0109a706df9ddf1fc56cf/servicex_app/servicex_app_test/resources/transformation/__init__.py -------------------------------------------------------------------------------- /servicex_app/servicex_app_test/resources/users/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssl-hep/ServiceX/74b29d758dfe8c78e1e0109a706df9ddf1fc56cf/servicex_app/servicex_app_test/resources/users/__init__.py -------------------------------------------------------------------------------- /servicex_app/servicex_app_test/resources/users/test_accept_user.py: -------------------------------------------------------------------------------- 1 | from servicex_app_test.web.web_test_base import WebTestBase 2 | 3 | 4 | class TestAcceptUser(WebTestBase): 5 | def test_accept_user(self, user, client): 6 | user.pending = True 7 | response = client.post('/accept', json={"email": user.email}) 8 | assert response.status_code == 200 9 | assert not user.pending 10 | assert user.save_to_db.called_once() 11 | 12 | def test_accept_user_missing(self, client, mocker): 13 | mocker.patch('servicex_app.models.UserModel.find_by_email', return_value=None) 14 | response = client.post('/accept', json={"email": 'janedoe@example.com'}) 15 | assert response.status_code == 404 16 | -------------------------------------------------------------------------------- /servicex_app/servicex_app_test/resources/users/test_all_users.py: -------------------------------------------------------------------------------- 1 | from flask.wrappers import Response 2 | 3 | from servicex_app_test.web.web_test_base import WebTestBase 4 | 5 | 6 | class TestAllUsers(WebTestBase): 7 | def test_get_users(self, client, mocker): 8 | resp_json = {'users': []} 9 | mock = mocker.patch('servicex_app.models.UserModel.return_all', 10 | return_value=resp_json) 11 | response: Response = client.get('/users') 12 | assert response.status_code == 200 13 | assert mock.called_once() 14 | assert response.json == resp_json 15 | 16 | def test_delete_users(self, client, mocker): 17 | resp_json = {'message': '0 row(s) deleted'} 18 | mock = mocker.patch('servicex_app.models.UserModel.delete_all', 19 | return_value=resp_json) 20 | response: Response = client.delete('/users') 21 | assert response.status_code == 200 22 | assert mock.called_once() 23 | assert response.json == resp_json 24 | -------------------------------------------------------------------------------- /servicex_app/servicex_app_test/resources/users/test_delete_user.py: -------------------------------------------------------------------------------- 1 | from flask.wrappers import Response 2 | 3 | from servicex_app_test.web.web_test_base import WebTestBase 4 | 5 | 6 | class TestDeleteUser(WebTestBase): 7 | def test_delete_user(self, mocker, client, user): 8 | user.id = 7 9 | resp_json = {'message': f'user {user.id} has been deleted'} 10 | resp: Response = client.delete(f'users/{user.id}') 11 | assert resp.status_code == 200 12 | assert resp.json == resp_json 13 | assert user.delete_from_db.called_once() 14 | 15 | def test_delete_user_missing(self, client): 16 | fake_user_id = 12345 17 | resp_json = {'message': f'user {fake_user_id} not found'} 18 | resp: Response = client.delete(f'users/{fake_user_id}') 19 | assert resp.status_code == 404 20 | assert resp.json == resp_json 21 | -------------------------------------------------------------------------------- /servicex_app/servicex_app_test/resources/users/test_pending_users.py: -------------------------------------------------------------------------------- 1 | from flask.wrappers import Response 2 | 3 | from servicex_app_test.web.web_test_base import WebTestBase 4 | 5 | 6 | class TestPendingUsers(WebTestBase): 7 | def test_get_pending_users(self, client, mocker): 8 | resp_json = {'users': []} 9 | mock = mocker.patch('servicex_app.models.UserModel.return_all_pending', 10 | return_value=resp_json) 11 | response: Response = client.get('/pending') 12 | assert response.status_code == 200 13 | assert mock.called_once() 14 | assert response.json == resp_json 15 | 16 | def test_delete_pending_users(self, client, mocker): 17 | resp_json = {'message': '0 row(s) deleted'} 18 | mock = mocker.patch('servicex_app.models.UserModel.delete_all_pending', 19 | return_value=resp_json) 20 | response: Response = client.delete('/pending') 21 | assert response.status_code == 200 22 | assert mock.called_once() 23 | assert response.json == resp_json 24 | -------------------------------------------------------------------------------- /servicex_app/servicex_app_test/test.config: -------------------------------------------------------------------------------- 1 | FOO = "bar" 2 | SECRET_VALUE = "blah" 3 | BOOL_VALUE = False 4 | VALID_DID_SCHEMES = "rucio" 5 | DID_FINDER_DEFAULT_SCHEME = "rucio" 6 | OBJECT_STORE_ENABLED = False 7 | TRANSFORMER_MANAGER_ENABLED = False 8 | RABBIT_MQ_URL = "amqp://foo.com" 9 | CODE_GEN_SERVICE_URL = "http://code.gen" 10 | ADVERTISED_HOSTNAME = "service.x" 11 | SQLALCHEMY_DATABASE_URI = 'sqlite:///:memory:' 12 | 13 | 14 | CODE_GEN_SERVICE_URLS = { 15 | 'atlasxaod': 'http://servicex-code-gen-atlasxaod:8000', 16 | 'cms': 'http://servicex-code-gen-cms:8000', 17 | 'python': 'http://servicex-code-gen-python:8000', 18 | 'uproot': 'http://servicex-code-gen-uproot:8000' 19 | } 20 | 21 | CODE_GEN_IMAGES = { 22 | 'atlasxaod': 'sslhep/servicex_code_gen_func_adl_xaod:develop', 23 | 'cms': 'sslhep/servicex_code_gen_cms_aod:develop', 24 | 'python': 'sslhep/servicex_code_gen_python:develop', 25 | 'uproot': 'sslhep/servicex_code_gen_func_adl_uproot:develop' 26 | } 27 | 28 | -------------------------------------------------------------------------------- /servicex_app/servicex_app_test/test_app_init.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import servicex_app 4 | 5 | 6 | class TestAppInit: 7 | def test_read_from_config(self): 8 | os.environ['APP_CONFIG_FILE'] = str(os.path.join(os.path.dirname(__file__), 9 | "test.config")) 10 | app = servicex_app.create_app() 11 | assert app.config['FOO'] == 'bar' 12 | assert app.config['SECRET_VALUE'] == 'blah' 13 | assert not app.config['BOOL_VALUE'] 14 | 15 | os.environ['SECRET_VALUE'] = 'shhh' 16 | os.environ['BOOL_VALUE'] = 'true' 17 | app_from_env = servicex_app.create_app() 18 | assert app_from_env.config['SECRET_VALUE'] == 'shhh' 19 | assert app_from_env.config['BOOL_VALUE'] 20 | -------------------------------------------------------------------------------- /servicex_app/servicex_app_test/test_mailgun_adaptor.py: -------------------------------------------------------------------------------- 1 | from flask import render_template 2 | 3 | from servicex_app.mailgun_adaptor import MailgunAdaptor 4 | from servicex_app_test.resource_test_base import ResourceTestBase 5 | 6 | 7 | class TestMailgunAdaptor(ResourceTestBase): 8 | def test_noop(self, mocker, client): 9 | import requests 10 | mock_post = mocker.patch.object(requests, "post") 11 | with client.application.app_context(): 12 | mailgun = MailgunAdaptor() 13 | mailgun.send('janedoe@example.com', 'welcome.html') 14 | mock_post.assert_not_called() 15 | 16 | def test_init(self): 17 | config = {'MAILGUN_API_KEY': 'key123', 'MAILGUN_DOMAIN': 'example.com'} 18 | client = self._test_client(extra_config=config) 19 | with client.application.app_context(): 20 | mailgun = MailgunAdaptor() 21 | assert mailgun.api_key == config['MAILGUN_API_KEY'] 22 | assert mailgun.domain == config['MAILGUN_DOMAIN'] 23 | 24 | def test_send(self, mocker): 25 | import requests 26 | mock_post = mocker.patch.object(requests, "post") 27 | config = {'MAILGUN_API_KEY': 'key123', 'MAILGUN_DOMAIN': 'example.com'} 28 | client = self._test_client(extra_config=config) 29 | with client.application.app_context(): 30 | mailgun = MailgunAdaptor() 31 | email, template_name = 'janedoe@example.com', 'welcome.html' 32 | mailgun.send(email, template_name) 33 | mock_post.assert_called_once() 34 | args, kwargs = mock_post.call_args 35 | endpoint_arg, data_arg = args 36 | assert data_arg['to'] == [email] 37 | assert data_arg['html'] == render_template(f"emails/{template_name}") 38 | -------------------------------------------------------------------------------- /servicex_app/servicex_app_test/web/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssl-hep/ServiceX/74b29d758dfe8c78e1e0109a706df9ddf1fc56cf/servicex_app/servicex_app_test/web/__init__.py -------------------------------------------------------------------------------- /servicex_app/servicex_app_test/web/test_about.py: -------------------------------------------------------------------------------- 1 | from flask import Response 2 | 3 | from .web_test_base import WebTestBase 4 | 5 | 6 | class TestAbout(WebTestBase): 7 | def test_about(self, client, captured_templates): 8 | response: Response = client.get('/about') 9 | assert response.status_code == 200 10 | template, context = captured_templates[0] 11 | assert template.name == "about.html" 12 | 13 | html = response.data.decode('utf-8') 14 | 15 | assert "atlasxaod" in html 16 | assert "sslhep/servicex_code_gen_func_adl_xaod:develop" in html 17 | 18 | assert "develop" in html 19 | -------------------------------------------------------------------------------- /servicex_app/servicex_app_test/web/test_api_token.py: -------------------------------------------------------------------------------- 1 | from flask import Response, url_for 2 | 3 | from .web_test_base import WebTestBase 4 | 5 | 6 | class TestApiToken(WebTestBase): 7 | def test_api_token(self, client, user, db): 8 | with client.session_transaction() as sess: 9 | sess['sub'] = user.sub 10 | user.refresh_token = 'jwt:refresh' 11 | response: Response = client.get(url_for('api_token')) 12 | assert user.refresh_token != 'jwt:refresh' 13 | assert db.session.commit.called_once() 14 | assert response.status_code == 302 15 | assert response.location == '/profile' 16 | -------------------------------------------------------------------------------- /servicex_app/servicex_app_test/web/test_global_dashboard.py: -------------------------------------------------------------------------------- 1 | from flask import Response, url_for 2 | from pytest import fixture 3 | 4 | from .web_test_base import WebTestBase 5 | 6 | 7 | class TestGlobalDashboard(WebTestBase): 8 | 9 | @fixture 10 | def mock_query(self, mocker): 11 | mock_tr = mocker.patch("servicex_app.web.dashboard.TransformRequest") 12 | return mock_tr.query.filter_by().order_by.return_value 13 | 14 | def test_get_empty_state(self, client, user, mock_query, captured_templates): 15 | pagination = mock_query.paginate(page=1, per_page=15, total=0, items=[]) 16 | mock_query.paginate.return_value = pagination 17 | response: Response = client.get(url_for('global-dashboard'), headers=self.fake_header()) 18 | assert response.status_code == 200 19 | template, context = captured_templates[0] 20 | assert template.name == "global_dashboard.html" 21 | assert context["pagination"] == pagination 22 | 23 | def test_get_with_results(self, client, user, mock_query, captured_templates): 24 | items = [self._test_transformation_req(id=i+1) for i in range(3)] 25 | pagination = mock_query.paginate(page=1, per_page=15, total=100, items=items) 26 | mock_query.paginate.return_value = pagination 27 | response: Response = client.get(url_for('global-dashboard'), headers=self.fake_header()) 28 | assert response.status_code == 200 29 | template, context = captured_templates[0] 30 | assert template.name == "global_dashboard.html" 31 | assert context["pagination"] == pagination 32 | -------------------------------------------------------------------------------- /servicex_app/servicex_app_test/web/test_home.py: -------------------------------------------------------------------------------- 1 | from flask import Response 2 | 3 | from .web_test_base import WebTestBase 4 | 5 | 6 | class TestHome(WebTestBase): 7 | def test_home(self, client, captured_templates): 8 | response: Response = client.get('/') 9 | assert response.status_code == 200 10 | template, context = captured_templates[0] 11 | assert template.name == "home.html" 12 | -------------------------------------------------------------------------------- /servicex_app/servicex_app_test/web/test_sign_in.py: -------------------------------------------------------------------------------- 1 | from flask import Response, url_for 2 | 3 | from .web_test_base import WebTestBase 4 | 5 | 6 | class TestSignIn(WebTestBase): 7 | def test_sign_in(self, client): 8 | response: Response = client.get(url_for('sign_in')) 9 | assert response.status_code == 302 10 | assert response.location == url_for('auth_callback') 11 | -------------------------------------------------------------------------------- /servicex_app/servicex_app_test/web/test_sign_out.py: -------------------------------------------------------------------------------- 1 | from urllib.parse import quote 2 | 3 | from flask import Response, url_for, session 4 | 5 | from .web_test_base import WebTestBase 6 | 7 | 8 | class TestSignOut(WebTestBase): 9 | def test_sign_out(self, mocker, oauth_client, oauth_session, client): 10 | oauth_tokens = self._oauth_tokens() 11 | with client.session_transaction() as sess: 12 | sess['tokens'] = oauth_tokens 13 | response: Response = client.get(url_for('sign_out')) 14 | relevant_tokens = [_[1] for _ in oauth_tokens.items() 15 | if _[0] in ('access_token', 'refresh_token')] 16 | calls = [ 17 | mocker.call('https://auth.globus.org/v2/oauth2/token/revoke', 18 | token=token_info) 19 | for token_info in relevant_tokens] 20 | assert len(oauth_session.mock_calls) == len(relevant_tokens) 21 | oauth_session.revoke_token.assert_has_calls(calls) 22 | assert not session.get('tokens') 23 | ga_logout_url = ''.join([ 24 | "https://auth.globus.org/v2/web/logout", 25 | f"?client={client.application.config['OAUTH_CLIENT_ID']}", 26 | f"&redirect_uri={url_for('home', _external=True)}", 27 | f"&redirect_name={quote('ServiceX Portal')}" 28 | ]) 29 | assert response.status_code == 302 30 | assert response.location == ga_logout_url 31 | -------------------------------------------------------------------------------- /servicex_app/servicex_app_test/web/test_transformation_request.py: -------------------------------------------------------------------------------- 1 | from flask import Response, url_for 2 | 3 | from pytest import fixture 4 | 5 | from servicex_app.models import TransformRequest 6 | from .web_test_base import WebTestBase 7 | 8 | 9 | class TestTransformationRequest(WebTestBase): 10 | endpoint = "transformation_request" 11 | module = "servicex_app.web.transformation_request" 12 | template_name = 'transformation_request.html' 13 | 14 | @fixture 15 | def mock_tr_cls(self, mocker): 16 | return mocker.patch(f"{self.module}.TransformRequest") 17 | 18 | @fixture 19 | def mock_tr(self, mock_tr_cls) -> TransformRequest: 20 | req = self._test_transformation_req() 21 | mock_tr_cls.lookup.return_value = req 22 | return req 23 | 24 | def test_get_by_primary_key(self, client, mock_tr: TransformRequest, captured_templates): 25 | resp: Response = client.get(url_for(self.endpoint, id_=mock_tr.id)) 26 | assert resp.status_code == 200 27 | template, context = captured_templates[0] 28 | assert template.name == self.template_name 29 | assert context["req"] == mock_tr 30 | 31 | def test_get_by_uuid(self, client, mock_tr: TransformRequest, captured_templates): 32 | resp: Response = client.get(url_for(self.endpoint, id_=mock_tr.request_id)) 33 | assert resp.status_code == 200 34 | template, context = captured_templates[0] 35 | assert template.name == self.template_name 36 | assert context["req"] == mock_tr 37 | 38 | def test_404(self, client, mock_tr_cls): 39 | mock_tr_cls.lookup.return_value = None 40 | resp: Response = client.get(url_for(self.endpoint, id_=1)) 41 | assert resp.status_code == 404 42 | -------------------------------------------------------------------------------- /servicex_app/servicex_app_test/web/test_view_profile.py: -------------------------------------------------------------------------------- 1 | from flask import Response, url_for 2 | 3 | from .web_test_base import WebTestBase 4 | 5 | 6 | class TestViewProfile(WebTestBase): 7 | def test_view_profile(self, user, client, captured_templates): 8 | with client.session_transaction() as sess: 9 | sess['sub'] = user.sub 10 | response: Response = client.get(url_for('profile')) 11 | assert response.status_code == 200 12 | template, context = captured_templates[0] 13 | assert template.name == "profile.html" 14 | assert context["user"] == user 15 | -------------------------------------------------------------------------------- /transformer_sidecar/.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 99 3 | exclude = 4 | .tox, 5 | __pycache__ 6 | validate_requests.py 7 | find_m.py 8 | messaging.py 9 | .local 10 | build 11 | migrations 12 | ./test_posix_vol/sample_files 13 | ./test_posix_vol/generated_code 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /transformer_sidecar/.github/workflows/buildpush.yaml: -------------------------------------------------------------------------------- 1 | name: Docker Hub 2 | 3 | on: 4 | push: 5 | pull_request: 6 | 7 | jobs: 8 | docker: 9 | runs-on: ubuntu-latest 10 | 11 | steps: 12 | - name: Checkout 13 | uses: actions/checkout@v1.2.0 14 | 15 | - name: Extract tag name 16 | shell: bash 17 | run: echo "##[set-output name=imagetag;]$(echo ${GITHUB_REF##*/})" 18 | id: extract_tag_name 19 | 20 | - name: Build Sidecar Transformer Image 21 | uses: elgohr/Publish-Docker-Github-Action@master 22 | with: 23 | name: sslhep/servicex_sidecar_transformer:${{ steps.extract_tag_name.outputs.imagetag }} 24 | username: ${{ secrets.DOCKER_USERNAME }} 25 | password: ${{ secrets.DOCKER_PASSWORD }} 26 | tag: "${GITHUB_REF##*/}" 27 | -------------------------------------------------------------------------------- /transformer_sidecar/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.10 2 | 3 | WORKDIR /servicex 4 | 5 | RUN python3 -m pip install --upgrade pip 6 | 7 | ENV POETRY_VERSION=2.1.1 8 | RUN pip install poetry==$POETRY_VERSION 9 | 10 | COPY pyproject.toml pyproject.toml 11 | COPY poetry.lock poetry.lock 12 | 13 | RUN poetry config virtualenvs.create false && \ 14 | poetry install --no-root --no-interaction --no-ansi 15 | 16 | 17 | RUN useradd -ms /bin/bash output -G sudo && passwd -d output 18 | RUN chgrp -R 0 /home/output && chmod -R g+rwX /home/output 19 | RUN useradd -g 0 -ms /bin/bash celery 20 | 21 | USER celery 22 | 23 | # Turn this on so that stdout isn't buffered - otherwise logs in kubectl don't 24 | # show up until much later! 25 | ENV PYTHONUNBUFFERED=1 26 | 27 | COPY scripts/ ./scripts/ 28 | # Copy over the source 29 | COPY src/ ./ 30 | 31 | ADD gai.conf /etc/gai.conf 32 | 33 | ENV X509_USER_PROXY=/tmp/grid-security/x509up 34 | ENV X509_CERT_DIR=/etc/grid-security/certificates 35 | ENV PYTHONPATH=/servicex 36 | 37 | -------------------------------------------------------------------------------- /transformer_sidecar/gai.conf: -------------------------------------------------------------------------------- 1 | label ::1/128 0 2 | label ::/0 1 3 | label 2002::/16 2 4 | label ::/96 3 5 | label ::ffff:0:0/96 4 -------------------------------------------------------------------------------- /transformer_sidecar/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "transformer-sidecar" 3 | version = "0.1.0" 4 | description = "" 5 | authors = ["Your Name "] 6 | readme = "README.md" 7 | packages = [{include = "transformer_sidecar"}] 8 | 9 | [tool.poetry.dependencies] 10 | python = "~3.10" 11 | Celery= "^5.4" 12 | urllib3 = "^2.0.7" 13 | psutil = "^5.9.3" 14 | retry = "^0.9.2" 15 | minio = "^7.1.12" 16 | python-logstash = "^0.4.8" 17 | 18 | uproot = "^5.6.0" 19 | fsspec = {extras = ["arrow"], version = "^2023.6.0"} 20 | tenacity = "^9.0.0" 21 | 22 | [tool.poetry.group.test] 23 | optional = true 24 | 25 | [tool.poetry.group.test.dependencies] 26 | flake8 = "^5.0.4" 27 | pytest-mock = "^3.10.0" 28 | pytest = "^7.2.0" 29 | coverage = "^6.5.0" 30 | requests = "^2.28.2" 31 | 32 | [build-system] 33 | requires = ["poetry-core"] 34 | build-backend = "poetry.core.masonry.api" 35 | -------------------------------------------------------------------------------- /transformer_sidecar/pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | pythonpath = ./src 3 | -------------------------------------------------------------------------------- /transformer_sidecar/scripts/proxy-exporter.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | proxydir=$(dirname ${X509_USER_PROXY}) 4 | 5 | if [[ ! -d $proxydir ]] 6 | then 7 | mkdir -p $proxydir 8 | fi 9 | 10 | while true; do 11 | 12 | while true; do 13 | cp /etc/grid-security-ro/x509up ${X509_USER_PROXY} 14 | RESULT=$? 15 | if [ $RESULT -eq 0 ]; then 16 | echo "INFO $INSTANCE_NAME xAOD-Transformer none Got proxy." 17 | chmod 600 ${X509_USER_PROXY} 18 | break 19 | else 20 | echo "WARNING $INSTANCE_NAME xAOD-Transformer none An issue encountered when getting proxy." 21 | sleep 5 22 | fi 23 | done 24 | 25 | # Refresh every hour 26 | sleep 3600 27 | 28 | done 29 | -------------------------------------------------------------------------------- /transformer_sidecar/src/transformer_sidecar/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2019, IRIS-HEP 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are met: 6 | # 7 | # * Redistributions of source code must retain the above copyright notice, this 8 | # list of conditions and the following disclaimer. 9 | # 10 | # * Redistributions in binary form must reproduce the above copyright notice, 11 | # this list of conditions and the following disclaimer in the documentation 12 | # and/or other materials provided with the distribution. 13 | # 14 | # * Neither the name of the copyright holder nor the names of its 15 | # contributors may be used to endorse or promote products derived from 16 | # this software without specific prior written permission. 17 | # 18 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /transformer_sidecar/test_posix_vol/.gitignore: -------------------------------------------------------------------------------- 1 | scripts/ 2 | sample_files/ -------------------------------------------------------------------------------- /transformer_sidecar/test_posix_vol/generated_code/transformer_capabilities.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "FuncADL based uproot transformer", 3 | "description": "Extracts data from flat ntuple style root files.", 4 | "limitations": "Would be good to note what isn't implemented", 5 | "file-formats": ["parquet", "root"], 6 | "stats-parser": "UprootStats", 7 | "language": "python", 8 | "command": "/generated/transform_single_file.py" 9 | } -------------------------------------------------------------------------------- /transformer_sidecar/test_posix_vol/transformer_capabilities.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Uproot transformer using native uproot arguments", 3 | "description": "Extracts data from flat ntuple style root files.", 4 | "limitations": "Would be good to note what isn't implemented", 5 | "file-formats": ["parquet"], 6 | "stats-parser": "UprootStats", 7 | "language": "python", 8 | "command": "/generated/transform_single_file.py" 9 | } -------------------------------------------------------------------------------- /transformer_sidecar/tests/test.root: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ssl-hep/ServiceX/74b29d758dfe8c78e1e0109a706df9ddf1fc56cf/transformer_sidecar/tests/test.root -------------------------------------------------------------------------------- /transformer_sidecar/tests/transformer_capabilities.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Uproot transformer using native uproot arguments", 3 | "description": "Extracts data from flat ntuple style root files.", 4 | "limitations": "Would be good to note what isn't implemented", 5 | "file-formats": ["parquet"], 6 | "stats-parser": "UprootStats", 7 | "language": "python", 8 | "command": "/generated/transform_single_file.py" 9 | } -------------------------------------------------------------------------------- /x509_secrets/.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | name: CI/CD 2 | 3 | on: 4 | push: 5 | branches: 6 | - "*" 7 | tags: 8 | - "*" 9 | 10 | pull_request: 11 | 12 | jobs: 13 | publish: 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/checkout@master 17 | 18 | - name: Extract tag name 19 | shell: bash 20 | run: echo "##[set-output name=imagetag;]$(echo ${GITHUB_REF##*/})" 21 | id: extract_tag_name 22 | 23 | - name: Build X509 Image 24 | uses: elgohr/Publish-Docker-Github-Action@master 25 | with: 26 | name: sslhep/x509-secrets:${{ steps.extract_tag_name.outputs.imagetag }} 27 | username: ${{ secrets.DOCKER_USERNAME }} 28 | password: ${{ secrets.DOCKER_PASSWORD }} 29 | tag: "${GITHUB_REF##*/}" 30 | -------------------------------------------------------------------------------- /x509_secrets/.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | secrets 3 | .python-version 4 | -------------------------------------------------------------------------------- /x509_secrets/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM sslhep/rucio-client:main 2 | 3 | USER root 4 | 5 | # Create app directory 6 | WORKDIR /usr/src/app 7 | 8 | # for CA certificates 9 | 10 | RUN mkdir -p /etc/grid-security/certificates /etc/grid-security/vomsdir 11 | 12 | RUN yum -y update 13 | 14 | ENV POETRY_VERSION=2.1.1 15 | RUN python3 -m pip install --upgrade pip 16 | 17 | RUN pip install poetry==$POETRY_VERSION 18 | COPY pyproject.toml pyproject.toml 19 | COPY poetry.lock poetry.lock 20 | 21 | RUN poetry config virtualenvs.create false && \ 22 | poetry install --no-root --no-interaction --no-ansi 23 | 24 | COPY . . 25 | 26 | # build 27 | RUN echo "Timestamp:" `date --utc` | tee /image-build-info.txt 28 | 29 | ENV X509_USER_PROXY=/tmp/x509up 30 | 31 | CMD ["sh", "-c", "python3", "x509_updater.py", "--voms", "$VOMS"] 32 | -------------------------------------------------------------------------------- /x509_secrets/README.md: -------------------------------------------------------------------------------- 1 | 2 | # X509 Proxy Generator 3 | This image passes a users grid cert and key to the VOMS Proxy server to generate 4 | an X509 proxy which is published into the cluster as a kubernetes secret. 5 | 6 | It accepts as runtime parameter the VOMS organization to use for validating 7 | the user. It also can be run inside docker (without a kubernetes cluster) for 8 | testing. 9 | 10 | To start docker container: 11 | ```bash 12 | docker run --rm -it \ 13 | -e VOMS=atlas \ 14 | --mount type=bind,source=$HOME/.globus,readonly,target=/etc/grid-certs \ 15 | --mount type=bind,source="$(pwd)"/secrets/secrets.txt,target=/servicex/secrets.txt \ 16 | --mount type=volume,source=x509,target=/etc/grid-security \ 17 | --name=x509-secrets sslhep/x509-secrets:develop 18 | ``` 19 | 20 | The environment var `VOMS` can be set to CMS if you wish to authenticate against 21 | that experiment. 22 | -------------------------------------------------------------------------------- /x509_secrets/pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "x509-secrets" 3 | version = "0.1.0" 4 | description = "" 5 | authors = ["Your Name "] 6 | readme = "README.md" 7 | packages = [{include = "x509_secrets"}] 8 | 9 | [tool.poetry.dependencies] 10 | python = "^3.9" 11 | kubernetes = "^25.3.0" 12 | urllib3 = "^2.2.2" 13 | 14 | 15 | [build-system] 16 | requires = ["poetry-core"] 17 | build-backend = "poetry.core.masonry.api" 18 | -------------------------------------------------------------------------------- /x509_secrets/tag_and_release.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # This script will create a release branch and freeze the dependencies 3 | git checkout develop 4 | git pull --ff-only origin develop 5 | 6 | git checkout -b v$1 7 | docker build -t sslhep/x509-secrets:stage . 8 | docker run --rm --entrypoint "pip" sslhep/x509-secrets:stage freeze > requirements.txt 9 | docker rmi sslhep/x509-secrets:stage 10 | git add requirements.txt 11 | git commit -m "Freeze dependencies for V$1" 12 | git push origin v$1 13 | --------------------------------------------------------------------------------