├── .coveralls.yml ├── .git-blame-ignore-revs ├── .github ├── CODEOWNERS ├── PULL_REQUEST_TEMPLATE.md ├── dependabot.yml ├── pr-labeler.yml └── workflows │ ├── auto-approve-dependabot-prs.yml │ ├── dependency-graph.yml │ ├── format.yaml │ ├── orch-build-tag-publish-and-run-tests.yml │ ├── orch-nightly-swat-tests.yml │ ├── pr-labeler.yml │ ├── trivy.yml │ └── unittest.yml ├── .gitignore ├── .java-version ├── .scala-steward.conf ├── .scalafmt.conf ├── CONTRIBUTING.md ├── DEVNOTES.md ├── Dockerfile ├── LICENSE.txt ├── NOTICE.txt ├── README.md ├── automation ├── Dockerfile-tests ├── README.md ├── build.sbt ├── project │ ├── Dependencies.scala │ ├── Settings.scala │ ├── build.properties │ └── plugins.sbt ├── render-local-env.sh ├── src │ └── test │ │ ├── resources │ │ ├── ADD_PARTICIPANTS.tsv │ │ ├── MEMBERSHIP_PARTICIPANT_SET.tsv │ │ └── UPDATE_PARTICIPANTS.tsv │ │ └── scala │ │ └── org │ │ └── broadinstitute │ │ └── dsde │ │ └── test │ │ ├── OrchConfig.scala │ │ └── api │ │ └── orch │ │ └── OrchestrationApiSpec.scala └── test.sh ├── benchmarks ├── project │ └── build.properties └── src │ └── main │ └── scala │ └── org │ └── broadinstitute │ └── dsde │ └── firecloud │ └── utils │ ├── TsvFileSupportBenchmark.scala │ └── TsvFormatterBenchmark.scala ├── build.sbt ├── jenkins └── jenkins_build.sh ├── local-dev ├── bin │ └── render └── templates │ ├── docker-rsync-local-orch.sh │ ├── firecloud-orchestration.conf │ ├── local-agora.conf │ ├── local-rawls.conf │ ├── local-sam.conf │ ├── local-thurloe.conf │ ├── mod_security_ignore.conf │ ├── oauth2.conf │ └── site.conf ├── project ├── Dependencies.scala ├── Merging.scala ├── Settings.scala ├── Testing.scala ├── Version.scala ├── build.properties └── plugins.sbt ├── script ├── build.sh ├── build_jar.sh └── create-configs.sh └── src ├── docker ├── clean_install.sh ├── install.sh └── run.sh ├── main ├── resources │ ├── adjectives_ab.txt │ ├── logback.xml │ ├── model.json │ ├── nouns_ab.txt │ ├── reference.conf │ └── swagger │ │ ├── README.md │ │ └── api-docs.yaml └── scala │ └── org │ └── broadinstitute │ └── dsde │ └── firecloud │ ├── Application.scala │ ├── Boot.scala │ ├── EntityService.scala │ ├── FireCloudApiService.scala │ ├── FireCloudConfig.scala │ ├── FireCloudException.scala │ ├── HealthChecks.scala │ ├── dataaccess │ ├── AgoraDAO.scala │ ├── CwdsDAO.scala │ ├── DisabledExternalCredsDAO.scala │ ├── ExternalCredsDAO.scala │ ├── GoogleServicesDAO.scala │ ├── HttpAgoraDAO.scala │ ├── HttpCwdsDAO.scala │ ├── HttpExternalCredsDAO.scala │ ├── HttpGoogleServicesDAO.scala │ ├── HttpRawlsDAO.scala │ ├── HttpSamDAO.scala │ ├── HttpShibbolethDAO.scala │ ├── HttpThurloeDAO.scala │ ├── RawlsDAO.scala │ ├── ReportsSubsystemStatus.scala │ ├── SamDAO.scala │ ├── ShibbolethDAO.scala │ └── ThurloeDAO.scala │ ├── filematch │ ├── FileMatcher.scala │ ├── FileMatchingOptions.scala │ ├── result │ │ ├── FailedMatchResult.scala │ │ ├── FileMatchResult.scala │ │ ├── PartialMatchResult.scala │ │ └── SuccessfulMatchResult.scala │ └── strategy │ │ ├── FileRecognitionStrategy.scala │ │ ├── IlluminaPairedEndStrategy.scala │ │ └── OntSingleReadStrategy.scala │ ├── model │ ├── DbGapPermission.scala │ ├── EntityUpdateDefinition.scala │ ├── ErrorReport.scala │ ├── ExternalCredsMessage.scala │ ├── JWT.scala │ ├── LinkedEraAccount.scala │ ├── ManagedGroup.scala │ ├── ModelJsonProtocol.scala │ ├── ModelSchema.scala │ ├── OrchMethodRepository.scala │ ├── PermissionReport.scala │ ├── Profile.scala │ ├── Project.scala │ ├── RegisterRequest.scala │ ├── SamResource.scala │ ├── SamUser.scala │ ├── SamUserAttributesRequest.scala │ ├── SamUserRegistrationRequest.scala │ ├── SamUserResponse.scala │ ├── SystemStatus.scala │ ├── UserInfo.scala │ ├── Workspace.scala │ └── package.scala │ ├── service │ ├── AgoraPermissionService.scala │ ├── AttributeSupport.scala │ ├── ExportEntitiesByTypeActor.scala │ ├── FireCloudDirectives.scala │ ├── FireCloudRequestBuilding.scala │ ├── ManagedGroupService.scala │ ├── NamespaceService.scala │ ├── NihService.scala │ ├── PerRequest.scala │ ├── PermissionReportService.scala │ ├── RegisterService.scala │ ├── StatusService.scala │ ├── TSVFileSupport.scala │ ├── UserService.scala │ └── WorkspaceService.scala │ ├── utils │ ├── DateUtils.scala │ ├── DisabledServiceFactory.scala │ ├── EnabledUserDirectives.scala │ ├── PerformanceLogging.scala │ ├── PermissionsSupport.scala │ ├── RestJsonClient.scala │ ├── StandardUserInfoDirectives.scala │ ├── StatusCodeUtils.scala │ ├── StreamingPassthrough.scala │ ├── TSVFormatter.scala │ ├── TSVParser.scala │ └── UserInfoDirectives.scala │ └── webservice │ ├── CookieAuthedApiService.scala │ ├── CromIamApiService.scala │ ├── EntityApiService.scala │ ├── ExportEntitiesApiService.scala │ ├── HealthApiService.scala │ ├── ManagedGroupApiService.scala │ ├── MethodConfigurationApiService.scala │ ├── MethodsApiService.scala │ ├── NamespaceApiService.scala │ ├── NihApiService.scala │ ├── OauthApiService.scala │ ├── PassthroughApiService.scala │ ├── RegisterApiService.scala │ ├── StaticNotebooksApiService.scala │ ├── StatusApiService.scala │ ├── UserApiService.scala │ └── WorkspaceApiService.scala └── test ├── resources ├── logback-test.xml ├── reference.conf └── testfiles │ ├── bagit │ ├── duplicate_participants_nested_testbag.zip │ ├── duplicate_samples_nested_testbag.zip │ ├── empty.zip │ ├── extra_file_nested_testbag.zip │ ├── flat_testbag.zip │ ├── nested_testbag.zip │ ├── not_a_zip.txt │ ├── nothingbag.zip │ ├── participants_only_flat_testbag.zip │ ├── samples_only_flat_testbag.zip │ └── testbag.zip │ └── tsv │ ├── ADD_PARTICIPANTS.txt │ ├── ADD_SAMPLES.txt │ ├── MEMBERSHIP_SAMPLE_SET.tsv │ ├── PARTICIPANTS_NO_PREFIX.txt │ ├── PARTICIPANTS_NO_PREFIX_OR_SUFFIX.txt │ ├── PARTICIPANTS_NO_SUFFIX.txt │ ├── TEST_INVALID.txt │ ├── TEST_INVALID_COLUMNS.txt │ └── UPDATE_SAMPLES.txt └── scala └── org └── broadinstitute └── dsde └── firecloud ├── EntityServiceSpec.scala ├── dataaccess ├── HttpCwdsDAOSpec.scala ├── HttpGoogleServicesDAOSpec.scala ├── MockAgoraDAO.scala ├── MockCwdsDAO.scala ├── MockRawlsDAO.scala ├── MockSamDAO.scala ├── MockShibbolethDAO.scala └── MockThurloeDAO.scala ├── filematch ├── FileMatcherSpec.scala └── strategy │ ├── IlluminaPairedEndStrategySpec.scala │ └── OntSingleReadStrategySpec.scala ├── mock ├── MockAgoraACLData.scala ├── MockGoogleServicesDAO.scala ├── MockTSV.scala ├── MockUtils.scala ├── MockWorkspaceServer.scala ├── ValidEntityCopyCallback.scala ├── ValidEntityDeleteCallback.scala └── ValidSubmissionCallback.scala ├── model ├── FlexibleModelSchemaSpec.scala ├── OrchMethodRepositorySpec.scala ├── ProfileSpec.scala └── SamResourceSpec.scala ├── service ├── AgoraACLTranslationSpec.scala ├── BaseServiceSpec.scala ├── EntitiesWithTypeServiceSpec.scala ├── ExportEntitiesByTypeServiceSpec.scala ├── FireCloudDirectivesSpec.scala ├── NihServiceSpec.scala ├── NihServiceUnitSpec.scala ├── RegisterServiceSpec.scala ├── ServiceSpec.scala ├── ServiceSpecSpec.scala ├── TSVFileSupportSpec.scala ├── UserServiceSpec.scala ├── WorkspaceServiceSpec.scala └── WorkspaceTagsServiceSpec.scala ├── utils ├── DisabledServiceFactoryTest.scala ├── EnabledUserDirectivesSpec.scala ├── PermissionsSupportSpec.scala ├── StatusCodeUtilsSpec.scala ├── StreamingPassthroughSpec.scala ├── TSVFormatterSpec.scala ├── TSVParserSpec.scala └── TestRequestBuilding.scala └── webservice ├── ApiServiceSpec.scala ├── CromIamApiServiceSpec.scala ├── EntityApiServiceSpec.scala ├── HealthApiServiceSpec.scala ├── ImportPermissionApiServiceSpec.scala ├── ManagedGroupApiServiceSpec.scala ├── MethodConfigurationApiServiceSpec.scala ├── MethodsApiServiceACLSpec.scala ├── MethodsApiServiceMultiACLSpec.scala ├── NamespaceApiServiceSpec.scala ├── NihApiServiceSpec.scala ├── PermissionReportApiSpec.scala ├── RegisterApiServiceSpec.scala ├── StatusApiServiceSpec.scala ├── UserApiServiceSpec.scala ├── WorkspaceApiServiceJobSpec.scala └── WorkspaceApiServiceSpec.scala /.coveralls.yml: -------------------------------------------------------------------------------- 1 | service_name: travis-ci -------------------------------------------------------------------------------- /.git-blame-ignore-revs: -------------------------------------------------------------------------------- 1 | # .git-blame-ignore-revs 2 | 3 | # scalafmt mass change 4 | 3686bb47a5b215733a46c3393a51e950fcaa635f 5 | 6 | # Scala Steward: Reformat with scalafmt 3.8.4 7 | c7a758f4143587c0355fee5580523cf7be0ead66 8 | 9 | # Scala Steward: Reformat with scalafmt 3.8.6 10 | 8f93851c78b3020fa73c0769470b8faddf4aa2f1 11 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @broadinstitute/dsp-core-services 2 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | \ 2 | 3 | Have you read [CONTRIBUTING.md](../CONTRIBUTING.md) lately? If not, do that first. 4 | 5 | I, the developer opening this PR, do solemnly pinky swear that: 6 | 7 | - [ ] I've followed [the instructions](https://github.com/broadinstitute/firecloud-orchestration/blob/develop/CONTRIBUTING.md#api-changes) if I've made any changes to the API, _especially_ if they're breaking changes 8 | - [ ] I've updated the [FISMA documentation](https://github.com/broadinstitute/firecloud-orchestration/blob/develop/CONTRIBUTING.md#fisma-documentation-changes) if I've made any security-related changes, including auth, encryption, or auditing 9 | 10 | In all cases: 11 | 12 | - [ ] Get two thumbsworth of review and PO signoff if necessary 13 | - [ ] Verify all tests go green 14 | - [ ] Squash and merge. Make sure your branch deletes; GitHub should do this for you. 15 | - [ ] Test this change deployed correctly and works on dev environment after deployment 16 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "github-actions" 4 | directory: "/" 5 | open-pull-requests-limit: 10 6 | reviewers: 7 | - "@broadinstitute/broad-core-services" 8 | commit-message: 9 | prefix: "[CORE-69]" 10 | schedule: 11 | interval: "weekly" 12 | day: "monday" 13 | time: "08:00" 14 | timezone: "America/New_York" 15 | groups: 16 | artifact-actions: 17 | patterns: 18 | - "actions/upload-artifact" 19 | - "actions/download-artifact" 20 | -------------------------------------------------------------------------------- /.github/pr-labeler.yml: -------------------------------------------------------------------------------- 1 | Scala_Steward: 'update/*' 2 | -------------------------------------------------------------------------------- /.github/workflows/auto-approve-dependabot-prs.yml: -------------------------------------------------------------------------------- 1 | name: Dependabot auto-approve 2 | on: pull_request 3 | 4 | permissions: 5 | pull-requests: write 6 | 7 | jobs: 8 | dependabot: 9 | runs-on: ubuntu-latest 10 | if: github.actor == 'dependabot[bot]' 11 | steps: 12 | - name: Dependabot metadata 13 | id: metadata 14 | uses: dependabot/fetch-metadata@v2 15 | with: 16 | github-token: "${{ secrets.GITHUB_TOKEN }}" 17 | - name: Approve a PR 18 | run: gh pr review --approve "$PR_URL" 19 | env: 20 | PR_URL: ${{ github.event.pull_request.html_url }} 21 | GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} 22 | -------------------------------------------------------------------------------- /.github/workflows/dependency-graph.yml: -------------------------------------------------------------------------------- 1 | name: Update Dependency Graph 2 | 3 | on: 4 | push: 5 | branches: 6 | - develop # default branch of the project 7 | 8 | jobs: 9 | update-graph: 10 | name: Update Dependency Graph 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v4 14 | - uses: sbt/setup-sbt@v1 15 | - uses: scalacenter/sbt-dependency-submission@v3 16 | with: 17 | ## Optional: Define the working directory of your build. 18 | ## It should contain the build.sbt file. 19 | working-directory: './' 20 | # Ignore dependencies for the benchmarking module; these are not used at runtime 21 | modules-ignore: bench_2.13 22 | -------------------------------------------------------------------------------- /.github/workflows/format.yaml: -------------------------------------------------------------------------------- 1 | name: Check formatting for modified files with scalafmt 2 | 3 | on: 4 | pull_request: 5 | paths-ignore: ['**.md'] 6 | 7 | jobs: 8 | format: 9 | 10 | runs-on: ubuntu-latest 11 | 12 | steps: 13 | 14 | - uses: actions/checkout@v4 15 | - uses: sbt/setup-sbt@v1 16 | with: 17 | fetch-depth: 2 18 | ref: ${{ github.event.pull_request.head.sha }} 19 | 20 | - name: Set up JDK 17 21 | uses: actions/setup-java@v4 22 | with: 23 | distribution: 'temurin' 24 | java-version: 17 25 | cache: sbt 26 | 27 | - name: Check formatting for modified files 28 | run: | 29 | sbt scalafmtCheckAll 30 | -------------------------------------------------------------------------------- /.github/workflows/pr-labeler.yml: -------------------------------------------------------------------------------- 1 | name: PR Labeler 2 | on: 3 | pull_request: 4 | types: [opened] 5 | 6 | jobs: 7 | pr-labeler: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: TimonVS/pr-labeler-action@v5 11 | with: 12 | configuration-path: .github/pr-labeler.yml # optional, .github/pr-labeler.yml is the default value 13 | env: 14 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 15 | -------------------------------------------------------------------------------- /.github/workflows/trivy.yml: -------------------------------------------------------------------------------- 1 | name: Trivy scan 2 | on: [pull_request] 3 | 4 | jobs: 5 | appsec-trivy: 6 | name: DSP AppSec Trivy check 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v4 10 | 11 | # The Dockerfile copies this, so it needs to exist for the build to succeed 12 | - run: touch FireCloud-Orchestration.jar 13 | 14 | # https://github.com/broadinstitute/dsp-appsec-trivy-action 15 | - uses: broadinstitute/dsp-appsec-trivy-action@v1 16 | -------------------------------------------------------------------------------- /.github/workflows/unittest.yml: -------------------------------------------------------------------------------- 1 | name: Scala CI 2 | 3 | on: 4 | pull_request: 5 | branches: [ develop ] 6 | push: 7 | paths-ignore: 8 | - 'README.md' 9 | branches: [ develop ] 10 | 11 | concurrency: 12 | group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} 13 | cancel-in-progress: true 14 | 15 | jobs: 16 | build: 17 | 18 | runs-on: ubuntu-latest 19 | 20 | steps: 21 | 22 | - uses: actions/checkout@v4 23 | - uses: sbt/setup-sbt@v1 24 | 25 | - name: Set up JDK 17 26 | uses: actions/setup-java@v4 27 | with: 28 | distribution: 'temurin' 29 | java-version: 17 30 | cache: 'sbt' 31 | 32 | - name: Run tests 33 | env: 34 | AGORA_URL_ROOT: http://localhost:8989 35 | RAWLS_URL_ROOT: http://localhost:8990 36 | THURLOE_URL_ROOT: http://localhost:8991 37 | FIRE_CLOUD_ID: 123 38 | run: sbt clean coverage test coverageReport 39 | 40 | - uses: codecov/codecov-action@v5 41 | if: ${{ always() }} 42 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.class 2 | *.log 3 | configure.rb 4 | config/ 5 | 6 | # sbt specific 7 | .sbtopts 8 | .bsp/ 9 | .cache/ 10 | .history/ 11 | .lib/ 12 | dist/* 13 | target/ 14 | lib_managed/ 15 | src_managed/ 16 | project/boot/ 17 | project/plugins/project/ 18 | 19 | # Scala-IDE specific 20 | .scala_dependencies 21 | .worksheet 22 | 23 | ### JetBrains template 24 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm 25 | 26 | *.iml 27 | 28 | ## Directory-based project format: 29 | .idea/ 30 | # if you remove the above rule, at least ignore the following: 31 | 32 | # User-specific stuff: 33 | .idea/workspace.xml 34 | .idea/tasks.xml 35 | .idea/dictionaries 36 | .idea/misc.xml 37 | .idea/modules/ 38 | 39 | # Sensitive or high-churn files: 40 | .idea/dataSources.ids 41 | .idea/dataSources.xml 42 | .idea/sqlDataSources.xml 43 | .idea/dynamic.xml 44 | .idea/uiDesigner.xml 45 | 46 | # Gradle: 47 | .idea/gradle.xml 48 | .idea/libraries 49 | 50 | # Mongo Explorer plugin: 51 | .idea/mongoSettings.xml 52 | 53 | ## File-based project format: 54 | *.ipr 55 | *.iws 56 | 57 | ## Plugin-specific files: 58 | 59 | # IntelliJ 60 | out/ 61 | 62 | # mpeltonen/sbt-idea plugin 63 | .idea_modules/ 64 | 65 | # JIRA plugin 66 | atlassian-ide-plugin.xml 67 | 68 | # Crashlytics plugin (for Android Studio and IntelliJ) 69 | com_crashlytics_export_strings.xml 70 | crashlytics.properties 71 | crashlytics-build.properties 72 | 73 | *.pid 74 | .DS_Store 75 | 76 | automation/test-reports/ 77 | automation/src/test/resources/** 78 | !automation/src/test/resources/PFBImportSpec-expected-entities.json 79 | !automation/src/test/resources/ADD_PARTICIPANTS.tsv 80 | !automation/src/test/resources/UPDATE_PARTICIPANTS.tsv 81 | !automation/src/test/resources/MEMBERSHIP_PARTICIPANT_SET.tsv 82 | 83 | mockserver* 84 | MockServer* 85 | -------------------------------------------------------------------------------- /.java-version: -------------------------------------------------------------------------------- 1 | 17 2 | -------------------------------------------------------------------------------- /.scalafmt.conf: -------------------------------------------------------------------------------- 1 | version = 3.8.6 2 | align = none 3 | align.openParenCallSite = true 4 | align.openParenDefnSite = true 5 | maxColumn = 120 6 | continuationIndent.defnSite = 2 7 | assumeStandardLibraryStripMargin = true 8 | danglingParentheses.preset = true 9 | rewrite.rules = [SortImports, RedundantBraces, RedundantParens, SortModifiers] 10 | docstrings.style = keep 11 | project.excludeFilters = [ 12 | Dependencies.scala, 13 | Settings.scala, 14 | build.sbt 15 | ] 16 | runner.dialect = scala213 17 | project.git = true 18 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | ## Opening a Pull Request 4 | 5 | You most likely want to do your work on a feature branch based on develop. There is no explicit naming convention; we usually use some combination of the JIRA issue number and something alluding to the work we're doing. 6 | 7 | When you open a pull request, add the JIRA issue number (e.g. `AJ-1234`) to the PR title. This will make a reference from JIRA to the GitHub issue. Add a brief description of your changes above the PR checkbox template. 8 | 9 | This is also a good opportunity to check that the acceptance criteria for your JIRA ticket exists and is met. Check with your PO if you have any questions there. You should also fill out a summary to go in the release notes and some instructions on what QA should be looking at. 10 | 11 | The checkboxes in the PR are important reminders to you, the developer. Please be conscientious and run through them when you open a PR. 12 | 13 | ## PR approval process 14 | 15 | If your PR is particularly complex it can be helpful to add some commentary, either in the description or line-by-line in the GitHub PR view. In the latter case, consider whether those comments should be in the code itself. 16 | 17 | You should get review from two people (either through GitHub's request-review feature, or by assigning the PR to them); one of them should probably be your tech lead, though ask. Do chase your reviewers (or find others) if they're slow; we don't like to let PRs linger. If you get PR feedback it's back to you to address it and then nudge your reviewers for re-review. 18 | 19 | Your PR is ready to merge when all the following things are true: 20 | 21 | 1. Two reviewers have thumbed your PR 22 | 2. If your change is user-facing, your PO has seen it and signed off 23 | 3. All tests pass 24 | 25 | ## API changes 26 | 27 | All changes to the API _must_ be documented in Swagger. 28 | 29 | ### Breaking API changes 30 | 31 | We strive to minimize breaking changes to the API, but sometimes they're unavoidable. In such cases we need to let our API consumers know ahead of time so their code doesn't suddenly stop working. 32 | 33 | If you're making breaking changes to the API, do the following: 34 | 35 | 0. Try really hard not to make a breaking change to the API. 36 | 1. Check in with Comms to explain the change and the impact on users. 37 | 2. Write an email to users explaining the change and send it least a few days BEFORE the release goes out. You'll need to explain the change and what users need to do to update their code. Get Comms signoff on the wording. 38 | 3. Get someone Suitable to send the email to api-users@firecloud.org. 39 | 40 | ## FISMA documentation changes 41 | 42 | If you're making changes to authentication, authorization, encryption, or audit trails, you should check in with Sarah Tahiri to see if our security documentation should be updated. 43 | -------------------------------------------------------------------------------- /DEVNOTES.md: -------------------------------------------------------------------------------- 1 | src/main/{packages}/ 2 | - dataaccess 3 | - DAOs, defined per artifact. For instance, one DAO for ES, one for Thurloe, one for mysql. 4 | - every DAO has an abstract parent, plus implementation classes for runtime instances. 5 | We will also have mock implementation classes that live in the src/test hierarchy 6 | - if a single DAO per artifact is too big, cut it up into multiple traits aggregated by a DAO class. 7 | - model 8 | - model classes. Case classes if possible. Little-to-no logic implementations; things like toString allowed 9 | - service 10 | - Actors that encapsulate business logic, defined per functional aspect. 11 | - Called by webservice; calls to DAOs 12 | - All methods should be testable; called by unit tests 13 | - For instance, there will be a Library service class, which may call both ES and Rawls DAOs 14 | - webservice 15 | - route definitions 16 | - minimal-to-none business logic; everything testable should be in a service 17 | - calls service classes, using PerRequest 18 | - startup class (Boot.scala, Agora.scala, Main.scala, etc - standardize name?) 19 | - instantiates runtime DAOs from config (group into an Application object) 20 | - defines the service constructors, which are used by webservices to create service instances 21 | - collects routes and starts webserver 22 | 23 | src/main/{packages}/ 24 | - dataaccess 25 | - mocks for DAO implementations 26 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM us.gcr.io/broad-dsp-gcr-public/base/jre:17-debian 2 | 3 | EXPOSE 8080 4 | 5 | RUN mkdir /orch 6 | COPY ./FireCloud-Orchestration*.jar /orch 7 | 8 | # 1. “Exec” form of CMD necessary to avoid “shell” form’s `sh` stripping 9 | # environment variables with periods in them, often used in DSP for Lightbend 10 | # config. 11 | # 2. Handling $JAVA_OPTS is necessary as long as firecloud-develop or the app’s 12 | # chart tries to set it. Apps that use devops’s foundation subchart don’t need 13 | # to handle this. 14 | # 3. The jar’s location and naming scheme in the filesystem is required by preflight 15 | # liquibase migrations in some app charts. Apps that expose liveness endpoints 16 | # may not need preflight liquibase migrations. 17 | # We use the “exec” form with `bash` to accomplish all of the above. 18 | CMD ["/bin/bash", "-c", "java $JAVA_OPTS -jar $(find /orch -name 'FireCloud-Orchestration*.jar')"] 19 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015, Broad Institute, Inc. 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | * Neither the name Broad Institute, Inc. nor the names of its 15 | contributors may be used to endorse or promote products derived from 16 | this software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE 28 | -------------------------------------------------------------------------------- /NOTICE.txt: -------------------------------------------------------------------------------- 1 | `adjectives_ab.txt` and `nouns_ab.txt` are modified versions of Aaron Bassett's pass phrase 2 | repo (https://github.com/aaronbassett/Pass-phrase) and are distributed with this 3 | software under the MIT License (https://aaron.mit-license.org/; see the README file). 4 | In accordance with that license, that software comes with the following notices: 5 | 6 | The MIT License (MIT) 7 | Copyright © 2019 Aaron Bassett, http://aaronbassett.com 8 | 9 | Permission is hereby granted, free of charge, to any person obtaining a copy 10 | of this software and associated documentation files (the “Software”), to deal 11 | in the Software without restriction, including without limitation the rights 12 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 13 | copies of the Software, and to permit persons to whom the Software is furnished 14 | to do so, subject to the following conditions: 15 | 16 | The above copyright notice and this permission notice shall be included in 17 | all copies or substantial portions of the Software. 18 | -------------------------------------------------------------------------------- /automation/Dockerfile-tests: -------------------------------------------------------------------------------- 1 | FROM sbtscala/scala-sbt:eclipse-temurin-17.0.15_6_1.11.1_2.13.16 2 | 3 | COPY src /app/src 4 | COPY test.sh /app 5 | COPY project /app/project 6 | COPY build.sbt /app 7 | 8 | WORKDIR /app 9 | 10 | ENTRYPOINT ["bash", "test.sh"] 11 | -------------------------------------------------------------------------------- /automation/README.md: -------------------------------------------------------------------------------- 1 | Quickstart: running integration tests locally on Mac/Docker 2 | 3 | ## Running in docker 4 | 5 | See [firecloud-automated-testing](https://github.com/broadinstitute/firecloud-automated-testing). 6 | 7 | 8 | ## Running tests from your local machine 9 | 10 | ### Set Up 11 | 12 | Render configs: 13 | ```bash 14 | ./render-local-env.sh [branch of firecloud-automated-testing] [vault token] [env] [service root] 15 | ``` 16 | 17 | **Arguments:** (arguments are positional) 18 | 19 | * branch of firecloud-automated-testing 20 | * Configs branch; defaults to `master` 21 | * Vault auth token 22 | * Defaults to reading it from the .vault-token via `$(cat ~/.vault-token)`. 23 | * env 24 | * Environment of your FiaB; defaults to `dev` 25 | * service root 26 | * the name of your local clone of firecloud-orchestration if not `firecloud-orchestration` 27 | 28 | ##### Testing against a live environment 29 | Be careful when testing against a persistent environment (dev, alpha, staging, etc) - be sure 30 | your test runs do not interrupt any other work happening in those environments, and be sure your 31 | test runs do not leave cruft behind. 32 | 33 | To render configs for a live environment: 34 | 1. Manually change `render-local-env.sh` line 15 from `FC_INSTANCE=fiab` to `FC_INSTANCE=live` 35 | 2. render configs with `./render-local-env.sh master $(cat ~/.vault-token) alpha`, replacing `alpha` with your target env 36 | 37 | TODO: update `render-local-env.sh` so it doesn't require manual code changes 38 | 39 | ##### Testing against your own FiaB 40 | 41 | TODO: cWDS-related tests (`AsyncImportSpec`) are nonfunctional against FiaBs without additional manual configuration 42 | 43 | To render configs to test against your own FiaB, accept all defaults: `./render-local-env.sh` 44 | 45 | ##### Using a local UI 46 | 47 | Set `LOCAL_UI=true` before calling `render-local-env.sh`: 48 | ```bash 49 | LOCAL_UI=true ./render-local-env.sh 50 | ``` 51 | 52 | ### Run tests 53 | 54 | #### From IntelliJ 55 | To run tests from IntelliJ, it is recommended to create a separate project for the `automation` folder instead of attempting to run tests from the `firecloud-orchestration` root folder. 56 | To do this, go to `File` -> `New` -> `Project from Existing Sources...` and navigate __into__ the `automation` folder, click `Open` to accept, and then import from SBT. 57 | 58 | Then, you may need to tweak your ScalaTest run configurations. In IntelliJ, go to `Run` > `Edit Configurations...`, select `ScalaTest` under `Defaults`, and: 59 | 60 | * make sure that there is a `Build` task configured to run before launch. 61 | * you may also need to check the `Use sbt` checkbox 62 | 63 | Now, simply open the test spec, right-click on the class name or a specific test string, and select `Run` or `Debug` as needed. 64 | A good one to start with is `OrchestrationApiSpec` to make sure your base configuration is correct. All test code lives in `automation/src/test/scala`. 65 | 66 | #### From the command line 67 | 68 | To run all tests: 69 | 70 | ```bash 71 | sbt test 72 | ``` 73 | 74 | To run a single suite: 75 | 76 | ```bash 77 | sbt "testOnly *OrchestrationApiSpec" 78 | ``` 79 | 80 | To run a single test within a suite: 81 | 82 | ```bash 83 | # matches test via substring 84 | sbt "testOnly *OrchestrationApiSpec -- -z \"not find a non-existent billing project\"" 85 | ``` 86 | 87 | For more information see [SBT's documentation](https://www.scala-sbt.org/1.x/docs/Testing.html) and [ScalaTest's User Guide](https://www.scalatest.org/user_guide). 88 | 89 | -------------------------------------------------------------------------------- /automation/build.sbt: -------------------------------------------------------------------------------- 1 | import Settings._ 2 | 3 | lazy val orchIntegration = project.in(file(".")) 4 | .settings(rootSettings:_*) 5 | 6 | version := "1.0" 7 | -------------------------------------------------------------------------------- /automation/project/Dependencies.scala: -------------------------------------------------------------------------------- 1 | import sbt._ 2 | 3 | object Dependencies { 4 | val scalaV = "2.13.16" 5 | 6 | val jacksonV = "2.19.0" 7 | val jacksonHotfixV = "2.19.0" // for when only some of the Jackson libs have hotfix releases 8 | val akkaV = "2.6.19" 9 | val akkaHttpV = "10.2.10" 10 | val workbenchLibsHash = "3e0cf25" 11 | 12 | val workbenchModelV = s"0.20-$workbenchLibsHash" 13 | val workbenchModel: ModuleID = "org.broadinstitute.dsde.workbench" %% "workbench-model" % workbenchModelV 14 | val excludeWorkbenchModel = ExclusionRule(organization = "org.broadinstitute.dsde.workbench", name = "workbench-model_" + scalaV) 15 | 16 | val workbenchGoogleV = s"0.33-$workbenchLibsHash" 17 | val workbenchGoogle: ModuleID = "org.broadinstitute.dsde.workbench" %% "workbench-google" % workbenchGoogleV excludeAll excludeWorkbenchModel 18 | val excludeWorkbenchGoogle = ExclusionRule(organization = "org.broadinstitute.dsde.workbench", name = "workbench-google_" + scalaV) 19 | 20 | val workbenchServiceTestV = s"5.0-$workbenchLibsHash" 21 | val workbenchServiceTest: ModuleID = "org.broadinstitute.dsde.workbench" %% "workbench-service-test" % workbenchServiceTestV % "test" classifier "tests" excludeAll (excludeWorkbenchGoogle, excludeWorkbenchModel) 22 | 23 | // Overrides for transitive dependencies. These apply - via Settings.scala - to all projects in this codebase. 24 | // These are overrides only; if the direct dependencies stop including any of these, they will not be included 25 | // by being listed here. 26 | // One reason to specify an override here is to avoid static-analysis security warnings. 27 | val transitiveDependencyOverrides: Seq[ModuleID] = Seq( 28 | "com.fasterxml.jackson.core" % "jackson-annotations" % jacksonV, 29 | "com.fasterxml.jackson.core" % "jackson-databind" % jacksonHotfixV, 30 | "com.fasterxml.jackson.core" % "jackson-core" % jacksonV, 31 | "io.grpc" % "grpc-xds" % "1.56.1", 32 | "org.typelevel" %% "cats-effect" % "3.4.11", 33 | "org.typelevel" %% "cats-core" % "2.10.0" 34 | ) 35 | 36 | val rootDependencies: Seq[ModuleID] = Seq( 37 | "com.fasterxml.jackson.module" %% "jackson-module-scala" % jacksonV, 38 | "net.virtual-void" %% "json-lenses" % "0.6.2" % "test", 39 | "ch.qos.logback" % "logback-classic" % "1.5.18", 40 | "com.typesafe.akka" %% "akka-http-core" % akkaHttpV, 41 | "com.typesafe.akka" %% "akka-stream-testkit" % akkaV, 42 | "com.typesafe.akka" %% "akka-http" % akkaHttpV, 43 | "com.typesafe.akka" %% "akka-http-spray-json" % akkaHttpV, 44 | "com.typesafe.akka" %% "akka-testkit" % akkaV % "test", 45 | "com.typesafe.akka" %% "akka-slf4j" % akkaV, 46 | "org.specs2" %% "specs2-core" % "4.15.0" % "test", 47 | "org.scalatest" %% "scalatest" % "3.2.19" % Test, 48 | "com.typesafe.scala-logging" %% "scala-logging" % "3.9.5", 49 | 50 | // required but not provided by workbench-google. 51 | // workbench-google specifies 7.0.1 52 | "net.logstash.logback" % "logstash-logback-encoder" % "7.1.1", 53 | 54 | workbenchServiceTest, 55 | workbenchModel, 56 | workbenchGoogle 57 | ) 58 | } 59 | -------------------------------------------------------------------------------- /automation/project/Settings.scala: -------------------------------------------------------------------------------- 1 | import Dependencies._ 2 | import sbt.Keys._ 3 | import sbt._ 4 | 5 | object Settings { 6 | 7 | // for org.broadinstitute.dsde.workbench modules 8 | val artifactory = "https://us-central1-maven.pkg.dev/dsp-artifact-registry/" 9 | val commonResolvers = List( 10 | "artifactory-releases" at artifactory + "libs-release", 11 | "artifactory-snapshots" at artifactory + "libs-snapshot" 12 | ) 13 | val proxyResolvers = List( 14 | "internal-maven-proxy" at artifactory + "maven-central" 15 | ) 16 | 17 | //coreDefaultSettings + defaultConfigs = the now deprecated defaultSettings 18 | val commonBuildSettings = Defaults.coreDefaultSettings ++ Defaults.defaultConfigs ++ Seq( 19 | javaOptions += "-Xmx2G", 20 | javacOptions ++= Seq("--release", "17") 21 | ) 22 | 23 | val commonCompilerSettings = Seq( 24 | "-unchecked", 25 | "-deprecation", 26 | "-feature", 27 | "-release:8", 28 | "-encoding", "utf8", "100" 29 | ) 30 | 31 | // test parameters explanation: 32 | // `-o` - causes test results to be written to the standard output 33 | // `F` - Display full stack traces 34 | // `D` - Display test duration after test name 35 | // (removed on April 22, 2018) `G` - show reminder of failed and canceled tests with full stack traces at the end of log file 36 | // `-fWD` - causes test results to be written to the summary.log with test duration but without colored text 37 | val testSettings = List( 38 | Test / testOptions += Tests.Argument("-oFD", "-u", "test-reports", "-fWD", "test-reports/TEST-summary.log") 39 | ) 40 | 41 | //common settings for all sbt subprojects 42 | val commonSettings = 43 | commonBuildSettings ++ testSettings ++ List( 44 | organization := "org.broadinstitute.dsde.firecloud", 45 | scalaVersion := "2.13.16", 46 | resolvers := proxyResolvers ++: resolvers.value ++: commonResolvers, 47 | scalacOptions ++= commonCompilerSettings, 48 | dependencyOverrides ++= transitiveDependencyOverrides 49 | ) 50 | 51 | //the full list of settings for the root project that's ultimately the one we build into a fat JAR and run 52 | //coreDefaultSettings (inside commonSettings) sets the project name, which we want to override, so ordering is important. 53 | //thus commonSettings needs to be added first. 54 | val rootSettings = commonSettings ++ List( 55 | name := "orchIntegration", 56 | libraryDependencies ++= rootDependencies 57 | ) 58 | } 59 | -------------------------------------------------------------------------------- /automation/project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version = 1.11.1 2 | -------------------------------------------------------------------------------- /automation/project/plugins.sbt: -------------------------------------------------------------------------------- 1 | logLevel := Level.Warn 2 | -------------------------------------------------------------------------------- /automation/render-local-env.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | ## Run from automation/ 5 | ## Clones the firecloud-automated-testing repo, pulls templatized configs, and renders them to src/test/resources 6 | 7 | # Defaults 8 | WORKING_DIR=$PWD 9 | VAULT_TOKEN=$(cat ~/.vault-token) 10 | FIRECLOUD_AUTOMATED_TESTING_BRANCH=master 11 | ENV=dev 12 | SERVICE=firecloud-orchestration 13 | LOCAL_UI=${LOCAL_UI:-false} # local ui defaults to false unless set in the env 14 | 15 | FC_INSTANCE=fiab 16 | if [ $LOCAL_UI = "true" ]; then 17 | FC_INSTANCE=local 18 | fi 19 | 20 | # Parameters 21 | FIRECLOUD_AUTOMATED_TESTING_BRANCH=${1:-$FIRECLOUD_AUTOMATED_TESTING_BRANCH} 22 | VAULT_TOKEN=${2:-$VAULT_TOKEN} 23 | ENV=${3:-$ENV} 24 | SERVICE_ROOT=${4:-$SERVICE} 25 | 26 | SCRIPT_ROOT=${SERVICE_ROOT}/automation 27 | 28 | confirm () { 29 | # call with a prompt string or use a default 30 | read -r -p "${1:-Are you sure?} [y/N] " response 31 | case $response in 32 | [yY]) 33 | shift 34 | $@ 35 | ;; 36 | *) 37 | ;; 38 | esac 39 | } 40 | 41 | # clone the firecloud-automated-testing repo 42 | clone_repo() { 43 | original_dir=$PWD 44 | cd ../.. 45 | echo "Currently in ${PWD}" 46 | confirm "OK to clone here?" git clone git@github.com:broadinstitute/firecloud-automated-testing.git 47 | cd $original_dir 48 | } 49 | 50 | pull_configs() { 51 | original_dir=$WORKING_DIR 52 | cd ../.. 53 | cd firecloud-automated-testing 54 | echo "Currently in ${PWD}" 55 | git stash 56 | git checkout ${FIRECLOUD_AUTOMATED_TESTING_BRANCH} 57 | git pull 58 | cd $original_dir 59 | } 60 | 61 | render_configs() { 62 | original_dir=$WORKING_DIR 63 | cd ../.. 64 | docker pull broadinstitute/dsde-toolbox:dev 65 | docker run -it --rm -e VAULT_TOKEN=${VAULT_TOKEN} \ 66 | -e ENVIRONMENT=${ENV} -e ROOT_DIR=${WORKING_DIR} -v $PWD/firecloud-automated-testing/configs:/input -v $PWD/$SCRIPT_ROOT:/output \ 67 | -e OUT_PATH=/output/src/test/resources -e INPUT_PATH=/input -e LOCAL_UI=$LOCAL_UI -e FC_INSTANCE=$FC_INSTANCE \ 68 | broadinstitute/dsde-toolbox:dev render-templates.sh 69 | 70 | # pull service-specific application.conf 71 | docker run -it --rm -e VAULT_TOKEN=${VAULT_TOKEN} \ 72 | -e ENVIRONMENT=${ENV} -e ROOT_DIR=${WORKING_DIR} -v $PWD/firecloud-automated-testing/configs/$SERVICE:/input -v $PWD/$SCRIPT_ROOT:/output \ 73 | -e OUT_PATH=/output/src/test/resources -e INPUT_PATH=/input -e LOCAL_UI=$LOCAL_UI -e FC_INSTANCE=$FC_INSTANCE \ 74 | broadinstitute/dsde-toolbox:dev render-templates.sh 75 | cd $original_dir 76 | } 77 | 78 | if [[ $PWD != *"${SCRIPT_ROOT}" ]]; then 79 | echo "Error: this script needs to be running from the ${SCRIPT_ROOT} directory!" 80 | exit 1 81 | fi 82 | confirm "Clone firecloud-automated-testing repo? Skip if you have already run this step before." clone_repo 83 | confirm "Checkout ${FIRECLOUD_AUTOMATED_TESTING_BRANCH}? This will stash local changes. If N, configs will be built from local changes." pull_configs 84 | render_configs 85 | -------------------------------------------------------------------------------- /automation/src/test/resources/ADD_PARTICIPANTS.tsv: -------------------------------------------------------------------------------- 1 | entity:participant_id 2 | participant_01 3 | participant_02 4 | participant_03 5 | participant_04 6 | participant_05 7 | participant_06 8 | participant_07 9 | participant_08 10 | -------------------------------------------------------------------------------- /automation/src/test/resources/MEMBERSHIP_PARTICIPANT_SET.tsv: -------------------------------------------------------------------------------- 1 | membership:participant_set_id participant 2 | your-setA-name participant_01 3 | your-setA-name participant_02 4 | your-set2-name participant_03 5 | your-set2-name participant_04 6 | -------------------------------------------------------------------------------- /automation/src/test/resources/UPDATE_PARTICIPANTS.tsv: -------------------------------------------------------------------------------- 1 | update:participant_id age 2 | participant_01 56 3 | participant_02 99 4 | participant_03 39 5 | participant_04 12 6 | -------------------------------------------------------------------------------- /automation/src/test/scala/org/broadinstitute/dsde/test/OrchConfig.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.test 2 | 3 | import org.broadinstitute.dsde.workbench.config.CommonConfig 4 | 5 | object OrchConfig extends CommonConfig { 6 | 7 | object Users extends CommonUsers { 8 | val tcgaJsonWebTokenKey = usersConfig.getString("tcgaJsonWebTokenKey") 9 | val targetJsonWebTokenKey = usersConfig.getString("targetJsonWebTokenKey") 10 | val targetAndTcgaJsonWebTokenKey = usersConfig.getString("targetAndTcgaJsonWebTokenKey") 11 | val genericJsonWebTokenKey = usersConfig.getString("genericJsonWebTokenKey") 12 | val tempSubjectId = usersConfig.getString("tempSubjectId") 13 | } 14 | 15 | } 16 | -------------------------------------------------------------------------------- /automation/test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | SBT_CMD=${1-"testOnly -- -l ProdTest"} 4 | echo $SBT_CMD 5 | 6 | set -o pipefail 7 | 8 | sbt -batch -Dheadless=true "${SBT_CMD}" 9 | TEST_EXIT_CODE=$? 10 | sbt clean 11 | 12 | if [[ $TEST_EXIT_CODE != 0 ]]; then exit $TEST_EXIT_CODE; fi 13 | -------------------------------------------------------------------------------- /benchmarks/project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=1.11.1 2 | -------------------------------------------------------------------------------- /benchmarks/src/main/scala/org/broadinstitute/dsde/firecloud/utils/TsvFileSupportBenchmark.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.utils 2 | 3 | import org.broadinstitute.dsde.firecloud.model.{EntityUpdateDefinition, FlexibleModelSchema, ModelSchema} 4 | import org.broadinstitute.dsde.firecloud.service.TSVFileSupport 5 | import org.broadinstitute.dsde.firecloud.utils.TsvFileSupportBenchmark.TsvData 6 | import org.openjdk.jmh.annotations.{Benchmark, Scope, State} 7 | import org.openjdk.jmh.infra.Blackhole 8 | 9 | object TsvFileSupportBenchmark { 10 | @State(Scope.Thread) 11 | class TsvData { 12 | val entityType: String = "sample" 13 | val memberTypeOpt: Option[String] = None 14 | // representation of the inbound TSV row 15 | val row: Seq[String] = Seq( 16 | "0005", 17 | "foo", 18 | "\"some\tquoted\tvalue\"", 19 | "42", 20 | "true", 21 | "-123.456", 22 | "gs://some-bucket/somefile.ext", 23 | """{"entityType":"targetType", "entityName":"targetName"}""", 24 | "[1,2,3,4,5]" 25 | ) 26 | val colInfo: Seq[(String, Option[String])] = Seq( 27 | ("sample_id", None), 28 | ("string", None), 29 | ("quotedstring", None), 30 | ("int", None), 31 | ("boolean", None), 32 | ("double", None), 33 | ("file", None), 34 | ("reference", None), 35 | ("array", None) 36 | ) 37 | val modelSchema: ModelSchema = FlexibleModelSchema 38 | val deleteEmptyValues: Boolean = false 39 | } 40 | } 41 | 42 | class TsvFileSupportBenchmark { 43 | 44 | @Benchmark 45 | def makeEntityRows(blackHole: Blackhole, tsvData: TsvData): EntityUpdateDefinition = { 46 | 47 | val result: EntityUpdateDefinition = TsvFileSupportHarness.setAttributesOnEntity(tsvData.entityType, 48 | tsvData.memberTypeOpt, 49 | tsvData.row, 50 | tsvData.colInfo, 51 | tsvData.modelSchema, 52 | tsvData.deleteEmptyValues 53 | ) 54 | blackHole.consume(result) 55 | result 56 | } 57 | 58 | } 59 | 60 | // helper object to get access to TSVFileSupport trait 61 | object TsvFileSupportHarness extends TSVFileSupport {} 62 | -------------------------------------------------------------------------------- /benchmarks/src/main/scala/org/broadinstitute/dsde/firecloud/utils/TsvFormatterBenchmark.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.utils 2 | 3 | import org.broadinstitute.dsde.firecloud.model.{FlexibleModelSchema, ModelSchema} 4 | import org.broadinstitute.dsde.firecloud.utils.TsvFormatterBenchmark.EntityData 5 | import org.broadinstitute.dsde.rawls.model._ 6 | import org.openjdk.jmh.annotations.{Benchmark, Scope, State} 7 | import org.openjdk.jmh.infra.Blackhole 8 | 9 | object TsvFormatterBenchmark { 10 | 11 | @State(Scope.Thread) 12 | class EntityData { 13 | val entityType: String = "sample" 14 | 15 | val model: ModelSchema = FlexibleModelSchema 16 | 17 | val headers: List[String] = List("sample_id", "col1", "col2", "fourth", "last") 18 | 19 | val entities: Seq[Entity] = Seq( 20 | Entity( 21 | "1", 22 | entityType, 23 | Map( 24 | AttributeName.withDefaultNS("col1") -> AttributeString("foo"), 25 | AttributeName.withDefaultNS("col2") -> AttributeBoolean(true), 26 | AttributeName.withDefaultNS("fourth") -> AttributeNumber(42), 27 | AttributeName.withDefaultNS("last") -> AttributeString("gs://some-bucket/somefile.ext") 28 | ) 29 | ), 30 | Entity( 31 | "0005", 32 | entityType, 33 | Map( 34 | AttributeName.withDefaultNS("col1") -> AttributeString("bar"), 35 | AttributeName.withDefaultNS("col2") -> AttributeBoolean(false), 36 | AttributeName.withDefaultNS("fourth") -> AttributeNumber(98.765), 37 | AttributeName.withDefaultNS("last") -> AttributeEntityReference("targetType", "targetName") 38 | ) 39 | ), 40 | Entity( 41 | "789", 42 | entityType, 43 | Map( 44 | AttributeName.withDefaultNS("col1") -> AttributeString("baz\tqux"), 45 | AttributeName.withDefaultNS("col2") -> AttributeBoolean(true), 46 | AttributeName.withDefaultNS("fourth") -> AttributeNumber(-123.45), 47 | AttributeName.withDefaultNS("last") -> AttributeValueList( 48 | Seq( 49 | AttributeString("gs://some-bucket/somefile1.ext"), 50 | AttributeString("gs://some-bucket/somefile2.ext"), 51 | AttributeString("gs://some-bucket/somefile3.ext") 52 | ) 53 | ) 54 | ) 55 | ) 56 | ) 57 | } 58 | 59 | } 60 | 61 | class TsvFormatterBenchmark { 62 | 63 | @Benchmark 64 | def makeEntityRows(blackHole: Blackhole, entityData: EntityData): List[List[String]] = { 65 | val result = 66 | TSVFormatter.makeEntityRows(entityData.entityType, entityData.entities, entityData.headers)(entityData.model) 67 | blackHole.consume(result) 68 | result 69 | } 70 | 71 | } 72 | -------------------------------------------------------------------------------- /build.sbt: -------------------------------------------------------------------------------- 1 | import Settings.* 2 | import Testing.* 3 | import pl.project13.scala.sbt.JmhPlugin 4 | import spray.revolver.RevolverPlugin 5 | 6 | lazy val root = project.in(file(".")) 7 | .settings(rootSettings *) 8 | .withTestSettings 9 | 10 | enablePlugins(RevolverPlugin) 11 | 12 | // JMH subproject for benchmarking 13 | lazy val bench = project.in(file("benchmarks")) 14 | .dependsOn(root % "compile->compile") 15 | .disablePlugins(RevolverPlugin) 16 | .enablePlugins(JmhPlugin) 17 | .settings( 18 | scalaVersion := (root / scalaVersion).value, 19 | // rewire tasks, so that 'bench/Jmh/run' automatically invokes 'bench/Jmh/compile' 20 | // and 'bench/Jmh/compile' invokes root's 'compile' 21 | Jmh / compile := (Jmh / compile).dependsOn(Compile / compile).value, 22 | Jmh / run := (Jmh / run).dependsOn(Jmh / compile).evaluated 23 | ) 24 | 25 | 26 | Revolver.enableDebugging(port = 5051, suspend = false) 27 | 28 | // When JAVA_OPTS are specified in the environment, they are usually meant for the application 29 | // itself rather than sbt, but they are not passed by default to the application, which is a forked 30 | // process. This passes them through to the "re-start" command, which is probably what a developer 31 | // would normally expect. 32 | reStart / javaOptions ++= sys.env("JAVA_OPTS").split(" ").toSeq 33 | -------------------------------------------------------------------------------- /jenkins/jenkins_build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eux 4 | SVCACCT_FILE="dspci-wb-gcr-service-account.json" 5 | GCR_SVCACCT_VAULT="secret/dsde/dsp-techops/common/$SVCACCT_FILE" 6 | VAULT_TOKEN=${VAULT_TOKEN:-$(cat /etc/vault-token-dsde)} 7 | 8 | docker run --rm -e VAULT_TOKEN=$VAULT_TOKEN \ 9 | broadinstitute/dsde-toolbox:latest vault read --format=json ${GCR_SVCACCT_VAULT} \ 10 | | jq .data > ${SVCACCT_FILE} 11 | 12 | ./script/build.sh jar -d push -g gcr.io/broad-dsp-gcr-public/${PROJECT} -k ${SVCACCT_FILE} 13 | 14 | # clean up 15 | rm -f ${SVCACCT_FILE} 16 | 17 | -------------------------------------------------------------------------------- /local-dev/bin/render: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -eo pipefail 4 | 5 | REPO_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../.." && pwd )" 6 | 7 | IMAGE_NAME="${IMAGE_NAME:-us-central1-docker.pkg.dev/dsp-artifact-registry/firecloud-develop-shim/firecloud-develop-shim}" 8 | IMAGE_TAG="${IMAGE_TAG:-latest}" 9 | IMAGE="${IMAGE_NAME}:${IMAGE_TAG}" 10 | 11 | PROJECT_ID="broad-dsde-dev" 12 | 13 | # Helper function to fetch secrets 14 | fetch_secret() { 15 | gcloud secrets versions access latest --project "$PROJECT_ID" --secret "$1" 16 | } 17 | mkdir -p config 18 | 19 | echo "Copying template files to config/ ..." 20 | #Files to copy directly to config## 21 | cp local-dev/templates/docker-rsync-local-orch.sh config/docker-rsync-local-orch.sh 22 | cp local-dev/templates/local-agora.conf config/local-agora.conf 23 | cp local-dev/templates/local-rawls.conf config/local-rawls.conf 24 | cp local-dev/templates/local-sam.conf config/local-sam.conf 25 | cp local-dev/templates/local-thurloe.conf config/local-thurloe.conf 26 | cp local-dev/templates/mod_security_ignore.conf config/mod_security_ignore.conf 27 | cp local-dev/templates/oauth2.conf config/oauth2.conf 28 | cp local-dev/templates/site.conf config/site.conf 29 | 30 | echo "Template files copied." 31 | 32 | echo "Fetching server certs..." 33 | gcloud container clusters get-credentials --zone us-central1-a --project broad-dsde-dev terra-dev 34 | kubectl -n local-dev get secrets local-dev-cert -o 'go-template={{index .data "tls.crt"}}' | base64 --decode > config/server.crt 35 | kubectl -n local-dev get secrets local-dev-cert -o 'go-template={{index .data "tls.key"}}' | base64 --decode > config/server.key 36 | kubectl -n local-dev get configmaps kube-root-ca.crt -o 'go-template={{ index .data "ca.crt" }}' > config/ca-bundle.crt 37 | echo "Certs fetched." 38 | 39 | echo "Fetching secrets from GSM..." 40 | ##Files to download from secrets## 41 | ##TODO move/get these secrets from GSM 42 | fetch_secret "firecloud-sa" > config/firecloud-account.json 43 | fetch_secret "firecloud-sa" > config/firecloud-account.pem 44 | fetch_secret "rawls-sa" > config/rawls-account.json 45 | fetch_secret "rawls-sa" > config/rawls-account.pem 46 | 47 | ## Secrets to pull for firecloud-orchestration.conf ## 48 | B2C_APPID_JSON=$(fetch_secret "b2c-application-id") 49 | RAWLS_SAKEY_JSON=$(fetch_secret "rawls-sa") 50 | FIRECLOUD_SECRETS=$(fetch_secret "firecloud-misc-secrets") 51 | FIRECLOUD_SAKEY_JSON=$(fetch_secret "firecloud-sa") 52 | 53 | # Extract the secret values using jq 54 | FIRECLOUD_CLIENT_EMAIL=$(echo "$FIRECLOUD_SAKEY_JSON" | jq -r '.client_email') 55 | RAWLS_CLIENT_EMAIL=$(echo "$RAWLS_SAKEY_JSON" | jq -r '.client_email') 56 | FIRECLOUD_ID=$(echo "$FIRECLOUD_SECRETS" | jq -r '.firecloud_id') 57 | B2C_APP_ID=$(echo "$B2C_APPID_JSON" | jq -r '.value') 58 | 59 | echo "Secrets fetched." 60 | # Define the template and output file paths 61 | ORCH_TEMPLATE_FILE="local-dev/templates/firecloud-orchestration.conf" 62 | ORCH_OUTPUT_FILE="config/firecloud-orchestration.conf" 63 | 64 | echo "Generating firecloud-orchestration.conf..." 65 | # Replace placeholders in the template file with the fetched secret values 66 | sed -e "s/{{ \$b2cAppId }}/$B2C_APP_ID/" \ 67 | -e "s/{{ \$firecloudSaKey.client_email }}/$FIRECLOUD_CLIENT_EMAIL/" \ 68 | -e "s/{{ \$rawlsSaKey.client_email }}/$RAWLS_CLIENT_EMAIL/" \ 69 | -e "s/{{ \$commonSecrets.firecloud_id }}/$FIRECLOUD_ID/" \ 70 | "$ORCH_TEMPLATE_FILE" > "$ORCH_OUTPUT_FILE" 71 | echo "firecloud-orchestration.conf generated." 72 | 73 | cat <> /etc/hosts" 78 | 79 | EOF 80 | 81 | -------------------------------------------------------------------------------- /local-dev/templates/local-agora.conf: -------------------------------------------------------------------------------- 1 | agora { 2 | baseUrl = "https://local.broadinstitute.org:30443" 3 | } 4 | -------------------------------------------------------------------------------- /local-dev/templates/local-rawls.conf: -------------------------------------------------------------------------------- 1 | rawls { 2 | baseUrl = "https://local.dsde-dev.broadinstitute.org:20443" 3 | } 4 | -------------------------------------------------------------------------------- /local-dev/templates/local-sam.conf: -------------------------------------------------------------------------------- 1 | sam { 2 | baseUrl = "https://local.broadinstitute.org:50443" 3 | } 4 | -------------------------------------------------------------------------------- /local-dev/templates/local-thurloe.conf: -------------------------------------------------------------------------------- 1 | thurloe { 2 | baseUrl = "https://local.broadinstitute.org:40443" 3 | } 4 | -------------------------------------------------------------------------------- /local-dev/templates/mod_security_ignore.conf: -------------------------------------------------------------------------------- 1 | SecRule REQUEST_URI "@contains importEntities" phase:1,id:'1000000001',nolog,allow,ctl:ruleEngine=Off,ctl:auditEngine=Off 2 | SecRule REQUEST_URI "@contains batchUpsert" phase:1,id:'1000000002',nolog,allow,ctl:ruleEngine=Off,ctl:auditEngine=Off 3 | SecRule REQUEST_URI "@contains loading..." phase:1,id:'1000000003',nolog,allow,ctl:ruleEngine=Off,ctl:auditEngine=Off 4 | -------------------------------------------------------------------------------- /local-dev/templates/oauth2.conf: -------------------------------------------------------------------------------- 1 | OAuth2Cache shm max_val_size=16384 2 | OAuth2TokenVerify metadata https://terradevb2c.b2clogin.com/terradevb2c.onmicrosoft.com/v2.0/.well-known/openid-configuration?p=b2c_1a_signup_signin_dev metadata.ssl_verify=true&verify.exp=required&verify.iat=skip 3 | OAuth2TokenVerify metadata https://terradevb2c.b2clogin.com/terradevb2c.onmicrosoft.com/v2.0/.well-known/openid-configuration?p=b2c_1a_signup_signin_billing_dev metadata.ssl_verify=true&verify.exp=required&verify.iat=skip 4 | OAuth2TokenVerify metadata https://terradevb2c.b2clogin.com/terradevb2c.onmicrosoft.com/v2.0/.well-known/openid-configuration?p=b2c_1a_signup_signin_tdr_dev metadata.ssl_verify=true&verify.exp=required&verify.iat=skip 5 | OAuth2TokenVerify metadata https://login.microsoftonline.com/common/v2.0/.well-known/openid-configuration metadata.ssl_verify=true&verify.exp=required&verify.iat=skip 6 | OAuth2TokenVerify metadata https://accounts.google.com/.well-known/openid-configuration metadata.ssl_verify=true&verify.exp=required&verify.iat=skip 7 | OAuth2TokenVerify introspect https://127.0.0.1/introspect/ introspect.ssl_verify=false&verify.exp=required&verify.iat=skip -------------------------------------------------------------------------------- /project/Merging.scala: -------------------------------------------------------------------------------- 1 | import sbtassembly.{MergeStrategy, PathList} 2 | 3 | object Merging { 4 | def customMergeStrategy(oldStrategy: (String) => MergeStrategy): (String => MergeStrategy) = { 5 | case x if x.endsWith("io.netty.versions.properties") => MergeStrategy.discard 6 | case x if x.contains("native-image/io.netty") => MergeStrategy.first 7 | 8 | // TODO: we no longer target Java 8, reassess this: 9 | // we target Java 8, which does not use module-info.class. Some dependencies (Jackson) cause assembly problems on module-info 10 | case x if x.endsWith("module-info.class") => MergeStrategy.discard 11 | 12 | case x if x.contains("javax/activation") => MergeStrategy.first 13 | case x if x.contains("javax/annotation") => MergeStrategy.first 14 | 15 | case x if x.endsWith("kotlin-stdlib.kotlin_module") => MergeStrategy.first 16 | case x if x.contains("bouncycastle") => MergeStrategy.first 17 | case x if x.endsWith("kotlin-stdlib-common.kotlin_module") => MergeStrategy.first 18 | case x if x.endsWith("okio.kotlin_module") => MergeStrategy.first 19 | case x if x.endsWith("arrow-git.properties") => MergeStrategy.concat 20 | 21 | // For the following error: 22 | // Error: (assembly) deduplicate: different file contents found in the following: 23 | // Error: /home/sbtuser/.cache/coursier/v1/https/repo1.maven.org/maven2/com/google/protobuf/protobuf-java/3.19.4/protobuf-java-3.19.4.jar:google/protobuf/struct.proto 24 | // Error: /home/sbtuser/.cache/coursier/v1/https/repo1.maven.org/maven2/com/typesafe/akka/akka-protobuf-v3_2.13/2.6.19/akka-protobuf-v3_2.13-2.6.19.jar:google/protobuf/struct.proto 25 | case PathList("google", "protobuf", _ @_*) => MergeStrategy.first 26 | case PathList("META-INF", "versions", "9", "OSGI-INF", "MANIFEST.MF") => MergeStrategy.first 27 | case PathList("META-INF", "spring", "aot.factories") => MergeStrategy.first 28 | case x => oldStrategy(x) 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /project/Settings.scala: -------------------------------------------------------------------------------- 1 | import Dependencies._ 2 | import Merging._ 3 | import Testing._ 4 | import Version._ 5 | import sbt.Keys._ 6 | import sbt._ 7 | import sbtassembly.AssemblyPlugin.autoImport._ 8 | import org.scalafmt.sbt.ScalafmtPlugin.autoImport.scalafmtFilter 9 | 10 | object Settings { 11 | 12 | val artifactory = "https://us-central1-maven.pkg.dev/dsp-artifact-registry/" 13 | 14 | val commonResolvers = List( 15 | "artifactory-releases" at artifactory + "libs-release", 16 | "artifactory-snapshots" at artifactory + "libs-snapshot", 17 | "jitpack.io" at "https://jitpack.io", 18 | "Akka library repository" at "https://repo.akka.io/maven" 19 | ) 20 | 21 | val proxyResolvers = List( 22 | "internal-maven-proxy" at artifactory + "maven-central" 23 | ) 24 | 25 | //coreDefaultSettings + defaultConfigs = the now deprecated defaultSettings 26 | val commonBuildSettings = Defaults.coreDefaultSettings ++ Defaults.defaultConfigs ++ Seq( 27 | javaOptions += "-Xmx2G", 28 | javacOptions ++= Seq("--release", "17") 29 | ) 30 | 31 | val commonCompilerSettings = Seq( 32 | "-unchecked", 33 | "-deprecation", 34 | "-feature", 35 | "-encoding", "utf8", 36 | "-release:8" 37 | ) 38 | 39 | //sbt assembly settings 40 | val commonAssemblySettings = Seq( 41 | assembly / assemblyMergeStrategy := customMergeStrategy((assembly / assemblyMergeStrategy).value), 42 | assembly / test := {} 43 | ) 44 | 45 | val scalafmtSettings = List( 46 | Global / excludeLintKeys += scalafmtFilter, 47 | Global / scalafmtFilter := "diff-ref=HEAD^" 48 | ) 49 | 50 | //common settings for all sbt subprojects 51 | val commonSettings = 52 | commonBuildSettings ++ commonAssemblySettings ++ commonTestSettings ++ scalafmtSettings ++ List( 53 | organization := "org.broadinstitute.dsde.firecloud", 54 | scalaVersion := "2.13.16", 55 | resolvers := proxyResolvers ++: resolvers.value ++: commonResolvers, 56 | scalacOptions ++= commonCompilerSettings, 57 | dependencyOverrides ++= transitiveDependencyOverrides 58 | ) 59 | 60 | //the full list of settings for the root project that's ultimately the one we build into a fat JAR and run 61 | //coreDefaultSettings (inside commonSettings) sets the project name, which we want to override, so ordering is important. 62 | //thus commonSettings needs to be added first. 63 | val rootSettings = commonSettings ++ List( 64 | name := "FireCloud-Orchestration", 65 | libraryDependencies ++= rootDependencies 66 | ) ++ commonAssemblySettings ++ rootVersionSettings 67 | 68 | 69 | } 70 | -------------------------------------------------------------------------------- /project/Testing.scala: -------------------------------------------------------------------------------- 1 | import sbt.Keys._ 2 | import sbt._ 3 | 4 | object Testing { 5 | 6 | def isIntegrationTest(name: String) = name contains "integrationtest" 7 | 8 | lazy val IntegrationTest = config("it") extend Test 9 | 10 | val commonTestSettings: Seq[Setting[_]] = List( 11 | Test / testOptions ++= Seq(Tests.Filter(s => !isIntegrationTest(s))), 12 | Test / testOptions += Tests.Argument("-oD"), // D = individual test durations 13 | IntegrationTest / testOptions := Seq(Tests.Filter(s => isIntegrationTest(s))), 14 | 15 | // ES client attempts to set the number of processors that Netty should use. 16 | // However, we've already initialized Netty elsewhere (mockserver, I assume), 17 | // so the call fails. Tell ES to skip attempting to set this value. 18 | Test / javaOptions += "-Des.set.netty.runtime.available.processors=false", 19 | Test / fork := true, 20 | Test / parallelExecution := false, 21 | IntegrationTest / fork := false // allow easy overriding of conf values via system props 22 | 23 | ) 24 | 25 | implicit class ProjectTestSettings(val project: Project) extends AnyVal { 26 | def withTestSettings: Project = project 27 | .configs(IntegrationTest) 28 | .settings(inConfig(IntegrationTest)(Defaults.testTasks): _*) 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /project/Version.scala: -------------------------------------------------------------------------------- 1 | import sbt.Keys._ 2 | import sbt._ 3 | 4 | import scala.sys.process._ 5 | 6 | object Version { 7 | val baseModelVersion = "0.1" 8 | 9 | def getVersionString = { 10 | def getLastModelCommitFromGit = s"""git rev-parse --short HEAD""" !! 11 | 12 | // either specify git model hash as an env var or derive it 13 | val lastModelCommit = sys.env.getOrElse("GIT_MODEL_HASH", getLastModelCommitFromGit).trim() 14 | val version = baseModelVersion + "-" + lastModelCommit 15 | 16 | // The project isSnapshot string passed in via command line settings, if desired. 17 | val isSnapshot = sys.props.getOrElse("project.isSnapshot", "true").toBoolean 18 | 19 | // For now, obfuscate SNAPSHOTs from sbt's developers: https://github.com/sbt/sbt/issues/2687#issuecomment-236586241 20 | if (isSnapshot) s"$version-SNAPSHOT" else version 21 | } 22 | 23 | val rootVersionSettings: Seq[Setting[_]] = 24 | Seq(version := getVersionString) 25 | } 26 | -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=1.11.1 2 | -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.3.1") 2 | 3 | addSbtPlugin("io.spray" % "sbt-revolver" % "0.10.0") 4 | 5 | addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.3.1") 6 | 7 | addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.4") 8 | 9 | addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.4.7") 10 | 11 | addDependencyTreePlugin 12 | -------------------------------------------------------------------------------- /script/build_jar.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This script provides an entry point to assemble the Sam jar file. 4 | # Used by the sam-build.yaml workflow in terra-github-workflows. 5 | # 6 | set -e 7 | 8 | # Set for versioning the jar 9 | GIT_MODEL_HASH=$(git log -n 1 --pretty=format:%h) 10 | 11 | docker run --rm -e GIT_MODEL_HASH=${GIT_MODEL_HASH} \ 12 | -v $PWD:/working \ 13 | -v jar-cache:/root/.ivy -v jar-cache:/root/.ivy2 \ 14 | -w /working \ 15 | sbtscala/scala-sbt:eclipse-temurin-17.0.15_6_1.11.1_2.13.16 /working/src/docker/clean_install.sh /working 16 | 17 | EXIT_CODE=$? 18 | 19 | if [ $EXIT_CODE != 0 ]; then 20 | echo "jar build exited with status $EXIT_CODE" 21 | exit $EXIT_CODE 22 | fi 23 | -------------------------------------------------------------------------------- /script/create-configs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euox pipefail 3 | IFS=$'\n\t' 4 | 5 | 6 | run_with_ui=${1:-'false'} 7 | 8 | docker run --rm -it -v "$PWD":/working broadinstitute/dsde-toolbox \ 9 | render-templates.sh local "$(<~/.vault-token)" 10 | 11 | if [ "$run_with_ui" = 'false' ]; then 12 | sed -i '' 's/"20080:80"/"80:80"/1' target/config/proxy-compose.yaml 13 | sed -i '' 's/"20443:443"/"443:443"/1' target/config/proxy-compose.yaml 14 | sed -i '' 's/basePath: \/service/basePath: \//1' src/main/resources/swagger/api-docs.yaml 15 | fi 16 | -------------------------------------------------------------------------------- /src/docker/clean_install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This script runs sbt assembly to produce a target jar file. 4 | # Used by build_jar.sh 5 | # chmod +x must be set for this script 6 | set -eux 7 | 8 | ORCH_DIR=$1 9 | cd $ORCH_DIR 10 | 11 | export SBT_OPTS="-Xms5g -Xmx5g -XX:MaxMetaspaceSize=5g" 12 | echo "starting sbt clean assembly ..." 13 | sbt 'set assembly / test := {}' clean assembly 14 | echo "... clean assembly complete, finding and moving jar ..." 15 | ORCH_JAR=$(find target | grep 'FireCloud-Orchestration.*\.jar') 16 | mv $ORCH_JAR . 17 | echo "... jar moved." 18 | -------------------------------------------------------------------------------- /src/docker/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # script to sbt build the orch jar 3 | 4 | set -e 5 | ORCH_DIR=$1 6 | cd $ORCH_DIR 7 | 8 | sbt -batch -d clean reload update compile 9 | sbt -batch test 10 | sbt --mem 2048 -batch assembly 11 | 12 | ORCH_JAR=$(find target | grep 'FireCloud-Orchestration.*\.jar') 13 | mv $ORCH_JAR . 14 | sbt clean 15 | -------------------------------------------------------------------------------- /src/docker/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euox pipefail 3 | IFS=$'\n\t' 4 | 5 | 6 | if [ -e /app/target/scala-2.13/FireCloud-Orchestration-assembly-*.jar ]; then 7 | exec java $JAVA_OPTS -jar /app/target/scala-2.13/FireCloud-Orchestration-assembly-*.jar 8 | else 9 | cd /app 10 | exec sbt '~ reStart' 11 | fi 12 | -------------------------------------------------------------------------------- /src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | [%level] [%d{HH:mm:ss.SSS}] [%thread] %logger{36} - %msg%n 8 | 9 | 10 | 11 | 12 | 13 | 14 | ERROR 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /src/main/resources/model.json: -------------------------------------------------------------------------------- 1 | { "schema" : { 2 | "participant": { 3 | "requiredAttributes": {}, 4 | "plural": "participants" 5 | }, 6 | "sample": { 7 | "requiredAttributes": { 8 | "participant": "participant" 9 | }, 10 | "plural": "samples" 11 | }, 12 | "pair": { 13 | "requiredAttributes": { 14 | "case_sample": "sample", 15 | "control_sample": "sample", 16 | "participant": "participant" 17 | }, 18 | "plural": "pairs" 19 | }, 20 | "participant_set": { 21 | "requiredAttributes": {}, 22 | "plural": "participant_sets", 23 | "memberType": "participant" 24 | }, 25 | "sample_set": { 26 | "requiredAttributes": {}, 27 | "plural": "sample_sets", 28 | "memberType": "sample" 29 | }, 30 | "pair_set": { 31 | "requiredAttributes": {}, 32 | "plural": "pair_sets", 33 | "memberType": "pair" 34 | } 35 | } } 36 | -------------------------------------------------------------------------------- /src/main/resources/swagger/README.md: -------------------------------------------------------------------------------- 1 | Guidelines For Swagger Editors 2 | ============================== 3 | 4 | `path`s and `definition`s are organized alphabetically. 5 | 6 | each `path` and `definition` should be separated by a newline. If you have multiple 7 | operations (e.g. GET and POST) for a single `path`, they should not be separated. 8 | 9 | `security` and `produces: application/json` are defined at the top level. You should only 10 | include those keys in your endpoints if you need to override the global definitions. 11 | 12 | We keep the entire swagger definition in a single file so it can be edited/validated via 13 | http://editor.swagger.io. When you make changes to swagger, you are responsible for validating 14 | the file. Warnings should be avoided; errors MUST be eliminated. 15 | 16 | Wherever possible, use `$ref:`s to centralize reusable blocks of code and reduce file size. 17 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/Application.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud 2 | 3 | import org.broadinstitute.dsde.firecloud.dataaccess._ 4 | 5 | /** 6 | * Created by davidan on 9/23/16. 7 | */ 8 | 9 | case class Application(agoraDAO: AgoraDAO, 10 | googleServicesDAO: GoogleServicesDAO, 11 | rawlsDAO: RawlsDAO, 12 | samDAO: SamDAO, 13 | thurloeDAO: ThurloeDAO, 14 | shibbolethDAO: ShibbolethDAO, 15 | cwdsDAO: CwdsDAO, 16 | ecmDAO: ExternalCredsDAO 17 | ) 18 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/FireCloudException.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud 2 | 3 | import org.broadinstitute.dsde.rawls.model.ErrorReport 4 | 5 | class FireCloudException(message: String = null, cause: Throwable = null) extends Exception(message, cause) 6 | 7 | class FireCloudExceptionWithErrorReport(val errorReport: ErrorReport) extends FireCloudException(errorReport.toString) 8 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/HealthChecks.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud 2 | 3 | import akka.actor.ActorSystem 4 | import akka.http.scaladsl.model.StatusCodes 5 | import com.typesafe.scalalogging.LazyLogging 6 | import org.broadinstitute.dsde.firecloud.dataaccess.ReportsSubsystemStatus 7 | import org.broadinstitute.dsde.firecloud.model.{AccessToken, RegistrationInfo, WorkbenchEnabled} 8 | import org.broadinstitute.dsde.workbench.util.health.Subsystems._ 9 | import org.broadinstitute.dsde.workbench.util.health.{SubsystemStatus, Subsystems} 10 | 11 | import scala.concurrent.{ExecutionContext, Future} 12 | 13 | object HealthChecks { 14 | val termsOfServiceUrl = "app.terra.bio/#terms-of-service" 15 | } 16 | 17 | class HealthChecks(app: Application)(implicit val system: ActorSystem, implicit val executionContext: ExecutionContext) 18 | extends LazyLogging { 19 | 20 | def healthMonitorChecks: () => Map[Subsystem, Future[SubsystemStatus]] = () => { 21 | val servicesToMonitor = Seq(app.rawlsDAO, app.samDAO, app.thurloeDAO) ++ 22 | Option.when(FireCloudConfig.Agora.enabled)(app.agoraDAO) ++ 23 | Option.when(FireCloudConfig.GoogleCloud.enabled)(app.googleServicesDAO) 24 | 25 | servicesToMonitor.map { subsystem => 26 | subsystem.serviceName -> subsystem.status 27 | }.toMap 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/dataaccess/AgoraDAO.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.dataaccess 2 | 3 | import org.broadinstitute.dsde.firecloud.model.OrchMethodRepository.{ 4 | AgoraEntityType, 5 | AgoraPermission, 6 | EntityAccessControlAgora, 7 | Method 8 | } 9 | import org.broadinstitute.dsde.firecloud.model.UserInfo 10 | import org.broadinstitute.dsde.rawls.model.ErrorReportSource 11 | import org.broadinstitute.dsde.workbench.util.health.Subsystems 12 | import org.broadinstitute.dsde.workbench.util.health.Subsystems.Subsystem 13 | 14 | import scala.concurrent.Future 15 | 16 | object AgoraDAO { 17 | lazy val serviceName = Subsystems.Agora 18 | } 19 | 20 | trait AgoraDAO extends ReportsSubsystemStatus { 21 | 22 | implicit val errorReportSource: ErrorReportSource = ErrorReportSource(AgoraDAO.serviceName.value) 23 | 24 | def getNamespacePermissions(ns: String, entity: String)(implicit userInfo: UserInfo): Future[List[AgoraPermission]] 25 | def postNamespacePermissions(ns: String, entity: String, perms: List[AgoraPermission])(implicit 26 | userInfo: UserInfo 27 | ): Future[List[AgoraPermission]] 28 | 29 | def getMultiEntityPermissions(entityType: AgoraEntityType.Value, entities: List[Method])(implicit 30 | userInfo: UserInfo 31 | ): Future[List[EntityAccessControlAgora]] 32 | 33 | def batchCreatePermissions(inputs: List[EntityAccessControlAgora])(implicit 34 | userInfo: UserInfo 35 | ): Future[List[EntityAccessControlAgora]] 36 | def getPermission(url: String)(implicit userInfo: UserInfo): Future[List[AgoraPermission]] 37 | def createPermission(url: String, agoraPermissions: List[AgoraPermission])(implicit 38 | userInfo: UserInfo 39 | ): Future[List[AgoraPermission]] 40 | 41 | override def serviceName: Subsystem = AgoraDAO.serviceName 42 | } 43 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/dataaccess/CwdsDAO.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.dataaccess 2 | 3 | import org.broadinstitute.dsde.firecloud.model.{AsyncImportRequest, CwdsListResponse, UserInfo} 4 | import org.databiosphere.workspacedata.client.ApiException 5 | import org.databiosphere.workspacedata.model.GenericJob 6 | object LegacyFileTypes { 7 | final val FILETYPE_PFB = "pfb" 8 | final val FILETYPE_TDR = "tdrexport" 9 | final val FILETYPE_RAWLS = "rawlsjson" 10 | } 11 | 12 | trait CwdsDAO { 13 | 14 | def isEnabled: Boolean 15 | 16 | def getSupportedFormats: List[String] 17 | 18 | @throws(classOf[ApiException]) 19 | def listJobsV1(workspaceId: String, runningOnly: Boolean)(implicit userInfo: UserInfo): List[CwdsListResponse] 20 | 21 | @throws(classOf[ApiException]) 22 | def getJobV1(workspaceId: String, jobId: String)(implicit userInfo: UserInfo): CwdsListResponse 23 | 24 | @throws(classOf[ApiException]) 25 | def importV1(workspaceId: String, asyncImportRequest: AsyncImportRequest)(implicit userInfo: UserInfo): GenericJob 26 | 27 | } 28 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/dataaccess/DisabledExternalCredsDAO.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.dataaccess 2 | 3 | import com.typesafe.scalalogging.LazyLogging 4 | import org.broadinstitute.dsde.firecloud.model.{LinkedEraAccount, UserInfo, WithAccessToken} 5 | 6 | import scala.concurrent.Future 7 | 8 | class DisabledExternalCredsDAO extends ExternalCredsDAO with LazyLogging { 9 | 10 | override def getLinkedAccount(userInfo: UserInfo): Future[Option[LinkedEraAccount]] = Future.successful { 11 | logger.warn("Getting Linked eRA Account from ECM, but ECM is disabled.") 12 | None 13 | } 14 | 15 | override def putLinkedEraAccount(linkedEraAccount: LinkedEraAccount, orchInfo: WithAccessToken): Future[Unit] = 16 | Future.successful { 17 | logger.warn("Putting Linked eRA Account to ECM, but ECM is disabled.") 18 | } 19 | 20 | override def deleteLinkedEraAccount(userInfo: UserInfo, orchInfo: WithAccessToken): Future[Unit] = 21 | Future.successful { 22 | logger.warn("Deleting Linked eRA Account from ECM, but ECM is disabled.") 23 | } 24 | 25 | override def getLinkedEraAccountForUsername(username: String, 26 | orchInfo: WithAccessToken 27 | ): Future[Option[LinkedEraAccount]] = Future.successful { 28 | logger.warn("Getting Linked eRA Account for username from ECM, but ECM is disabled.") 29 | None 30 | } 31 | 32 | override def getActiveLinkedEraAccounts(orchInfo: WithAccessToken): Future[Seq[LinkedEraAccount]] = 33 | Future.successful { 34 | logger.warn("Getting Active Linked eRA Accounts from ECM, but ECM is disabled.") 35 | Seq.empty 36 | } 37 | 38 | override def getVisas(provider: String, 39 | userId: String, 40 | issuer: String, 41 | visaType: String, 42 | orchInfo: WithAccessToken 43 | ): Future[Seq[AnyRef]] = 44 | Future.successful { 45 | logger.warn("Getting Visas from ECM, but ECM is disabled.") 46 | Seq.empty 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/dataaccess/ExternalCredsDAO.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.dataaccess 2 | 3 | import org.broadinstitute.dsde.firecloud.model.{LinkedEraAccount, UserInfo, WithAccessToken} 4 | import org.databiosphere.workspacedata.client.ApiException 5 | 6 | import scala.concurrent.Future 7 | 8 | trait ExternalCredsDAO { 9 | 10 | @throws(classOf[ApiException]) 11 | def getLinkedAccount(userInfo: UserInfo): Future[Option[LinkedEraAccount]] 12 | 13 | @throws(classOf[ApiException]) 14 | def putLinkedEraAccount(linkedEraAccount: LinkedEraAccount, orchInfo: WithAccessToken): Future[Unit] 15 | 16 | @throws(classOf[ApiException]) 17 | def deleteLinkedEraAccount(userInfo: UserInfo, orchInfo: WithAccessToken): Future[Unit] 18 | 19 | @throws(classOf[ApiException]) 20 | def getLinkedEraAccountForUsername(username: String, orchInfo: WithAccessToken): Future[Option[LinkedEraAccount]] 21 | 22 | @throws(classOf[ApiException]) 23 | def getActiveLinkedEraAccounts(orchInfo: WithAccessToken): Future[Seq[LinkedEraAccount]] 24 | 25 | @throws(classOf[ApiException]) 26 | def getVisas(provider: String, 27 | userId: String, 28 | issuer: String, 29 | visaType: String, 30 | orchInfo: WithAccessToken 31 | ): Future[Seq[AnyRef]] 32 | } 33 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/dataaccess/GoogleServicesDAO.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.dataaccess 2 | 3 | import better.files.File 4 | import org.broadinstitute.dsde.rawls.model.ErrorReportSource 5 | import org.broadinstitute.dsde.workbench.model.google.{GcsBucketName, GcsObjectName, GcsPath} 6 | import org.broadinstitute.dsde.workbench.util.health.Subsystems.Subsystem 7 | import org.broadinstitute.dsde.workbench.util.health.{SubsystemStatus, Subsystems} 8 | 9 | import java.io.InputStream 10 | import scala.concurrent.Future 11 | 12 | object GoogleServicesDAO { 13 | lazy val serviceName = Subsystems.GoogleBuckets 14 | } 15 | 16 | trait GoogleServicesDAO extends ReportsSubsystemStatus { 17 | 18 | implicit val errorReportSource: ErrorReportSource = ErrorReportSource(GoogleServicesDAO.serviceName.value) 19 | 20 | def getAdminUserAccessToken: String 21 | def getBucketObjectAsInputStream(bucketName: String, objectKey: String): InputStream 22 | def getObjectResourceUrl(bucketName: String, objectKey: String): String 23 | 24 | def writeObjectAsRawlsSA(bucketName: GcsBucketName, objectKey: GcsObjectName, objectContents: Array[Byte]): GcsPath 25 | def writeObjectAsRawlsSA(bucketName: GcsBucketName, objectKey: GcsObjectName, tempFile: File): GcsPath 26 | 27 | def deleteGoogleGroup(groupEmail: String): Unit 28 | def createGoogleGroup(groupName: String): Option[String] 29 | def addMemberToAnonymizedGoogleGroup(groupName: String, targetUserEmail: String): Option[String] 30 | 31 | def status: Future[SubsystemStatus] 32 | override def serviceName: Subsystem = GoogleServicesDAO.serviceName 33 | 34 | def publishMessages(fullyQualifiedTopic: String, messages: Seq[String]): Future[Unit] 35 | 36 | def listBucket(bucketName: GcsBucketName, prefix: Option[String], recursive: Boolean): List[GcsObjectName] 37 | } 38 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/dataaccess/HttpShibbolethDAO.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.dataaccess 2 | 3 | import akka.actor.ActorSystem 4 | import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport 5 | import akka.stream.Materializer 6 | import org.broadinstitute.dsde.firecloud.FireCloudConfig 7 | import org.broadinstitute.dsde.firecloud.utils.RestJsonClient 8 | 9 | import scala.concurrent.{ExecutionContext, Future} 10 | 11 | class HttpShibbolethDAO(implicit val system: ActorSystem, 12 | implicit val materializer: Materializer, 13 | implicit val executionContext: ExecutionContext 14 | ) extends ShibbolethDAO 15 | with RestJsonClient 16 | with SprayJsonSupport { 17 | 18 | override def getPublicKey(): Future[String] = { 19 | val publicKeyUrl = FireCloudConfig.Shibboleth.publicKeyUrl 20 | 21 | unAuthedRequestToObject[String](Get(publicKeyUrl)) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/dataaccess/ReportsSubsystemStatus.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.dataaccess 2 | 3 | import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport 4 | import org.broadinstitute.dsde.workbench.util.health.SubsystemStatus 5 | import org.broadinstitute.dsde.workbench.util.health.Subsystems.Subsystem 6 | 7 | import scala.concurrent.Future 8 | 9 | /** 10 | * Created by anichols on 4/21/17. 11 | */ 12 | trait ReportsSubsystemStatus extends SprayJsonSupport { 13 | 14 | def status: Future[SubsystemStatus] 15 | 16 | def serviceName: Subsystem 17 | } 18 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/dataaccess/ShibbolethDAO.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.dataaccess 2 | 3 | import org.broadinstitute.dsde.rawls.model.ErrorReportSource 4 | 5 | import scala.concurrent.Future 6 | 7 | trait ShibbolethDAO { 8 | 9 | implicit val errorReportSource: ErrorReportSource = ErrorReportSource("shibboleth") 10 | 11 | def getPublicKey(): Future[String] 12 | 13 | } 14 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/dataaccess/ThurloeDAO.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.dataaccess 2 | 3 | import com.typesafe.scalalogging.LazyLogging 4 | import org.broadinstitute.dsde.firecloud.model.{BasicProfile, ProfileWrapper, UserInfo, WithAccessToken} 5 | import org.broadinstitute.dsde.rawls.model.ErrorReportSource 6 | import org.broadinstitute.dsde.workbench.util.health.Subsystems 7 | import org.broadinstitute.dsde.workbench.util.health.Subsystems.Subsystem 8 | 9 | import scala.concurrent.Future 10 | import scala.util.Try 11 | 12 | /** 13 | * Created by mbemis on 10/21/16. 14 | */ 15 | object ThurloeDAO { 16 | lazy val serviceName = Subsystems.Thurloe 17 | } 18 | 19 | trait ThurloeDAO extends LazyLogging with ReportsSubsystemStatus { 20 | 21 | implicit val errorReportSource: ErrorReportSource = ErrorReportSource(ThurloeDAO.serviceName.value) 22 | 23 | def getAllKVPs(forUserId: String, callerToken: WithAccessToken): Future[Option[ProfileWrapper]] 24 | def getAllUserValuesForKey(key: String): Future[Map[String, String]] 25 | def saveProfile(userInfo: UserInfo, profile: BasicProfile): Future[Unit] 26 | 27 | /** 28 | * Save KVPs for myself - the KVPs will be saved to the same user that authenticates the call. 29 | * @param userInfo contains the userid for which to save KVPs and that user's auth token 30 | * @param keyValues the KVPs to save 31 | * @return success/failure of save 32 | */ 33 | def saveKeyValues(userInfo: UserInfo, keyValues: Map[String, String]): Future[Try[Unit]] 34 | 35 | /** 36 | * Save KVPs for a different user - the KVPs will be saved to the "forUserId" user, 37 | * but the call to Thurloe will be authenticated as the "callerToken" user. 38 | * @param forUserId the userid of the user for which to save KVPs 39 | * @param callerToken auth token of the user making the call 40 | * @param keyValues the KVPs to save 41 | * @return success/failure of save 42 | */ 43 | def saveKeyValues(forUserId: String, callerToken: WithAccessToken, keyValues: Map[String, String]): Future[Try[Unit]] 44 | 45 | def bulkUserQuery(userIds: List[String], keySelection: List[String]): Future[List[ProfileWrapper]] 46 | 47 | def deleteKeyValue(forUserId: String, keyName: String, callerToken: WithAccessToken): Future[Try[Unit]] 48 | 49 | override def serviceName: Subsystem = ThurloeDAO.serviceName 50 | } 51 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/filematch/FileMatchingOptions.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.filematch 2 | 3 | import spray.json.DefaultJsonProtocol.jsonFormat4 4 | import spray.json.RootJsonFormat 5 | import spray.json.DefaultJsonProtocol._ 6 | 7 | /** 8 | * Request payload, specified by end users, to control file-matching functionality 9 | * @param prefix bucket prefix in which to list files 10 | * @param read1Name name for the "read1" column 11 | * @param read2Name name for the "read2" column 12 | * @param recursive should bucket-listing be recursive? 13 | */ 14 | case class FileMatchingOptions(prefix: String, 15 | read1Name: Option[String] = None, 16 | read2Name: Option[String] = None, 17 | recursive: Option[Boolean] = None 18 | ) 19 | 20 | object FileMatchingOptionsFormat { 21 | implicit val fileMatchingOptionsFormat: RootJsonFormat[FileMatchingOptions] = jsonFormat4(FileMatchingOptions) 22 | } 23 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/filematch/result/FailedMatchResult.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.filematch.result 2 | 3 | import com.google.common.annotations.VisibleForTesting 4 | 5 | import java.nio.file.Path 6 | 7 | /** 8 | * FileMatchResult indicating that the file did not hit on any known pattern. 9 | */ 10 | case class FailedMatchResult(firstFile: Path) extends FileMatchResult {} 11 | 12 | @VisibleForTesting 13 | object FailedMatchResult { 14 | def fromString(firstFile: String): FailedMatchResult = 15 | FailedMatchResult(new java.io.File(firstFile).toPath) 16 | } 17 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/filematch/result/FileMatchResult.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.filematch.result 2 | 3 | import java.nio.file.Path 4 | 5 | /** 6 | * Marker trait for failed/partial/successful file-matching results 7 | */ 8 | trait FileMatchResult { 9 | def firstFile: Path 10 | } 11 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/filematch/result/PartialMatchResult.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.filematch.result 2 | 3 | import com.google.common.annotations.VisibleForTesting 4 | 5 | import java.nio.file.Path 6 | 7 | /** 8 | * FileMatchResult indicating that the file successfully hit a known pattern, but no paired file could be found. 9 | */ 10 | case class PartialMatchResult(firstFile: Path, id: String) extends FileMatchResult {} 11 | 12 | @VisibleForTesting 13 | object PartialMatchResult { 14 | def fromStrings(firstFile: String, id: String): PartialMatchResult = 15 | PartialMatchResult(new java.io.File(firstFile).toPath, id) 16 | } 17 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/filematch/result/SuccessfulMatchResult.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.filematch.result 2 | 3 | import com.google.common.annotations.VisibleForTesting 4 | 5 | import java.nio.file.Path 6 | 7 | /** 8 | * FileMatchResult indicating that the file successfully hit a known pattern. 9 | */ 10 | case class SuccessfulMatchResult(firstFile: Path, secondFile: Path, id: String) extends FileMatchResult { 11 | // convert this SuccessfulMatchResult to a PartialMatchResult 12 | def toPartial: PartialMatchResult = PartialMatchResult(firstFile, id) 13 | } 14 | 15 | @VisibleForTesting 16 | object SuccessfulMatchResult { 17 | def fromStrings(firstFile: String, secondFile: String, id: String): SuccessfulMatchResult = 18 | SuccessfulMatchResult(new java.io.File(firstFile).toPath, new java.io.File(secondFile).toPath, id) 19 | } 20 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/filematch/strategy/FileRecognitionStrategy.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.filematch.strategy 2 | 3 | import org.broadinstitute.dsde.firecloud.filematch.result.FileMatchResult 4 | 5 | import java.nio.file.Path 6 | 7 | /** 8 | * Marker trait representing file-naming conventions used for pairing matched reads. 9 | */ 10 | trait FileRecognitionStrategy { 11 | 12 | def matchFirstFile(path: Path): FileMatchResult 13 | 14 | } 15 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/filematch/strategy/IlluminaPairedEndStrategy.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.filematch.strategy 2 | 3 | import org.broadinstitute.dsde.firecloud.filematch.result.{FailedMatchResult, FileMatchResult, SuccessfulMatchResult} 4 | import org.broadinstitute.dsde.firecloud.filematch.strategy.IlluminaPairedEndStrategy.FILE_ENDINGS 5 | 6 | import java.nio.file.Path 7 | 8 | /** 9 | * Naming conventions for Illumina single end and paired end read patterns. Examples of files recognized: 10 | * 11 | * Sample1_01.fastq.gz -> Sample1_02.fastq.gz 12 | * sample01_1.fastq.gz -> sample01_2.fastq.gz 13 | * sample01_R1.fastq.gz -> sample01_R2.fastq.gz 14 | * sample01_F.fastq.gz -> sample01_R.fastq.gz 15 | * sample01_R1.fastq -> sample01_R2.fastq 16 | * SampleName_S1_L001_R1_001.fastq.gz -> SampleName_S1_L001_R2_001.fastq.gz 17 | */ 18 | class IlluminaPairedEndStrategy extends FileRecognitionStrategy { 19 | override def matchFirstFile(path: Path): FileMatchResult = { 20 | // search known patterns for a "read1" file 21 | val foundMatch = FILE_ENDINGS.find { case (key, _) => path.toString.endsWith(key) } 22 | 23 | foundMatch match { 24 | // we found a "read1" 25 | case Some((key, value)) => 26 | // generate the id: strip the suffix from the filename. 27 | val id = path.getFileName.toString.replace(key, "") 28 | // generate the second filename: replace the first suffix with the second suffix 29 | val secondFile = new java.io.File(path.toString.replace(key, value)) 30 | SuccessfulMatchResult(path, secondFile.toPath, id) 31 | 32 | // the file is not recognized 33 | case None => FailedMatchResult(path) 34 | } 35 | } 36 | } 37 | 38 | object IlluminaPairedEndStrategy { 39 | // if the first file ends with ${key}, then the second file should end with ${value} 40 | val FILE_ENDINGS: Map[String, String] = Map( 41 | "_01.fastq.gz" -> "_02.fastq.gz", 42 | "_1.fastq.gz" -> "_2.fastq.gz", 43 | "_R1.fastq.gz" -> "_R2.fastq.gz", 44 | "_F.fastq.gz" -> "_R.fastq.gz", 45 | "_R1.fastq" -> "_R2.fastq", 46 | "_R1_001.fastq.gz" -> "_R2_001.fastq.gz" 47 | ) 48 | } 49 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/filematch/strategy/OntSingleReadStrategy.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.filematch.strategy 2 | 3 | import org.broadinstitute.dsde.firecloud.filematch.result.{FailedMatchResult, FileMatchResult, PartialMatchResult} 4 | import OntSingleReadStrategy.PATTERN 5 | 6 | import java.nio.file.Path 7 | import scala.util.matching.Regex 8 | 9 | class OntSingleReadStrategy extends FileRecognitionStrategy { 10 | 11 | override def matchFirstFile(path: Path): FileMatchResult = 12 | path.getFileName.toString match { 13 | case PATTERN(id) => PartialMatchResult(path, id) 14 | case _ => FailedMatchResult(path) 15 | } 16 | } 17 | 18 | object OntSingleReadStrategy { 19 | // if the file contains barcode### and ends with .fastq or .fastq.gz, it's an ONT single read file 20 | val PATTERN: Regex = """(?i)(.*barcode\d+).*\.fastq(?:.gz)?$""".r 21 | } 22 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/model/DbGapPermission.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.model 2 | 3 | import org.broadinstitute.dsde.workbench.model.ValueObject 4 | 5 | case class PhsId(value: String) extends ValueObject 6 | case class ConsentGroup(value: String) extends ValueObject 7 | case class DbGapPermission(phsId: PhsId, consentGroup: ConsentGroup) 8 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/model/EntityUpdateDefinition.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.model 2 | 3 | import org.broadinstitute.dsde.rawls.model.Attribute 4 | import spray.json._ 5 | 6 | /** 7 | * Created by tsharpe on 7/28/15. 8 | */ 9 | 10 | case class EntityUpdateDefinition(name: String, entityType: String, operations: Seq[Map[String, Attribute]]) 11 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/model/ErrorReport.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.model 2 | 3 | import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport 4 | import akka.http.scaladsl.model.{HttpEntity, HttpResponse, StatusCode} 5 | import akka.http.scaladsl.unmarshalling.Unmarshal 6 | import akka.stream.Materializer 7 | import org.broadinstitute.dsde.firecloud.service.PerRequest.RequestComplete 8 | import org.broadinstitute.dsde.rawls.model.WorkspaceJsonSupport._ 9 | import org.broadinstitute.dsde.rawls.model.{ErrorReport, ErrorReportSource} 10 | 11 | import scala.concurrent.{ExecutionContext, Future} 12 | import scala.util.Try 13 | 14 | object ErrorReportExtensions { 15 | object FCErrorReport extends SprayJsonSupport { 16 | 17 | def apply( 18 | response: HttpResponse 19 | )(implicit ers: ErrorReportSource, executionContext: ExecutionContext, mat: Materializer): Future[ErrorReport] = 20 | // code prior to creation of this error report may have already consumed the response entity 21 | 22 | response.entity match { 23 | case HttpEntity.Strict(contentType, data) => 24 | val entityString = data.decodeString(java.nio.charset.Charset.defaultCharset()) 25 | Unmarshal(entityString).to[ErrorReport].map { re => 26 | new ErrorReport(ers.source, re.message, Option(response.status), Seq(re), Seq.empty, None) 27 | } recover { case _ => 28 | new ErrorReport(ers.source, entityString, Option(response.status), Seq.empty, Seq.empty, None) 29 | } 30 | case _ => 31 | val fallbackMessage = Try(response.toString()).toOption.getOrElse("Unexpected error") 32 | Future.successful( 33 | new ErrorReport(ers.source, fallbackMessage, Option(response.status), Seq.empty, Seq.empty, None) 34 | ) 35 | } 36 | // 37 | // Unmarshal(response).to[ErrorReport].map { re => 38 | // new ErrorReport(ers.source, re.message, Option(response.status), Seq(re), Seq.empty, None) 39 | // } recoverWith { 40 | // case _ => Unmarshal(response).to[String].map { message => 41 | // new ErrorReport(ers.source, message, Option(response.status), Seq.empty, Seq.empty, None) 42 | // } 43 | // } 44 | } 45 | } 46 | 47 | object RequestCompleteWithErrorReport extends SprayJsonSupport { 48 | 49 | def apply(statusCode: StatusCode, message: String) = 50 | RequestComplete(statusCode, ErrorReport(statusCode, message)) 51 | 52 | def apply(statusCode: StatusCode, message: String, throwable: Throwable) = 53 | RequestComplete(statusCode, ErrorReport(statusCode, message, throwable)) 54 | 55 | def apply(statusCode: StatusCode, message: String, causes: Seq[ErrorReport]) = 56 | RequestComplete(statusCode, ErrorReport(statusCode, message, causes)) 57 | } 58 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/model/ExternalCredsMessage.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.model 2 | 3 | import io.circe.Decoder 4 | 5 | case class ExternalCredsMessage(providerName: String, userId: String) 6 | 7 | object ExternalCredsMessage { 8 | implicit val externalCredsMessageDecoder: Decoder[ExternalCredsMessage] = 9 | Decoder.forProduct2("providerName", "userId")(ExternalCredsMessage.apply) 10 | } 11 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/model/JWT.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.model 2 | 3 | case class JWTWrapper( 4 | jwt: String 5 | ) 6 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/model/LinkedEraAccount.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.model 2 | 3 | import bio.terra.externalcreds.model.AdminLinkInfo 4 | import org.joda.time.{DateTime, Instant} 5 | 6 | object LinkedEraAccount { 7 | def apply(samUserId: String, nihLink: NihLink): LinkedEraAccount = 8 | LinkedEraAccount(samUserId, nihLink.linkedNihUsername, Instant.ofEpochSecond(nihLink.linkExpireTime).toDateTime) 9 | 10 | def apply(adminLinkInfo: AdminLinkInfo): LinkedEraAccount = 11 | LinkedEraAccount(adminLinkInfo.getUserId, 12 | adminLinkInfo.getLinkedExternalId, 13 | new DateTime(adminLinkInfo.getLinkExpireTime) 14 | ) 15 | 16 | def unapply(linkedEraAccount: LinkedEraAccount): AdminLinkInfo = 17 | new AdminLinkInfo() 18 | .userId(linkedEraAccount.userId) 19 | .linkedExternalId(linkedEraAccount.linkedExternalId) 20 | .linkExpireTime(linkedEraAccount.linkExpireTime.toDate) 21 | } 22 | 23 | case class LinkedEraAccount(userId: String, linkedExternalId: String, linkExpireTime: DateTime) 24 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/model/ManagedGroup.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.model 2 | 3 | import org.broadinstitute.dsde.workbench.model.WorkbenchEmail 4 | 5 | /** 6 | * Created by mbemis on 3/29/18. 7 | */ 8 | 9 | object ManagedGroupRoles { 10 | sealed trait ManagedGroupRole { 11 | override def toString: String = 12 | this match { 13 | case Admin => "admin" 14 | case Member => "member" 15 | case AdminNotifier => "admin-notifier" 16 | case _ => throw new Exception(s"invalid ManagedGroupRole [$this]") 17 | } 18 | 19 | def withName(name: String): ManagedGroupRole = ManagedGroupRoles.withName(name) 20 | } 21 | 22 | // we'll match on singular and plural for these roles because there's some inconsistency introduced 23 | // between orch and sam. this maintains backwards compatibility and as a bonus is a bit more user-friendly 24 | def withName(name: String): ManagedGroupRole = name.toLowerCase match { 25 | case role if role matches "(?i)admin(s?$)" => Admin 26 | case role if role matches "(?i)member(s?$)" => Member 27 | case role if role matches "(?i)admin-notifier(s?$)" => AdminNotifier 28 | case _ => throw new Exception(s"invalid ManagedGroupRole [$name]") 29 | } 30 | 31 | case object Admin extends ManagedGroupRole 32 | case object Member extends ManagedGroupRole 33 | case object AdminNotifier extends ManagedGroupRole 34 | 35 | val membershipRoles: Set[ManagedGroupRole] = Set(Admin, Member) 36 | } 37 | 38 | case class FireCloudManagedGroup(adminsEmails: List[WorkbenchEmail], 39 | membersEmails: List[WorkbenchEmail], 40 | groupEmail: WorkbenchEmail 41 | ) 42 | case class FireCloudManagedGroupMembership(groupName: String, groupEmail: String, role: String) 43 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/model/PermissionReport.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.model 2 | 3 | import org.broadinstitute.dsde.firecloud.model.OrchMethodRepository.EntityAccessControl 4 | import org.broadinstitute.dsde.rawls.model.AccessEntry 5 | 6 | /** 7 | * Created by davidan on 7/5/17. 8 | */ 9 | case class PermissionReport( 10 | workspaceACL: Map[String, AccessEntry], 11 | referencedMethods: Seq[EntityAccessControl] 12 | ) 13 | 14 | case class PermissionReportRequest( 15 | users: Option[Seq[String]], 16 | configs: Option[Seq[OrchMethodConfigurationName]] 17 | ) 18 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/model/Project.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.model 2 | 3 | import org.broadinstitute.dsde.firecloud.FireCloudException 4 | import org.broadinstitute.dsde.rawls.model.{RawlsBillingProjectName, RawlsEnumeration, RawlsUserEmail} 5 | 6 | object Project { 7 | 8 | // following are horribly copied-and-pasted from rawls core, since they're not available as shared models 9 | case class CreateRawlsBillingProjectFullRequest(projectName: String, billingAccount: String) 10 | 11 | case class RawlsBillingProjectMembership(projectName: RawlsBillingProjectName, 12 | role: ProjectRoles.ProjectRole, 13 | creationStatus: CreationStatuses.CreationStatus, 14 | message: Option[String] = None 15 | ) 16 | 17 | case class RawlsBillingProjectMember(email: RawlsUserEmail, role: ProjectRoles.ProjectRole) 18 | 19 | object CreationStatuses { 20 | sealed trait CreationStatus extends RawlsEnumeration[CreationStatus] { 21 | override def toString = toName(this) 22 | 23 | override def withName(name: String): CreationStatus = CreationStatuses.withName(name) 24 | } 25 | 26 | def toName(status: CreationStatus): String = status match { 27 | case Creating => "Creating" 28 | case Ready => "Ready" 29 | case Error => "Error" 30 | } 31 | 32 | def withName(name: String): CreationStatus = name.toLowerCase match { 33 | case "creating" => Creating 34 | case "ready" => Ready 35 | case "error" => Error 36 | case _ => throw new FireCloudException(s"invalid CreationStatus [${name}]") 37 | } 38 | 39 | case object Creating extends CreationStatus 40 | case object Ready extends CreationStatus 41 | case object Error extends CreationStatus 42 | 43 | val all: Set[CreationStatus] = Set(Creating, Ready, Error) 44 | val terminal: Set[CreationStatus] = Set(Ready, Error) 45 | } 46 | 47 | object ProjectRoles { 48 | sealed trait ProjectRole extends RawlsEnumeration[ProjectRole] { 49 | override def toString = toName(this) 50 | 51 | override def withName(name: String): ProjectRole = ProjectRoles.withName(name) 52 | } 53 | 54 | def toName(role: ProjectRole): String = role match { 55 | case Owner => "Owner" 56 | case User => "User" 57 | } 58 | 59 | def withName(name: String): ProjectRole = name.toLowerCase match { 60 | case "owner" => Owner 61 | case "user" => User 62 | case _ => throw new FireCloudException(s"invalid ProjectRole [${name}]") 63 | } 64 | 65 | case object Owner extends ProjectRole 66 | case object User extends ProjectRole 67 | 68 | val all: Set[ProjectRole] = Set(Owner, User) 69 | } 70 | // END copy/paste from rawls 71 | } 72 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/model/RegisterRequest.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.model 2 | 3 | case class RegisterRequest(acceptsTermsOfService: Boolean, profile: BasicProfile) 4 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/model/SamResource.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.model 2 | 3 | import org.broadinstitute.dsde.workbench.model.{ValueObject, WorkbenchGroupName} 4 | 5 | object SamResource { 6 | 7 | case class ResourceId(value: String) extends ValueObject 8 | case class AccessPolicyName(value: String) extends ValueObject 9 | case class UserPolicy(resourceId: ResourceId, 10 | public: Boolean, 11 | accessPolicyName: AccessPolicyName, 12 | missingAuthDomainGroups: Set[WorkbenchGroupName], 13 | authDomainGroups: Set[WorkbenchGroupName] 14 | ) 15 | } 16 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/model/SamUser.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.model 2 | 3 | import org.broadinstitute.dsde.workbench.model.{AzureB2CId, GoogleSubjectId, WorkbenchEmail, WorkbenchUserId} 4 | 5 | import java.time.Instant 6 | 7 | case class SamUser(id: WorkbenchUserId, 8 | googleSubjectId: Option[GoogleSubjectId], 9 | email: WorkbenchEmail, 10 | azureB2CId: Option[AzureB2CId], 11 | enabled: Boolean, 12 | createdAt: Instant, 13 | registeredAt: Option[Instant], 14 | updatedAt: Instant 15 | ) 16 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/model/SamUserAttributesRequest.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.model 2 | 3 | case class SamUserAttributesRequest(marketingConsent: Option[Boolean]) 4 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/model/SamUserRegistrationRequest.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.model 2 | 3 | import org.broadinstitute.dsde.workbench.model.ErrorReport 4 | 5 | case class SamUserRegistrationRequest( 6 | acceptsTermsOfService: Boolean, 7 | userAttributes: SamUserAttributesRequest 8 | ) 9 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/model/SamUserResponse.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.model 2 | 3 | import org.broadinstitute.dsde.workbench.model.{AzureB2CId, GoogleSubjectId, WorkbenchEmail, WorkbenchUserId} 4 | 5 | import java.time.Instant 6 | 7 | final case class SamUserResponse( 8 | id: WorkbenchUserId, 9 | googleSubjectId: Option[GoogleSubjectId], 10 | email: WorkbenchEmail, 11 | azureB2CId: Option[AzureB2CId], 12 | allowed: Boolean, 13 | createdAt: Instant, 14 | registeredAt: Option[Instant], 15 | updatedAt: Instant 16 | ) {} 17 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/model/SystemStatus.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.model 2 | 3 | /** 4 | * Created by anichols on 4/7/17. 5 | */ 6 | case class ThurloeStatus(status: String, error: Option[String]) 7 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/model/UserInfo.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.model 2 | 3 | import akka.http.scaladsl.model.headers.OAuth2BearerToken 4 | 5 | import scala.util.Try 6 | 7 | /** 8 | * Created by dvoet on 7/21/15. 9 | * 10 | * Copied wholesale from rawls on 15-Oct-2015, commit a9664c9f08d0681d6647e6611fd0c785aa8aa24a 11 | * 12 | * modified to also include sub, retrieved from the OIDC_CLAIM_sub header. 13 | * I could have removed the userEmail, accessToken, and accessTokenExpiresIn because we don't 14 | * use those in orchestration. However, they're quite lightweight, and I've left them in to keep diffs between 15 | * orchestration and rawls as clean as possible. 16 | * 17 | * Amended 11/16/2016: 18 | * Added trait WithAccessToken, which is extended by the existing UserInfo class as well as a new AccessToken class 19 | * This is so that we can use AccessToken for cookies we get from the browser, which do not come with the fields in 20 | * UserInfo. 21 | */ 22 | 23 | trait WithAccessToken { val accessToken: OAuth2BearerToken } 24 | 25 | /** 26 | * Represents an authenticated user. 27 | * @param userEmail the user's email address. Resolved to the owner if the request is from a pet. 28 | * @param accessToken the user's access token. Either a B2C JWT or a Google opaque token. 29 | * @param accessTokenExpiresIn number of seconds until the access token expires. 30 | * @param id the user id. Either a Google id (numeric) or a B2C id (uuid). 31 | * @param googleAccessTokenThroughB2C if this is a Google login through B2C, contains the opaque 32 | * Google access token. Empty otherwise. 33 | */ 34 | case class UserInfo(userEmail: String, 35 | accessToken: OAuth2BearerToken, 36 | accessTokenExpiresIn: Long, 37 | id: String, 38 | googleAccessTokenThroughB2C: Option[OAuth2BearerToken] = None 39 | ) extends WithAccessToken { 40 | def isB2C: Boolean = 41 | // B2C ids are uuids, while google ids are numeric 42 | Try(BigInt(id)).isFailure 43 | } 44 | 45 | object UserInfo { 46 | def apply(accessToken: String, subjectId: String): UserInfo = 47 | UserInfo("", OAuth2BearerToken(accessToken), -1, subjectId) 48 | } 49 | 50 | case class AccessToken(accessToken: OAuth2BearerToken) extends WithAccessToken 51 | object AccessToken { 52 | def apply(tokenStr: String) = new AccessToken(OAuth2BearerToken(tokenStr)) 53 | } 54 | 55 | // response from Google has other fields, but these are the ones we care about 56 | case class OAuthUser(sub: String, email: String) 57 | 58 | case class RegistrationInfo(userInfo: WorkbenchUserInfo, 59 | enabled: WorkbenchEnabled, 60 | messages: Option[List[String]] = None 61 | ) 62 | case class RegistrationInfoV2(userSubjectId: String, userEmail: String, enabled: Boolean) 63 | 64 | case class UserIdInfo(userSubjectId: String, userEmail: String, googleSubjectId: String) 65 | 66 | case class WorkbenchUserInfo(userSubjectId: String, userEmail: String) 67 | case class WorkbenchEnabled(google: Boolean, ldap: Boolean, allUsersGroup: Boolean) 68 | case class WorkbenchEnabledV2(enabled: Boolean, inAllUsersGroup: Boolean, inGoogleProxyGroup: Boolean) 69 | 70 | // indicates whether or not the user can import (workflow|data|etc) into a workspace - the user 71 | // must have either a writable workspace or the ability to create a workspace (ready billing project) 72 | case class UserImportPermission(billingProject: Boolean, writableWorkspace: Boolean) 73 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/model/package.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud 2 | 3 | import akka.http.scaladsl.model.{ContentTypes, HttpEntity, HttpResponse, StatusCodes} 4 | import akka.http.scaladsl.server.RejectionHandler 5 | import org.broadinstitute.dsde.rawls.model.{ErrorReport, ErrorReportSource} 6 | 7 | import scala.language.implicitConversions 8 | import scala.util.{Failure, Success, Try} 9 | 10 | package object model { 11 | implicit val errorReportSource: ErrorReportSource = ErrorReportSource("FireCloud") 12 | 13 | import org.broadinstitute.dsde.rawls.model.WorkspaceJsonSupport._ 14 | import spray.json._ 15 | 16 | /* 17 | Rejection handler: if the response from the rejection is not already json, make it json. 18 | */ 19 | implicit val defaultErrorReportRejectionHandler: RejectionHandler = RejectionHandler.default.mapRejectionResponse { 20 | case resp @ HttpResponse(statusCode, _, ent: HttpEntity.Strict, _) => 21 | // since all Akka default rejection responses are Strict this will handle all rejections 22 | val entityString = ent.data.utf8String 23 | Try(entityString.parseJson) match { 24 | case Success(_) => 25 | resp 26 | case Failure(_) => 27 | // N.B. this handler previously manually escaped double quotes in the entityString. We don't need to do that, 28 | // since the .toJson below handles escaping internally. 29 | resp.withEntity( 30 | HttpEntity(ContentTypes.`application/json`, ErrorReport(statusCode, entityString).toJson.prettyPrint) 31 | ) 32 | } 33 | } 34 | 35 | /* 36 | N.B. This file previously contained two rejection handlers. The second was specific for 37 | MalformedRequestContentRejection and produced almost exactly the same result as defaultErrorReportRejectionHandler 38 | above (minor toString differences in the error message itself). I have removed that extraneous handler 39 | to simplify routing and debugging. 40 | */ 41 | 42 | } 43 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/service/AttributeSupport.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.service 2 | 3 | import org.broadinstitute.dsde.rawls.model._ 4 | import org.broadinstitute.dsde.rawls.model.Attributable._ 5 | import org.broadinstitute.dsde.rawls.model.AttributeUpdateOperations._ 6 | 7 | /** 8 | * Created by putnam on 11/1/16. 9 | */ 10 | trait AttributeSupport { 11 | 12 | /** 13 | * given a set of existing attributes and a set of new attributes, calculate the attribute operations 14 | * that need to be performed 15 | */ 16 | def generateAttributeOperations(existingAttrs: AttributeMap, 17 | newAttrs: AttributeMap, 18 | attributeFilter: AttributeName => Boolean 19 | ): Seq[AttributeUpdateOperation] = { 20 | val oldKeys = existingAttrs.keySet.filter(attributeFilter) 21 | val newFields = newAttrs.filter { case (name: AttributeName, _) => attributeFilter(name) } 22 | 23 | // remove any attributes that currently exist on the workspace, but are not in the user's packet 24 | // for any array attributes, we remove them and recreate them entirely. Add the array attrs. 25 | val keysToRemove: Set[AttributeName] = 26 | oldKeys.diff(newFields.keySet) ++ newFields.filter(_._2.isInstanceOf[AttributeList[_]]).keySet 27 | val removeOperations = keysToRemove.map(RemoveAttribute).toSeq 28 | 29 | val updateOperations = newFields.toSeq flatMap { 30 | case (key, value: AttributeValue) => Seq(AddUpdateAttribute(key, value)) 31 | case (key, value: AttributeEntityReference) => Seq(AddUpdateAttribute(key, value)) 32 | 33 | case (key, value: AttributeList[Attribute @unchecked]) => value.list.map(x => AddListMember(key, x)) 34 | } 35 | 36 | // handle removals before upserts 37 | removeOperations ++ updateOperations 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/service/FireCloudDirectives.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.service 2 | 3 | import akka.http.scaladsl.client.RequestBuilding 4 | import akka.http.scaladsl.model.Uri 5 | import akka.http.scaladsl.server.{Directives, Route} 6 | import org.broadinstitute.dsde.firecloud.utils.RestJsonClient 7 | import org.parboiled.common.FileUtils 8 | 9 | import scala.util.Try 10 | 11 | object FireCloudDirectiveUtils { 12 | def encodeUri(path: String): String = { 13 | val pattern = """(https|http)://([^/\r\n]+?)(:\d+)?(/[^\r\n]*)?""".r 14 | 15 | def toUri(url: String) = url match { 16 | case pattern(theScheme, theHost, thePort, thePath) => 17 | val p: Int = Try(thePort.replace(":", "").toInt).toOption.getOrElse(0) 18 | Uri.from(scheme = theScheme, port = p, host = theHost, path = thePath) 19 | } 20 | toUri(path).toString 21 | } 22 | } 23 | 24 | trait FireCloudDirectives extends Directives with RequestBuilding with RestJsonClient { 25 | 26 | def encodeUri(path: String): String = FireCloudDirectiveUtils.encodeUri(path) 27 | 28 | } 29 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/service/FireCloudRequestBuilding.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.service 2 | 3 | import org.broadinstitute.dsde.firecloud.FireCloudConfig 4 | import org.broadinstitute.dsde.firecloud.dataaccess.HttpGoogleServicesDAO 5 | import org.broadinstitute.dsde.firecloud.dataaccess.HttpGoogleServicesDAO._ 6 | import org.broadinstitute.dsde.firecloud.model.WithAccessToken 7 | import akka.http.scaladsl.client.RequestBuilding 8 | import akka.http.scaladsl.model.HttpRequest 9 | import akka.http.scaladsl.model.headers.{Authorization, HttpCredentials, OAuth2BearerToken, RawHeader} 10 | import akka.http.scaladsl.server.RequestContext 11 | 12 | trait FireCloudRequestBuilding extends RequestBuilding { 13 | 14 | val fireCloudHeader = RawHeader("X-FireCloud-Id", FireCloudConfig.FireCloud.fireCloudId) 15 | 16 | def authHeaders(credentials: Option[HttpCredentials]): HttpRequest => HttpRequest = 17 | credentials match { 18 | // if we have authorization credentials, apply them to the outgoing request 19 | case Some(c) => addCredentials(c) ~> addFireCloudCredentials 20 | // else, noop. But the noop needs to return an identity function in order to compile. 21 | // alternately, we could throw an error here, since we assume some authorization should exist. 22 | case None => (r: HttpRequest) => r ~> addFireCloudCredentials 23 | } 24 | 25 | def authHeaders(requestContext: RequestContext): HttpRequest => HttpRequest = { 26 | // inspect headers for a pre-existing Authorization: header 27 | val authorizationHeader: Option[HttpCredentials] = requestContext.request.headers collectFirst { 28 | case Authorization(h) => h 29 | } 30 | authHeaders(authorizationHeader) 31 | } 32 | 33 | def authHeaders(accessToken: WithAccessToken): HttpRequest => HttpRequest = 34 | authHeaders(Some(accessToken.accessToken)) 35 | 36 | // with great power comes great responsibility! 37 | def addAdminCredentials = addCredentials(OAuth2BearerToken(HttpGoogleServicesDAO.getAdminUserAccessToken)) 38 | 39 | def addFireCloudCredentials = addHeader(fireCloudHeader) 40 | 41 | } 42 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/service/NamespaceService.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.service 2 | 3 | import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport 4 | import akka.http.scaladsl.model.StatusCodes._ 5 | import org.broadinstitute.dsde.firecloud.dataaccess.AgoraDAO 6 | import org.broadinstitute.dsde.firecloud.model.OrchMethodRepository.{AgoraPermission, FireCloudPermission} 7 | import org.broadinstitute.dsde.firecloud.model.ModelJsonProtocol.impFireCloudPermission 8 | import org.broadinstitute.dsde.firecloud.model.{RequestCompleteWithErrorReport, UserInfo} 9 | import org.broadinstitute.dsde.firecloud.service.PerRequest.{PerRequestMessage, RequestComplete} 10 | import org.broadinstitute.dsde.firecloud.{Application, FireCloudExceptionWithErrorReport} 11 | import spray.json.DefaultJsonProtocol._ 12 | 13 | import scala.concurrent.{ExecutionContext, Future} 14 | 15 | object NamespaceService { 16 | def constructor(app: Application)(userInfo: UserInfo)(implicit executionContext: ExecutionContext) = 17 | new NamespaceService(userInfo, app.agoraDAO) 18 | } 19 | 20 | class NamespaceService(protected val argUserInfo: UserInfo, val agoraDAO: AgoraDAO)(implicit 21 | protected val executionContext: ExecutionContext 22 | ) extends SprayJsonSupport { 23 | 24 | implicit val userInfo: UserInfo = argUserInfo 25 | 26 | def getFireCloudPermissions(ns: String, entity: String): Future[PerRequestMessage] = { 27 | val agoraPermissions = agoraDAO.getNamespacePermissions(ns, entity) 28 | delegatePermissionsResponse(agoraPermissions) 29 | } 30 | 31 | def postFireCloudPermissions(ns: String, 32 | entity: String, 33 | permissions: List[FireCloudPermission] 34 | ): Future[PerRequestMessage] = { 35 | val agoraPermissionsToPost = permissions map { permission => AgoraPermissionService.toAgoraPermission(permission) } 36 | val agoraPermissionsPosted = agoraDAO.postNamespacePermissions(ns, entity, agoraPermissionsToPost) 37 | delegatePermissionsResponse(agoraPermissionsPosted) 38 | } 39 | 40 | private def delegatePermissionsResponse(agoraPerms: Future[List[AgoraPermission]]): Future[PerRequestMessage] = 41 | agoraPerms map { perms => 42 | RequestComplete(OK, perms map AgoraPermissionService.toFireCloudPermission) 43 | } recover { 44 | case e: FireCloudExceptionWithErrorReport => 45 | // RequestComplete(e.errorReport.statusCode.getOrElse(InternalServerError), e.errorReport) 46 | RequestComplete(e.errorReport.statusCode.getOrElse(InternalServerError)) 47 | case e: Throwable => 48 | RequestCompleteWithErrorReport(InternalServerError, e.getMessage) 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/service/PerRequest.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.service 2 | 3 | import akka.http.scaladsl.marshalling.{Marshaller, ToResponseMarshaller} 4 | import akka.http.scaladsl.model.{HttpHeader, StatusCodes} 5 | import org.broadinstitute.dsde.rawls.model.ErrorReport 6 | 7 | import scala.concurrent.ExecutionContext 8 | 9 | object PerRequest { 10 | 11 | implicit def requestCompleteMarshaller(implicit 12 | executionContext: ExecutionContext 13 | ): ToResponseMarshaller[PerRequestMessage] = Marshaller { _: ExecutionContext => 14 | { 15 | case requestComplete @ RequestComplete(errorReport: ErrorReport) => 16 | requestComplete 17 | .marshaller(requestComplete.response) 18 | .map(_.map(_.map(_.withStatus(errorReport.statusCode.getOrElse(StatusCodes.InternalServerError))))) 19 | case requestComplete: RequestComplete[_] => 20 | requestComplete.marshaller(requestComplete.response) 21 | 22 | case requestComplete @ RequestCompleteWithHeaders(errorReport: ErrorReport, _) => 23 | requestComplete 24 | .marshaller(requestComplete.response) 25 | .map( 26 | _.map( 27 | _.map( 28 | _.mapHeaders(_ ++ requestComplete.headers) 29 | .withStatus(errorReport.statusCode.getOrElse(StatusCodes.InternalServerError)) 30 | ) 31 | ) 32 | ) 33 | case requestComplete: RequestCompleteWithHeaders[_] => 34 | requestComplete 35 | .marshaller(requestComplete.response) 36 | .map(_.map(_.map(_.mapHeaders(_ ++ requestComplete.headers)))) 37 | } 38 | } 39 | 40 | sealed trait PerRequestMessage 41 | 42 | /** 43 | * Report complete, follows same pattern as spray.routing.RequestContext.complete; examples of how to call 44 | * that method should apply here too. E.g. even though this method has only one parameter, it can be called 45 | * with 2 where the first is a StatusCode: RequestComplete(StatusCode.Created, response) 46 | */ 47 | case class RequestComplete[T](response: T)(implicit val marshaller: ToResponseMarshaller[T]) extends PerRequestMessage 48 | 49 | /** 50 | * Report complete with response headers. To response with a special status code the first parameter can be a 51 | * tuple where the first element is StatusCode: RequestCompleteWithHeaders((StatusCode.Created, results), header). 52 | * Note that this is here so that RequestComplete above can behave like spray.routing.RequestContext.complete. 53 | */ 54 | case class RequestCompleteWithHeaders[T](response: T, headers: HttpHeader*)(implicit 55 | val marshaller: ToResponseMarshaller[T] 56 | ) extends PerRequestMessage 57 | 58 | /** allows for pattern matching with extraction of marshaller */ 59 | private object RequestComplete_ { 60 | def unapply[T >: Any](requestComplete: RequestComplete[T]) = Some( 61 | (requestComplete.response, requestComplete.marshaller) 62 | ) 63 | } 64 | 65 | /** allows for pattern matching with extraction of marshaller */ 66 | private object RequestCompleteWithHeaders_ { 67 | def unapply[T >: Any](requestComplete: RequestCompleteWithHeaders[T]) = Some( 68 | (requestComplete.response, requestComplete.headers, requestComplete.marshaller) 69 | ) 70 | } 71 | 72 | } 73 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/service/StatusService.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.service 2 | 3 | import akka.actor.ActorRef 4 | import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport 5 | import akka.http.scaladsl.model.StatusCodes 6 | import akka.pattern._ 7 | import akka.util.Timeout 8 | import org.broadinstitute.dsde.firecloud.service.PerRequest.{PerRequestMessage, RequestComplete} 9 | import org.broadinstitute.dsde.workbench.util.health.HealthMonitor.GetCurrentStatus 10 | import org.broadinstitute.dsde.workbench.util.health.StatusCheckResponse 11 | import org.broadinstitute.dsde.workbench.util.health.StatusJsonSupport.StatusCheckResponseFormat 12 | 13 | import scala.concurrent.{ExecutionContext, Future} 14 | import scala.concurrent.duration._ 15 | 16 | /** 17 | * Created by anichols on 4/5/17. 18 | */ 19 | object StatusService { 20 | def constructor(healthMonitor: ActorRef)()(implicit executionContext: ExecutionContext): StatusService = 21 | new StatusService(healthMonitor) 22 | } 23 | 24 | class StatusService(val healthMonitor: ActorRef)(implicit protected val executionContext: ExecutionContext) 25 | extends SprayJsonSupport { 26 | implicit val timeout: Timeout = Timeout(1.minute) // timeout for the ask to healthMonitor for GetCurrentStatus 27 | 28 | def collectStatusInfo(): Future[PerRequestMessage] = 29 | (healthMonitor ? GetCurrentStatus).mapTo[StatusCheckResponse].map { statusCheckResponse => 30 | // if we've successfully reached this point, always return a 200, so the load balancers 31 | // don't think orchestration is down. the statusCheckResponse will still contain ok: true|false 32 | // in its payload, depending on the status of subsystems. 33 | RequestComplete(StatusCodes.OK, statusCheckResponse) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/utils/DateUtils.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.utils 2 | 3 | import org.joda.time.{DateTime, Hours, Seconds} 4 | 5 | object DateUtils { 6 | 7 | val EPOCH = 1000L 8 | 9 | def nowPlus30Days: Long = 10 | nowDateTime.plusDays(30).getMillis / EPOCH 11 | 12 | def nowMinus30Days: Long = 13 | nowDateTime.minusDays(30).getMillis / EPOCH 14 | 15 | def nowPlus24Hours: Long = 16 | nowDateTime.plusHours(24).getMillis / EPOCH 17 | 18 | def nowMinus24Hours: Long = 19 | nowDateTime.minusHours(24).getMillis / EPOCH 20 | 21 | def nowPlus1Hour: Long = 22 | nowDateTime.plusHours(1).getMillis / EPOCH 23 | 24 | def nowMinus1Hour: Long = 25 | nowDateTime.minusHours(1).getMillis / EPOCH 26 | 27 | def hoursSince(seconds: Long): Int = 28 | Hours.hoursBetween(dtFromSeconds(seconds), nowDateTime).getHours 29 | 30 | def hoursUntil(seconds: Long): Int = 31 | Hours.hoursBetween(nowDateTime, dtFromSeconds(seconds)).getHours 32 | 33 | def secondsSince(seconds: Long): Int = 34 | Seconds.secondsBetween(dtFromSeconds(seconds), nowDateTime).getSeconds 35 | 36 | def now: Long = 37 | nowDateTime.getMillis / EPOCH 38 | 39 | def nowDateTime: DateTime = 40 | dtFromMillis(System.currentTimeMillis()) 41 | 42 | def dtFromMillis(millis: Long): DateTime = 43 | new DateTime(millis) 44 | 45 | def dtFromSeconds(seconds: Long): DateTime = 46 | new DateTime(seconds * EPOCH) 47 | 48 | } 49 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/utils/DisabledServiceFactory.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.utils 2 | 3 | import java.lang.reflect.Proxy 4 | import scala.reflect.{classTag, ClassTag} 5 | 6 | object DisabledServiceFactory { 7 | 8 | /** 9 | * Create a new instance of a service that throws UnsupportedOperationException for all methods 10 | * unless the method is boolean isEnabled(), in which case return false. 11 | * Implemented using a dynamic proxy. 12 | * @tparam T the type of the service, must be a trait 13 | * @return a new instance of the service that throws UnsupportedOperationException for all methods 14 | */ 15 | def newDisabledService[T: ClassTag]: T = 16 | Proxy 17 | .newProxyInstance( 18 | classTag[T].runtimeClass.getClassLoader, 19 | Array(classTag[T].runtimeClass), 20 | (_, method, _) => 21 | if ( 22 | method.getName 23 | .equals("isEnabled") && method.getParameterCount == 0 && method.getReturnType == classOf[Boolean] 24 | ) 25 | false 26 | else 27 | throw new UnsupportedOperationException(s"${method.toGenericString} is disabled.") 28 | ) 29 | .asInstanceOf[T] 30 | } 31 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/utils/PerformanceLogging.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.utils 2 | 3 | import java.time.{Instant, LocalDateTime, ZoneId} 4 | import java.time.format.DateTimeFormatter 5 | 6 | import com.typesafe.scalalogging.Logger 7 | import org.slf4j.LoggerFactory 8 | 9 | trait PerformanceLogging { 10 | 11 | val perfLogger: Logger = Logger(LoggerFactory.getLogger("PerformanceLogging")) 12 | val systemTimeZone = ZoneId.systemDefault() 13 | val dateFormatter = DateTimeFormatter.ISO_LOCAL_TIME 14 | 15 | def perfmsg(caller: String, msg: String, start: Instant, finish: Instant): String = { 16 | lazy val elapsed = finish.toEpochMilli - start.toEpochMilli 17 | 18 | lazy val startStr = LocalDateTime.ofInstant(start, systemTimeZone).format(dateFormatter) 19 | lazy val finishStr = LocalDateTime.ofInstant(finish, systemTimeZone).format(dateFormatter) 20 | 21 | s"[$caller] took [$elapsed] ms, start [$startStr], finish [$finishStr]: $msg" 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/utils/StandardUserInfoDirectives.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.utils 2 | 3 | import akka.http.scaladsl.model.headers.OAuth2BearerToken 4 | import akka.http.scaladsl.server.Directive1 5 | import akka.http.scaladsl.server.Directives.{headerValueByName, optionalHeaderValueByName} 6 | import org.broadinstitute.dsde.firecloud.model.UserInfo 7 | 8 | trait StandardUserInfoDirectives extends UserInfoDirectives { 9 | 10 | // The OAUTH2_CLAIM_google_id header is populated when a user signs in to Google via B2C. 11 | // If present, use that value instead of the B2C id for backwards compatibility. 12 | def requireUserInfo(): Directive1[UserInfo] = ( 13 | headerValueByName("OIDC_access_token") & 14 | headerValueByName("OIDC_CLAIM_user_id") & 15 | headerValueByName("OIDC_CLAIM_expires_in") & 16 | headerValueByName("OIDC_CLAIM_email") & 17 | optionalHeaderValueByName("OAUTH2_CLAIM_google_id") & 18 | optionalHeaderValueByName("OAUTH2_CLAIM_idp_access_token") 19 | ) tmap { case (token, userId, expiresIn, email, googleIdOpt, googleTokenOpt) => 20 | UserInfo(email, 21 | OAuth2BearerToken(token), 22 | expiresIn.toLong, 23 | googleIdOpt.getOrElse(userId), 24 | googleTokenOpt.map(OAuth2BearerToken) 25 | ) 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/utils/StatusCodeUtils.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.utils 2 | 3 | import akka.http.scaladsl.model.{StatusCode, StatusCodes} 4 | 5 | import scala.util.Try 6 | 7 | trait StatusCodeUtils { 8 | 9 | /** 10 | * Safely translates an integer to a StatusCode. This method avoids the RuntimeException 11 | * thrown by StatusCode.int2StatusCode when supplied with an unknown code and returns 12 | * a default status code instead. 13 | * 14 | * @param intCode the integer value to translate 15 | * @param default the code to return if the integer value is unknown; 16 | * defaults to Internal Server Error 17 | * @return the final status code 18 | */ 19 | def statusCodeFrom(intCode: Int, default: Option[StatusCode] = None): StatusCode = 20 | Try(StatusCode.int2StatusCode(intCode)).getOrElse( 21 | default.getOrElse( 22 | StatusCodes.custom(intCode, 23 | reason = s"unknown status $intCode", 24 | defaultMessage = s"unknown status $intCode", 25 | isSuccess = false, 26 | allowsEntity = true 27 | ) 28 | ) 29 | ) 30 | 31 | } 32 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/utils/TSVParser.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.utils 2 | 3 | import java.io.StringReader 4 | import scala.jdk.CollectionConverters._ 5 | import com.univocity.parsers.csv.{CsvParser, CsvParserSettings} 6 | 7 | case class TSVLoadFile( 8 | firstColumnHeader: String, // The first header column, used to determine the type of entities being imported 9 | headers: Seq[String], // All the headers 10 | tsvData: Seq[Seq[String]] // List of rows of the TSV, broken out into fields 11 | ) 12 | 13 | object TSVParser { 14 | final val DELIMITER = '\t' 15 | 16 | private def makeParser = { 17 | // Note we're using a CsvParser with a tab delimiter rather a TsvParser. 18 | // This is because the CSV formatter handles quotations correctly while the TSV formatter doesn't. 19 | // See https://github.com/uniVocity/univocity-parsers#csv-format 20 | val settings = new CsvParserSettings 21 | // Automatically detect what the line separator is (e.g. \n for Unix, \r\n for Windows). 22 | settings.setLineSeparatorDetectionEnabled(true) 23 | settings.setMaxColumns(1024) 24 | // 64 mb in bytes/4 (assumes 4 bytes per character) 25 | settings.setMaxCharsPerColumn(16777216) 26 | settings.getFormat.setDelimiter(DELIMITER) 27 | settings.setErrorContentLength(16384) 28 | // By default, the CsvParser returns null for missing fields, however the application expects the 29 | // empty string. These replace all nulls with the empty string. 30 | settings.setNullValue("") 31 | settings.setEmptyValue("") 32 | new CsvParser(settings) 33 | } 34 | 35 | private def parseLine(tsvLine: Array[String], tsvLineNo: Int, nCols: Int): List[String] = { 36 | if (tsvLine.length != nCols) { 37 | throw new RuntimeException(s"TSV parsing error in line $tsvLineNo: wrong number of fields") 38 | } 39 | tsvLine.toList 40 | } 41 | 42 | def parse(tsvString: String): TSVLoadFile = 43 | makeParser.parseAll(new StringReader(tsvString)).asScala.toList match { 44 | case h :: t => 45 | val tsvData = t.zipWithIndex.map { case (line, idx) => parseLine(line, idx, h.length) } 46 | // for user-friendliness, we are lenient and ignore any lines that are either just a newline, 47 | // or consist only of delimiters (tabs) but have no data. 48 | // we implement this by checking to see if any of the line's values is non-empty. If the line 49 | // consists only of delimiters, all values will be empty. 50 | // NB: CsvParserSettings.setSkipEmptyLines, setIgnoreTrailingWhitespaces, and setIgnoreLeadingWhitespaces 51 | // do not help with this use case, so we write our own implementation. 52 | val validData = tsvData.collect { 53 | case hasValues if hasValues.exists(_.nonEmpty) => hasValues 54 | } 55 | TSVLoadFile(h.head, h.toList, validData) 56 | case _ => throw new RuntimeException("TSV parsing error: no header") 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/utils/UserInfoDirectives.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.utils 2 | 3 | import akka.http.scaladsl.server.Directive1 4 | import org.broadinstitute.dsde.firecloud.model.UserInfo 5 | 6 | import scala.concurrent.ExecutionContext 7 | 8 | /** 9 | * Directives to get user information. 10 | * 11 | * Copied wholesale from rawls on 15-Oct-2015, commit a9664c9f08d0681d6647e6611fd0c785aa8aa24a 12 | */ 13 | trait UserInfoDirectives { 14 | def requireUserInfo(): Directive1[UserInfo] 15 | } 16 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/webservice/CromIamApiService.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.webservice 2 | 3 | import org.broadinstitute.dsde.firecloud.FireCloudConfig 4 | import org.broadinstitute.dsde.firecloud.service.{FireCloudDirectives, FireCloudRequestBuilding} 5 | import org.broadinstitute.dsde.firecloud.utils.{StandardUserInfoDirectives, StreamingPassthrough} 6 | import akka.http.scaladsl.model.{HttpMethods, Uri} 7 | import akka.http.scaladsl.server.Route 8 | 9 | trait CromIamApiService 10 | extends FireCloudRequestBuilding 11 | with FireCloudDirectives 12 | with StandardUserInfoDirectives 13 | with StreamingPassthrough { 14 | 15 | lazy val workflowRoot: String = FireCloudConfig.CromIAM.authUrl + "/workflows/v1" 16 | lazy val engineRoot: String = FireCloudConfig.CromIAM.baseUrl + "/engine/v1" 17 | lazy val rawlsWorkflowRoot: String = FireCloudConfig.Rawls.authUrl + "/workflows" 18 | 19 | // This is the subset of CromIAM endpoints required for Job Manager. Orchestration is acting as a proxy between 20 | // CromIAM and Job Manager as of February 2019. 21 | // Adam Nichols, 2019-02-04 22 | 23 | val localBase = s"/api/workflows/v1" 24 | 25 | /* 26 | NOTE: The rawls routes are a bit different from the cromiam routes. While it does carry the same "/api/workflows" base path, 27 | there are caveats to its redirect that need to be taken into account 28 | 1) The remote authority isn't CromIAM's baseUrl but rather Rawls' baseUrl 29 | 2) The local path is defined as "/workflows/{version}/{id}/backend/metadata/{operation}, but Rawls endpoint is 30 | defined as /workflows/{id}/genomics/{operation}, meaning that path reconstruction is necessary 31 | Since there's only one such route, it was simpler to have an explicit route defined for his edge case and have it evaluated 32 | before the rest of the workflow routes. 33 | */ 34 | val rawlsServiceRoute: Route = 35 | pathPrefix("workflows" / Segment / Segment / "backend" / "metadata" / Segments) { 36 | (version, workflowId, operationSegments) => 37 | val suffix = operationSegments.mkString("/") 38 | streamingPassthroughWithPathRedirect(Uri.Path(localBase) -> Uri(rawlsWorkflowRoot), 39 | s"/${workflowId}/genomics/${suffix}" 40 | ) 41 | } 42 | 43 | val cromIamServiceRoutes: Route = 44 | pathPrefix("workflows" / Segment) { _ => 45 | streamingPassthrough(Uri.Path(localBase) -> Uri(workflowRoot)) 46 | } 47 | 48 | val cromIamApiServiceRoutes = rawlsServiceRoute ~ cromIamServiceRoutes 49 | 50 | val cromIamEngineRoutes: Route = 51 | pathPrefix("engine" / Segment) { _ => 52 | streamingPassthrough(Uri.Path("/engine/v1") -> Uri(engineRoot)) 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/webservice/EntityApiService.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.webservice 2 | 3 | import akka.http.scaladsl.server.Route 4 | import org.broadinstitute.dsde.firecloud.model.ModelJsonProtocol._ 5 | import org.broadinstitute.dsde.firecloud.model._ 6 | import org.broadinstitute.dsde.firecloud.service.{FireCloudDirectives, FireCloudRequestBuilding} 7 | import org.broadinstitute.dsde.firecloud.utils.{RestJsonClient, StandardUserInfoDirectives, StreamingPassthrough} 8 | import org.broadinstitute.dsde.firecloud.{EntityService, FireCloudConfig} 9 | import org.broadinstitute.dsde.rawls.model.{EntityCopyDefinition, WorkspaceName} 10 | import org.slf4j.LoggerFactory 11 | 12 | import scala.concurrent.ExecutionContext 13 | import scala.util.Try 14 | 15 | trait EntityApiService 16 | extends FireCloudDirectives 17 | with StreamingPassthrough 18 | with FireCloudRequestBuilding 19 | with StandardUserInfoDirectives 20 | with RestJsonClient { 21 | 22 | implicit val executionContext: ExecutionContext 23 | lazy val log = LoggerFactory.getLogger(getClass) 24 | 25 | val entityServiceConstructor: (ModelSchema) => EntityService 26 | 27 | def entityRoutes: Route = 28 | pathPrefix("api") { 29 | pathPrefix("workspaces" / Segment / Segment) { (workspaceNamespace, workspaceName) => 30 | val baseRawlsEntitiesUrl = FireCloudConfig.Rawls.entityPathFromWorkspace(workspaceNamespace, workspaceName) 31 | path("entities_with_type") { 32 | get { 33 | requireUserInfo() { userInfo => 34 | // TODO: the model schema doesn't matter for this one. Ideally, make it Optional 35 | complete { 36 | entityServiceConstructor(FlexibleModelSchema).getEntitiesWithType(workspaceNamespace, 37 | workspaceName, 38 | userInfo 39 | ) 40 | } 41 | } 42 | } 43 | } ~ 44 | pathPrefix("entities") { 45 | path("copy") { 46 | post { 47 | requireUserInfo() { userInfo => 48 | parameter(Symbol("linkExistingEntities").?) { linkExistingEntities => 49 | entity(as[EntityCopyWithoutDestinationDefinition]) { copyRequest => 50 | val linkExistingEntitiesBool = 51 | Try(linkExistingEntities.getOrElse("false").toBoolean).getOrElse(false) 52 | val copyMethodConfig = new EntityCopyDefinition( 53 | sourceWorkspace = copyRequest.sourceWorkspace, 54 | destinationWorkspace = WorkspaceName(workspaceNamespace, workspaceName), 55 | entityType = copyRequest.entityType, 56 | entityNames = copyRequest.entityNames 57 | ) 58 | val extReq = Post(FireCloudConfig.Rawls.workspacesEntitiesCopyUrl(linkExistingEntitiesBool), 59 | copyMethodConfig 60 | ) 61 | 62 | complete(userAuthedRequest(extReq)(userInfo)) 63 | } 64 | } 65 | } 66 | } 67 | } 68 | } 69 | } 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/webservice/HealthApiService.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.webservice 2 | 3 | import akka.http.scaladsl.model.StatusCodes.{OK, ServiceUnavailable} 4 | import akka.http.scaladsl.server.Route 5 | import org.broadinstitute.dsde.firecloud.service.FireCloudDirectives 6 | import org.slf4j.LoggerFactory 7 | 8 | import scala.concurrent.ExecutionContext 9 | 10 | trait HealthApiService extends FireCloudDirectives { 11 | 12 | implicit val executionContext: ExecutionContext 13 | lazy val log = LoggerFactory.getLogger(getClass) 14 | 15 | val healthServiceRoutes: Route = 16 | path("health") { 17 | complete(OK) 18 | } ~ 19 | path("error") { 20 | complete(ServiceUnavailable) 21 | } 22 | 23 | } 24 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/webservice/ManagedGroupApiService.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.webservice 2 | 3 | import akka.http.scaladsl.client.RequestBuilding 4 | import akka.http.scaladsl.server.{Directives, Route} 5 | import org.broadinstitute.dsde.firecloud.model._ 6 | import org.broadinstitute.dsde.firecloud.service._ 7 | import org.broadinstitute.dsde.firecloud.utils.{StandardUserInfoDirectives, UserInfoDirectives} 8 | import org.broadinstitute.dsde.workbench.model.{WorkbenchEmail, WorkbenchGroupName} 9 | import org.slf4j.LoggerFactory 10 | 11 | import scala.concurrent.ExecutionContext 12 | 13 | trait ManagedGroupApiService extends Directives with RequestBuilding with StandardUserInfoDirectives { 14 | 15 | implicit val executionContext: ExecutionContext 16 | 17 | lazy val log = LoggerFactory.getLogger(getClass) 18 | 19 | val managedGroupServiceConstructor: (WithAccessToken) => ManagedGroupService 20 | 21 | val managedGroupServiceRoutes: Route = requireUserInfo() { userInfo => 22 | pathPrefix("api") { 23 | pathPrefix("groups") { 24 | pathEnd { 25 | get { 26 | complete(managedGroupServiceConstructor(userInfo).listGroups()) 27 | } 28 | } ~ 29 | pathPrefix(Segment) { groupName => 30 | pathEnd { 31 | get { 32 | complete(managedGroupServiceConstructor(userInfo).listGroupMembers(WorkbenchGroupName(groupName))) 33 | } ~ 34 | post { 35 | complete(managedGroupServiceConstructor(userInfo).createGroup(WorkbenchGroupName(groupName))) 36 | } ~ 37 | delete { 38 | complete(managedGroupServiceConstructor(userInfo).deleteGroup(WorkbenchGroupName(groupName))) 39 | } 40 | } ~ 41 | path("requestAccess") { 42 | post { 43 | complete { 44 | managedGroupServiceConstructor(userInfo).requestGroupAccess(WorkbenchGroupName(groupName)) 45 | } 46 | } 47 | } ~ 48 | path(Segment / Segment) { (role, email) => 49 | put { 50 | complete { 51 | managedGroupServiceConstructor(userInfo).addGroupMember(WorkbenchGroupName(groupName), 52 | ManagedGroupRoles.withName(role), 53 | WorkbenchEmail(email) 54 | ) 55 | } 56 | } ~ 57 | delete { 58 | complete { 59 | managedGroupServiceConstructor(userInfo).removeGroupMember(WorkbenchGroupName(groupName), 60 | ManagedGroupRoles.withName(role), 61 | WorkbenchEmail(email) 62 | ) 63 | } 64 | } 65 | } 66 | } 67 | } 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/webservice/NamespaceApiService.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.webservice 2 | 3 | import akka.http.scaladsl.client.RequestBuilding 4 | import akka.http.scaladsl.server.{Directives, Route} 5 | import org.broadinstitute.dsde.firecloud.model.OrchMethodRepository.FireCloudPermission 6 | import org.broadinstitute.dsde.firecloud.model.ModelJsonProtocol._ 7 | import org.broadinstitute.dsde.firecloud.model.UserInfo 8 | import org.broadinstitute.dsde.firecloud.service.NamespaceService 9 | import org.broadinstitute.dsde.firecloud.utils.StandardUserInfoDirectives 10 | import spray.json.DefaultJsonProtocol._ 11 | 12 | import scala.concurrent.ExecutionContext 13 | 14 | trait NamespaceApiService extends Directives with RequestBuilding with StandardUserInfoDirectives { 15 | 16 | implicit val executionContext: ExecutionContext 17 | 18 | val namespaceServiceConstructor: UserInfo => NamespaceService 19 | 20 | val namespaceRoutes: Route = 21 | pathPrefix("api" / "methods|configurations".r / Segment / "permissions") { (agoraEntity, namespace) => 22 | requireUserInfo() { userInfo => 23 | get { 24 | complete(namespaceServiceConstructor(userInfo).getFireCloudPermissions(namespace, agoraEntity)) 25 | } ~ 26 | post { 27 | // explicitly pull in the json-extraction error handler from ModelJsonProtocol 28 | handleRejections(entityExtractionRejectionHandler) { 29 | entity(as[List[FireCloudPermission]]) { permissions => 30 | complete { 31 | namespaceServiceConstructor(userInfo).postFireCloudPermissions(namespace, agoraEntity, permissions) 32 | } 33 | } 34 | } 35 | } 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/webservice/NihApiService.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.webservice 2 | 3 | import akka.http.scaladsl.client.RequestBuilding 4 | import akka.http.scaladsl.model.StatusCodes 5 | import akka.http.scaladsl.server.{Directives, Route} 6 | import org.broadinstitute.dsde.firecloud.model.ModelJsonProtocol._ 7 | import org.broadinstitute.dsde.firecloud.model._ 8 | import org.broadinstitute.dsde.firecloud.service.NihService 9 | import org.broadinstitute.dsde.firecloud.utils.{EnabledUserDirectives, StandardUserInfoDirectives} 10 | import org.slf4j.LoggerFactory 11 | 12 | import scala.concurrent.ExecutionContext 13 | 14 | trait NihApiService extends Directives with RequestBuilding with EnabledUserDirectives with StandardUserInfoDirectives { 15 | 16 | implicit val executionContext: ExecutionContext 17 | lazy val log = LoggerFactory.getLogger(getClass) 18 | 19 | val nihServiceConstructor: () => NihService 20 | 21 | val syncRoute: Route = 22 | path("sync_whitelist" / Segment) { whitelistName => 23 | post { 24 | complete(nihServiceConstructor().syncAllowlistAllUsers(whitelistName)) 25 | } 26 | } ~ path("sync_whitelist") { 27 | post { 28 | complete(nihServiceConstructor().syncAllNihAllowlistsAllUsers()) 29 | } 30 | } 31 | 32 | val nihRoutes: Route = 33 | requireUserInfo() { userInfo => 34 | requireEnabledUser(userInfo) { 35 | pathPrefix("nih") { 36 | // api/nih/callback: accept JWT, update linkage + lastlogin 37 | path("callback") { 38 | post { 39 | entity(as[JWTWrapper]) { jwtWrapper => 40 | complete(nihServiceConstructor().updateNihLinkAndSyncSelf(userInfo, jwtWrapper)) 41 | } 42 | } 43 | } ~ 44 | path("status") { 45 | complete(nihServiceConstructor().getNihStatus(userInfo)) 46 | } ~ 47 | path("account") { 48 | delete { 49 | complete { 50 | nihServiceConstructor().unlinkNihAccountAndSyncSelf(userInfo).map(_ => StatusCodes.NoContent) 51 | } 52 | } 53 | } 54 | } 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/webservice/OauthApiService.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.webservice 2 | 3 | import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport 4 | import akka.http.scaladsl.model.StatusCodes 5 | import akka.http.scaladsl.server.Route 6 | import org.broadinstitute.dsde.firecloud.service.FireCloudDirectives 7 | import org.broadinstitute.dsde.firecloud.service.PerRequest.RequestComplete 8 | import org.broadinstitute.dsde.firecloud.utils.StandardUserInfoDirectives 9 | import spray.json.DefaultJsonProtocol._ 10 | 11 | import scala.concurrent.ExecutionContext 12 | 13 | /** 14 | * These two routes are no-ops. The backend behavior they previously supported is no longer necessary: 15 | * - reporting on how old the user's refresh token is; 16 | * - retrieving a new oauth token from Google and sending it to Rawls to store 17 | * 18 | * In the spirit of backwards compatibility, we are not (yet) removing these APIs. Instead, we hardcode 19 | * them to respond as if they were successful, without performing any backend work. 20 | */ 21 | trait OauthApiService extends FireCloudDirectives with StandardUserInfoDirectives with SprayJsonSupport { 22 | 23 | implicit val executionContext: ExecutionContext 24 | 25 | val oauthRoutes: Route = 26 | path("handle-oauth-code") { 27 | post { 28 | complete(StatusCodes.NoContent) 29 | } 30 | } ~ 31 | path("api" / "refresh-token-status") { 32 | get { 33 | requireUserInfo() { _ => 34 | complete(RequestComplete(StatusCodes.OK, Map("requiresRefresh" -> false))) 35 | } 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/webservice/PassthroughApiService.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.webservice 2 | 3 | import akka.http.scaladsl.server.{Directives, Route} 4 | import org.broadinstitute.dsde.firecloud.FireCloudConfig 5 | import org.broadinstitute.dsde.firecloud.utils.StreamingPassthrough 6 | 7 | trait PassthroughApiService extends Directives with StreamingPassthrough { 8 | 9 | private lazy val agora = FireCloudConfig.Agora.baseUrl 10 | private lazy val cromiam = FireCloudConfig.CromIAM.baseUrl 11 | private lazy val rawls = FireCloudConfig.Rawls.baseUrl 12 | private lazy val sam = FireCloudConfig.Sam.baseUrl 13 | 14 | val passthroughRoutes: Route = concat( 15 | // Agora 16 | pathPrefix("api" / "configurations")(streamingPassthrough(s"$agora/api/v1/configurations")), 17 | pathPrefix("api" / "methods")(streamingPassthrough(s"$agora/api/v1/methods")), 18 | pathPrefix("ga4gh")(streamingPassthrough(s"$agora/ga4gh")), 19 | // CromIAM 20 | pathPrefix("api" / "womtool")(streamingPassthrough(s"$cromiam/api/womtool")), 21 | // Rawls 22 | pathPrefix("api" / "inputsOutputs")(streamingPassthrough(s"$rawls/api/methodconfigs/inputsOutputs")), 23 | pathPrefix("api" / "profile" / "billing")(streamingPassthrough(s"$rawls/api/user/billing")), 24 | pathPrefix("api" / "template")(streamingPassthrough(s"$rawls/api/methodconfigs/template")), 25 | pathPrefix("version" / "executionEngine")(streamingPassthrough(s"$rawls/version/executionEngine")), 26 | // Sam 27 | pathPrefix("api" / "proxyGroup")(streamingPassthrough(s"$sam/api/google/user/proxyGroup")), 28 | pathPrefix("register" / "user")(streamingPassthrough(s"$sam/register/user")), 29 | pathPrefix("register")(streamingPassthrough(s"$sam/register/user")), 30 | pathPrefix("tos")(streamingPassthrough(s"$sam/tos")), 31 | 32 | // any /api routes not otherwise defined will pass through to Rawls 33 | pathPrefix("api")(streamingPassthrough(s"$rawls/api")) 34 | ) 35 | 36 | } 37 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/webservice/RegisterApiService.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.webservice 2 | 3 | import akka.http.scaladsl.client.RequestBuilding 4 | import akka.http.scaladsl.model.HttpMethods.{DELETE, GET, POST} 5 | import org.broadinstitute.dsde.firecloud.model.ModelJsonProtocol._ 6 | import org.broadinstitute.dsde.firecloud.model._ 7 | import org.broadinstitute.dsde.firecloud.service.{FireCloudDirectives, RegisterService} 8 | import org.broadinstitute.dsde.firecloud.utils.{EnabledUserDirectives, StandardUserInfoDirectives} 9 | import spray.json.DefaultJsonProtocol._ 10 | import akka.http.scaladsl.server.Route 11 | import org.broadinstitute.dsde.firecloud.service.RegisterService.{ 12 | samTosBaseUrl, 13 | samTosDetailsUrl, 14 | samTosStatusUrl, 15 | samTosTextUrl 16 | } 17 | 18 | import scala.concurrent.ExecutionContext 19 | 20 | trait RegisterApiService 21 | extends FireCloudDirectives 22 | with EnabledUserDirectives 23 | with RequestBuilding 24 | with StandardUserInfoDirectives { 25 | 26 | implicit val executionContext: ExecutionContext 27 | 28 | val registerServiceConstructor: () => RegisterService 29 | 30 | val v1RegisterRoutes: Route = 31 | pathPrefix("users" / "v1" / "registerWithProfile") { 32 | post { 33 | requireUserInfo() { userInfo => 34 | entity(as[RegisterRequest]) { registerRequest => 35 | complete { 36 | registerServiceConstructor().createUserWithProfile(userInfo, registerRequest) 37 | } 38 | } 39 | } 40 | } 41 | } 42 | 43 | val registerRoutes: Route = 44 | pathPrefix("register") { 45 | path("profile") { 46 | post { 47 | requireUserInfo() { userInfo => 48 | entity(as[BasicProfile]) { basicProfile => 49 | complete(registerServiceConstructor().createUpdateProfile(userInfo, basicProfile)) 50 | } 51 | } 52 | } 53 | } 54 | } 55 | 56 | val profileRoutes: Route = 57 | pathPrefix("profile") { 58 | path("preferences") { 59 | post { 60 | requireUserInfo() { userInfo => 61 | requireEnabledUser(userInfo) { 62 | entity(as[Map[String, String]]) { preferences => 63 | complete(registerServiceConstructor().updateProfilePreferences(userInfo, preferences)) 64 | } 65 | } 66 | } 67 | } 68 | } 69 | } 70 | 71 | } 72 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/webservice/StaticNotebooksApiService.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.webservice 2 | 3 | import akka.http.scaladsl.server.Route 4 | import org.broadinstitute.dsde.firecloud.FireCloudConfig 5 | import org.broadinstitute.dsde.firecloud.service._ 6 | import org.broadinstitute.dsde.firecloud.utils.{RestJsonClient, StandardUserInfoDirectives} 7 | 8 | import scala.concurrent.ExecutionContext 9 | 10 | trait StaticNotebooksApiService extends FireCloudDirectives with StandardUserInfoDirectives with RestJsonClient { 11 | 12 | implicit val executionContext: ExecutionContext 13 | 14 | val calhounStaticNotebooksRoot: String = FireCloudConfig.StaticNotebooks.baseUrl 15 | val calhounStaticNotebooksURL: String = s"$calhounStaticNotebooksRoot/api/convert" 16 | 17 | val staticNotebooksRoutes: Route = 18 | path("staticNotebooks" / "convert") { 19 | requireUserInfo() { userInfo => 20 | post { requestContext => 21 | // call Calhoun and pass its response back to our own caller 22 | // can't use passthrough() here because that demands a JSON response 23 | // and we expect this to return text/html 24 | val extReq = Post(calhounStaticNotebooksURL, requestContext.request.entity) 25 | userAuthedRequest(extReq)(userInfo).flatMap { resp => 26 | requestContext.complete(resp) 27 | } 28 | } 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/main/scala/org/broadinstitute/dsde/firecloud/webservice/StatusApiService.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.webservice 2 | 3 | import java.text.SimpleDateFormat 4 | 5 | import akka.http.scaladsl.client.RequestBuilding 6 | import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport 7 | import akka.http.scaladsl.model.StatusCodes 8 | import akka.http.scaladsl.server.{Directives, Route} 9 | import org.broadinstitute.dsde.firecloud.service._ 10 | 11 | import scala.concurrent.ExecutionContext 12 | import spray.json.{JsObject, JsString} 13 | import spray.json.DefaultJsonProtocol._ 14 | 15 | object BuildTimeVersion { 16 | val version = Option(getClass.getPackage.getImplementationVersion) 17 | val versionJson = JsObject(Map("version" -> JsString(version.getOrElse("n/a")))) 18 | } 19 | 20 | trait StatusApiService extends Directives with RequestBuilding with SprayJsonSupport { 21 | 22 | final private val dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ") 23 | implicit val executionContext: ExecutionContext 24 | 25 | val statusServiceConstructor: () => StatusService 26 | 27 | val statusRoutes: Route = 28 | path("status") { 29 | get { 30 | complete(statusServiceConstructor().collectStatusInfo()) 31 | } 32 | } ~ 33 | path("version") { 34 | get { requestContext => 35 | requestContext.complete(StatusCodes.OK, BuildTimeVersion.versionJson) 36 | } 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | [%level] [%d{HH:mm:ss.SSS}] [%thread] %logger{36} - %msg%n 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /src/test/resources/reference.conf: -------------------------------------------------------------------------------- 1 | include "src/main/resources/reference.conf" 2 | 3 | akka { 4 | loglevel = "OFF" 5 | } 6 | 7 | auth { 8 | googleClientId = "dummy" 9 | googleSecretsJson = """{"web":{"auth_provider_x509_cert_url":"","auth_uri":"","client_id":"","client_secret":"","javascript_origins":[],"redirect_uris":[],"token_uri":""}}""" 10 | pemFile = "/dev/null" 11 | pemFileClientId = "dummy" 12 | jsonFile = "/dev/null" 13 | rawlsPemFile = "/dev/null" 14 | rawlsPemFileClientId = "dummy" 15 | 16 | swaggerRealm = "broad-dsde-dev" 17 | } 18 | 19 | agora { 20 | baseUrl = "http://localhost:8989" 21 | } 22 | 23 | rawls { 24 | baseUrl = "http://localhost:8990" 25 | } 26 | 27 | thurloe { 28 | baseUrl = "http://localhost:8991" 29 | } 30 | 31 | sam { 32 | baseUrl = "http://localhost:8994" 33 | } 34 | 35 | cromiam { 36 | baseUrl = "http://localhost:8995" 37 | } 38 | 39 | cwds { 40 | baseUrl = "https://local.broadinstitute.org:8996" 41 | enabled = true 42 | supportedFormats = ["pfb","tdrexport","rawlsjson"] 43 | bucketName = "cwds-testconf-bucketname" 44 | } 45 | 46 | externalCreds { 47 | baseUrl = "https://externalcreds.dsde-dev.broadinstitute.org" 48 | enabled = true 49 | } 50 | 51 | firecloud { 52 | baseUrl = "https://local.broadinstitute.org" 53 | portalUrl = "https://local.broadinstitute.org" 54 | fireCloudId = "123" 55 | serviceProject = "test-project" 56 | supportDomain = "test-domain.org" 57 | supportPrefix = "ag-test-" 58 | userAdminAccount = "fake-admin@fake.firecloud.org" 59 | } 60 | 61 | nih { 62 | whitelistBucket = "firecloud-whitelist-dev" 63 | whitelists = { 64 | "TARGET" { 65 | "fileName":"target-whitelist.txt", 66 | "rawlsGroup":"TARGET-dbGaP-Authorized", 67 | "phsId":"phs002499", 68 | "consentGroup":"c1" 69 | }, 70 | "TCGA" { 71 | "fileName":"tcga-whitelist.txt", 72 | "rawlsGroup":"TCGA-dbGaP-Authorized", 73 | "phsId":"phs002555", 74 | "consentGroup":"c1" 75 | }, 76 | "RAS" { 77 | "fileName":"dbgap_phs002409_c1_whitelist.txt", 78 | "rawlsGroup":"dbgap_phs002409_c1", 79 | "phsId":"phs002409", 80 | "consentGroup":"c1" 81 | } 82 | "BROKEN" { 83 | "fileName":"broken-whitelist.txt", 84 | "rawlsGroup":"this-doesnt-matter", 85 | "phsId":"phs001234", 86 | "consentGroup":"c2" 87 | } 88 | "DISABLED" { 89 | "fileName":"disabled-whitelist.txt", 90 | "rawlsGroup":"this-doesnt-matter", 91 | "phsId":"phs101234", 92 | "consentGroup":"c2" 93 | "disabled": true 94 | } 95 | } 96 | rasIssuer = "https://stsstg.nih.gov" 97 | rasVisaType = "https://ras.nih.gov/visas/v1.1" 98 | dbGapPermissionAndGroup = [ 99 | {phsId = "phs002409", consentGroup = "c1", groupName = "dbgap_phs002409_c1"}, 100 | {phsId = "phs002410", consentGroup = "c1", groupName = "dbgap_phs002410_c1"}, 101 | ] 102 | } 103 | 104 | # these are the settings currently used at runtime; copy them here 105 | # so we're testing under similar conditions. 106 | spray.can.host-connector { 107 | max-connections = 4 108 | max-retries = 5 109 | pipelining = off 110 | } 111 | 112 | notification { 113 | fullyQualifiedNotificationTopic = "dummy" 114 | } 115 | 116 | firecloud { 117 | max-filematching-bucket-files = 25000 118 | } 119 | -------------------------------------------------------------------------------- /src/test/resources/testfiles/bagit/duplicate_participants_nested_testbag.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/broadinstitute/firecloud-orchestration/e9656b185e1e113bfa4526bcdd0918a22f64cc6e/src/test/resources/testfiles/bagit/duplicate_participants_nested_testbag.zip -------------------------------------------------------------------------------- /src/test/resources/testfiles/bagit/duplicate_samples_nested_testbag.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/broadinstitute/firecloud-orchestration/e9656b185e1e113bfa4526bcdd0918a22f64cc6e/src/test/resources/testfiles/bagit/duplicate_samples_nested_testbag.zip -------------------------------------------------------------------------------- /src/test/resources/testfiles/bagit/empty.zip: -------------------------------------------------------------------------------- 1 | PK -------------------------------------------------------------------------------- /src/test/resources/testfiles/bagit/extra_file_nested_testbag.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/broadinstitute/firecloud-orchestration/e9656b185e1e113bfa4526bcdd0918a22f64cc6e/src/test/resources/testfiles/bagit/extra_file_nested_testbag.zip -------------------------------------------------------------------------------- /src/test/resources/testfiles/bagit/flat_testbag.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/broadinstitute/firecloud-orchestration/e9656b185e1e113bfa4526bcdd0918a22f64cc6e/src/test/resources/testfiles/bagit/flat_testbag.zip -------------------------------------------------------------------------------- /src/test/resources/testfiles/bagit/nested_testbag.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/broadinstitute/firecloud-orchestration/e9656b185e1e113bfa4526bcdd0918a22f64cc6e/src/test/resources/testfiles/bagit/nested_testbag.zip -------------------------------------------------------------------------------- /src/test/resources/testfiles/bagit/not_a_zip.txt: -------------------------------------------------------------------------------- 1 | This is a text file, not a zip file. -------------------------------------------------------------------------------- /src/test/resources/testfiles/bagit/nothingbag.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/broadinstitute/firecloud-orchestration/e9656b185e1e113bfa4526bcdd0918a22f64cc6e/src/test/resources/testfiles/bagit/nothingbag.zip -------------------------------------------------------------------------------- /src/test/resources/testfiles/bagit/participants_only_flat_testbag.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/broadinstitute/firecloud-orchestration/e9656b185e1e113bfa4526bcdd0918a22f64cc6e/src/test/resources/testfiles/bagit/participants_only_flat_testbag.zip -------------------------------------------------------------------------------- /src/test/resources/testfiles/bagit/samples_only_flat_testbag.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/broadinstitute/firecloud-orchestration/e9656b185e1e113bfa4526bcdd0918a22f64cc6e/src/test/resources/testfiles/bagit/samples_only_flat_testbag.zip -------------------------------------------------------------------------------- /src/test/resources/testfiles/bagit/testbag.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/broadinstitute/firecloud-orchestration/e9656b185e1e113bfa4526bcdd0918a22f64cc6e/src/test/resources/testfiles/bagit/testbag.zip -------------------------------------------------------------------------------- /src/test/resources/testfiles/tsv/ADD_PARTICIPANTS.txt: -------------------------------------------------------------------------------- 1 | entity:participant_id 2 | participant_01 3 | participant_02 4 | participant_03 5 | participant_04 6 | participant_05 7 | participant_06 8 | participant_07 9 | participant_08 -------------------------------------------------------------------------------- /src/test/resources/testfiles/tsv/ADD_SAMPLES.txt: -------------------------------------------------------------------------------- 1 | entity:sample_id participant_id sample_type 2 | sample_01 participant_01 primary_solid_tumor 3 | sample_02 participant_01 blood_derived_normal 4 | sample_03 participant_02 primary_solid_tumor 5 | sample_04 participant_02 blood-derived_normal -------------------------------------------------------------------------------- /src/test/resources/testfiles/tsv/MEMBERSHIP_SAMPLE_SET.tsv: -------------------------------------------------------------------------------- 1 | membership:sample_set_id sample 2 | your-setA-name your-entity1-id 3 | your-setA-name your-entity2-id 4 | your-set2-name your-entity2-id 5 | your-set2-name your-entity3-id 6 | -------------------------------------------------------------------------------- /src/test/resources/testfiles/tsv/PARTICIPANTS_NO_PREFIX.txt: -------------------------------------------------------------------------------- 1 | participant_id 2 | participant_01 3 | participant_02 4 | participant_03 5 | participant_04 6 | participant_05 7 | participant_06 8 | participant_07 9 | participant_08 -------------------------------------------------------------------------------- /src/test/resources/testfiles/tsv/PARTICIPANTS_NO_PREFIX_OR_SUFFIX.txt: -------------------------------------------------------------------------------- 1 | participant 2 | participant_01 3 | participant_02 4 | participant_03 5 | participant_04 6 | participant_05 7 | participant_06 8 | participant_07 9 | participant_08 -------------------------------------------------------------------------------- /src/test/resources/testfiles/tsv/PARTICIPANTS_NO_SUFFIX.txt: -------------------------------------------------------------------------------- 1 | entity:participant 2 | participant_01 3 | participant_02 4 | participant_03 5 | participant_04 6 | participant_05 7 | participant_06 8 | participant_07 9 | participant_08 -------------------------------------------------------------------------------- /src/test/resources/testfiles/tsv/TEST_INVALID.txt: -------------------------------------------------------------------------------- 1 | entity:sample_id participant_id sample_type TCGA-5M-AAT4-01A TCGA-5M-AAT4 primary_solid_tumor TCGA-5M-AAT4-10A TCGA-5M-AAT4 blood_derived_normal TCGA-NH-A8F8-01A TCGA-NH-A8F8 primary_solid_tumor TCGA-NH-A8F8-10A TCGA-NH-A8F8 blood-derived_normal -------------------------------------------------------------------------------- /src/test/resources/testfiles/tsv/TEST_INVALID_COLUMNS.txt: -------------------------------------------------------------------------------- 1 | bad-prefix:participant_id col1 col2 2 | TCGA-5M-AAT4-01A TCGA-5M-AAT4 primary_solid_tumor 3 | TCGA-5M-AAT4-10A TCGA-5M-AAT4 blood_derived_normal 4 | TCGA-NH-A8F8-01A TCGA-NH-A8F8 primary_solid_tumor 5 | TCGA-NH-A8F8-10A TCGA-NH-A8F8 blood-derived_normal -------------------------------------------------------------------------------- /src/test/resources/testfiles/tsv/UPDATE_SAMPLES.txt: -------------------------------------------------------------------------------- 1 | update:sample_id participant_id sample_type new_attribute 2 | sample_01 participant_01 primary_solid_tumor new_attribute_01 3 | sample_02 participant_01 new_sample_type new_attribute_02 4 | sample_03 participant_02 primary_solid_tumor new_attribute_03 5 | sample_04 participant_02 blood-derived_normal new_attribute_04 -------------------------------------------------------------------------------- /src/test/scala/org/broadinstitute/dsde/firecloud/dataaccess/HttpGoogleServicesDAOSpec.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.dataaccess 2 | 3 | import akka.actor.ActorSystem 4 | import cats.effect.{IO, Resource} 5 | import com.google.cloud.storage.{BlobInfo, Storage, StorageException} 6 | import com.google.cloud.storage.Storage.BlobWriteOption 7 | import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper 8 | import org.broadinstitute.dsde.workbench.google2.{GoogleStorageInterpreter, GoogleStorageService} 9 | import org.broadinstitute.dsde.workbench.model.google.{GcsBucketName, GcsObjectName, GcsPath} 10 | import org.mockito.ArgumentMatchers.any 11 | import org.mockito.Mockito.when 12 | import org.scalatest.flatspec.AnyFlatSpec 13 | import org.scalatest.matchers.should.Matchers 14 | import org.scalatest.PrivateMethodTester 15 | import org.scalatestplus.mockito.MockitoSugar.mock 16 | import org.typelevel.log4cats.SelfAwareStructuredLogger 17 | import org.typelevel.log4cats.slf4j.Slf4jLogger 18 | 19 | import java.nio.charset.StandardCharsets 20 | import cats.effect.std.Semaphore 21 | import com.typesafe.config.ConfigFactory 22 | import fs2.Stream 23 | import scala.concurrent.duration.Duration 24 | import scala.concurrent.Await 25 | 26 | class HttpGoogleServicesDAOSpec extends AnyFlatSpec with Matchers with PrivateMethodTester { 27 | 28 | val testProject = "broad-dsde-dev" 29 | implicit val system: ActorSystem = ActorSystem("HttpGoogleCloudStorageDAOSpec") 30 | import system.dispatcher 31 | val gcsDAO = new HttpGoogleServicesDAO() 32 | 33 | behavior of "HttpGoogleServicesDAO" 34 | 35 | it should "return GcsPath for a successful object upload" in { 36 | // create local storage service 37 | val db = LocalStorageHelper.getOptions().getService() 38 | val localStorage = storageResource(db) 39 | 40 | val bucketName = GcsBucketName("some-bucket") 41 | val objectName = GcsObjectName("folder/object.json") 42 | 43 | val objectContents = "Hello world".getBytes(StandardCharsets.UTF_8) 44 | 45 | assertResult(GcsPath(bucketName, objectName)) { 46 | gcsDAO.streamUploadObject(localStorage, bucketName, objectName, Stream.emits(objectContents).covary[IO]) 47 | } 48 | } 49 | 50 | it should "throw error for an unsuccessful object upload" in { 51 | // under the covers, Storage.writer is the Google library method that gets called. So, mock that 52 | // and force it to throw 53 | val mockedException = new StorageException(418, "intentional unit test failure") 54 | val throwingStorageHelper = mock[Storage] 55 | when(throwingStorageHelper.writer(any[BlobInfo], any[BlobWriteOption])) 56 | .thenThrow(mockedException) 57 | val localStorage = storageResource(throwingStorageHelper) 58 | 59 | val bucketName = GcsBucketName("some-bucket") 60 | val objectName = GcsObjectName("folder/object.json") 61 | 62 | val objectContents = "Hello world".getBytes(StandardCharsets.UTF_8) 63 | 64 | val caught = intercept[StorageException] { 65 | gcsDAO.streamUploadObject(localStorage, bucketName, objectName, Stream.emits(objectContents).covary[IO]) 66 | } 67 | 68 | assertResult(mockedException) { 69 | caught 70 | } 71 | } 72 | 73 | private def storageResource(backingStore: Storage): Resource[IO, GoogleStorageService[IO]] = { 74 | // create local storage service 75 | implicit val logger: SelfAwareStructuredLogger[IO] = Slf4jLogger.getLogger[IO] 76 | import cats.effect.unsafe.implicits.global 77 | 78 | val semaphore = Semaphore[IO](1).unsafeRunSync() 79 | Resource.pure[IO, GoogleStorageService[IO]](GoogleStorageInterpreter[IO](backingStore, Some(semaphore))) 80 | } 81 | 82 | } 83 | -------------------------------------------------------------------------------- /src/test/scala/org/broadinstitute/dsde/firecloud/dataaccess/MockAgoraDAO.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.dataaccess 2 | 3 | import akka.http.scaladsl.model.Uri 4 | import org.broadinstitute.dsde.firecloud.mock.MockAgoraACLData 5 | import org.broadinstitute.dsde.firecloud.model.OrchMethodRepository.{ 6 | ACLNames, 7 | AgoraPermission, 8 | EntityAccessControlAgora, 9 | Method 10 | } 11 | import org.broadinstitute.dsde.firecloud.model.UserInfo 12 | import org.broadinstitute.dsde.workbench.util.health.SubsystemStatus 13 | 14 | import scala.concurrent.ExecutionContext.Implicits.global 15 | import scala.concurrent.Future 16 | 17 | object MockAgoraDAO { 18 | def agoraPermission = AgoraPermission( 19 | user = Some("test-user@broadinstitute.org"), 20 | roles = Some(ACLNames.ListOwner) 21 | ) 22 | } 23 | 24 | class MockAgoraDAO extends AgoraDAO { 25 | 26 | override def getNamespacePermissions(ns: String, entity: String)(implicit 27 | userInfo: UserInfo 28 | ): Future[List[AgoraPermission]] = 29 | Future(List(MockAgoraDAO.agoraPermission)) 30 | 31 | override def postNamespacePermissions(ns: String, entity: String, perms: List[AgoraPermission])(implicit 32 | userInfo: UserInfo 33 | ): Future[List[AgoraPermission]] = 34 | Future(List(MockAgoraDAO.agoraPermission)) 35 | 36 | override def getMultiEntityPermissions( 37 | entityType: _root_.org.broadinstitute.dsde.firecloud.model.OrchMethodRepository.AgoraEntityType.Value, 38 | entities: List[Method] 39 | )(implicit userInfo: UserInfo) = 40 | Future(List.empty[EntityAccessControlAgora]) 41 | 42 | def status: Future[SubsystemStatus] = 43 | Future(SubsystemStatus(ok = true, None)) 44 | 45 | override def batchCreatePermissions(inputs: List[EntityAccessControlAgora])(implicit 46 | userInfo: UserInfo 47 | ): Future[List[EntityAccessControlAgora]] = 48 | Future.successful(MockAgoraACLData.multiUpsertResponse) 49 | 50 | override def getPermission(url: String)(implicit userInfo: UserInfo): Future[List[AgoraPermission]] = { 51 | val pathString = Uri(url).path.toString() 52 | 53 | val rawData = if (pathString.endsWith(MockAgoraACLData.standardPermsPath)) { 54 | MockAgoraACLData.standardAgora 55 | } else if (pathString.endsWith(MockAgoraACLData.withEdgeCasesPath)) { 56 | MockAgoraACLData.edgesAgora 57 | } else { 58 | List.empty 59 | } 60 | 61 | // methods endpoints return the mock data in reverse order - this way we can differentiate methods vs. configs 62 | if (pathString.startsWith("/api/v1/methods/")) { 63 | Future.successful(rawData.reverse) 64 | } else { 65 | Future.successful(rawData) 66 | } 67 | 68 | } 69 | 70 | override def createPermission(url: String, agoraPermissions: List[AgoraPermission])(implicit 71 | userInfo: UserInfo 72 | ): Future[List[AgoraPermission]] = { 73 | 74 | val pathString = Uri(url).path.toString() 75 | 76 | // carried over from the previous mockserver: methods returns a bad, unparsable response from Agora 77 | // this allows us to test orch's handling of bad responses 78 | val rawData = if (pathString.endsWith(MockAgoraACLData.standardPermsPath)) { 79 | if (pathString.startsWith("/api/v1/methods")) { 80 | MockAgoraACLData.edgesAgora 81 | } else { 82 | MockAgoraACLData.standardAgora 83 | } 84 | } else { 85 | List.empty 86 | } 87 | 88 | Future.successful(rawData) 89 | 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /src/test/scala/org/broadinstitute/dsde/firecloud/dataaccess/MockCwdsDAO.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.dataaccess 2 | 3 | import akka.http.scaladsl.model.StatusCodes.{BadRequest, EnhanceYourCalm, Forbidden, UnavailableForLegalReasons} 4 | import org.broadinstitute.dsde.firecloud.dataaccess.LegacyFileTypes.{FILETYPE_PFB, FILETYPE_RAWLS, FILETYPE_TDR} 5 | import org.broadinstitute.dsde.firecloud.model.{AsyncImportRequest, CwdsListResponse, UserInfo} 6 | import org.broadinstitute.dsde.rawls.model.ErrorReportSource 7 | import org.databiosphere.workspacedata.client.ApiException 8 | import org.databiosphere.workspacedata.model.GenericJob 9 | import org.databiosphere.workspacedata.model.GenericJob.{JobTypeEnum, StatusEnum} 10 | 11 | import java.time.OffsetDateTime 12 | import java.util.UUID 13 | 14 | class MockCwdsDAO( 15 | enabled: Boolean = true, 16 | supportedFormats: List[String] = List("pfb", "tdrexport", "rawlsjson") 17 | ) extends HttpCwdsDAO(enabled, supportedFormats) { 18 | implicit val errorReportSource: ErrorReportSource = ErrorReportSource( 19 | "MockCWDS" 20 | ) 21 | override def listJobsV1(workspaceId: String, runningOnly: Boolean)(implicit 22 | userInfo: UserInfo 23 | ): List[CwdsListResponse] = List() 24 | 25 | override def getJobV1(workspaceId: String, jobId: String)(implicit 26 | userInfo: UserInfo 27 | ): CwdsListResponse = 28 | CwdsListResponse(jobId, "ReadyForUpsert", "pfb", None) 29 | 30 | override def importV1( 31 | workspaceId: String, 32 | importRequest: AsyncImportRequest 33 | )(implicit userInfo: UserInfo): GenericJob = 34 | importRequest.filetype match { 35 | case FILETYPE_PFB | FILETYPE_TDR | FILETYPE_RAWLS => 36 | if (importRequest.url.contains("forbidden")) 37 | throw new ApiException( 38 | Forbidden.intValue, 39 | "Missing Authorization: Bearer token in header" 40 | ) 41 | else if (importRequest.url.contains("bad.request")) 42 | throw new ApiException( 43 | BadRequest.intValue, 44 | "Bad request as reported by cwds" 45 | ) 46 | else if (importRequest.url.contains("its.lawsuit.time")) 47 | throw new ApiException( 48 | UnavailableForLegalReasons.intValue, 49 | "cwds message" 50 | ) 51 | else if (importRequest.url.contains("good")) makeJob(workspaceId) 52 | else 53 | throw new ApiException( 54 | EnhanceYourCalm.intValue, 55 | "enhance your calm" 56 | ) 57 | case _ => ??? 58 | } 59 | 60 | private def makeJob( 61 | workspaceId: String 62 | ) = { 63 | val genericJob: GenericJob = new GenericJob 64 | genericJob.setJobId(UUID.randomUUID()) 65 | genericJob.setStatus(StatusEnum.RUNNING) 66 | genericJob.setJobType(JobTypeEnum.DATA_IMPORT) 67 | // will this cause a problem in tests? Some test data has non-UUIDs. 68 | genericJob.setInstanceId(UUID.fromString(workspaceId)) 69 | genericJob.setCreated(OffsetDateTime.now()) 70 | genericJob.setUpdated(OffsetDateTime.now()) 71 | genericJob 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/test/scala/org/broadinstitute/dsde/firecloud/dataaccess/MockShibbolethDAO.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.dataaccess 2 | 3 | import scala.concurrent.Future 4 | 5 | class MockShibbolethDAO extends ShibbolethDAO { 6 | val publicKey = 7 | "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsDPAkAwpWiO2659gPsIj\nzx9IypuiInn2F4IaCCJSxtjqRNw5g6QPJeMVjmnn3jT8CCMzvoOIOq8n7rmyog/p\npjJpq4AcVA0GjV8Nz7cWF/VwR+e/mN5CGvY4OfnCTBi5PUmywGLZMcJNhcbnka69\nexL18WwnM0d6/A/LYcmCQcI+YuakDksGAdrOn74WOrKQFa78SVOnB6Mfpf65rmu7\nTMQ66JBUuM2vIW+P1p4//+9MBSKUoGyXkbOsykBc1XYn/lLRoDCf2onYDTGjdILh\n7eSXdi6+VzgQ7j3hdkSRSj+mN2Vmq/AEWHd1lc/OQDMcRcEnRPyhwny9VW0gehyt\nWwIDAQAB\n-----END PUBLIC KEY-----" 8 | override def getPublicKey(): Future[String] = 9 | Future.successful(publicKey) 10 | } 11 | -------------------------------------------------------------------------------- /src/test/scala/org/broadinstitute/dsde/firecloud/filematch/strategy/IlluminaPairedEndStrategySpec.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.filematch.strategy 2 | 3 | import org.broadinstitute.dsde.firecloud.filematch.result.{FailedMatchResult, FileMatchResult, SuccessfulMatchResult} 4 | import org.scalatest.freespec.AnyFreeSpec 5 | import org.scalatest.matchers.should.Matchers 6 | 7 | class IlluminaPairedEndStrategySpec extends AnyFreeSpec with Matchers { 8 | 9 | val strategy = new IlluminaPairedEndStrategy 10 | 11 | /** 12 | * Naming conventions for Illumina single end and paired end read patterns. Examples of files recognized: 13 | * 14 | * SampleName_S1_L001_R1_001.fastq.gz -> SampleName_S1_L001_R2_001.fastq.gz 15 | */ 16 | 17 | // set of input, expected test cases 18 | val recognizedTestCases: Map[String, FileMatchResult] = Map( 19 | "Sample1_01.fastq.gz" -> SuccessfulMatchResult(toPath("Sample1_01.fastq.gz"), 20 | toPath("Sample1_02.fastq.gz"), 21 | "Sample1" 22 | ), 23 | "someSubdirectory/Sample1_01.fastq.gz" -> SuccessfulMatchResult(toPath("someSubdirectory/Sample1_01.fastq.gz"), 24 | toPath("someSubdirectory/Sample1_02.fastq.gz"), 25 | "Sample1" 26 | ), 27 | "sample42_1.fastq.gz" -> SuccessfulMatchResult(toPath("sample42_1.fastq.gz"), 28 | toPath("sample42_2.fastq.gz"), 29 | "sample42" 30 | ), 31 | "sample01_R1.fastq.gz" -> SuccessfulMatchResult(toPath("sample01_R1.fastq.gz"), 32 | toPath("sample01_R2.fastq.gz"), 33 | "sample01" 34 | ), 35 | "/foo/bar/789a_F.fastq.gz" -> SuccessfulMatchResult(toPath("/foo/bar/789a_F.fastq.gz"), 36 | toPath("/foo/bar/789a_R.fastq.gz"), 37 | "789a" 38 | ), 39 | "sample01_R1.fastq" -> SuccessfulMatchResult(toPath("sample01_R1.fastq"), toPath("sample01_R2.fastq"), "sample01"), 40 | "SampleName_S1_L001_R1_001.fastq.gz" -> SuccessfulMatchResult(toPath("SampleName_S1_L001_R1_001.fastq.gz"), 41 | toPath("SampleName_S1_L001_R2_001.fastq.gz"), 42 | "SampleName_S1_L001" 43 | ) 44 | ) 45 | 46 | val unrecognizedInputs: List[String] = 47 | List("my-cat-picture.png", "Sample1_01.fastq.gz/is/a/bad/directory/name", "Sample1_01.fasta.gz", "Sample1_01.bam") 48 | 49 | "IlluminaPairedEndStrategy" - { 50 | recognizedTestCases foreach { case (inputString, expectedMatchResult) => 51 | s"should hit on recognized input file $inputString" in { 52 | val matchResult = strategy.matchFirstFile(toPath(inputString)) 53 | matchResult shouldBe expectedMatchResult 54 | } 55 | } 56 | 57 | unrecognizedInputs foreach { inputString => 58 | s"should miss on unrecognized input file $inputString" in { 59 | val matchResult = strategy.matchFirstFile(toPath(inputString)) 60 | matchResult shouldBe FailedMatchResult(toPath(inputString)) 61 | } 62 | } 63 | 64 | } 65 | 66 | private def toPath(input: String) = new java.io.File(input).toPath 67 | 68 | } 69 | -------------------------------------------------------------------------------- /src/test/scala/org/broadinstitute/dsde/firecloud/mock/MockAgoraACLData.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.mock 2 | 3 | import org.broadinstitute.dsde.firecloud.model.OrchMethodRepository.ACLNames._ 4 | import org.broadinstitute.dsde.firecloud.model.OrchMethodRepository.{ 5 | AgoraPermission, 6 | EntityAccessControlAgora, 7 | FireCloudPermission, 8 | Method 9 | } 10 | 11 | /** 12 | * Created by davidan on 10/29/15. 13 | */ 14 | object MockAgoraACLData { 15 | 16 | val standardPermsPath = "/ns/standard/1/permissions" 17 | val withEdgeCasesPath = "/ns/edges/1/permissions" 18 | 19 | private val email = Some("davidan@broadinstitute.org") 20 | 21 | // FC PERMISSIONS 22 | private val ownerFC = FireCloudPermission("owner@broadinstitute.org", Owner) 23 | private val readerFC = FireCloudPermission("reader@broadinstitute.org", Reader) 24 | private val noAccessFC = FireCloudPermission("noaccess@broadinstitute.org", NoAccess) 25 | // AGORA PERMISSIONS 26 | private val allAgora = AgoraPermission(Some("owner@broadinstitute.org"), Some(ListAll)) 27 | private val ownerAgora = AgoraPermission(Some("owner@broadinstitute.org"), Some(ListOwner)) 28 | private val readerAgora = AgoraPermission(Some("reader@broadinstitute.org"), Some(ListReader)) 29 | private val noAccessAgora = AgoraPermission(Some("noaccess@broadinstitute.org"), Some(ListNoAccess)) 30 | // AGORA EDGE CASES 31 | private val partialsAgora = AgoraPermission(Some("agora-partial@broadinstitute.org"), Some(List("Read", "Write"))) 32 | private val extrasAgora = 33 | AgoraPermission(Some("agora-extras@broadinstitute.org"), Some(ListOwner ++ List("Extra", "Permissions"))) 34 | private val emptyAgora = AgoraPermission(Some("agora-empty@broadinstitute.org"), Some(List(""))) 35 | private val noneAgora = AgoraPermission(Some("agora-none@broadinstitute.org"), None) 36 | private val emptyUserAgora = AgoraPermission(Some(""), Some(ListOwner)) 37 | private val noneUserAgora = AgoraPermission(None, Some(ListOwner)) 38 | 39 | // standardAgora translates to standardFC. 40 | // standardFC translates to translatedStandardAgora 41 | val standardAgora = List(allAgora, readerAgora, ownerAgora, noAccessAgora) 42 | val standardFC = List(ownerFC, readerFC, ownerFC, noAccessFC) 43 | 44 | val translatedStandardAgora = List(ownerAgora, readerAgora, ownerAgora, noAccessAgora) 45 | 46 | val edgesAgora = 47 | standardAgora ++ List(partialsAgora, extrasAgora, emptyAgora, noneAgora, emptyUserAgora, noneUserAgora) 48 | 49 | // multi-permissions endpoint response 50 | val multiUpsertResponse: List[EntityAccessControlAgora] = List( 51 | EntityAccessControlAgora(Method(Some("ns1"), Some("n1"), Some(1)), 52 | Seq(AgoraPermission(Some("user1@example.com"), Some(ListAll))), 53 | None 54 | ), 55 | EntityAccessControlAgora(Method(Some("ns2"), Some("n2"), Some(2)), 56 | Seq(AgoraPermission(Some("user2@example.com"), Some(ListReader))), 57 | None 58 | ), 59 | EntityAccessControlAgora(Method(Some("ns3"), Some("n3"), Some(3)), 60 | Seq.empty[AgoraPermission], 61 | Some("this is an error message") 62 | ) 63 | ) 64 | 65 | } 66 | -------------------------------------------------------------------------------- /src/test/scala/org/broadinstitute/dsde/firecloud/mock/MockUtils.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.mock 2 | 3 | import java.text.SimpleDateFormat 4 | import java.util.Date 5 | 6 | import org.broadinstitute.dsde.rawls.model.ErrorReport 7 | import org.mockserver.model.Header 8 | import akka.http.scaladsl.model.StatusCode 9 | 10 | object MockUtils { 11 | 12 | val isoDateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZZ") 13 | val authHeader = new Header("Authorization", "Bearer mF_9.B5f-4.1JqM") 14 | val header = new Header("Content-Type", "application/json") 15 | val workspaceServerPort = 8990 16 | val methodsServerPort = 8989 17 | val thurloeServerPort = 8991 18 | val consentServerPort = 8992 19 | val ontologyServerPort = 8993 20 | val samServerPort = 8994 21 | val cromiamServerPort = 8995 22 | 23 | def randomPositiveInt(): Int = 24 | scala.util.Random.nextInt(9) + 1 25 | 26 | def randomAlpha(): String = { 27 | val chars = ('a' to 'z') ++ ('A' to 'Z') ++ ('0' to '9') 28 | randomStringFromCharList(randomPositiveInt(), chars) 29 | } 30 | 31 | def randomBoolean(): Boolean = 32 | scala.util.Random.nextBoolean() 33 | 34 | def randomStringFromCharList(length: Int, chars: Seq[Char]): String = { 35 | val sb = new StringBuilder 36 | for (i <- 1 to length) { 37 | val randomNum = util.Random.nextInt(chars.length) 38 | sb.append(chars(randomNum)) 39 | } 40 | sb.toString() 41 | } 42 | 43 | def isoDate(): String = 44 | isoDateFormat.format(new Date()) 45 | 46 | def rawlsErrorReport(statusCode: StatusCode) = 47 | ErrorReport("Rawls", "dummy text", Option(statusCode), Seq(), Seq(), None) 48 | 49 | def randomElement[A](list: List[A]): A = 50 | list(scala.util.Random.nextInt(list.length)) 51 | 52 | } 53 | -------------------------------------------------------------------------------- /src/test/scala/org/broadinstitute/dsde/firecloud/mock/ValidEntityCopyCallback.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.mock 2 | 3 | import org.broadinstitute.dsde.firecloud.mock.MockUtils._ 4 | import org.broadinstitute.dsde.firecloud.model.ModelJsonProtocol._ 5 | import org.broadinstitute.dsde.rawls.model.EntityCopyDefinition 6 | import org.mockserver.mock.action.ExpectationResponseCallback 7 | import org.mockserver.model.HttpResponse._ 8 | import org.mockserver.model.{HttpRequest, HttpResponse} 9 | import akka.http.scaladsl.model.StatusCodes._ 10 | import spray.json._ 11 | 12 | class ValidEntityCopyCallback extends ExpectationResponseCallback { 13 | 14 | override def handle(httpRequest: HttpRequest): HttpResponse = { 15 | 16 | val copyRequest = httpRequest.getBodyAsString.parseJson.convertTo[EntityCopyDefinition] 17 | 18 | (copyRequest.sourceWorkspace.namespace, copyRequest.destinationWorkspace.name) match { 19 | case (x: String, y: String) if x == "broad-dsde-dev" && y == "valid" => 20 | response() 21 | .withHeaders(header) 22 | .withStatusCode(Created.intValue) 23 | case _ => 24 | response() 25 | .withHeaders(header) 26 | .withStatusCode(NotFound.intValue) 27 | .withBody(MockUtils.rawlsErrorReport(NotFound).toJson.compactPrint) 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/test/scala/org/broadinstitute/dsde/firecloud/mock/ValidEntityDeleteCallback.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.mock 2 | 3 | import org.broadinstitute.dsde.firecloud.mock.MockUtils._ 4 | import org.broadinstitute.dsde.firecloud.model.EntityId 5 | import org.broadinstitute.dsde.firecloud.model.ModelJsonProtocol._ 6 | import spray.json.DefaultJsonProtocol._ 7 | import org.mockserver.mock.action.ExpectationResponseCallback 8 | import org.mockserver.model.HttpResponse._ 9 | import org.mockserver.model.{HttpRequest, HttpResponse} 10 | import akka.http.scaladsl.model.StatusCodes._ 11 | import spray.json._ 12 | 13 | import scala.util.Try 14 | 15 | class ValidEntityDeleteCallback extends ExpectationResponseCallback { 16 | 17 | val validEntities = Set(EntityId("sample", "id"), EntityId("sample", "bar")) 18 | 19 | override def handle(httpRequest: HttpRequest): HttpResponse = { 20 | val deleteRequest = Try(httpRequest.getBodyAsString.parseJson.convertTo[Set[EntityId]]) 21 | 22 | if (deleteRequest.isSuccess && deleteRequest.get.subsetOf(validEntities)) { 23 | response() 24 | .withHeaders(header) 25 | .withStatusCode(NoContent.intValue) 26 | } else { 27 | response() 28 | .withHeaders(header) 29 | .withStatusCode(BadRequest.intValue) 30 | .withBody(MockUtils.rawlsErrorReport(BadRequest).toJson.compactPrint) 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/test/scala/org/broadinstitute/dsde/firecloud/mock/ValidSubmissionCallback.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.mock 2 | 3 | import org.broadinstitute.dsde.firecloud.mock.MockUtils._ 4 | import org.broadinstitute.dsde.firecloud.model.ModelJsonProtocol._ 5 | import org.broadinstitute.dsde.firecloud.model.OrchSubmissionRequest 6 | import org.mockserver.mock.action.ExpectationResponseCallback 7 | import org.mockserver.model.HttpResponse._ 8 | import org.mockserver.model.{HttpRequest, HttpResponse} 9 | import akka.http.scaladsl.model.StatusCodes._ 10 | import spray.json._ 11 | 12 | class ValidSubmissionCallback extends ExpectationResponseCallback { 13 | 14 | override def handle(httpRequest: HttpRequest): HttpResponse = { 15 | 16 | val jsonAst = httpRequest.getBodyAsString.parseJson 17 | val submission = jsonAst.convertTo[OrchSubmissionRequest] 18 | submission match { 19 | case x 20 | if x.entityName.isDefined && 21 | x.entityType.isDefined && 22 | x.expression.isDefined && 23 | x.useCallCache.isDefined && 24 | x.deleteIntermediateOutputFiles.isDefined && 25 | x.workflowFailureMode.isDefined && 26 | x.methodConfigurationName.isDefined && 27 | x.methodConfigurationNamespace.isDefined => 28 | response() 29 | .withHeaders(header) 30 | .withStatusCode(OK.intValue) 31 | .withBody(MockWorkspaceServer.mockValidSubmission.toJson.prettyPrint) 32 | case _ => 33 | response() 34 | .withHeaders(header) 35 | .withStatusCode(BadRequest.intValue) 36 | .withBody(MockUtils.rawlsErrorReport(BadRequest).toJson.compactPrint) 37 | } 38 | 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /src/test/scala/org/broadinstitute/dsde/firecloud/model/FlexibleModelSchemaSpec.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.model 2 | 3 | import org.scalatest.freespec.AnyFreeSpec 4 | 5 | class FlexibleModelSchemaSpec extends AnyFreeSpec { 6 | 7 | val schema = FlexibleModelSchema 8 | 9 | "ModelSchema.isAttributeArray" - { 10 | 11 | "should be false for various scalars" in { 12 | List("", "-1", "0", "1", "hello", "{}", "true", "false", "null", "123.45", "[", "]", "][", "[-]") foreach { 13 | input => 14 | withClue(s"for input '$input', should return false") { 15 | assert(!schema.isAttributeArray(input)) 16 | } 17 | } 18 | } 19 | 20 | "should be true for various arrays" in { 21 | List( 22 | "[]", 23 | "[1]", 24 | "[1,2,3]", 25 | "[true]", 26 | "[true,false]", 27 | "[true,1,null]", 28 | """["foo"]""", 29 | """["foo","bar"]""", 30 | """["white", "space"]""", 31 | """["foo",1,true,null]""", 32 | """[{}, [{},{},{"a":"b"},true], "foo"]""" 33 | ) foreach { input => 34 | withClue(s"for input '$input', should return true") { 35 | assert(schema.isAttributeArray(input)) 36 | } 37 | } 38 | } 39 | 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/test/scala/org/broadinstitute/dsde/firecloud/model/OrchMethodRepositorySpec.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.model 2 | 3 | import org.broadinstitute.dsde.firecloud.model.OrchMethodRepository.{ACLNames, FireCloudPermission} 4 | import org.scalatest.freespec.AnyFreeSpec 5 | import org.scalatest.matchers.should.Matchers 6 | 7 | class OrchMethodRepositorySpec extends AnyFreeSpec with Matchers { 8 | 9 | "FireCloudPermission" - { 10 | "Correctly formed permissions should validate" - { 11 | "Valid email user" in { 12 | val permission = FireCloudPermission(user = "test@broadinstitute.org", role = ACLNames.Owner) 13 | permission shouldNot be(null) 14 | } 15 | "Public user" in { 16 | val permission = FireCloudPermission(user = "public", role = ACLNames.Owner) 17 | permission shouldNot be(null) 18 | } 19 | } 20 | 21 | "Incorrectly formed permissions should not validate" - { 22 | "Empty email" in { 23 | val ex = intercept[IllegalArgumentException] { 24 | val permission = FireCloudPermission(user = "", role = ACLNames.Owner) 25 | } 26 | ex shouldNot be(null) 27 | } 28 | "Invalid email" in { 29 | val ex = intercept[IllegalArgumentException] { 30 | val permission = FireCloudPermission(user = "in valid at email.com", role = ACLNames.Owner) 31 | } 32 | ex shouldNot be(null) 33 | } 34 | "Invalid role" in { 35 | val ex = intercept[IllegalArgumentException] { 36 | val permission = FireCloudPermission(user = "test@broadinstitute.org", role = ACLNames.ListNoAccess.head) 37 | } 38 | ex shouldNot be(null) 39 | } 40 | } 41 | 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /src/test/scala/org/broadinstitute/dsde/firecloud/model/SamResourceSpec.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.model 2 | 3 | import org.broadinstitute.dsde.firecloud.model.SamResource.{AccessPolicyName, ResourceId, UserPolicy} 4 | import org.scalatest.freespec.AnyFreeSpec 5 | import org.scalatest.matchers.should.Matchers 6 | import spray.json._ 7 | import org.broadinstitute.dsde.firecloud.model.ModelJsonProtocol._ 8 | import spray.json.DefaultJsonProtocol._ 9 | 10 | class SamResourceSpec extends AnyFreeSpec with Matchers { 11 | 12 | val userPolicyJSON = """ 13 | | { 14 | | "resourceId": "8011932d-d76e-4c5d-9f66-1538d86a683b", 15 | | "public": true, 16 | | "accessPolicyName": "reader", 17 | | "missingAuthDomainGroups": [], 18 | | "authDomainGroups": [] 19 | | } 20 | """.stripMargin 21 | 22 | val userPolicyListJSON = 23 | """ 24 | | [{ 25 | | "resourceId": "8011932d-d76e-4c5d-9f66-1538d86a683b", 26 | | "public": true, 27 | | "accessPolicyName": "reader", 28 | | "missingAuthDomainGroups": [], 29 | | "authDomainGroups": [] 30 | | }, 31 | | { 32 | | "resourceId": "195feff3-d4b0-43df-9d0d-d49eda2036eb", 33 | | "public": false, 34 | | "accessPolicyName": "owner", 35 | | "missingAuthDomainGroups": [], 36 | | "authDomainGroups": [] 37 | | }, 38 | | { 39 | | "resourceId": "a2e2a933-76ed-4679-a3c1-fcec146441b5", 40 | | "public": false, 41 | | "accessPolicyName": "owner", 42 | | "missingAuthDomainGroups": [], 43 | | "authDomainGroups": [] 44 | | }] 45 | """.stripMargin 46 | 47 | "UserPolicy JSON" - { 48 | 49 | "should convert to UserPolicy object" in { 50 | val jsobj: JsValue = JsonParser(userPolicyJSON) 51 | val userPolicy: UserPolicy = jsobj.convertTo[UserPolicy] 52 | assert(userPolicy.public) 53 | assertResult("reader")(userPolicy.accessPolicyName.value) 54 | } 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /src/test/scala/org/broadinstitute/dsde/firecloud/service/BaseServiceSpec.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.service 2 | 3 | import akka.http.scaladsl.server.ExceptionHandler 4 | import org.broadinstitute.dsde.firecloud.dataaccess._ 5 | import org.broadinstitute.dsde.firecloud.mock.MockGoogleServicesDAO 6 | import org.broadinstitute.dsde.firecloud.{Application, FireCloudApiService} 7 | import org.scalatest.BeforeAndAfter 8 | 9 | class BaseServiceSpec extends ServiceSpec with BeforeAndAfter { 10 | 11 | // this gets fed into sealRoute so that exceptions are handled the same in tests as in real life 12 | implicit val exceptionHandler: ExceptionHandler = FireCloudApiService.exceptionHandler 13 | 14 | val agoraDao: MockAgoraDAO = new MockAgoraDAO 15 | val googleServicesDao: MockGoogleServicesDAO = new MockGoogleServicesDAO 16 | val rawlsDao: MockRawlsDAO = new MockRawlsDAO 17 | val samDao: MockSamDAO = new MockSamDAO 18 | val thurloeDao: MockThurloeDAO = new MockThurloeDAO 19 | val shibbolethDao: MockShibbolethDAO = new MockShibbolethDAO 20 | val cwdsDao: CwdsDAO = new MockCwdsDAO 21 | val ecmDao: ExternalCredsDAO = new DisabledExternalCredsDAO 22 | 23 | val app: Application = 24 | new Application(agoraDao, googleServicesDao, rawlsDao, samDao, thurloeDao, shibbolethDao, cwdsDao, ecmDao) 25 | 26 | } 27 | -------------------------------------------------------------------------------- /src/test/scala/org/broadinstitute/dsde/firecloud/service/EntitiesWithTypeServiceSpec.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.service 2 | 3 | import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport 4 | import org.broadinstitute.dsde.firecloud.{EntityService, FireCloudConfig} 5 | import org.broadinstitute.dsde.firecloud.model._ 6 | import org.broadinstitute.dsde.rawls.model._ 7 | import akka.http.scaladsl.model.StatusCodes._ 8 | import akka.http.scaladsl.server.Route.{seal => sealRoute} 9 | import spray.json.DefaultJsonProtocol._ 10 | import org.broadinstitute.dsde.firecloud.model.ModelJsonProtocol._ 11 | import org.broadinstitute.dsde.firecloud.webservice.EntityApiService 12 | 13 | import scala.concurrent.ExecutionContext 14 | 15 | class EntitiesWithTypeServiceSpec extends BaseServiceSpec with EntityApiService with SprayJsonSupport { 16 | 17 | override val executionContext: ExecutionContext = scala.concurrent.ExecutionContext.Implicits.global 18 | 19 | val entityServiceConstructor: (ModelSchema) => EntityService = EntityService.constructor(app) 20 | 21 | val validFireCloudPath = 22 | FireCloudConfig.Rawls.authPrefix + FireCloudConfig.Rawls.workspacesPath + "/broad-dsde-dev/valid/" 23 | val invalidFireCloudPath = 24 | FireCloudConfig.Rawls.authPrefix + FireCloudConfig.Rawls.workspacesPath + "/broad-dsde-dev/invalid/" 25 | 26 | "EntityService-EntitiesWithType" - { 27 | 28 | "when calling GET on a valid entities_with_type path" - { 29 | "valid list of entity types are returned" in { 30 | val path = validFireCloudPath + "entities_with_type" 31 | Get(path) ~> dummyUserIdHeaders("1234") ~> sealRoute(entityRoutes) ~> check { 32 | status should be(OK) 33 | val entities = responseAs[List[Entity]] 34 | entities shouldNot be(empty) 35 | } 36 | } 37 | } 38 | 39 | "when calling GET on an invalid entities_with_type path" - { 40 | "server error is returned" in { 41 | val path = invalidFireCloudPath + "entities_with_type" 42 | Get(path) ~> dummyUserIdHeaders("1234") ~> sealRoute(entityRoutes) ~> check { 43 | status should be(NotFound) 44 | errorReportCheck("Rawls", NotFound) 45 | } 46 | } 47 | } 48 | 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/test/scala/org/broadinstitute/dsde/firecloud/service/FireCloudDirectivesSpec.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.service 2 | 3 | import akka.http.scaladsl.testkit.ScalatestRouteTest 4 | import org.scalatest.freespec.AnyFreeSpec 5 | 6 | import scala.concurrent.ExecutionContext 7 | 8 | class FireCloudDirectivesSpec extends AnyFreeSpec with ScalatestRouteTest with FireCloudDirectives { 9 | 10 | override val executionContext: ExecutionContext = scala.concurrent.ExecutionContext.Implicits.global 11 | 12 | "FireCloudDirectives" - { 13 | "Unencoded passthrough URLs" - { 14 | "should be encoded" in { 15 | val unencoded = "http://abc.com/path with spaces/" 16 | val encoded = encodeUri(unencoded) 17 | assert(encoded.equals("http://abc.com/path%20with%20spaces/")) 18 | } 19 | } 20 | "Encoded passthrough URLs" - { 21 | "should not be re-encoded" in { 22 | val unencoded = "https://abc.com/path%20with%20spaces/" 23 | val encoded = encodeUri(unencoded) 24 | assert(encoded.equals("https://abc.com/path%20with%20spaces/")) 25 | } 26 | } 27 | "Passthrough URLs with no parameters" - { 28 | "should not break during encoding" in { 29 | val unencoded = "http://abc.com/path" 30 | val encoded = encodeUri(unencoded) 31 | assert(encoded.equals("http://abc.com/path")) 32 | } 33 | } 34 | "URL with port specified" - { 35 | "should not break during encoding" in { 36 | val unencoded = "http://abc.com:8080/" 37 | val encoded = encodeUri(unencoded) 38 | assert(encoded.equals("http://abc.com:8080/")) 39 | } 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/test/scala/org/broadinstitute/dsde/firecloud/service/ServiceSpec.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.service 2 | 3 | import org.broadinstitute.dsde.rawls.model.ErrorReport 4 | import org.broadinstitute.dsde.firecloud.model.ModelJsonProtocol._ 5 | import org.broadinstitute.dsde.firecloud.utils.TestRequestBuilding 6 | import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport 7 | import org.scalatest.concurrent.ScalaFutures 8 | import org.scalatest.freespec.AnyFreeSpec 9 | import org.scalatest.matchers.should.Matchers 10 | import akka.http.scaladsl.model.HttpMethods._ 11 | import akka.http.scaladsl.model.HttpMethod 12 | import akka.http.scaladsl.model.StatusCode 13 | import akka.http.scaladsl.server._ 14 | import akka.http.scaladsl.server.Directives._ 15 | import akka.stream.ActorMaterializer 16 | 17 | import akka.http.scaladsl.server.Route 18 | import akka.http.scaladsl.testkit.{RouteTestTimeout, ScalatestRouteTest} 19 | import akka.testkit.TestKitBase 20 | 21 | import scala.concurrent.duration._ 22 | 23 | // common Service Spec to be inherited by service tests 24 | trait ServiceSpec 25 | extends AnyFreeSpec 26 | with ScalaFutures 27 | with ScalatestRouteTest 28 | with Matchers 29 | with TestRequestBuilding 30 | with TestKitBase { 31 | 32 | implicit val routeTestTimeout: RouteTestTimeout = RouteTestTimeout(5.seconds) 33 | 34 | val allHttpMethods = Seq(CONNECT, DELETE, GET, HEAD, PATCH, POST, PUT, TRACE) 35 | 36 | def allHttpMethodsExcept(method: HttpMethod, methods: HttpMethod*): Seq[HttpMethod] = allHttpMethodsExcept( 37 | method +: methods 38 | ) 39 | def allHttpMethodsExcept(methods: Seq[HttpMethod]): Seq[HttpMethod] = allHttpMethods.diff(methods) 40 | 41 | // is the response an ErrorReport with the given Source and StatusCode 42 | def errorReportCheck(source: String, statusCode: StatusCode): Unit = { 43 | val report = responseAs[ErrorReport] 44 | report.source should be(source) 45 | report.statusCode.get should be(statusCode) 46 | } 47 | 48 | def checkIfPassedThrough(route: Route, method: HttpMethod, uri: String, toBeHandled: Boolean): Unit = 49 | new RequestBuilder(method)(uri) ~> dummyAuthHeaders ~> route ~> check { 50 | handled should be(toBeHandled) 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/test/scala/org/broadinstitute/dsde/firecloud/service/ServiceSpecSpec.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.service 2 | 3 | import akka.http.scaladsl.model.HttpMethods._ 4 | 5 | final class ServiceSpecSpec extends ServiceSpec { 6 | "allHttpMethodsExcept() works" in { 7 | allHttpMethodsExcept(GET) should be(Seq(CONNECT, DELETE, HEAD, PATCH, POST, PUT, TRACE)) 8 | allHttpMethodsExcept(DELETE, POST) should be(Seq(CONNECT, GET, HEAD, PATCH, PUT, TRACE)) 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /src/test/scala/org/broadinstitute/dsde/firecloud/utils/DisabledServiceFactoryTest.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.utils 2 | 3 | import org.broadinstitute.dsde.firecloud.dataaccess.CwdsDAO 4 | import org.broadinstitute.dsde.firecloud.model.UserInfo 5 | import org.scalatest.freespec.AnyFreeSpec 6 | import org.scalatest.matchers.should.Matchers 7 | 8 | class DisabledServiceFactoryTest extends AnyFreeSpec with Matchers { 9 | "DisabledServiceFactory" - { 10 | "newDisabledService" - { 11 | "should return a new instance of the service that throws UnsupportedOperationException for all methods except isEnabled" in { 12 | implicit val userInfo: UserInfo = null 13 | 14 | val disabledService = DisabledServiceFactory.newDisabledService[CwdsDAO] 15 | assertThrows[UnsupportedOperationException] { 16 | disabledService.getSupportedFormats 17 | } 18 | assertThrows[UnsupportedOperationException] { 19 | disabledService.listJobsV1("workspaceId", runningOnly = true) 20 | } 21 | assertThrows[UnsupportedOperationException] { 22 | disabledService.getJobV1("workspaceId", "jobId") 23 | } 24 | assertThrows[UnsupportedOperationException] { 25 | disabledService.importV1("workspaceId", null) 26 | } 27 | } 28 | 29 | "should return a new instance of the service that returns false for isEnabled" in { 30 | val disabledService = DisabledServiceFactory.newDisabledService[CwdsDAO] 31 | disabledService.isEnabled shouldBe false 32 | } 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/test/scala/org/broadinstitute/dsde/firecloud/utils/PermissionsSupportSpec.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.utils 2 | 3 | import org.broadinstitute.dsde.firecloud.dataaccess.{MockRawlsDAO, MockSamDAO, RawlsDAO, SamDAO} 4 | import org.broadinstitute.dsde.firecloud.model.UserInfo 5 | import org.broadinstitute.dsde.firecloud.service.PerRequest.RequestComplete 6 | import org.broadinstitute.dsde.firecloud.{FireCloudException, FireCloudExceptionWithErrorReport} 7 | import org.broadinstitute.dsde.workbench.model.WorkbenchGroupName 8 | import org.scalatest.freespec.AnyFreeSpecLike 9 | import akka.http.scaladsl.model.StatusCodes 10 | 11 | import scala.concurrent.duration.{Duration, SECONDS} 12 | import scala.concurrent.{Await, ExecutionContext, Future} 13 | 14 | class PermissionsSupportSpec extends PermissionsSupport with AnyFreeSpecLike { 15 | protected val rawlsDAO: RawlsDAO = new MockRawlsDAO 16 | protected val samDao: SamDAO = new PermissionsSupportMockSamDAO 17 | implicit protected val executionContext: ExecutionContext = scala.concurrent.ExecutionContext.Implicits.global 18 | 19 | val dur: Duration = Duration(60, SECONDS) 20 | 21 | "tryIsGroupMember" - { 22 | "should return true if user is a member" in { 23 | assert(Await.result(tryIsGroupMember(UserInfo("", "alice"), "apples"), dur)) 24 | assert(Await.result(tryIsGroupMember(UserInfo("", "bob"), "bananas"), dur)) 25 | } 26 | "should return false if user is not a member" in { 27 | assert(!Await.result(tryIsGroupMember(UserInfo("", "alice"), "bananas"), dur)) 28 | assert(!Await.result(tryIsGroupMember(UserInfo("", "bob"), "apples"), dur)) 29 | } 30 | "should catch and wrap source exceptions" in { 31 | val ex = intercept[FireCloudExceptionWithErrorReport] { 32 | Await.result(tryIsGroupMember(UserInfo("", "failme"), "anygroup"), Duration.Inf) 33 | } 34 | assert(ex.errorReport.message == "Unable to query for group membership status.") 35 | } 36 | } 37 | 38 | "asGroupMember" - { 39 | "should allow inner function to succeed if user is a member" in { 40 | implicit val userInfo = UserInfo("", "alice") 41 | def command = asGroupMember("apples")(Future.successful(RequestComplete(StatusCodes.OK))) 42 | val x = Await.result(command, dur) 43 | assertResult(RequestComplete(StatusCodes.OK))(x) 44 | } 45 | "should throw FireCloudExceptionWithErrorReport if user is not a member" in { 46 | implicit val userInfo = UserInfo("", "bob") 47 | def command = asGroupMember("apples")(Future.successful(RequestComplete(StatusCodes.OK))) 48 | val x = intercept[FireCloudExceptionWithErrorReport] { 49 | Await.result(command, dur) 50 | } 51 | assertResult(Some(StatusCodes.Forbidden))(x.errorReport.statusCode) 52 | assertResult("You must be in the appropriate group.")(x.errorReport.message) 53 | } 54 | } 55 | } 56 | 57 | class PermissionsSupportMockSamDAO extends MockSamDAO { 58 | private val groupMap = Map( 59 | "apples" -> Seq("alice"), 60 | "bananas" -> Seq("bob") 61 | ) 62 | 63 | override def isGroupMember(groupName: WorkbenchGroupName, userInfo: UserInfo): Future[Boolean] = 64 | userInfo.id match { 65 | case "failme" => Future.failed(new Exception("intentional exception for unit tests")) 66 | case _ => Future.successful(groupMap.getOrElse(groupName.value, Seq.empty[String]).contains(userInfo.id)) 67 | } 68 | 69 | } 70 | -------------------------------------------------------------------------------- /src/test/scala/org/broadinstitute/dsde/firecloud/utils/StatusCodeUtilsSpec.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.utils 2 | 3 | import akka.http.scaladsl.model.StatusCode 4 | import akka.http.scaladsl.model.StatusCodes._ 5 | import org.scalatest.flatspec.AnyFlatSpec 6 | import org.scalatest.matchers.should.Matchers.convertToAnyShouldWrapper 7 | 8 | class StatusCodeUtilsSpec extends AnyFlatSpec with StatusCodeUtils { 9 | 10 | behavior of "statusCodeFrom" 11 | 12 | val expectedCases: Map[Int, StatusCode] = Map( 13 | 200 -> OK, 14 | 404 -> NotFound, 15 | 503 -> ServiceUnavailable 16 | ) 17 | 18 | expectedCases.foreach { case (intCode, statusCode) => 19 | it should s"handle known code $intCode" in { 20 | statusCodeFrom(intCode) shouldBe statusCode 21 | } 22 | } 23 | 24 | val unknownCodes: List[Int] = List(-1, 0, 42, 222, 555) 25 | 26 | unknownCodes.foreach { intCode => 27 | it should s"create a custom status code $intCode for unknown values" in { 28 | val actual = statusCodeFrom(intCode) 29 | actual.intValue() shouldBe intCode 30 | actual.isSuccess() shouldBe false 31 | actual.defaultMessage() shouldBe s"unknown status $intCode" 32 | } 33 | } 34 | 35 | unknownCodes.foreach { intCode => 36 | it should s"default unknown code $intCode to the caller-supplied default" in { 37 | statusCodeFrom(intCode, Option(ImATeapot)) shouldBe ImATeapot 38 | } 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /src/test/scala/org/broadinstitute/dsde/firecloud/utils/TestRequestBuilding.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.utils 2 | 3 | import akka.http.scaladsl.model.HttpRequest 4 | import akka.http.scaladsl.model.headers.{Cookie, OAuth2BearerToken, RawHeader} 5 | import org.broadinstitute.dsde.firecloud.service.FireCloudRequestBuilding 6 | 7 | trait TestRequestBuilding extends FireCloudRequestBuilding { 8 | 9 | val dummyToken: String = "mF_9.B5f-4.1JqM" 10 | 11 | def dummyAuthHeaders: RequestTransformer = 12 | addCredentials(OAuth2BearerToken(dummyToken)) 13 | 14 | def dummyUserIdHeaders(userId: String, 15 | token: String = "access_token", 16 | email: String = "random@site.com" 17 | ): WithTransformerConcatenation[HttpRequest, HttpRequest] = 18 | addCredentials(OAuth2BearerToken(token)) ~> 19 | addHeader(RawHeader("OIDC_CLAIM_user_id", userId)) ~> 20 | addHeader(RawHeader("OIDC_access_token", token)) ~> 21 | addHeader(RawHeader("OIDC_CLAIM_email", email)) ~> 22 | addHeader(RawHeader("OIDC_CLAIM_expires_in", "100000")) 23 | 24 | def dummyCookieAuthHeaders: RequestTransformer = 25 | addHeader(Cookie("FCtoken", dummyToken)) 26 | } 27 | -------------------------------------------------------------------------------- /src/test/scala/org/broadinstitute/dsde/firecloud/webservice/ApiServiceSpec.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.webservice 2 | 3 | import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport 4 | import akka.http.scaladsl.testkit.{RouteTestTimeout, ScalatestRouteTest} 5 | import org.broadinstitute.dsde.firecloud.Application 6 | import org.broadinstitute.dsde.firecloud.dataaccess._ 7 | import org.broadinstitute.dsde.firecloud.mock.MockGoogleServicesDAO 8 | import org.broadinstitute.dsde.firecloud.service.NihService 9 | import org.broadinstitute.dsde.firecloud.utils.TestRequestBuilding 10 | import org.scalatest.flatspec.AnyFlatSpec 11 | import org.scalatest.matchers.should.Matchers 12 | 13 | import scala.concurrent.duration._ 14 | 15 | /** 16 | * Created by mbemis on 3/1/17. 17 | */ 18 | 19 | // common trait to be inherited by API service tests 20 | trait ApiServiceSpec 21 | extends AnyFlatSpec 22 | with Matchers 23 | with ScalatestRouteTest 24 | with SprayJsonSupport 25 | with TestRequestBuilding { 26 | // increase the timeout for ScalatestRouteTest from the default of 1 second, otherwise 27 | // intermittent failures occur on requests not completing in time 28 | implicit val routeTestTimeout: RouteTestTimeout = RouteTestTimeout(5.seconds) 29 | 30 | def actorRefFactory = system 31 | 32 | trait ApiServices extends NihApiService { 33 | val agoraDao: MockAgoraDAO 34 | val googleDao: MockGoogleServicesDAO 35 | val rawlsDao: MockRawlsDAO 36 | val samDao: MockSamDAO 37 | val thurloeDao: MockThurloeDAO 38 | val shibbolethDao: ShibbolethDAO 39 | val cwdsDao: CwdsDAO 40 | val ecmDao: ExternalCredsDAO 41 | 42 | def actorRefFactory = system 43 | 44 | val nihServiceConstructor = NihService.constructor( 45 | new Application(agoraDao, googleDao, rawlsDao, samDao, thurloeDao, shibbolethDao, cwdsDao, ecmDao) 46 | ) _ 47 | 48 | } 49 | 50 | // lifted from rawls. prefer this to using theSameElementsAs directly, because its functionality depends on whitespace 51 | def assertSameElements[T](expected: IterableOnce[T], actual: IterableOnce[T]): Unit = 52 | expected.iterator.to(Iterable) should contain theSameElementsAs actual.iterator.to(Iterable) 53 | 54 | } 55 | -------------------------------------------------------------------------------- /src/test/scala/org/broadinstitute/dsde/firecloud/webservice/HealthApiServiceSpec.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.webservice 2 | 3 | import akka.http.scaladsl.model.StatusCodes._ 4 | import akka.http.scaladsl.server.Route.{seal => sealRoute} 5 | import org.broadinstitute.dsde.firecloud.service.ServiceSpec 6 | 7 | import scala.concurrent.ExecutionContext 8 | 9 | class HealthApiServiceSpec extends ServiceSpec with HealthApiService { 10 | 11 | def actorRefFactory = system 12 | 13 | override val executionContext: ExecutionContext = scala.concurrent.ExecutionContext.Implicits.global 14 | 15 | "HealthApiService" - { 16 | "when GET-ting the health service endpoint" - { 17 | "OK response is returned" in 18 | Get("/health") ~> sealRoute(healthServiceRoutes) ~> check { 19 | status should equal(OK) 20 | } 21 | "Service Unavailable response is returned" in 22 | Get("/error") ~> sealRoute(healthServiceRoutes) ~> check { 23 | status should equal(ServiceUnavailable) 24 | } 25 | } 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /src/test/scala/org/broadinstitute/dsde/firecloud/webservice/MethodsApiServiceMultiACLSpec.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.webservice 2 | 3 | import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport 4 | import akka.http.scaladsl.model.HttpMethods 5 | import akka.http.scaladsl.model.StatusCodes._ 6 | import akka.http.scaladsl.server.Route.{seal => sealRoute} 7 | import org.broadinstitute.dsde.firecloud.model.OrchMethodRepository._ 8 | import org.broadinstitute.dsde.firecloud.model.ModelJsonProtocol.impMethodAclPair 9 | import org.broadinstitute.dsde.firecloud.model.UserInfo 10 | import org.broadinstitute.dsde.firecloud.service.{AgoraPermissionService, BaseServiceSpec, ServiceSpec} 11 | import org.broadinstitute.dsde.rawls.model.MethodRepoMethod 12 | import spray.json.DefaultJsonProtocol._ 13 | import spray.json._ 14 | 15 | import scala.concurrent.ExecutionContext 16 | 17 | class MethodsApiServiceMultiACLSpec 18 | extends BaseServiceSpec 19 | with ServiceSpec 20 | with MethodsApiService 21 | with SprayJsonSupport { 22 | 23 | def actorRefFactory = system 24 | 25 | override val executionContext: ExecutionContext = scala.concurrent.ExecutionContext.Implicits.global 26 | 27 | val agoraPermissionService: (UserInfo) => AgoraPermissionService = AgoraPermissionService.constructor(app) 28 | 29 | val localMethodPermissionsPath = s"/$localMethodsPath/permissions" 30 | 31 | // most of the functionality of this endpoint either exists in Agora or is unit-tested elsewhere. 32 | // here, we just test the routing and basic input/output of the endpoint. 33 | 34 | "Methods Repository multi-ACL upsert endpoint" - { 35 | "when testing DELETE, GET, POST methods on the multi-permissions path" - { 36 | "NotFound is returned" in { 37 | List(HttpMethods.DELETE, HttpMethods.GET, HttpMethods.POST) foreach { method => 38 | new RequestBuilder(method)(localMethodPermissionsPath) ~> sealRoute(methodsApiServiceRoutes) ~> check { 39 | status should equal(MethodNotAllowed) 40 | } 41 | } 42 | } 43 | } 44 | 45 | "when sending valid input" - { 46 | "returns OK and translates responses" in { 47 | val payload = Seq( 48 | MethodAclPair(MethodRepoMethod("ns1", "n1", 1), Seq(FireCloudPermission("user1@example.com", "OWNER"))), 49 | MethodAclPair(MethodRepoMethod("ns2", "n2", 2), Seq(FireCloudPermission("user2@example.com", "READER"))) 50 | ) 51 | Put(localMethodPermissionsPath, payload) ~> dummyUserIdHeaders("MethodsApiServiceMultiACLSpec") ~> sealRoute( 52 | methodsApiServiceRoutes 53 | ) ~> check { 54 | status should equal(OK) 55 | 56 | val resp = responseAs[Seq[MethodAclPair]] 57 | assert(resp.nonEmpty) 58 | } 59 | } 60 | } 61 | 62 | // BAD INPUTS 63 | "when posting malformed data" - { 64 | "BadRequest is returned" in 65 | // endpoint expects a JsArray; send it a JsObject and expect BadRequest. 66 | Put(localMethodPermissionsPath, JsObject(Map("foo" -> JsString("bar")))) ~> dummyAuthHeaders ~> sealRoute( 67 | methodsApiServiceRoutes 68 | ) ~> check { 69 | status should equal(BadRequest) 70 | } 71 | } 72 | 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /src/test/scala/org/broadinstitute/dsde/firecloud/webservice/NamespaceApiServiceSpec.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.webservice 2 | 3 | import akka.http.scaladsl.model.HttpMethods 4 | import akka.http.scaladsl.model.StatusCodes._ 5 | import akka.http.scaladsl.server.Route.{seal => sealRoute} 6 | import org.broadinstitute.dsde.firecloud.dataaccess.MockAgoraDAO 7 | import org.broadinstitute.dsde.firecloud.model.OrchMethodRepository.FireCloudPermission 8 | import org.broadinstitute.dsde.firecloud.model.ModelJsonProtocol._ 9 | import org.broadinstitute.dsde.firecloud.model.UserInfo 10 | import org.broadinstitute.dsde.firecloud.service.{AgoraPermissionService, BaseServiceSpec, NamespaceService} 11 | import spray.json.DefaultJsonProtocol._ 12 | 13 | import scala.concurrent.ExecutionContext 14 | 15 | class NamespaceApiServiceSpec extends BaseServiceSpec with NamespaceApiService { 16 | 17 | val namespaceServiceConstructor: (UserInfo) => NamespaceService = NamespaceService.constructor(app) 18 | 19 | override val executionContext: ExecutionContext = scala.concurrent.ExecutionContext.Implicits.global 20 | 21 | val urls = List("/api/methods/namespace/permissions", "/api/configurations/namespace/permissions") 22 | 23 | val fcPermissions = List(AgoraPermissionService.toFireCloudPermission(MockAgoraDAO.agoraPermission)) 24 | 25 | "NamespaceApiService" - { 26 | 27 | "when calling GET on a namespace permissions path" - { 28 | "a valid list of FireCloud permissions is returned" in { 29 | urls map { url => 30 | Get(url) ~> dummyUserIdHeaders("1234") ~> sealRoute(namespaceRoutes) ~> check { 31 | status should equal(OK) 32 | val permissions = responseAs[List[FireCloudPermission]] 33 | permissions should be(fcPermissions) 34 | } 35 | } 36 | } 37 | } 38 | 39 | "when calling POST on a namespace permissions path" - { 40 | "a valid FireCloud permission is returned" in { 41 | urls map { url => 42 | Post(url, fcPermissions) ~> dummyUserIdHeaders("1234") ~> sealRoute(namespaceRoutes) ~> check { 43 | status should equal(OK) 44 | val permissions = responseAs[List[FireCloudPermission]] 45 | permissions should be(fcPermissions) 46 | } 47 | } 48 | } 49 | } 50 | 51 | "when calling PUT or DELETE on a namespace permissions path" - { 52 | "a Method Not Allowed response is returned" in { 53 | urls map { url => 54 | List(HttpMethods.PUT, HttpMethods.DELETE) map { method => 55 | new RequestBuilder(method)(url, fcPermissions) ~> dummyUserIdHeaders("1234") ~> sealRoute( 56 | namespaceRoutes 57 | ) ~> check { 58 | status should equal(MethodNotAllowed) 59 | } 60 | } 61 | } 62 | } 63 | } 64 | 65 | } 66 | 67 | } 68 | -------------------------------------------------------------------------------- /src/test/scala/org/broadinstitute/dsde/firecloud/webservice/StatusApiServiceSpec.scala: -------------------------------------------------------------------------------- 1 | package org.broadinstitute.dsde.firecloud.webservice 2 | 3 | import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport 4 | import org.broadinstitute.dsde.firecloud.HealthChecks 5 | import org.broadinstitute.dsde.firecloud.service.{BaseServiceSpec, StatusService} 6 | import org.broadinstitute.dsde.workbench.util.health.StatusJsonSupport.StatusCheckResponseFormat 7 | import org.broadinstitute.dsde.workbench.util.health.Subsystems._ 8 | import org.broadinstitute.dsde.workbench.util.health.{HealthMonitor, StatusCheckResponse} 9 | import akka.http.scaladsl.model.HttpMethods.GET 10 | import akka.http.scaladsl.model.StatusCodes.OK 11 | 12 | import scala.concurrent.ExecutionContext 13 | import scala.concurrent.duration._ 14 | 15 | /* We don't do much testing of the HealthMonitor itself, because that's tested as part of 16 | workbench-libs. Here, we test routing, de/serialization, and the config we send into 17 | the HealthMonitor. 18 | */ 19 | class StatusApiServiceSpec extends BaseServiceSpec with StatusApiService with SprayJsonSupport { 20 | 21 | def actorRefFactory = system 22 | 23 | override val executionContext: ExecutionContext = scala.concurrent.ExecutionContext.Implicits.global 24 | 25 | val healthMonitorChecks = new HealthChecks(app).healthMonitorChecks 26 | val healthMonitor = 27 | system.actorOf(HealthMonitor.props(healthMonitorChecks().keySet)(healthMonitorChecks), "health-monitor") 28 | val monitorSchedule = 29 | system.scheduler.scheduleWithFixedDelay(Duration.Zero, 1.second, healthMonitor, HealthMonitor.CheckAll) 30 | 31 | override def beforeAll() = 32 | // wait for the healthMonitor to start up ... 33 | Thread.sleep(3000) 34 | 35 | override def afterAll() = 36 | monitorSchedule.cancel() 37 | 38 | override val statusServiceConstructor: () => StatusService = StatusService.constructor(healthMonitor) 39 | 40 | val statusPath = "/status" 41 | 42 | "Status endpoint" - { 43 | allHttpMethodsExcept(GET) foreach { method => 44 | s"should reject ${method.toString} method" in 45 | new RequestBuilder(method)(statusPath) ~> statusRoutes ~> check { 46 | assert(!handled) 47 | } 48 | } 49 | "should return OK for an unauthenticated GET" in 50 | Get(statusPath) ~> statusRoutes ~> check { 51 | assert(status == OK) 52 | } 53 | "should deserialize to a StatusCheckResponse" in 54 | Get(statusPath) ~> statusRoutes ~> check { 55 | responseAs[StatusCheckResponse] 56 | } 57 | "should contain all the subsystems we care about" in 58 | Get(statusPath) ~> statusRoutes ~> check { 59 | val statusCheckResponse = responseAs[StatusCheckResponse] 60 | // changing the values of expectedSystems may affect the orch liveness probe 61 | // https://github.com/broadinstitute/terra-helmfile/blob/master/charts/firecloudorch/templates/probe/configmap.yaml 62 | val expectedSystems = Set(Agora, GoogleBuckets, Rawls, Sam, Thurloe) 63 | assertResult(expectedSystems)(statusCheckResponse.systems.keySet) 64 | } 65 | } 66 | 67 | } 68 | --------------------------------------------------------------------------------