├── .codecov.yml ├── .codespellrc ├── .eslintrc.cjs ├── .flake8 ├── .github └── workflows │ ├── codespell.yml │ ├── linter_checks.yml │ ├── pypi_release.yaml │ ├── tests.yml │ └── vercel_deploy.yaml ├── .gitignore ├── .vercelignore ├── LICENSE ├── README.md ├── _vercel_dev.json ├── _vercel_prod.json ├── api └── index.py ├── devel └── troubleshoot_remote_read_speed.py ├── doc ├── about_the_api.md ├── compute_resource_prerequisites.md ├── create_dendro_app.md ├── for_developers.md ├── host_compute_resource.md ├── iac_aws_batch.md └── letter_count │ ├── .gitignore │ ├── README.md │ ├── input.txt │ ├── main.py │ ├── output.json │ ├── sample_context_1.yaml │ ├── sample_context_2.yaml │ ├── sample_input.txt │ ├── sample_output_1.json │ ├── sample_output_2.json │ └── spec.json ├── iac └── aws_batch │ ├── .gitignore │ ├── README.md │ ├── __init__.py │ ├── app.py │ ├── aws_batch │ ├── __init__.py │ ├── aws_batch_stack.py │ └── stack_config.py │ ├── cdk.json │ ├── devel │ └── create_ami_map.py │ ├── requirements-dev.txt │ ├── requirements.txt │ ├── source.bat │ └── tests │ ├── __init__.py │ └── unit │ ├── __init__.py │ └── test_aws_batch_stack.py ├── index.html ├── package.json ├── public ├── dendro.png ├── help │ ├── common.md │ ├── dandiset.md │ ├── dandisets.md │ ├── home.md │ ├── project-dandi-import.md │ ├── project-processors.md │ ├── project-project-files.md │ ├── project-project-home.md │ ├── project-project-jobs.md │ └── project-project-linked-analysis.md └── scripts │ ├── load_electrical_series.py │ └── load_spike_sorting.py ├── python ├── .gitignore ├── README.md ├── dendro │ ├── __init__.py │ ├── api_helpers │ │ ├── __init__.py │ │ ├── clients │ │ │ ├── MockMongoClient.py │ │ │ ├── __init__.py │ │ │ ├── _get_mongo_client.py │ │ │ ├── _remove_id_field.py │ │ │ ├── db.py │ │ │ └── pubsub.py │ │ ├── core │ │ │ ├── __init__.py │ │ │ ├── _create_random_id.py │ │ │ ├── _get_project_role.py │ │ │ ├── _hide_secret_params_in_job.py │ │ │ ├── _model_dump.py │ │ │ └── settings.py │ │ ├── routers │ │ │ ├── __init__.py │ │ │ ├── client │ │ │ │ ├── __init__.py │ │ │ │ └── router.py │ │ │ ├── common.py │ │ │ ├── compute_resource │ │ │ │ ├── __init__.py │ │ │ │ └── router.py │ │ │ ├── gui │ │ │ │ ├── __init__.py │ │ │ │ ├── _authenticate_gui_request.py │ │ │ │ ├── compute_resource_routes.py │ │ │ │ ├── create_job_route.py │ │ │ │ ├── file_routes.py │ │ │ │ ├── find_routes.py │ │ │ │ ├── github_auth_routes.py │ │ │ │ ├── job_routes.py │ │ │ │ ├── project_routes.py │ │ │ │ ├── router.py │ │ │ │ ├── script_routes.py │ │ │ │ ├── usage_routes.py │ │ │ │ └── user_routes.py │ │ │ └── processor │ │ │ │ ├── __init__.py │ │ │ │ └── router.py │ │ └── services │ │ │ ├── __init__.py │ │ │ ├── _create_output_file.py │ │ │ ├── _crypto_keys.py │ │ │ ├── _remove_detached_files_and_jobs.py │ │ │ ├── gui │ │ │ ├── __init__.py │ │ │ ├── create_job.py │ │ │ ├── delete_project.py │ │ │ ├── get_compute_resource_user_usage.py │ │ │ └── set_file.py │ │ │ └── processor │ │ │ ├── __init__.py │ │ │ ├── _get_fsbucket_signed_upload_url.py │ │ │ ├── _get_signed_upload_url.py │ │ │ ├── get_upload_url.py │ │ │ └── update_job_status.py │ ├── aws_batch │ │ ├── __init__.py │ │ └── aws_batch_job_definition.py │ ├── cli.py │ ├── client │ │ ├── Project.py │ │ ├── __init__.py │ │ ├── _create_batch_id.py │ │ ├── _interim │ │ │ ├── NwbRecording.py │ │ │ ├── NwbSorting.py │ │ │ └── __init__.py │ │ ├── _upload_blob.py │ │ ├── set_file.py │ │ └── submit_job.py │ ├── common │ │ ├── __init__.py │ │ ├── _api_request.py │ │ ├── _crypto_keys.py │ │ └── dendro_types.py │ ├── compute_resource │ │ ├── AppManager.py │ │ ├── ComputeResourceException.py │ │ ├── JobManager.py │ │ ├── PubsubClient.py │ │ ├── SlurmJobHandler.py │ │ ├── __init__.py │ │ ├── _run_job_in_aws_batch.py │ │ ├── _start_job.py │ │ ├── register_compute_resource.py │ │ └── start_compute_resource.py │ ├── internal_job_monitoring │ │ ├── __init__.py │ │ ├── common.py │ │ ├── console_output_monitor.py │ │ ├── job_status_monitor.py │ │ └── resource_utilization_monitor.py │ ├── mock.py │ ├── sdk │ │ ├── App.py │ │ ├── AppProcessor.py │ │ ├── FileManifest.py │ │ ├── InputFile.py │ │ ├── InputFolder.py │ │ ├── Job.py │ │ ├── OutputFile.py │ │ ├── OutputFolder.py │ │ ├── ProcessorBase.py │ │ ├── __init__.py │ │ ├── _load_spec_from_uri.py │ │ ├── _make_spec_file.py │ │ ├── _run_job_child_process.py │ │ ├── _run_job_parent_process.py │ │ ├── _test_app_processor.py │ │ └── get_project_file_from_uri.py │ └── version.txt ├── mock-output-file.txt ├── pytest.ini ├── setup.py └── tests │ ├── mock_app │ └── main.py │ ├── mock_app_2 │ └── main.py │ ├── test_api_request_failures.py │ ├── test_client.py │ ├── test_compute_resource.py │ ├── test_crypto_keys.py │ ├── test_github_auth_route.py │ ├── test_integration.py │ ├── test_misc.py │ └── test_sdk.py ├── requirements.txt ├── src ├── ApiKeysWindow │ └── ApiKeysWindow.tsx ├── App.css ├── App.tsx ├── ApplicationBar.tsx ├── ComputeResourceNameDisplay.tsx ├── DendroContext │ └── DendroContext.tsx ├── GitHub │ ├── GitHubAuthPage.tsx │ ├── GitHubLoginWindow.tsx │ ├── GitHubLoginWindowOld.tsx │ └── PersonalAccessTokenWindow.tsx ├── GithubAuth │ ├── GithubAuthContext.ts │ ├── GithubAuthSetup.tsx │ ├── getGithubAuthFromLocalStorage.ts │ ├── useGithubAuth.ts │ └── useSetupGithubAuth.ts ├── HelpPanel │ └── HelpPanel.tsx ├── MainWindow.tsx ├── Markdown │ └── Markdown.tsx ├── RecentProjectsPanel │ └── RecentProjectsPanel.tsx ├── RemoteH5File │ ├── RemoteH5File.ts │ ├── RemoteH5Worker.js │ ├── h5wasm │ │ ├── file_handlers.js │ │ ├── h5wasm_license.txt │ │ ├── hdf5_hl.js │ │ ├── hdf5_util_jfm.js │ │ ├── readme.txt │ │ └── wasmBinaryFile.js │ └── helpers.ts ├── TabWidget │ ├── TabWidget.tsx │ └── TabWidgetTabBar.tsx ├── UserIdComponent.tsx ├── confirm_prompt_alert.ts ├── dbInterface │ ├── dbInterface.ts │ └── getAuthorizationHeaderForUrl.ts ├── index.css ├── main.tsx ├── nh5.ts ├── pages │ ├── AboutPage │ │ └── AboutPage.tsx │ ├── AdminPage │ │ └── AdminPage.tsx │ ├── ComputeResourcePage │ │ ├── ComputeResourceAppsTable.tsx │ │ ├── ComputeResourceAppsTableMenuBar.tsx │ │ ├── ComputeResourcePage.tsx │ │ └── NewAppWindow.tsx │ ├── ComputeResourcesPage │ │ ├── ComputeResourcesContext.tsx │ │ ├── ComputeResourcesPage.css │ │ ├── ComputeResourcesPage.tsx │ │ └── ComputeResourcesTable.tsx │ ├── DandiBrowser │ │ ├── DandiBrowser.tsx │ │ ├── DandisetView.tsx │ │ ├── SearchResults.tsx │ │ ├── formatByteCount.ts │ │ ├── types.ts │ │ └── useProjectsForTag.ts │ ├── HomePage │ │ ├── HomePage.css │ │ └── HomePage.tsx │ ├── ImportDandiAssetPage │ │ └── ImportDandiAssetPage.tsx │ ├── ProjectPage │ │ ├── ComputeResourceAppsTable2 │ │ │ ├── ComputeResourceAppsTable2.tsx │ │ │ └── ComputeResourceAppsTableMenuBar2.tsx │ │ ├── ComputeResourceSection.tsx │ │ ├── ComputeResourceUsageComponent │ │ │ └── ComputeResourceUsageComponent.tsx │ │ ├── DandiUpload │ │ │ ├── DandiUploadWindow.tsx │ │ │ └── prepareDandiUploadTask.ts │ │ ├── EditJobDefinitionWindow │ │ │ └── EditJobDefinitionWindow.tsx │ │ ├── FileActions.tsx │ │ ├── FileBrowser │ │ │ ├── DropdownMenu.css │ │ │ ├── DropdownMenu.tsx │ │ │ ├── FileBrowser2.tsx │ │ │ ├── FileBrowserMenuBar.tsx │ │ │ ├── FileBrowserTable.tsx │ │ │ ├── file-browser-table.css │ │ │ └── formatByteCount.ts │ │ ├── FileView │ │ │ ├── ElectricalSeriesSection │ │ │ │ ├── ElectricalSeriesSection.tsx │ │ │ │ └── LoadElectricalSeriesScriptWindow.tsx │ │ │ ├── FigurlFileView.tsx │ │ │ ├── FileView.tsx │ │ │ ├── FileViewTable.tsx │ │ │ ├── FolderFileView.tsx │ │ │ ├── Nh5FileView.tsx │ │ │ ├── NwbFileView.tsx │ │ │ ├── OtherFileView.tsx │ │ │ └── SpikeSortingOutputSection │ │ │ │ ├── LoadSpikeSortingInScriptWindow.tsx │ │ │ │ └── SpikeSortingOutputSection.tsx │ │ ├── GenerateSpikeSortingSummaryWindow │ │ │ └── GenerateSpikeSortingSummaryWindow.tsx │ │ ├── JobView │ │ │ ├── JobView.tsx │ │ │ └── OutputsTable.tsx │ │ ├── JobsWindow │ │ │ ├── JobsTable.tsx │ │ │ ├── JobsTableMenuBar.tsx │ │ │ └── JobsWindow.tsx │ │ ├── LoadNwbInPythonWindow │ │ │ └── LoadNwbInPythonWindow.tsx │ │ ├── MearecGenerateTemplatesWindow │ │ │ └── MearecGenerateTemplatesWindow.tsx │ │ ├── Processor.css │ │ ├── ProcessorsView.tsx │ │ ├── ProjectAnalysis │ │ │ ├── AnalysisSourceClient.ts │ │ │ ├── AnalysisSourceFileBrowser.tsx │ │ │ ├── AnalysisSourceFileView.css │ │ │ ├── AnalysisSourceFileView.tsx │ │ │ ├── ClonedRepo.ts │ │ │ ├── ProjectAnalysis.tsx │ │ │ └── fs.ts │ │ ├── ProjectFiles.tsx │ │ ├── ProjectHome.tsx │ │ ├── ProjectJobs.tsx │ │ ├── ProjectPage.tsx │ │ ├── ProjectPageContext.tsx │ │ ├── ProjectScripts.tsx │ │ ├── ProjectSettingsWindow.tsx │ │ ├── ResourceUtilizationView │ │ │ ├── LogPlot.tsx │ │ │ └── ResourceUtilizationView.tsx │ │ ├── RunBatchSpikeSortingWindow │ │ │ ├── ElectrodeGeometryView.tsx │ │ │ ├── ElectrodeGeometryWidget.tsx │ │ │ ├── LeftColumn.tsx │ │ │ ├── RequiredResourcesEditor.tsx │ │ │ ├── RightColumn.tsx │ │ │ ├── RunBatchSpikeSortingWindow.tsx │ │ │ ├── SelectProcessorComponent.tsx │ │ │ └── getDefaultRequiredResources.ts │ │ ├── RunFileActionWindow │ │ │ └── RunFileActionWindow.tsx │ │ ├── UploadSmalFileWindow │ │ │ └── UploadSmallFileWindow.tsx │ │ ├── openFilesInNeurosift.ts │ │ └── scripts │ │ │ ├── CodeEditor.tsx │ │ │ ├── RunScript │ │ │ ├── RunScriptWindow.tsx │ │ │ ├── RunScriptWorkerTypes.ts │ │ │ └── runScriptWorker.ts │ │ │ ├── ScriptView.tsx │ │ │ ├── ScriptsTable.tsx │ │ │ └── ScriptsTableMenuBar.tsx │ ├── ProjectsPage │ │ ├── ProjectsPage.css │ │ ├── ProjectsPage.tsx │ │ ├── ProjectsTable.tsx │ │ └── useProjectsForUser.ts │ └── RegisterComputeResourcePage │ │ └── RegisterComputeResourcePage.tsx ├── plugins │ ├── DendroFrontendPlugin.ts │ ├── initializePlugins.ts │ └── mearec │ │ └── mearecPlugin.ts ├── pubnub │ └── pubnub.ts ├── scientific-table.css ├── table1.css ├── timeStrings.ts ├── types │ ├── dendro-types.ts │ └── validateObject.ts ├── useRoute.ts ├── useWindowDimensions.ts └── vite-env.d.ts ├── tsconfig.json ├── tsconfig.node.json ├── vercel.json ├── vercel.json.readme.md ├── vite.config.ts └── yarn.lock /.codecov.yml: -------------------------------------------------------------------------------- 1 | coverage: 2 | status: 3 | project: 4 | default: 5 | informational: true 6 | patch: 7 | default: 8 | informational: true -------------------------------------------------------------------------------- /.codespellrc: -------------------------------------------------------------------------------- 1 | [codespell] 2 | skip = .git,*.pdf,*.svg,*.lock,*.css,.codespellrc,input.txt 3 | check-hidden = true 4 | # ignore-regex = 5 | ignore-words-list = te,nd 6 | -------------------------------------------------------------------------------- /.eslintrc.cjs: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | "env": { 3 | "browser": true, 4 | "es2021": true 5 | }, 6 | "extends": [ 7 | "eslint:recommended", 8 | "plugin:react/recommended", 9 | "plugin:@typescript-eslint/recommended", 10 | "plugin:react-hooks/recommended" 11 | ], 12 | "overrides": [ 13 | ], 14 | "parser": "@typescript-eslint/parser", 15 | "parserOptions": { 16 | "ecmaVersion": "latest", 17 | "sourceType": "module" 18 | }, 19 | "plugins": [ 20 | "react", 21 | "@typescript-eslint", 22 | "react-hooks" 23 | ], 24 | "rules": { 25 | "react/react-in-jsx-scope": "off", 26 | "@typescript-eslint/no-explicit-any": "off", 27 | "@typescript-eslint/no-empty-function": "off" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | extend-ignore = E124,E128,E301,E302,E305,E402,E501,E261,W504 3 | # E124: closing bracket does not match visual indentation 4 | # E128: continuation line under-indented for visual indent 5 | # E301: expected 1 blank line, found 0 6 | # E302: expected 2 blank lines, found 1 7 | # E305: expected 2 blank lines after class or function definition, found 1 8 | # E402: module level import not at top of file 9 | # E501: line too long (82 > 79 characters) 10 | # E261: at least two spaces before inline comment 11 | # W504: line break after binary operator -------------------------------------------------------------------------------- /.github/workflows/codespell.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: Codespell 3 | 4 | on: 5 | push: 6 | pull_request: 7 | 8 | permissions: 9 | contents: read 10 | 11 | jobs: 12 | codespell: 13 | name: Check for spelling errors 14 | runs-on: ubuntu-latest 15 | 16 | steps: 17 | - name: Checkout 18 | uses: actions/checkout@v3 19 | - name: Codespell 20 | uses: codespell-project/actions-codespell@v2 21 | -------------------------------------------------------------------------------- /.github/workflows/linter_checks.yml: -------------------------------------------------------------------------------- 1 | name: testing 2 | 3 | on: 4 | push: 5 | paths: 6 | - 'python/**' 7 | pull_request: 8 | paths: 9 | - 'python/**' 10 | workflow_dispatch: 11 | 12 | jobs: 13 | build: 14 | runs-on: ubuntu-latest 15 | name: Linter checks 16 | steps: 17 | - uses: actions/checkout@v1 18 | - name: Install 19 | run: cd python && pip install -e .[compute_resource] 20 | - name: Install packages needed for tests 21 | run: pip install pytest pytest-asyncio pytest-cov pyright boto3 kachery_cloud flake8 22 | - name: Install additional packages used by api_helpers 23 | run: cd python && pip install -e .[api] 24 | - name: Install additional optional packages 25 | run: pip install GPUtil 26 | # - name: Install packages needed by iac/aws_batch 27 | # run: cd iac/aws_batch && pip install -r requirements.txt 28 | - name: Run linter checks 29 | run: cd python && flake8 --config ../.flake8 && pyright 30 | # - name: Run linter checks for iac 31 | # run: cd iac && flake8 --config ../.flake8 && pyright 32 | -------------------------------------------------------------------------------- /.github/workflows/pypi_release.yaml: -------------------------------------------------------------------------------- 1 | name: PyPI release 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | paths: 8 | # only run on changes to python/dendro/version.txt 9 | - python/dendro/version.txt 10 | # manual trigger 11 | workflow_dispatch: 12 | 13 | jobs: 14 | pypi-release: 15 | name: PyPI release 16 | runs-on: ubuntu-latest 17 | steps: 18 | - 19 | uses: actions/checkout@v3 20 | - 21 | name: Set up Python 22 | uses: actions/setup-python@v3 23 | with: 24 | python-version: "3.10" 25 | - 26 | name: Install 27 | run: cd python && pip install -e .[compute_resource] 28 | - 29 | name: Install packages needed for tests 30 | run: pip install pytest pytest-asyncio pytest-cov boto3 kachery_cloud 31 | - 32 | name: Run non-api tests 33 | run: cd python && pytest -m "not api" tests/ # make sure we are not depending on any of the additional packages in requirements.txt 34 | - 35 | name: Install packages needed for api tests 36 | run: cd python && pip install -e .[api] 37 | - 38 | name: Install other packages needed for api tests 39 | run: pip install httpx 40 | - 41 | name: Run tests and collect coverage 42 | run: cd python && pytest tests/ 43 | - 44 | name: Install dependencies 45 | run: | 46 | python -m pip install --upgrade pip 47 | pip install build 48 | pip install twine 49 | - 50 | name: Build and publish to PyPI 51 | run: | 52 | cd python 53 | python -m build 54 | twine upload dist/* 55 | env: 56 | TWINE_USERNAME: __token__ 57 | TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} 58 | - 59 | name: Tag the release using version from python/dendro/version.txt 60 | run: | 61 | git config --global user.email "jmagland@flatironinstitute.org" 62 | git config --global user.name "Jeremy Magland" 63 | git tag -a v$(cat python/dendro/version.txt | tr -d '[:space:]') -m "v$(cat python/dendro/version.txt | tr -d '[:space:]')" 64 | git push origin --tags 65 | env: 66 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 67 | 68 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | paths: 6 | - 'python/**' 7 | pull_request: 8 | paths: 9 | - 'python/**' 10 | workflow_dispatch: 11 | 12 | jobs: 13 | build: 14 | runs-on: ubuntu-latest 15 | name: Tests 16 | steps: 17 | - uses: actions/checkout@v1 18 | 19 | # Install dendro and packages needed for tests 20 | - name: Install 21 | run: cd python && pip install -e .[compute_resource] 22 | - name: Install packages needed for tests 23 | run: pip install pytest pytest-asyncio pytest-cov boto3 kachery_cloud 24 | 25 | # Run non-api tests 26 | - name: Run non-api tests 27 | run: cd python && pytest -m "not api" tests/ # make sure we are not depending on any of the additional packages in requirements.txt 28 | - name: Install packages needed for api tests 29 | 30 | # Install packages needed for api tests 31 | run: cd python && pip install -e .[api] 32 | - name: Install other packages needed for api tests 33 | run: pip install httpx 34 | 35 | # Run full tests and collect coverage 36 | - name: Run tests and collect coverage 37 | run: cd python && pytest --cov dendro --cov-report=xml --cov-report=term tests/ 38 | 39 | # Try with pydantic v1 (no coverage this time) 40 | - name: Try with pydantic v1 41 | run: pip install pydantic==1.9.2 # support versions >= 1.9.2 42 | - name: Run tests 43 | run: cd python && pytest tests/ 44 | 45 | - uses: codecov/codecov-action@v3 46 | with: 47 | token: ${{ secrets.CODECOV_TOKEN }} 48 | fail_ci_if_error: false 49 | file: ./python/coverage.xml 50 | flags: unittests -------------------------------------------------------------------------------- /.github/workflows/vercel_deploy.yaml: -------------------------------------------------------------------------------- 1 | name: Vercel deploy 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | paths: 8 | # only run workflow when changes are made to these files 9 | - package.json 10 | # manual trigger 11 | workflow_dispatch: 12 | 13 | jobs: 14 | vercel-deploy: 15 | name: Vercel deploy 16 | runs-on: ubuntu-latest 17 | steps: 18 | - uses: actions/checkout@v2 19 | - 20 | name: Install Vercel CLI 21 | run: npm install --global vercel@latest 22 | - 23 | name: Set vercel.json for production 24 | run: cp _vercel_prod.json vercel.json 25 | - 26 | # Not sure why this is needed, but it is 27 | name: Create .vercel/project.json 28 | run: | 29 | mkdir -p .vercel 30 | echo '{"projectId":"${{ secrets.VERCEL_PROJECT_ID }}", "orgId":"${{ secrets.VERCEL_ORG_ID }}"}' > .vercel/project.json 31 | - 32 | name: Pull Vercel Environment Information 33 | run: vercel pull --yes --environment=production --token=${{ secrets.VERCEL_TOKEN }} 34 | - 35 | name: Build Project Artifacts 36 | run: vercel --prod --token=${{ secrets.VERCEL_TOKEN }} 37 | 38 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | slurm_batch_* 2 | slurm_group_assignments 3 | tmp 4 | __pycache__ 5 | 6 | # Logs 7 | logs 8 | *.log 9 | npm-debug.log* 10 | yarn-debug.log* 11 | yarn-error.log* 12 | pnpm-debug.log* 13 | lerna-debug.log* 14 | 15 | node_modules 16 | dist 17 | dist-ssr 18 | *.local 19 | 20 | # Editor directories and files 21 | .vscode/* 22 | !.vscode/extensions.json 23 | .idea 24 | .DS_Store 25 | *.suo 26 | *.ntvs* 27 | *.njsproj 28 | *.sln 29 | *.sw? 30 | .vercel 31 | -------------------------------------------------------------------------------- /.vercelignore: -------------------------------------------------------------------------------- 1 | __pycache__ -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![PyPI version](https://badge.fury.io/py/dendro.svg)](https://badge.fury.io/py/dendro) 2 | [![testing](https://github.com/flatironinstitute/dendro/actions/workflows/tests.yml/badge.svg)](https://github.com/flatironinstitute/dendro/actions/workflows/tests.yml) 3 | [![codecov](https://codecov.io/gh/flatironinstitute/dendro/graph/badge.svg?token=B2DUYR34RZ)](https://codecov.io/gh/flatironinstitute/dendro) 4 | 5 | NOTE: [Here is the more recent version of Dendro](https://github.com/magland/dendro) 6 | 7 | # BELOW is OLD/OBSOLETE information 8 | 9 | ## Getting started 10 | 11 | [View the documentation](https://flatironinstitute.github.io/dendro-docs) 12 | 13 | [Visit the live site](https://dendro.vercel.app/) 14 | 15 | ### LICENSE 16 | 17 | Apache 2.0 18 | 19 | ## Authors 20 | 21 | Jeremy Magland (Flatiron Institute) and Luiz Tauffer (CatalystNeuro) 22 | 23 | In collaboration with Ben Dichter (CatalystNeuro) 24 | 25 | -------------------------------------------------------------------------------- /_vercel_dev.json: -------------------------------------------------------------------------------- 1 | { 2 | "rewrites": [ 3 | { 4 | "source": "/api/(.*)", 5 | "destination": "/api/index.py" 6 | }, 7 | { 8 | "source": "/docs", 9 | "destination": "/api/index.py" 10 | }, 11 | { 12 | "source": "/redoc", 13 | "destination": "/api/index.py" 14 | }, 15 | { 16 | "source": "/openapi.json", 17 | "destination": "/api/index.py" 18 | } 19 | ] 20 | } -------------------------------------------------------------------------------- /_vercel_prod.json: -------------------------------------------------------------------------------- 1 | { 2 | "rewrites": [ 3 | { 4 | "source": "/api/(.*)", 5 | "destination": "/api/index.py" 6 | }, 7 | { 8 | "source": "/docs", 9 | "destination": "/api/index.py" 10 | }, 11 | { 12 | "source": "/redoc", 13 | "destination": "/api/index.py" 14 | }, 15 | { 16 | "source": "/openapi.json", 17 | "destination": "/api/index.py" 18 | }, 19 | { 20 | "source": "/(.*)", 21 | "destination": "/index.html" 22 | } 23 | ] 24 | } -------------------------------------------------------------------------------- /api/index.py: -------------------------------------------------------------------------------- 1 | from fastapi import FastAPI 2 | 3 | # Here's the reason that all the other Python files are in ../python/dendro/api_helpers 4 | # I was noticing very long build times (~15 minutes)... 5 | # Apparently, vercel treats every .py file in /api as a lambda function. 6 | # So it was building each and every one of them, even though index.py should be the only one. 7 | # See https://github.com/orgs/vercel/discussions/46 8 | 9 | import os 10 | thisdir = os.path.dirname(os.path.realpath(__file__)) 11 | 12 | import sys 13 | print(f'This dir: {thisdir}') 14 | sys.path.append(thisdir + "/../python") 15 | from dendro.api_helpers.routers.processor.router import router as processor_router 16 | from dendro.api_helpers.routers.compute_resource.router import router as compute_resource_router 17 | from dendro.api_helpers.routers.client.router import router as client_router 18 | from dendro.api_helpers.routers.gui.router import router as gui_router 19 | 20 | from fastapi.middleware.cors import CORSMiddleware 21 | 22 | 23 | app = FastAPI() 24 | 25 | # Set up CORS 26 | origins = [ 27 | "http://localhost:3000", 28 | "http://localhost:3001", 29 | "http://localhost:5173", 30 | "http://localhost:4200", 31 | "https://dendro.vercel.app", 32 | "https://flatironinstitute.github.io", 33 | "https://neurosift.app", 34 | "https://dendro-arc.vercel.app" 35 | ] 36 | 37 | app.add_middleware( 38 | CORSMiddleware, 39 | allow_origins=origins, 40 | allow_credentials=True, 41 | allow_methods=["*"], 42 | allow_headers=["*"], 43 | ) 44 | 45 | # requests from a processing job 46 | app.include_router(processor_router, prefix="/api/processor", tags=["Processor"]) 47 | 48 | # requests from a compute resource 49 | app.include_router(compute_resource_router, prefix="/api/compute_resource", tags=["Compute Resource"]) 50 | 51 | # requests from a client (usually Python) 52 | app.include_router(client_router, prefix="/api/client", tags=["Client"]) 53 | 54 | # requests from the GUI 55 | app.include_router(gui_router, prefix="/api/gui", tags=["GUI"]) 56 | -------------------------------------------------------------------------------- /devel/troubleshoot_remote_read_speed.py: -------------------------------------------------------------------------------- 1 | import time 2 | import h5py 3 | import remfile 4 | 5 | url = 'https://dandiarchive-embargo.s3.amazonaws.com/000620/blobs/8ce/bee/8cebeede-f6e8-4bd5-a307-ed3c852269bb?response-content-disposition=attachment%3B%20filename%3D%22sub-Elgar_ecephys.nwb%22&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAUBRWC5GAEKH3223E%2F20231101%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20231101T153921Z&X-Amz-Expires=3600&X-Amz-SignedHeaders=host&X-Amz-Signature=0f41fc34bb4c1759489fca53c439359bfb450efac1f1780672c480a1d127569c' 6 | 7 | # open the remote file 8 | f = h5py.File(remfile.File(url, verbose=True), 'r') 9 | 10 | # load the neurodata object 11 | X = f['/acquisition/ElectricalSeriesAP'] 12 | 13 | starting_time = X['starting_time'][()] 14 | rate = X['starting_time'].attrs['rate'] 15 | data = X['data'] 16 | 17 | print(f'starting_time: {starting_time}') 18 | print(f'rate: {rate}') 19 | print(f'data shape: {data.shape}') 20 | 21 | timer = time.time() 22 | x = data[:30000 * 60,0:10] 23 | elapsed = time.time() - timer 24 | print(f'elapsed (sec): {elapsed}') 25 | 26 | print(x.shape) -------------------------------------------------------------------------------- /doc/about_the_api.md: -------------------------------------------------------------------------------- 1 | # About the Dendro API 2 | 3 | The Dendro API is implemented using Python with [FastAPI](https://fastapi.tiangolo.com/), deployed as serverless functions on [Vercel](https://vercel.com/about). It is divided into four sections: GUI, Compute Resource, Processor, and Client. 4 | 5 | Here is the [ReDoc-generated documentation](https://dendro.vercel.app/redoc) and the [Swagger UI-generated documentation](https://dendro.vercel.app/docs). 6 | 7 | Here is the source code: [index.py](../api/index.py) and [api_helpers](../python/dendro/api_helpers). 8 | 9 | 10 | ## GUI 11 | 12 | The GUI API receives requests from the [web interface](https://dendro.vercel.app). Operations include 13 | 14 | * Getting and creating projects 15 | * Getting and creating project files 16 | * Getting and creating project jobs 17 | * Registering and managing compute resources, including configuring apps. 18 | * Getting pub/sub information 19 | * Authenticating via GitHub 20 | 21 | All write operations and some read operations require authentication via GitHub. 22 | 23 | ## Compute Resource 24 | 25 | The compute resource API receives requests from compute resources. Operations include 26 | 27 | * Getting the list of apps configured on the web interface 28 | * Getting pub/sub information 29 | * Getting a list of unfinished jobs for the compute resource, including their private keys 30 | * Setting the compute resource spec (on startup of the compute resource) 31 | 32 | All operations require authentication via signatures created using the secret compute resource private key, which is associated with the public compute resource ID. 33 | 34 | ## Processor 35 | 36 | The processor API receives requests from processing jobs. Operations include 37 | 38 | * Getting detailed information about a processing job 39 | * Getting the status of a processing job (to see if it has been canceled) 40 | * Setting the status of a processing job (running, completed, failed, ...) 41 | * Getting presigned upload URLs for uploading outputs and console logs 42 | 43 | All operations are authenticated using the job private key 44 | 45 | ## Client 46 | 47 | The client API receives requests from Python clients. Operations include 48 | 49 | * Getting the files and jobs associated with a project 50 | 51 | At this time, no authentication is required. 52 | -------------------------------------------------------------------------------- /doc/compute_resource_prerequisites.md: -------------------------------------------------------------------------------- 1 | # Dendro compute resource prerequisites 2 | 3 | Suppose you have a Linux machine and would like to use it as a dendro compute resource. Here are the recommended steps to prepare it with the necessary software. 4 | 5 | ## Use Linux 6 | 7 | Any distribution should do. 8 | 9 | ## Conda / Miniconda 10 | 11 | I recommend using Miniconda, but you can use other conda solutions, virtualenv, etc. 12 | 13 | Follow these instructions: https://docs.conda.io/projects/miniconda/en/latest/miniconda-install.html 14 | 15 | This will involve downloading the Linux installation script file (usually the "Miniconda3 Linux 64-bit" option) and running it using bash. 16 | 17 | Create a new conda environment. I'd recommend using Python 3.9, but you can more recent versions as well. 18 | 19 | ```bash 20 | conda create -n processing python=3.9 21 | ``` 22 | 23 | You can replace "processing" with any name you like. To use this environment run the following each time you open your terminal 24 | 25 | ```bash 26 | conda activate processing 27 | ``` 28 | 29 | or add this command to your ~/.bashrc file to automatically start in this environment each time you open a new terminal. 30 | 31 | ## Install docker or apptainer 32 | 33 | To use your computer as a dendro compute resource, you'll most likely need to install either docker or apptainer (or singularity). Docker is simpler to install, whereas apptainer is better for shared environments or compute clusters. 34 | 35 | To install docker server (or docker engine): https://docs.docker.com/engine/install/ 36 | 37 | To get started with apptainer: https://apptainer.org/ -------------------------------------------------------------------------------- /doc/host_compute_resource.md: -------------------------------------------------------------------------------- 1 | # Hosting a Dendro compute resource 2 | 3 | Each Dendro project comes equipped with a dedicated compute resource for executing analysis jobs. The default setting uses a compute resource provided by the authors with limitations on CPU, memory, and concurrent jobs, shared among all users. This public resource should only be used for testing with small jobs. Contact one of the authors if you would like to run more intensive processing or configure your own compute resources. 4 | 5 | Prerequisites 6 | 7 | * Python >= 3.9 8 | * Docker or apptainer (or singularity >= 3.11) 9 | 10 | Clone this repo, then 11 | 12 | ```bash 13 | # install 14 | cd dendro/python 15 | pip install -e .[compute_resource] 16 | ``` 17 | 18 | ```bash 19 | # Initialize (one time) 20 | export COMPUTE_RESOURCE_DIR=/some/path 21 | export CONTAINER_METHOD=apptainer # or docker (or singularity) 22 | cd $COMPUTE_RESOURCE_DIR 23 | dendro register-compute-resource 24 | # Open the provided link in a browser and log in using GitHub 25 | ``` 26 | 27 | ```bash 28 | # Start the compute resource 29 | cd $COMPUTE_RESOURCE_DIR 30 | dendro start-compute-resource 31 | # Leave this open in a terminal. It is recommended that you use a terminal multiplexer like tmux or screen. 32 | ``` 33 | 34 | In the web interface, go to settings for your project, and select your compute resource. New jobs submitted within your project will now use your compute resource for analysis jobs. 35 | 36 | ## Configuring apps for your compute resource 37 | 38 | In order to run jobs with your compute resource, you will need to configure apps to use it. 39 | 40 | In the web interface, click on the appropriate link to manage your compute resource. You will then be able to add apps to your compute resource by entering the information (see below for available apps). 41 | 42 | :warning: After you make changes to your compute resource on the web interface, reload the page so that your changes will take effect. 43 | 44 | The following are available apps that you can configure 45 | 46 | | App name | Spec URI | 47 | | -------- | --------------- | 48 | | mountainsort5 | https://github.com/scratchrealm/pc-spike-sorting/blob/main/mountainsort5/spec.json | 49 | | kilosort3 | https://github.com/scratchrealm/pc-spike-sorting/blob/main/kilosort3/spec.json | 50 | | kilosort2_5 | https://github.com/scratchrealm/pc-spike-sorting/blob/main/kilosort2_5/spec.json | 51 | | spike-sorting_utils | https://github.com/scratchrealm/pc-spike-sorting/blob/main/spike_sorting_utils/spec.json | 52 | | dandi-upload | https://github.com/scratchrealm/pc-spike-sorting/blob/main/dandi_upload/spec.json | 53 | 54 | ## Submitting jobs to AWS Batch 55 | 56 | [See iac_aws_batch](./iac_aws_batch.md) -------------------------------------------------------------------------------- /doc/letter_count/.gitignore: -------------------------------------------------------------------------------- 1 | input.txt 2 | output.json -------------------------------------------------------------------------------- /doc/letter_count/README.md: -------------------------------------------------------------------------------- 1 | # Dendro app: letter_count 2 | 3 | This is an example Dendro processing app for the tutorial. -------------------------------------------------------------------------------- /doc/letter_count/main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import os 4 | import json 5 | from dendro.sdk import App, BaseModel, Field, ProcessorBase, InputFile, OutputFile 6 | 7 | 8 | app = App( 9 | 'letter_count', 10 | description="Example Dendro processing app for tutorial", 11 | app_image="", 12 | app_executable=os.path.abspath(__file__) 13 | ) 14 | 15 | description = """ 16 | This is the a processor in the letter_count app. It counts the number of times a particular letter appears in a text file and produces and JSON file with the result. 17 | """ 18 | 19 | class LetterCountProcessorContext(BaseModel): 20 | input: InputFile = Field(description='Input text file') 21 | output: OutputFile = Field(description='Output JSON file') 22 | letter: str = Field(description='Letter to count') 23 | 24 | class LetterCountProcessor(ProcessorBase): 25 | name = 'letter_count' 26 | label = 'Letter Count' 27 | description = description 28 | tags = ['tutorial'] 29 | attributes = {'wip': True} 30 | 31 | @staticmethod 32 | def run(context: LetterCountProcessorContext): 33 | input_fname = 'input.txt' 34 | output_fname = 'output.json' 35 | context.input.download(input_fname) 36 | letter = context.letter 37 | 38 | with open(input_fname, 'r') as f: 39 | text = f.read() 40 | count = text.count(letter) 41 | 42 | output = { 43 | 'count': count 44 | } 45 | with open(output_fname, 'w') as f: 46 | f.write(json.dumps(output)) 47 | context.output.upload(output_fname) 48 | 49 | app.add_processor(LetterCountProcessor) 50 | 51 | if __name__ == '__main__': 52 | app.run() 53 | -------------------------------------------------------------------------------- /doc/letter_count/output.json: -------------------------------------------------------------------------------- 1 | {"count": 110} -------------------------------------------------------------------------------- /doc/letter_count/sample_context_1.yaml: -------------------------------------------------------------------------------- 1 | input: 2 | url: https://filesamples.com/samples/document/txt/sample3.txt 3 | output: 4 | output_file_name: sample_output_1.json 5 | letter: "d" -------------------------------------------------------------------------------- /doc/letter_count/sample_context_2.yaml: -------------------------------------------------------------------------------- 1 | input: 2 | local_file_name: sample_input.txt 3 | output: 4 | output_file_name: sample_output_2.json 5 | letter: "d" -------------------------------------------------------------------------------- /doc/letter_count/sample_input.txt: -------------------------------------------------------------------------------- 1 | This is a sample input file. 2 | Here are 7 letters: ddddddd -------------------------------------------------------------------------------- /doc/letter_count/sample_output_1.json: -------------------------------------------------------------------------------- 1 | {"count": 110} -------------------------------------------------------------------------------- /doc/letter_count/sample_output_2.json: -------------------------------------------------------------------------------- 1 | {"count": 7} -------------------------------------------------------------------------------- /doc/letter_count/spec.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "letter_count", 3 | "description": "Example Dendro processing app for tutorial", 4 | "appImage": "", 5 | "appExecutable": "/home/magland/src/dendro/doc/letter_count/main.py", 6 | "executable": "/home/magland/src/dendro/doc/letter_count/main.py", 7 | "processors": [ 8 | { 9 | "name": "letter_count", 10 | "description": "\nThis is the a processor in the letter_count app. It counts the number of times a particular letter appears in a text file and produces and JSON file with the result.\n", 11 | "label": "Letter Count", 12 | "inputs": [ 13 | { 14 | "name": "input", 15 | "description": "Input text file" 16 | } 17 | ], 18 | "outputs": [ 19 | { 20 | "name": "output", 21 | "description": "Output JSON file" 22 | } 23 | ], 24 | "parameters": [ 25 | { 26 | "name": "letter", 27 | "description": "Letter to count", 28 | "type": "str" 29 | } 30 | ], 31 | "attributes": [ 32 | { 33 | "name": "wip", 34 | "value": true 35 | } 36 | ], 37 | "tags": [ 38 | { 39 | "tag": "tutorial" 40 | } 41 | ] 42 | } 43 | ] 44 | } -------------------------------------------------------------------------------- /iac/aws_batch/.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | package-lock.json 3 | .pytest_cache 4 | *.egg-info 5 | 6 | # Byte-compiled / optimized / DLL files 7 | __pycache__/ 8 | *.py[cod] 9 | *$py.class 10 | 11 | # Environments 12 | .env 13 | .venv 14 | env/ 15 | venv/ 16 | ENV/ 17 | env.bak/ 18 | venv.bak/ 19 | 20 | # CDK Context & Staging files 21 | .cdk.staging/ 22 | cdk.out/ 23 | cdk.context.json -------------------------------------------------------------------------------- /iac/aws_batch/README.md: -------------------------------------------------------------------------------- 1 | 2 | # Welcome to your CDK Python project! 3 | 4 | You should explore the contents of this project. It demonstrates a CDK app with an instance of a stack (`aws_batch_stack`) 5 | which contains an Amazon SQS queue that is subscribed to an Amazon SNS topic. 6 | 7 | The `cdk.json` file tells the CDK Toolkit how to execute your app. 8 | 9 | This project is set up like a standard Python project. The initialization process also creates 10 | a virtualenv within this project, stored under the .venv directory. To create the virtualenv 11 | it assumes that there is a `python3` executable in your path with access to the `venv` package. 12 | If for any reason the automatic creation of the virtualenv fails, you can create the virtualenv 13 | manually once the init process completes. 14 | 15 | To manually create a virtualenv on MacOS and Linux: 16 | 17 | ``` 18 | $ python3 -m venv .venv 19 | ``` 20 | 21 | After the init process completes and the virtualenv is created, you can use the following 22 | step to activate your virtualenv. 23 | 24 | ``` 25 | $ source .venv/bin/activate 26 | ``` 27 | 28 | If you are a Windows platform, you would activate the virtualenv like this: 29 | 30 | ``` 31 | % .venv\Scripts\activate.bat 32 | ``` 33 | 34 | Once the virtualenv is activated, you can install the required dependencies. 35 | 36 | ``` 37 | $ pip install -r requirements.txt 38 | ``` 39 | 40 | At this point you can now synthesize the CloudFormation template for this code. 41 | 42 | ``` 43 | $ cdk synth 44 | ``` 45 | 46 | You can now begin exploring the source code, contained in the hello directory. 47 | There is also a very trivial test included that can be run like this: 48 | 49 | ``` 50 | $ pytest 51 | ``` 52 | 53 | To add additional dependencies, for example other CDK libraries, just add to 54 | your requirements.txt file and rerun the `pip install -r requirements.txt` 55 | command. 56 | 57 | ## Useful commands 58 | 59 | * `cdk ls` list all stacks in the app 60 | * `cdk synth` emits the synthesized CloudFormation template 61 | * `cdk deploy` deploy this stack to your default AWS account/region 62 | * `cdk diff` compare deployed stack with current state 63 | * `cdk docs` open CDK documentation 64 | 65 | Enjoy! 66 | -------------------------------------------------------------------------------- /iac/aws_batch/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flatironinstitute/dendro-old/ad3211be0d92a255360801eef68be4f5aca7201e/iac/aws_batch/__init__.py -------------------------------------------------------------------------------- /iac/aws_batch/app.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import aws_cdk as cdk 4 | 5 | from aws_batch.aws_batch_stack import AwsBatchStack 6 | 7 | 8 | app = cdk.App() 9 | 10 | aws_batch_stack = AwsBatchStack( 11 | scope=app, 12 | create_efs=False, 13 | ) 14 | 15 | app.synth() 16 | -------------------------------------------------------------------------------- /iac/aws_batch/aws_batch/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flatironinstitute/dendro-old/ad3211be0d92a255360801eef68be4f5aca7201e/iac/aws_batch/aws_batch/__init__.py -------------------------------------------------------------------------------- /iac/aws_batch/aws_batch/stack_config.py: -------------------------------------------------------------------------------- 1 | stack_id = "DendroBatchStack" 2 | 3 | batch_service_role_id = f"{stack_id}-BatchServiceRole" 4 | ecs_instance_role_id = f"{stack_id}-EcsInstanceRole" 5 | batch_jobs_access_role_id = f"{stack_id}-BatchJobsAccessRole" 6 | # efs_file_system_id = f"{stack_id}-EfsFileSystem" 7 | vpc_id = f"{stack_id}-Vpc" 8 | security_group_id = f"{stack_id}-SecurityGroup" 9 | launch_template_id = f"{stack_id}-LaunchTemplate" 10 | compute_env_gpu_id = f"{stack_id}-compute-env-gpu" 11 | compute_env_cpu_id = f"{stack_id}-compute-env-cpu" 12 | job_queue_gpu_id = f"{stack_id}-job-queue-gpu" 13 | job_queue_cpu_id = f"{stack_id}-job-queue-cpu" 14 | -------------------------------------------------------------------------------- /iac/aws_batch/cdk.json: -------------------------------------------------------------------------------- 1 | { 2 | "app": "python3 app.py", 3 | "watch": { 4 | "include": [ 5 | "**" 6 | ], 7 | "exclude": [ 8 | "README.md", 9 | "cdk*.json", 10 | "requirements*.txt", 11 | "source.bat", 12 | "**/__init__.py", 13 | "python/__pycache__", 14 | "tests" 15 | ] 16 | }, 17 | "context": { 18 | "@aws-cdk/aws-lambda:recognizeLayerVersion": true, 19 | "@aws-cdk/core:checkSecretUsage": true, 20 | "@aws-cdk/core:target-partitions": [ 21 | "aws", 22 | "aws-cn" 23 | ], 24 | "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true, 25 | "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true, 26 | "@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true, 27 | "@aws-cdk/aws-iam:minimizePolicies": true, 28 | "@aws-cdk/core:validateSnapshotRemovalPolicy": true, 29 | "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true, 30 | "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true, 31 | "@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true, 32 | "@aws-cdk/aws-apigateway:disableCloudWatchRole": true, 33 | "@aws-cdk/core:enablePartitionLiterals": true, 34 | "@aws-cdk/aws-events:eventsTargetQueueSameAccount": true, 35 | "@aws-cdk/aws-iam:standardizedServicePrincipals": true, 36 | "@aws-cdk/aws-ecs:disableExplicitDeploymentControllerForCircuitBreaker": true, 37 | "@aws-cdk/aws-iam:importedRoleStackSafeDefaultPolicyName": true, 38 | "@aws-cdk/aws-s3:serverAccessLogsUseBucketPolicy": true, 39 | "@aws-cdk/aws-route53-patters:useCertificate": true, 40 | "@aws-cdk/customresources:installLatestAwsSdkDefault": false, 41 | "@aws-cdk/aws-rds:databaseProxyUniqueResourceName": true, 42 | "@aws-cdk/aws-codedeploy:removeAlarmsFromDeploymentGroup": true, 43 | "@aws-cdk/aws-apigateway:authorizerChangeDeploymentLogicalId": true, 44 | "@aws-cdk/aws-ec2:launchTemplateDefaultUserData": true, 45 | "@aws-cdk/aws-secretsmanager:useAttachedSecretResourcePolicyForSecretTargetAttachments": true, 46 | "@aws-cdk/aws-redshift:columnId": true, 47 | "@aws-cdk/aws-stepfunctions-tasks:enableEmrServicePolicyV2": true, 48 | "@aws-cdk/aws-ec2:restrictDefaultSecurityGroup": true, 49 | "@aws-cdk/aws-apigateway:requestValidatorUniqueId": true, 50 | "@aws-cdk/aws-kms:aliasNameRef": true, 51 | "@aws-cdk/aws-autoscaling:generateLaunchTemplateInsteadOfLaunchConfig": true, 52 | "@aws-cdk/core:includePrefixInUniqueNameGeneration": true, 53 | "@aws-cdk/aws-efs:denyAnonymousAccess": true, 54 | "@aws-cdk/aws-opensearchservice:enableOpensearchMultiAzWithStandby": true, 55 | "@aws-cdk/aws-lambda-nodejs:useLatestRuntimeVersion": true, 56 | "@aws-cdk/aws-efs:mountTargetOrderInsensitiveLogicalId": true 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /iac/aws_batch/devel/create_ami_map.py: -------------------------------------------------------------------------------- 1 | import json 2 | import subprocess 3 | 4 | 5 | def main(): 6 | regions = [ 7 | 'af-south-1', 8 | 'ap-south-1', 'ap-southeast-1', 'ap-southeast-2', 'ap-northeast-1', 'ap-northeast-2', 'ap-south-1', 'ap-east-1', 9 | 'ca-central-1', 10 | 'eu-west-1', 'eu-west-2', 'eu-west-3', 'eu-north-1', 'eu-south-1', 'eu-central-1', 11 | 'me-south-1', 12 | 'sa-east-1', 13 | 'us-east-1', 'us-east-2', 'us-west-1', 'us-west-2' 14 | ] 15 | ami_map = {} 16 | for region in regions: 17 | print(f'Region: {region}') 18 | cmd = f'aws ssm get-parameter --name /aws/service/ecs/optimized-ami/amazon-linux-2/gpu/recommended --region {region} --output json' 19 | # cmd = f'aws ssm get-parameter --name /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended --region {region} --output json' 20 | print(cmd) 21 | try: 22 | output = subprocess.check_output(cmd, shell=True) 23 | output = json.loads(output) 24 | yy = json.loads(output['Parameter']['Value']) 25 | image_id = yy['image_id'] 26 | print(f'Image ID: {image_id}') 27 | ami_map[region] = image_id 28 | except Exception as e: 29 | print(f'Error: {e}') 30 | print('') 31 | print('') 32 | continue 33 | print('') 34 | print('~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~') 35 | print(json.dumps(ami_map, indent=4)) 36 | 37 | if __name__ == '__main__': 38 | main() 39 | -------------------------------------------------------------------------------- /iac/aws_batch/requirements-dev.txt: -------------------------------------------------------------------------------- 1 | pytest==6.2.5 2 | -------------------------------------------------------------------------------- /iac/aws_batch/requirements.txt: -------------------------------------------------------------------------------- 1 | aws-cdk-lib==2.106.1 2 | constructs>=10.0.0,<11.0.0 3 | -------------------------------------------------------------------------------- /iac/aws_batch/source.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | rem The sole purpose of this script is to make the command 4 | rem 5 | rem source .venv/bin/activate 6 | rem 7 | rem (which activates a Python virtualenv on Linux or Mac OS X) work on Windows. 8 | rem On Windows, this command just runs this batch file (the argument is ignored). 9 | rem 10 | rem Now we don't need to document a Windows command for activating a virtualenv. 11 | 12 | echo Executing .venv\Scripts\activate.bat for you 13 | .venv\Scripts\activate.bat 14 | -------------------------------------------------------------------------------- /iac/aws_batch/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flatironinstitute/dendro-old/ad3211be0d92a255360801eef68be4f5aca7201e/iac/aws_batch/tests/__init__.py -------------------------------------------------------------------------------- /iac/aws_batch/tests/unit/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flatironinstitute/dendro-old/ad3211be0d92a255360801eef68be4f5aca7201e/iac/aws_batch/tests/unit/__init__.py -------------------------------------------------------------------------------- /iac/aws_batch/tests/unit/test_aws_batch_stack.py: -------------------------------------------------------------------------------- 1 | # import aws_cdk as core 2 | # import aws_cdk.assertions as assertions 3 | # from aws_batch.aws_batch_stack import AwsBatchStack 4 | 5 | 6 | # def test_sqs_queue_created(): 7 | # app = core.App() 8 | # stack = AwsBatchStack(app, "aws-batch") 9 | # template = assertions.Template.from_stack(stack) 10 | 11 | # template.has_resource_properties("AWS::SQS::Queue", { 12 | # "VisibilityTimeout": 300 13 | # }) 14 | 15 | 16 | # def test_sns_topic_created(): 17 | # app = core.App() 18 | # stack = AwsBatchStack(app, "aws-batch") 19 | # template = assertions.Template.from_stack(stack) 20 | 21 | # template.resource_count_is("AWS::SNS::Topic", 1) 22 | -------------------------------------------------------------------------------- /index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | dendro 8 | 9 | 10 |
11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dendro", 3 | "private": true, 4 | "version": "0.2.15", 5 | "scripts": { 6 | "dev": "vite", 7 | "build": "tsc && vite build", 8 | "lint": "eslint src --ext ts,tsx --report-unused-disable-directives --max-warnings 0", 9 | "preview": "vite preview", 10 | "deploy": "cp _vercel_prod.json vercel.json && vercel --prod && cp _vercel_dev.json vercel.json" 11 | }, 12 | "dependencies": { 13 | "@emotion/react": "^11.10.5", 14 | "@emotion/styled": "^11.10.5", 15 | "@fi-sci/electrode-geometry": "~0.0.1", 16 | "@fi-sci/misc": "~0.0.1", 17 | "@fi-sci/modal-window": "~0.0.1", 18 | "@fi-sci/splitter": "~0.0.1", 19 | "@fortawesome/fontawesome-svg-core": "^6.3.0", 20 | "@fortawesome/free-brands-svg-icons": "^6.3.0", 21 | "@fortawesome/free-solid-svg-icons": "^6.4.0", 22 | "@fortawesome/react-fontawesome": "^0.2.0", 23 | "@isomorphic-git/lightning-fs": "^4.6.0", 24 | "@monaco-editor/react": "^4.6.0", 25 | "@mui/icons-material": "^5.11.16", 26 | "@mui/material": "^5.13.0", 27 | "dompurify": "^3.0.8", 28 | "github-markdown-css": "^5.2.0", 29 | "isomorphic-git": "^1.25.2", 30 | "monaco-editor": "^0.46.0", 31 | "nunjucks": "^3.2.4", 32 | "plotly.js": "^2.27.1", 33 | "pubnub": "^7.4.1", 34 | "react": "^18.2.0", 35 | "react-dom": "^18.2.0", 36 | "react-draggable": "^4.4.5", 37 | "react-ipynb-renderer": "^2.1.4", 38 | "react-markdown": "^8.0.7", 39 | "react-plotly.js": "^2.6.0", 40 | "react-router-dom": "^6.11.1", 41 | "react-syntax-highlighter": "^15.5.0", 42 | "rehype-mathjax": "^4.0.2", 43 | "rehype-raw": "^6.1.1", 44 | "remark-gfm": "^3.0.1", 45 | "remark-math": "^5.1.1" 46 | }, 47 | "devDependencies": { 48 | "@types/dompurify": "^3.0.5", 49 | "@types/node": "^20.6.1", 50 | "@types/nunjucks": "^3.2.5", 51 | "@types/pubnub": "^7.3.4", 52 | "@types/react": "^18.0.28", 53 | "@types/react-dom": "^18.0.11", 54 | "@types/react-plotly.js": "^2.6.3", 55 | "@types/react-syntax-highlighter": "^15.5.9", 56 | "@typescript-eslint/eslint-plugin": "^5.57.1", 57 | "@typescript-eslint/parser": "^5.57.1", 58 | "@vitejs/plugin-react": "^4.0.0", 59 | "eslint": "^8.38.0", 60 | "eslint-plugin-react": "^7.32.2", 61 | "eslint-plugin-react-hooks": "^4.6.0", 62 | "eslint-plugin-react-refresh": "^0.3.4", 63 | "typescript": "^5.0.2", 64 | "vite": "^4.3.2" 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /public/dendro.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flatironinstitute/dendro-old/ad3211be0d92a255360801eef68be4f5aca7201e/public/dendro.png -------------------------------------------------------------------------------- /public/help/common.md: -------------------------------------------------------------------------------- 1 | Use the buttons in the top menu bar to configure compute resources, manage projects, set secret keys, or sign in. 2 | 3 | [Dendro documentation](https://flatironinstitute.github.io/dendro-docs/docs/intro) -------------------------------------------------------------------------------- /public/help/dandiset.md: -------------------------------------------------------------------------------- 1 | # Dandiset 2 | 3 | You are viewing Dandiset {{ route.dandisetId }}. 4 | 5 | To import files from this dandiset, you will need to create an associated project, or open an existing one. 6 | 7 | {% if signedIn %} 8 | {% else %} 9 | **To create a new project, you'll need to sign in.** 10 | {% endif %} 11 | 12 | 13 | {% if staging %} 14 | You are viewing the staging site. 15 | {% endif %} 16 | -------------------------------------------------------------------------------- /public/help/dandisets.md: -------------------------------------------------------------------------------- 1 | # Browse dandisets 2 | 3 | Search for and select a dandiset. 4 | 5 | 6 | {% if staging %} 7 | You are viewing the staging site. If you need to access embargoed datasets, configure your API key first. 8 | 9 | See [Dandi Archive (staging)](https://gui-staging.dandiarchive.org/). 10 | {% else %} 11 | 12 | See [Dandi Archive](https://dandiarchive.org/). 13 | 14 | {% endif %} 15 | 16 | -------------------------------------------------------------------------------- /public/help/home.md: -------------------------------------------------------------------------------- 1 | # Dendro home 2 | 3 | Dendro is a web application and compute framework aimed at researchers who want to manage and analyze neurophysiology data. -------------------------------------------------------------------------------- /public/help/project-dandi-import.md: -------------------------------------------------------------------------------- 1 | # Import files from DANDI 2 | 3 | To import files to this project, use the checkboxes to select the files and then click "Import selected assets". The import may take some time depending on the number of files selected. No data will actually be transferred as the Dendro files are only pointers to the resources. After the files have been imported, they will be visible in the Files tab. -------------------------------------------------------------------------------- /public/help/project-processors.md: -------------------------------------------------------------------------------- 1 | # Project processors 2 | 3 | You are viewing the processors for this project. These are determined based on the compute resource you have configured for the project. To change the compute resource, go to the project home tab and click the Settings button. -------------------------------------------------------------------------------- /public/help/project-project-files.md: -------------------------------------------------------------------------------- 1 | # Project files 2 | 3 | You are viewing the files for this project. -------------------------------------------------------------------------------- /public/help/project-project-home.md: -------------------------------------------------------------------------------- 1 | # Project home 2 | 3 | You are viewing a project. Use the tabs on the left to navigate to the project files and jobs or to import files from DANDI. -------------------------------------------------------------------------------- /public/help/project-project-jobs.md: -------------------------------------------------------------------------------- 1 | # Project jobs 2 | 3 | You are viewing the jobs for this project. -------------------------------------------------------------------------------- /public/scripts/load_electrical_series.py: -------------------------------------------------------------------------------- 1 | import h5py 2 | import dendro.client as prc 3 | from dendro.client._interim import NwbRecording 4 | import remfile 5 | 6 | 7 | # Load project {{ project.projectName }} 8 | project = prc.load_project('{{ project.projectId }}') 9 | 10 | # Lazy load {{ fileName }} 11 | nwb_file = remfile.File(project.get_file('{{ fileName }}')) 12 | h5_file = h5py.File(nwb_file, 'r') 13 | 14 | # Create a recording object 15 | recording = NwbRecording(h5_file, electrical_series_path='{{ electricalSeriesPath }}') 16 | 17 | # Get recording information 18 | duration_sec = recording.get_duration() 19 | sampling_frequency = recording.get_sampling_frequency() 20 | num_channels = recording.get_num_channels() 21 | 22 | print(f'Duration (sec): {duration_sec}') 23 | print(f'Sampling frequency (Hz): {sampling_frequency}') 24 | print(f'Number of channels: {num_channels}') 25 | 26 | # Load the first 1000 frames of the recording 27 | traces = recording.get_traces(start_frame=0, end_frame=1000) 28 | 29 | print(f'Traces shape for first 1000 frames: {traces.shape}') 30 | -------------------------------------------------------------------------------- /public/scripts/load_spike_sorting.py: -------------------------------------------------------------------------------- 1 | import dendro.client as prc 2 | from dendro.client._interim import NwbSorting 3 | import remfile 4 | 5 | 6 | # Load project {{ project.projectName }} 7 | project = prc.load_project('{{ project.projectId }}') 8 | 9 | # Lazy load {{ fileName }} 10 | nwb_file = remfile.File(project.get_file('{{ fileName }}')) 11 | 12 | # Create a recording object 13 | sorting = NwbSorting(nwb_file) 14 | 15 | # Get sorting information 16 | unit_ids = sorting.get_unit_ids() 17 | sampling_frequency = sorting.get_sampling_frequency() 18 | 19 | print(f'Unit ids: {unit_ids}') 20 | print(f'Sampling frequency (Hz): {sampling_frequency}') 21 | 22 | spike_train_1 = sorting.get_unit_spike_train(unit_id=unit_ids[0]) 23 | print(f'Number of events for unit {unit_ids[0]}: {len(spike_train_1)}') 24 | -------------------------------------------------------------------------------- /python/.gitignore: -------------------------------------------------------------------------------- 1 | _dendro 2 | file_cache 3 | script_jobs 4 | example-data 5 | tmp 6 | .dendro-compute-resource-node.yaml 7 | 8 | # Byte-compiled / optimized / DLL files 9 | __pycache__/ 10 | *.py[cod] 11 | *$py.class 12 | 13 | # C extensions 14 | *.so 15 | 16 | # Distribution / packaging 17 | .Python 18 | build/ 19 | develop-eggs/ 20 | dist/ 21 | downloads/ 22 | eggs/ 23 | .eggs/ 24 | lib/ 25 | lib64/ 26 | parts/ 27 | sdist/ 28 | var/ 29 | wheels/ 30 | pip-wheel-metadata/ 31 | share/python-wheels/ 32 | *.egg-info/ 33 | .installed.cfg 34 | *.egg 35 | MANIFEST 36 | 37 | # PyInstaller 38 | # Usually these files are written by a python script from a template 39 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 40 | *.manifest 41 | *.spec 42 | 43 | # Installer logs 44 | pip-log.txt 45 | pip-delete-this-directory.txt 46 | 47 | # Unit test / coverage reports 48 | htmlcov/ 49 | .tox/ 50 | .nox/ 51 | .coverage 52 | .coverage.* 53 | .cache 54 | nosetests.xml 55 | coverage.xml 56 | *.cover 57 | *.py,cover 58 | .hypothesis/ 59 | .pytest_cache/ 60 | 61 | # Translations 62 | *.mo 63 | *.pot 64 | 65 | # Django stuff: 66 | *.log 67 | local_settings.py 68 | db.sqlite3 69 | db.sqlite3-journal 70 | 71 | # Flask stuff: 72 | instance/ 73 | .webassets-cache 74 | 75 | # Scrapy stuff: 76 | .scrapy 77 | 78 | # Sphinx documentation 79 | docs/_build/ 80 | 81 | # PyBuilder 82 | target/ 83 | 84 | # Jupyter Notebook 85 | .ipynb_checkpoints 86 | 87 | # IPython 88 | profile_default/ 89 | ipython_config.py 90 | 91 | # pyenv 92 | .python-version 93 | 94 | # pipenv 95 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 96 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 97 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 98 | # install all needed dependencies. 99 | #Pipfile.lock 100 | 101 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 102 | __pypackages__/ 103 | 104 | # Celery stuff 105 | celerybeat-schedule 106 | celerybeat.pid 107 | 108 | # SageMath parsed files 109 | *.sage.py 110 | 111 | # Environments 112 | .env 113 | .venv 114 | env/ 115 | venv/ 116 | ENV/ 117 | env.bak/ 118 | venv.bak/ 119 | 120 | # Spyder project settings 121 | .spyderproject 122 | .spyproject 123 | 124 | # Rope project settings 125 | .ropeproject 126 | 127 | # mkdocs documentation 128 | /site 129 | 130 | # mypy 131 | .mypy_cache/ 132 | .dmypy.json 133 | dmypy.json 134 | 135 | # Pyre type checker 136 | .pyre/ 137 | -------------------------------------------------------------------------------- /python/README.md: -------------------------------------------------------------------------------- 1 | # dendro -------------------------------------------------------------------------------- /python/dendro/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from pydantic import VERSION as pydantic_version # noqa 4 | 5 | # In the future we may want to intercept and override the pydantic BaseModel 6 | from pydantic import BaseModel, Field # noqa 7 | 8 | # read the version from thisdir/version.txt 9 | thisdir = os.path.dirname(os.path.realpath(__file__)) 10 | with open(os.path.join(thisdir, 'version.txt')) as f: 11 | __version__ = f.read().strip() 12 | -------------------------------------------------------------------------------- /python/dendro/api_helpers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flatironinstitute/dendro-old/ad3211be0d92a255360801eef68be4f5aca7201e/python/dendro/api_helpers/__init__.py -------------------------------------------------------------------------------- /python/dendro/api_helpers/clients/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flatironinstitute/dendro-old/ad3211be0d92a255360801eef68be4f5aca7201e/python/dendro/api_helpers/clients/__init__.py -------------------------------------------------------------------------------- /python/dendro/api_helpers/clients/_get_mongo_client.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from ..core.settings import get_settings 3 | from .MockMongoClient import MockMongoClient 4 | from ...mock import using_mock 5 | 6 | _globals = {"mock_mongo_client": None} 7 | 8 | 9 | # pyright: reportGeneralTypeIssues=false 10 | def _get_mongo_client(): 11 | # We want one async mongo client per event loop 12 | loop = asyncio.get_event_loop() 13 | if hasattr(loop, "_mongo_client"): 14 | return loop._mongo_client # type: ignore 15 | 16 | mongo_uri = get_settings().MONGO_URI 17 | 18 | # If we're using a mock client, return it 19 | if using_mock(): 20 | client = _globals["mock_mongo_client"] # type: ignore 21 | if client is None: 22 | client = MockMongoClient() 23 | _globals["mock_mongo_client"] = client # type: ignore 24 | else: # pragma: no cover 25 | # Otherwise, create a new client 26 | assert mongo_uri is not None, "MONGO_URI environment variable not set" 27 | from motor.motor_asyncio import AsyncIOMotorClient 28 | 29 | client = AsyncIOMotorClient(mongo_uri) 30 | 31 | # Store the client on the event loop 32 | setattr(loop, "_mongo_client", client) 33 | 34 | return client 35 | 36 | 37 | def _clear_mock_mongo_databases(): 38 | client: MockMongoClient = _globals["mock_mongo_client"] # type: ignore 39 | if client is not None: 40 | client.clear_databases() 41 | -------------------------------------------------------------------------------- /python/dendro/api_helpers/clients/_remove_id_field.py: -------------------------------------------------------------------------------- 1 | def _remove_id_field(obj): 2 | if obj is None: 3 | return 4 | if '_id' in obj: 5 | del obj['_id'] 6 | -------------------------------------------------------------------------------- /python/dendro/api_helpers/clients/pubsub.py: -------------------------------------------------------------------------------- 1 | import json 2 | import aiohttp 3 | import urllib.parse 4 | from ..core.settings import get_settings 5 | from ...mock import using_mock 6 | 7 | 8 | class PubsubError(Exception): 9 | pass 10 | 11 | async def publish_pubsub_message(*, channel: str, message: dict): 12 | settings = get_settings() 13 | # see https://www.pubnub.com/docs/sdks/rest-api/publish-message-to-channel 14 | sub_key = settings.PUBNUB_SUBSCRIBE_KEY 15 | pub_key = settings.PUBNUB_PUBLISH_KEY 16 | uuid = 'dendro' 17 | # payload is url encoded json 18 | payload = json.dumps(message) 19 | payload = urllib.parse.quote(payload) 20 | url = f"https://ps.pndsn.com/publish/{pub_key}/{sub_key}/0/{channel}/0/{payload}?uuid={uuid}" 21 | 22 | headers = { 23 | 'Accept': 'application/json' 24 | } 25 | 26 | if using_mock(): 27 | # don't actually publish the message for the mock case 28 | return True 29 | 30 | # async http get request 31 | async with aiohttp.ClientSession() as session: # pragma: no cover 32 | async with session.get(url, headers=headers) as resp: 33 | if resp.status != 200: 34 | raise PubsubError(f"Error publishing to pubsub: {resp.status} {resp.text}") 35 | return True 36 | -------------------------------------------------------------------------------- /python/dendro/api_helpers/core/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flatironinstitute/dendro-old/ad3211be0d92a255360801eef68be4f5aca7201e/python/dendro/api_helpers/core/__init__.py -------------------------------------------------------------------------------- /python/dendro/api_helpers/core/_create_random_id.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | 3 | 4 | def _create_random_id(length: int) -> str: 5 | # Generate a random UUID 6 | full_uuid = uuid.uuid4() 7 | 8 | # Convert to a string and take the first [length] characters 9 | short_id = str(full_uuid).replace('-', '')[:length] 10 | 11 | return short_id 12 | -------------------------------------------------------------------------------- /python/dendro/api_helpers/core/_get_project_role.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | from typing import Union 4 | from ...common.dendro_types import DendroProject 5 | 6 | 7 | def _get_project_role(project: DendroProject, user_id: Union[str, None]) -> str: 8 | ADMIN_USER_IDS_JSON = os.getenv('ADMIN_USER_IDS', '[]') 9 | ADMIN_USER_IDS = json.loads(ADMIN_USER_IDS_JSON) 10 | if user_id: 11 | if user_id in ADMIN_USER_IDS: 12 | return 'admin' 13 | if project.ownerId == user_id: 14 | return 'admin' 15 | user = next((x for x in project.users if x.userId == user_id), None) 16 | if user: 17 | return user.role 18 | if project.publiclyReadable: 19 | return 'viewer' 20 | else: 21 | return 'none' 22 | 23 | def _project_has_user(project: DendroProject, user_id: Union[str, None]) -> bool: 24 | if not user_id: 25 | return False 26 | if project.ownerId == user_id: 27 | return True 28 | user = next((x for x in project.users if x.userId == user_id), None) 29 | if user: 30 | return True 31 | return False 32 | 33 | def _check_user_can_read_project(project: DendroProject, user_id: Union[str, None]): 34 | if not _get_project_role(project, user_id) in ['admin', 'editor', 'viewer']: 35 | raise Exception('User does not have read permission for this project') 36 | 37 | def _check_user_can_edit_project(project: DendroProject, user_id: Union[str, None]): 38 | if not _get_project_role(project, user_id) in ['admin', 'editor']: 39 | raise Exception('User does not have edit permission for this project') 40 | 41 | def _check_user_is_project_admin(project: DendroProject, user_id: Union[str, None]): 42 | if not _get_project_role(project, user_id) == 'admin': 43 | raise Exception('User does not have admin permission for this project') 44 | -------------------------------------------------------------------------------- /python/dendro/api_helpers/core/_hide_secret_params_in_job.py: -------------------------------------------------------------------------------- 1 | from ...common.dendro_types import DendroJob 2 | 3 | 4 | def _hide_secret_params_in_job(job: DendroJob): 5 | for param in job.inputParameters: 6 | if param.secret: 7 | param.value = None 8 | -------------------------------------------------------------------------------- /python/dendro/api_helpers/core/_model_dump.py: -------------------------------------------------------------------------------- 1 | def _model_dump(model, exclude_none=False): 2 | # handle both pydantic v1 and v2 3 | if hasattr(model, 'model_dump'): 4 | return model.model_dump(exclude_none=exclude_none) 5 | else: 6 | return model.dict(exclude_none=exclude_none) 7 | -------------------------------------------------------------------------------- /python/dendro/api_helpers/core/settings.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | import os 3 | 4 | # Note: BaseSettings is no longer a part of pydantic package, and I didn't want to add a dependency on pydantic-settings 5 | # So I'm using BaseModel instead 6 | 7 | 8 | class Settings: 9 | def __init__(self) -> None: 10 | # General app config 11 | self.MONGO_URI: Optional[str] = os.environ.get("MONGO_URI") 12 | 13 | self.PUBNUB_SUBSCRIBE_KEY: Optional[str] = os.environ.get("VITE_PUBNUB_SUBSCRIBE_KEY") 14 | self.PUBNUB_PUBLISH_KEY: Optional[str] = os.environ.get("PUBNUB_PUBLISH_KEY") 15 | 16 | self.GITHUB_CLIENT_ID: Optional[str] = os.environ.get("VITE_GITHUB_CLIENT_ID") 17 | self.GITHUB_CLIENT_SECRET: Optional[str] = os.environ.get("GITHUB_CLIENT_SECRET") 18 | 19 | self.DEFAULT_COMPUTE_RESOURCE_ID: Optional[str] = os.environ.get("VITE_DEFAULT_COMPUTE_RESOURCE_ID") 20 | 21 | self.OUTPUT_BUCKET_BASE_URL: Optional[str] = os.environ.get("OUTPUT_BUCKET_BASE_URL") 22 | self.OUTPUT_BUCKET_URI: Optional[str] = os.environ.get("OUTPUT_BUCKET_URI", None) 23 | self.OUTPUT_BUCKET_CREDENTIALS: Optional[str] = os.environ.get("OUTPUT_BUCKET_CREDENTIALS", None) 24 | self.FSBUCKET_SECRET_KEY: Optional[str] = os.environ.get("FSBUCKET_SECRET_KEY", None) 25 | 26 | def get_settings(): 27 | return Settings() 28 | -------------------------------------------------------------------------------- /python/dendro/api_helpers/routers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flatironinstitute/dendro-old/ad3211be0d92a255360801eef68be4f5aca7201e/python/dendro/api_helpers/routers/__init__.py -------------------------------------------------------------------------------- /python/dendro/api_helpers/routers/client/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flatironinstitute/dendro-old/ad3211be0d92a255360801eef68be4f5aca7201e/python/dendro/api_helpers/routers/client/__init__.py -------------------------------------------------------------------------------- /python/dendro/api_helpers/routers/common.py: -------------------------------------------------------------------------------- 1 | import traceback 2 | from fastapi import HTTPException 3 | from functools import wraps 4 | 5 | 6 | class AuthException(Exception): 7 | pass 8 | 9 | def api_route_wrapper(route_func): 10 | @wraps(route_func) 11 | async def wrapper(*args, **kwargs): 12 | try: 13 | return await route_func(*args, **kwargs) 14 | except AuthException as ae: 15 | raise HTTPException(status_code=401, detail=str(ae)) 16 | except Exception as e: 17 | traceback.print_exc() 18 | raise HTTPException(status_code=500, detail=str(e)) from e 19 | return wrapper 20 | -------------------------------------------------------------------------------- /python/dendro/api_helpers/routers/compute_resource/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flatironinstitute/dendro-old/ad3211be0d92a255360801eef68be4f5aca7201e/python/dendro/api_helpers/routers/compute_resource/__init__.py -------------------------------------------------------------------------------- /python/dendro/api_helpers/routers/gui/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flatironinstitute/dendro-old/ad3211be0d92a255360801eef68be4f5aca7201e/python/dendro/api_helpers/routers/gui/__init__.py -------------------------------------------------------------------------------- /python/dendro/api_helpers/routers/gui/create_job_route.py: -------------------------------------------------------------------------------- 1 | from typing import Union 2 | from fastapi import APIRouter, Header 3 | 4 | from ._authenticate_gui_request import _authenticate_gui_request 5 | from ...services.gui.create_job import create_job 6 | from ..common import api_route_wrapper 7 | 8 | from ....common.dendro_types import CreateJobRequest, CreateJobResponse 9 | 10 | 11 | router = APIRouter() 12 | 13 | # create job 14 | 15 | @router.post("/jobs") 16 | @api_route_wrapper 17 | async def create_job_handler( 18 | data: CreateJobRequest, 19 | github_access_token: Union[str, None] = Header(None), 20 | dendro_api_key: Union[str, None] = Header(None), 21 | force_require_approval: bool = False 22 | ) -> CreateJobResponse: 23 | # authenticate the request 24 | user_id = await _authenticate_gui_request( 25 | github_access_token=github_access_token, 26 | dendro_api_key=dendro_api_key, 27 | raise_on_not_authenticated=True 28 | ) 29 | assert user_id 30 | 31 | # parse the request 32 | project_id = data.projectId 33 | processor_name = data.processorName 34 | input_files_from_request = data.inputFiles 35 | output_files_from_request = data.outputFiles 36 | input_parameters = data.inputParameters 37 | processor_spec = data.processorSpec 38 | batch_id = data.batchId 39 | dandi_api_key = data.dandiApiKey 40 | required_resources = data.requiredResources 41 | run_method = data.runMethod 42 | 43 | job_id = await create_job( 44 | project_id=project_id, 45 | processor_name=processor_name, 46 | input_files_from_request=input_files_from_request, 47 | output_files_from_request=output_files_from_request, 48 | input_parameters=input_parameters, 49 | processor_spec=processor_spec, 50 | batch_id=batch_id, 51 | user_id=user_id, 52 | dandi_api_key=dandi_api_key, 53 | required_resources=required_resources, 54 | run_method=run_method, 55 | pending_approval=True if force_require_approval else None # None means determine automatically 56 | ) 57 | 58 | return CreateJobResponse( 59 | jobId=job_id, 60 | success=True 61 | ) 62 | -------------------------------------------------------------------------------- /python/dendro/api_helpers/routers/gui/find_routes.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | from fastapi import APIRouter 3 | from ..common import api_route_wrapper 4 | from .... import BaseModel 5 | from ....common.dendro_types import DendroProject, DendroFile 6 | from ...clients.db import fetch_files_with_content_string, fetch_project, fetch_files_with_metadata 7 | 8 | 9 | router = APIRouter() 10 | 11 | # find projects 12 | class FindProjectsRequest(BaseModel): 13 | fileUrl: str 14 | 15 | class CreateProjectResponse(BaseModel): 16 | projects: List[DendroProject] 17 | success: bool 18 | 19 | @router.post("/find_projects") 20 | @api_route_wrapper 21 | async def find_projects(data: FindProjectsRequest) -> CreateProjectResponse: 22 | files = await fetch_files_with_content_string(f'url:{data.fileUrl}') 23 | project_ids: List[str] = [] 24 | for file in files: 25 | if file.projectId not in project_ids: 26 | project_ids.append(file.projectId) 27 | projects: List[DendroProject] = [] 28 | for project_id in project_ids: 29 | p = await fetch_project(project_id) 30 | if p is not None: 31 | projects.append(p) 32 | return CreateProjectResponse(projects=projects, success=True) 33 | 34 | # find files with metadata 35 | class FindFilesWithMetadataRequest(BaseModel): 36 | query: dict 37 | 38 | class FindFilesWithMetadataResponse(BaseModel): 39 | files: List[DendroFile] 40 | 41 | @router.post("/find_files_with_metadata") 42 | @api_route_wrapper 43 | async def find_files_with_metadata(data: FindFilesWithMetadataRequest) -> FindFilesWithMetadataResponse: 44 | files = await fetch_files_with_metadata(data.query) 45 | return FindFilesWithMetadataResponse(files=files) 46 | -------------------------------------------------------------------------------- /python/dendro/api_helpers/routers/gui/github_auth_routes.py: -------------------------------------------------------------------------------- 1 | from .... import BaseModel 2 | from fastapi import APIRouter 3 | import aiohttp 4 | from ...core.settings import get_settings 5 | 6 | 7 | router = APIRouter() 8 | 9 | class GithubAuthError(Exception): 10 | pass 11 | 12 | # github auth 13 | class GithubAuthResponse(BaseModel): 14 | access_token: str 15 | 16 | @router.get("/github_auth/{code}") 17 | async def github_auth(code) -> GithubAuthResponse: 18 | settings = get_settings() 19 | GITHUB_CLIENT_ID = settings.GITHUB_CLIENT_ID 20 | GITHUB_CLIENT_SECRET = settings.GITHUB_CLIENT_SECRET 21 | assert GITHUB_CLIENT_ID, 'Env var not set: VITE_GITHUB_CLIENT_ID' 22 | assert GITHUB_CLIENT_SECRET, 'Env var not set: GITHUB_CLIENT_SECRET' 23 | url = f'https://github.com/login/oauth/access_token?client_id={GITHUB_CLIENT_ID}&client_secret={GITHUB_CLIENT_SECRET}&code={code}' 24 | headers = { 25 | 'accept': 'application/json' 26 | } 27 | async with aiohttp.ClientSession() as session: 28 | async with session.get(url, headers=headers) as resp: 29 | r = await resp.json() 30 | if 'access_token' in r: 31 | return GithubAuthResponse(access_token=r['access_token']) # pragma: no cover 32 | elif 'error' in r: 33 | raise GithubAuthError(f'Error in github oauth response: {r["error"]}') 34 | else: 35 | raise Exception('No access_token in github oauth response.') # pragma: no cover 36 | -------------------------------------------------------------------------------- /python/dendro/api_helpers/routers/gui/router.py: -------------------------------------------------------------------------------- 1 | from fastapi import APIRouter 2 | from .create_job_route import router as create_job_router 3 | from .project_routes import router as project_router 4 | from .compute_resource_routes import router as compute_resource_router 5 | from .file_routes import router as file_router 6 | from .job_routes import router as job_router 7 | from .script_routes import router as script_router 8 | from .github_auth_routes import router as github_auth_router 9 | from .user_routes import router as user_router 10 | from .usage_routes import router as usage_router 11 | from .find_routes import router as find_router 12 | 13 | router = APIRouter() 14 | 15 | router.include_router(create_job_router) 16 | router.include_router(project_router, prefix="/projects") 17 | router.include_router(compute_resource_router, prefix="/compute_resources") 18 | router.include_router(file_router) 19 | router.include_router(job_router, prefix="/jobs") 20 | router.include_router(script_router, prefix="/scripts") 21 | router.include_router(github_auth_router) 22 | router.include_router(user_router, prefix="/users") 23 | router.include_router(usage_router, prefix="/usage") 24 | router.include_router(find_router) 25 | -------------------------------------------------------------------------------- /python/dendro/api_helpers/routers/gui/usage_routes.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | from fastapi import APIRouter, Header 4 | from .... import BaseModel 5 | from ....common.dendro_types import ComputeResourceUserUsage 6 | from ._authenticate_gui_request import _authenticate_gui_request 7 | from ..common import api_route_wrapper 8 | from ...services.gui.get_compute_resource_user_usage import get_compute_resource_user_usage 9 | 10 | 11 | router = APIRouter() 12 | 13 | # get usage 14 | class GetUsageResponse(BaseModel): 15 | usage: ComputeResourceUserUsage 16 | success: bool 17 | 18 | @router.get("/compute_resource/{compute_resource_id}/user/{user_id}") 19 | @api_route_wrapper 20 | async def get_usage(compute_resource_id, user_id, github_access_token: str = Header(...)): 21 | # url decode the %7C in the user_id 22 | user_id = user_id.replace('%7C', '|') 23 | 24 | # authenticate the request 25 | auth_user_id = await _authenticate_gui_request(github_access_token=github_access_token, raise_on_not_authenticated=True) 26 | assert auth_user_id 27 | 28 | if user_id != auth_user_id: 29 | ADMIN_USER_IDS_JSON = os.getenv('ADMIN_USER_IDS', '[]') 30 | ADMIN_USER_IDS = json.loads(ADMIN_USER_IDS_JSON) 31 | 32 | if user_id not in ADMIN_USER_IDS: 33 | raise Exception(f'User is not admin and cannot view usage for other users ({user_id} != {auth_user_id})') 34 | 35 | # get usage 36 | usage = await get_compute_resource_user_usage(compute_resource_id=compute_resource_id, user_id=user_id) 37 | 38 | return GetUsageResponse(usage=usage, success=True) 39 | -------------------------------------------------------------------------------- /python/dendro/api_helpers/routers/gui/user_routes.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | import uuid 3 | from fastapi import APIRouter, Header 4 | from pydantic import BaseModel 5 | from ..common import api_route_wrapper 6 | from ._authenticate_gui_request import _authenticate_gui_request 7 | from ...clients.db import set_dendro_api_key_for_user 8 | 9 | 10 | router = APIRouter() 11 | 12 | class CreateDendroApiKeyResponse(BaseModel): 13 | dendroApiKey: str 14 | success: bool 15 | 16 | @router.post("/{user_id}/dendro_api_key") 17 | @api_route_wrapper 18 | async def create_dendro_api_key(user_id, github_access_token: str = Header(...)): 19 | # authenticate the request 20 | user_id = await _authenticate_gui_request(github_access_token=github_access_token, raise_on_not_authenticated=True) 21 | assert user_id 22 | 23 | # create the api key as the sha1 hash a of uuid 24 | new_dendro_api_key = hashlib.sha1(str(uuid.uuid4()).encode('utf-8')).hexdigest() 25 | 26 | # set the api key for the user 27 | await set_dendro_api_key_for_user(user_id, new_dendro_api_key) 28 | 29 | return CreateDendroApiKeyResponse(dendroApiKey=new_dendro_api_key, success=True) 30 | -------------------------------------------------------------------------------- /python/dendro/api_helpers/routers/processor/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flatironinstitute/dendro-old/ad3211be0d92a255360801eef68be4f5aca7201e/python/dendro/api_helpers/routers/processor/__init__.py -------------------------------------------------------------------------------- /python/dendro/api_helpers/services/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flatironinstitute/dendro-old/ad3211be0d92a255360801eef68be4f5aca7201e/python/dendro/api_helpers/services/__init__.py -------------------------------------------------------------------------------- /python/dendro/api_helpers/services/_remove_detached_files_and_jobs.py: -------------------------------------------------------------------------------- 1 | from ..clients._get_mongo_client import _get_mongo_client 2 | from ..clients._remove_id_field import _remove_id_field 3 | from ...common.dendro_types import DendroFile, DendroJob 4 | 5 | 6 | async def _remove_detached_files_and_jobs(project_id: str): 7 | client = _get_mongo_client() 8 | files_collection = client['dendro']['files'] 9 | jobs_collection = client['dendro']['jobs'] 10 | 11 | files = await files_collection.find({ 12 | 'projectId': project_id 13 | }).to_list(length=None) # type: ignore 14 | jobs = await jobs_collection.find({ 15 | 'projectId': project_id, 16 | 'deleted': {'$ne': True} 17 | }).to_list(length=None) # type: ignore 18 | 19 | for file in files: 20 | _remove_id_field(file) 21 | for job in jobs: 22 | _remove_id_field(job) 23 | files = [DendroFile(**x) for x in files] 24 | jobs = [DendroJob(**x) for x in jobs] 25 | 26 | something_changed = True 27 | while something_changed: 28 | file_ids = set(x.fileId for x in files) 29 | job_ids = set(x.jobId for x in jobs) 30 | 31 | job_ids_to_delete = set() 32 | for job in jobs: 33 | if any(x not in file_ids for x in job.inputFileIds): 34 | job_ids_to_delete.add(job.jobId) 35 | if job.outputFileIds: 36 | if any(x not in file_ids for x in job.outputFileIds): 37 | job_ids_to_delete.add(job.jobId) 38 | file_ids_to_delete = set() 39 | for file in files: 40 | if file.jobId: 41 | if file.jobId not in job_ids: 42 | file_ids_to_delete.add(file.fileId) 43 | something_changed = False 44 | if len(job_ids_to_delete) > 0: 45 | something_changed = True 46 | # Let's actually delete them rather than just marking them as deleted 47 | jobs_collection.delete_many({ 48 | 'jobId': {'$in': list(job_ids_to_delete)} 49 | }) 50 | # for job_id in job_ids_to_delete: 51 | # await jobs_collection.update_one({ 52 | # 'jobId': job_id 53 | # }, { 54 | # '$set': { 55 | # 'deleted': True 56 | # } 57 | # }) 58 | jobs = [x for x in jobs if x.jobId not in job_ids_to_delete] 59 | if len(file_ids_to_delete) > 0: 60 | something_changed = True 61 | await files_collection.delete_many({ 62 | 'fileId': {'$in': list(file_ids_to_delete)} 63 | }) 64 | files = [x for x in files if x.fileId not in file_ids_to_delete] 65 | -------------------------------------------------------------------------------- /python/dendro/api_helpers/services/gui/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flatironinstitute/dendro-old/ad3211be0d92a255360801eef68be4f5aca7201e/python/dendro/api_helpers/services/gui/__init__.py -------------------------------------------------------------------------------- /python/dendro/api_helpers/services/gui/delete_project.py: -------------------------------------------------------------------------------- 1 | from ....common.dendro_types import DendroProject 2 | from ...clients.db import delete_all_files_in_project, delete_all_jobs_in_project, delete_project as db_delete_project 3 | 4 | 5 | async def delete_project(project: DendroProject): 6 | await delete_all_files_in_project(project.projectId) 7 | await delete_all_jobs_in_project(project.projectId) 8 | await db_delete_project(project.projectId) 9 | return None 10 | -------------------------------------------------------------------------------- /python/dendro/api_helpers/services/gui/get_compute_resource_user_usage.py: -------------------------------------------------------------------------------- 1 | from ....common.dendro_types import ComputeResourceUserUsage 2 | from ...clients.db import fetch_jobs_including_deleted 3 | 4 | 5 | async def get_compute_resource_user_usage(*, compute_resource_id: str, user_id: str) -> ComputeResourceUserUsage: 6 | jobs_including_deleted = await fetch_jobs_including_deleted(compute_resource_id=compute_resource_id, user_id=user_id) 7 | return ComputeResourceUserUsage( 8 | computeResourceId=compute_resource_id, 9 | userId=user_id, 10 | jobsIncludingDeleted=jobs_including_deleted 11 | ) 12 | -------------------------------------------------------------------------------- /python/dendro/api_helpers/services/gui/set_file.py: -------------------------------------------------------------------------------- 1 | import time 2 | from typing import Union 3 | from ...clients.db import fetch_file, delete_file, insert_file, update_project, update_file_metadata 4 | from ....common.dendro_types import DendroFile 5 | from ...core._create_random_id import _create_random_id 6 | from .._remove_detached_files_and_jobs import _remove_detached_files_and_jobs 7 | 8 | 9 | async def set_file( 10 | user_id: str, 11 | project_id: str, 12 | file_name: str, 13 | content: str, # for example, url:https://... 14 | job_id: Union[str, None], 15 | size: int, 16 | metadata: dict, 17 | is_folder: bool = False 18 | ): 19 | existing_file = await fetch_file(project_id, file_name) 20 | if existing_file is not None: 21 | await delete_file(project_id, file_name) 22 | deleted_old_file = True 23 | else: 24 | deleted_old_file = False 25 | 26 | new_file = DendroFile( 27 | projectId=project_id, 28 | fileId=_create_random_id(8), 29 | userId=user_id, 30 | fileName=file_name, 31 | size=size, 32 | timestampCreated=time.time(), 33 | content=content, 34 | metadata=metadata, 35 | isFolder=is_folder, 36 | jobId=job_id 37 | ) 38 | await insert_file(new_file) 39 | 40 | if deleted_old_file: 41 | await _remove_detached_files_and_jobs(project_id) 42 | 43 | await update_project( 44 | project_id=project_id, 45 | update={ 46 | 'timestampModified': time.time() 47 | } 48 | ) 49 | 50 | return new_file.fileId 51 | 52 | async def set_file_metadata( 53 | project_id: str, 54 | file_name: str, 55 | metadata: dict 56 | ): 57 | existing_file = await fetch_file(project_id, file_name) 58 | if existing_file is None: 59 | raise Exception(f"Cannot set metadata. File {file_name} not found in project {project_id}") 60 | await update_file_metadata( 61 | project_id=project_id, 62 | file_id=existing_file.fileId, 63 | metadata=metadata 64 | ) 65 | -------------------------------------------------------------------------------- /python/dendro/api_helpers/services/processor/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flatironinstitute/dendro-old/ad3211be0d92a255360801eef68be4f5aca7201e/python/dendro/api_helpers/services/processor/__init__.py -------------------------------------------------------------------------------- /python/dendro/api_helpers/services/processor/_get_fsbucket_signed_upload_url.py: -------------------------------------------------------------------------------- 1 | from typing import Optional 2 | import time 3 | import hashlib 4 | 5 | 6 | def _get_fsbucket_signed_upload_url( 7 | *, fsbucket_api_url, secret_key: str, object_key: str, size: Optional[int] = None 8 | ): 9 | # expires in one hour 10 | expires = int(time.time()) + 3600 11 | signature = _create_signature( 12 | secret_key=secret_key, object_key=object_key, expires=expires, method="PUT" 13 | ) 14 | url = f"{fsbucket_api_url}/{object_key}?signature={signature}&expires={expires}" 15 | return url 16 | 17 | 18 | def _create_signature(*, secret_key: str, object_key: str, expires: int, method: str): 19 | path = f'/{object_key}' 20 | string_to_sign = f"{method}\n{path}\n{expires}\n{secret_key}" 21 | hash = hashlib.sha256() 22 | hash.update(string_to_sign.encode("utf-8")) 23 | return hash.hexdigest() 24 | -------------------------------------------------------------------------------- /python/dendro/api_helpers/services/processor/_get_signed_upload_url.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, Dict, Any 2 | import boto3 3 | from boto3.session import Config 4 | import json 5 | 6 | 7 | async def _get_signed_upload_url(*, 8 | bucket_uri: str, 9 | bucket_credentials: str, 10 | object_key: str, 11 | size: Optional[int] = None 12 | ): 13 | creds = json.loads(bucket_credentials) 14 | access_key_id = creds['accessKeyId'] 15 | secret_access_key = creds['secretAccessKey'] 16 | endpoint = creds.get('endpoint', None) 17 | 18 | region_name = _get_region_name_from_uri(bucket_uri) 19 | bucket_name = _get_bucket_name_from_uri(bucket_uri) 20 | 21 | s3_client = boto3.client( 22 | 's3', 23 | aws_access_key_id=access_key_id, 24 | aws_secret_access_key=secret_access_key, 25 | endpoint_url=endpoint, 26 | region_name=region_name, 27 | config=Config(signature_version='s3v4') 28 | ) 29 | 30 | params: Dict[str, Any] = { 31 | 'Bucket': bucket_name, 32 | 'Key': object_key 33 | } 34 | if size is not None: 35 | params['ContentLength'] = size 36 | 37 | return s3_client.generate_presigned_url( 38 | 'put_object', 39 | Params=params, 40 | ExpiresIn=30 * 60 # 30 minutes 41 | ) 42 | 43 | def _get_bucket_name_from_uri(bucket_uri: str) -> str: 44 | if not bucket_uri: 45 | return '' 46 | return bucket_uri.split('?')[0].split('/')[2] 47 | 48 | def _get_region_name_from_uri(bucket_uri: str) -> str: 49 | # for example: s3://bucket-name?region=us-west-2 50 | if not bucket_uri: 51 | return '' 52 | default_region = 'auto' # for cloudflare 53 | a = bucket_uri.split('?') 54 | if len(a) == 1: 55 | return default_region 56 | b = a[1].split('&') 57 | c = [x for x in b if x.startswith('region=')] 58 | if len(c) == 0: 59 | return default_region 60 | d = c[0].split('=') 61 | if len(d) != 2: 62 | return default_region 63 | return d[1] 64 | -------------------------------------------------------------------------------- /python/dendro/aws_batch/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flatironinstitute/dendro-old/ad3211be0d92a255360801eef68be4f5aca7201e/python/dendro/aws_batch/__init__.py -------------------------------------------------------------------------------- /python/dendro/client/__init__.py: -------------------------------------------------------------------------------- 1 | from .Project import Project, load_project # noqa: F401 2 | from .submit_job import submit_job, SubmitJobInputFile, SubmitJobOutputFile, SubmitJobParameter # noqa: F401 3 | from .set_file import set_file, set_file_metadata # noqa: F401 4 | from ..common.dendro_types import DendroJobRequiredResources # noqa: F401 5 | from ._upload_blob import upload_bytes_blob, upload_file_blob, upload_json_blob, upload_text_blob # noqa: F401 6 | from ._create_batch_id import create_batch_id # noqa: F401 7 | -------------------------------------------------------------------------------- /python/dendro/client/_create_batch_id.py: -------------------------------------------------------------------------------- 1 | import random 2 | 3 | 4 | def create_batch_id() -> str: 5 | """Create a new batch ID string. 6 | 7 | This is a random string that can be used when submitting multiple jobs. 8 | """ 9 | choices = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789' 10 | return ''.join(random.choices(choices, k=8)) 11 | -------------------------------------------------------------------------------- /python/dendro/client/_interim/NwbSorting.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import h5py 3 | 4 | 5 | def NwbSorting(file): 6 | h5_file = h5py.File(file, 'r') 7 | 8 | # Load unit IDs 9 | ids: np.ndarray = h5_file['units']['id'][:] # type: ignore 10 | 11 | # Load spike times index 12 | spike_times_index: np.ndarray = h5_file['units']['spike_times_index'][:] # type: ignore 13 | 14 | # Load spike times 15 | spike_times: np.ndarray = h5_file['units']['spike_times'][:] # type: ignore 16 | 17 | units_dict = {} 18 | sampling_frequency = 30000 # TODO: get this from the NWB file 19 | for i in range(len(ids)): 20 | if i == 0: 21 | s = spike_times[0:spike_times_index[0]] 22 | else: 23 | s = spike_times[spike_times_index[i - 1]:spike_times_index[i]] 24 | units_dict[ids[i]] = (s * sampling_frequency).astype(np.int32) 25 | sorting = _numpy_sorting_from_dict([units_dict], sampling_frequency=sampling_frequency) 26 | return sorting 27 | 28 | def _numpy_sorting_from_dict(units_dict_list, *, sampling_frequency): 29 | import spikeinterface as si # type: ignore 30 | try: 31 | # different versions of spikeinterface 32 | # see: https://github.com/SpikeInterface/spikeinterface/issues/2083 33 | sorting = si.NumpySorting.from_dict( # type: ignore 34 | units_dict_list, sampling_frequency=sampling_frequency # type: ignore 35 | ) 36 | except: # noqa 37 | sorting = si.NumpySorting.from_unit_dict( # type: ignore 38 | units_dict_list, sampling_frequency=sampling_frequency # type: ignore 39 | ) 40 | return sorting 41 | -------------------------------------------------------------------------------- /python/dendro/client/_interim/__init__.py: -------------------------------------------------------------------------------- 1 | from .NwbRecording import NwbRecording # noqa: F401 2 | from .NwbSorting import NwbSorting # noqa: F401 3 | -------------------------------------------------------------------------------- /python/dendro/common/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flatironinstitute/dendro-old/ad3211be0d92a255360801eef68be4f5aca7201e/python/dendro/common/__init__.py -------------------------------------------------------------------------------- /python/dendro/compute_resource/ComputeResourceException.py: -------------------------------------------------------------------------------- 1 | class ComputeResourceException(Exception): 2 | pass 3 | -------------------------------------------------------------------------------- /python/dendro/compute_resource/PubsubClient.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | import queue 3 | from pubnub.pnconfiguration import PNConfiguration 4 | from pubnub.callbacks import SubscribeCallback 5 | from pubnub.pubnub import PubNub 6 | 7 | class MySubscribeCallback(SubscribeCallback): 8 | def __init__(self, message_queue: queue.Queue, compute_resource_id: str): 9 | self._message_queue = message_queue 10 | self._compute_resource_id = compute_resource_id 11 | def message(self, pubnub, message): 12 | msg = message.message 13 | if msg.get('computeResourceId', None) == self._compute_resource_id: 14 | self._message_queue.put(msg) 15 | 16 | class PubsubClient: 17 | def __init__(self, *, 18 | pubnub_subscribe_key: str, 19 | pubnub_channel: str, 20 | pubnub_user: str, 21 | compute_resource_id: str 22 | ): 23 | self._message_queue = queue.Queue() 24 | pnconfig = PNConfiguration() 25 | pnconfig.subscribe_key = pubnub_subscribe_key # type: ignore (not sure why we need to type ignore this) 26 | pnconfig.user_id = pubnub_user 27 | pnconfig.uuid = compute_resource_id 28 | self._pubnub = PubNub(pnconfig) 29 | self._listener = MySubscribeCallback(message_queue=self._message_queue, compute_resource_id=compute_resource_id) 30 | self._pubnub.add_listener(self._listener) 31 | self._pubnub.subscribe().channels([pubnub_channel]).execute() 32 | def take_messages(self) -> List[dict]: 33 | ret = [] 34 | while True: 35 | try: 36 | msg = self._message_queue.get(block=False) 37 | ret.append(msg) 38 | except queue.Empty: 39 | break 40 | return ret 41 | def close(self): 42 | self._pubnub.unsubscribe_all() 43 | self._pubnub.stop() 44 | self._pubnub.remove_listener(self._listener) 45 | # unfortunately this doesn't actually kill the thread 46 | # I submitted a ticket to pubnub about this 47 | # and they acknowledged that it's a problem 48 | # but they don't seem to be fixing it 49 | -------------------------------------------------------------------------------- /python/dendro/compute_resource/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flatironinstitute/dendro-old/ad3211be0d92a255360801eef68be4f5aca7201e/python/dendro/compute_resource/__init__.py -------------------------------------------------------------------------------- /python/dendro/internal_job_monitoring/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flatironinstitute/dendro-old/ad3211be0d92a255360801eef68be4f5aca7201e/python/dendro/internal_job_monitoring/__init__.py -------------------------------------------------------------------------------- /python/dendro/internal_job_monitoring/common.py: -------------------------------------------------------------------------------- 1 | import os 2 | from ..common._api_request import _processor_get_api_request 3 | 4 | 5 | def _get_upload_url(*, job_id: str, job_private_key: str, output_name: str) -> str: 6 | """Get a signed upload URL for the output (console or resource log) of a job""" 7 | url_path = f'/api/processor/jobs/{job_id}/outputs/{output_name}/upload_url' 8 | headers = { 9 | 'job-private-key': job_private_key 10 | } 11 | res = _processor_get_api_request( 12 | url_path=url_path, 13 | headers=headers 14 | ) 15 | return res['uploadUrl'] 16 | 17 | def _process_is_alive(pid: str) -> bool: 18 | """ 19 | Check if a process is alive. 20 | """ 21 | try: 22 | os.kill(int(pid), 0) 23 | return True 24 | except OSError: 25 | return False 26 | -------------------------------------------------------------------------------- /python/dendro/internal_job_monitoring/job_status_monitor.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | from .common import _process_is_alive 4 | from ..common._api_request import _processor_get_api_request 5 | 6 | 7 | def job_status_monitor(parent_pid: str): 8 | """ 9 | Monitor job status to see if the job was canceled. 10 | """ 11 | job_id = os.environ.get('JOB_ID', None) 12 | if job_id is None: 13 | raise KeyError('JOB_ID is not set') 14 | job_private_key = os.environ.get('JOB_PRIVATE_KEY', None) 15 | if job_private_key is None: 16 | raise KeyError('JOB_PRIVATE_KEY is not set') 17 | cancel_out_file = os.environ.get('CANCEL_OUT_FILE', None) 18 | if cancel_out_file is None: 19 | raise KeyError('CANCEL_OUT_FILE is not set') 20 | 21 | last_check_timestamp = 0 22 | overall_timer = time.time() 23 | 24 | while True: 25 | if not _process_is_alive(parent_pid): 26 | print(f'Parent process {parent_pid} is no longer alive. Exiting.') 27 | break 28 | 29 | elapsed_since_check = time.time() - last_check_timestamp 30 | overall_elapsed = time.time() - overall_timer 31 | if overall_elapsed < 60: 32 | interval = 10 33 | elif overall_elapsed < 60 * 5: 34 | interval = 30 35 | elif overall_elapsed < 60 * 20: 36 | interval = 60 37 | else: 38 | interval = 120 39 | if elapsed_since_check >= interval: 40 | last_check_timestamp = time.time() 41 | try: 42 | status = _get_job_status(job_id=job_id, job_private_key=job_private_key) 43 | if status != 'running': 44 | print(f'Job status is {status}. Canceling.') 45 | with open(cancel_out_file, 'w') as f: 46 | if isinstance(status, str): 47 | f.write(status) 48 | else: 49 | f.write('0') 50 | break 51 | else: 52 | print('Job status is running') 53 | except: # noqa 54 | # maybe there was a network error 55 | print('Error getting job status') 56 | 57 | time.sleep(1) 58 | 59 | def _get_job_status(*, job_id: str, job_private_key: str) -> str: 60 | """Get a job status from the dendro API""" 61 | url_path = f'/api/processor/jobs/{job_id}/status' 62 | headers = { 63 | 'job-private-key': job_private_key 64 | } 65 | res = _processor_get_api_request( 66 | url_path=url_path, 67 | headers=headers 68 | ) 69 | return res['status'] 70 | -------------------------------------------------------------------------------- /python/dendro/mock.py: -------------------------------------------------------------------------------- 1 | _globals = { 2 | 'use_mock': False 3 | } 4 | 5 | def using_mock() -> bool: 6 | return _globals['use_mock'] 7 | 8 | def set_use_mock(use_mock: bool): 9 | _globals['use_mock'] = use_mock 10 | -------------------------------------------------------------------------------- /python/dendro/sdk/FileManifest.py: -------------------------------------------------------------------------------- 1 | from typing import List, Union 2 | from pydantic import BaseModel 3 | 4 | 5 | class FileManifestFile(BaseModel): 6 | name: str 7 | url: str 8 | size: Union[int, None] = None 9 | 10 | class FileManifest(BaseModel): 11 | files: List[FileManifestFile] 12 | -------------------------------------------------------------------------------- /python/dendro/sdk/ProcessorBase.py: -------------------------------------------------------------------------------- 1 | from typing import Any, List 2 | 3 | class ProcessorBase: 4 | name: str 5 | label: str 6 | description: str 7 | tags: List[str] 8 | attributes: dict 9 | 10 | @staticmethod 11 | def run( 12 | context: Any 13 | ): 14 | raise NotImplementedError() 15 | -------------------------------------------------------------------------------- /python/dendro/sdk/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | 3 | from .InputFile import InputFile 4 | from .InputFolder import InputFolder 5 | from .OutputFile import OutputFile 6 | from .OutputFolder import OutputFolder 7 | from .App import App 8 | 9 | from .get_project_file_from_uri import get_project_file_from_uri 10 | 11 | from .ProcessorBase import ProcessorBase 12 | 13 | from .. import BaseModel, Field 14 | -------------------------------------------------------------------------------- /python/dendro/sdk/_load_spec_from_uri.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import os 3 | import json 4 | import tempfile 5 | 6 | def _load_spec_from_uri(uri: str) -> dict: 7 | # Convert github blob URL to raw URL 8 | if (uri.startswith('https://github.com/')) and ('/blob/' in uri): 9 | raw_url = uri.replace('github.com', 'raw.githubusercontent.com').replace('/blob/', '/') + f'?cachebust={os.urandom(16).hex()}' 10 | print(f'URL: {raw_url}') 11 | else: 12 | raw_url = uri 13 | 14 | if raw_url.startswith('file://'): 15 | # Read the content from a local file 16 | with open(raw_url[len('file://'):], 'r', encoding='utf-8') as file: 17 | content = file.read() 18 | else: 19 | # Download the content 20 | response = requests.get(raw_url, timeout=60) 21 | response.raise_for_status() 22 | content = response.text 23 | 24 | # Save to a temporary file 25 | with tempfile.NamedTemporaryFile(delete=False, mode='w', suffix='.json') as temp_file: 26 | temp_file.write(content) 27 | temp_file_path = temp_file.name 28 | 29 | # Read the JSON content from the file 30 | with open(temp_file_path, 'r', encoding='utf-8') as file: 31 | data = json.load(file) 32 | 33 | # Clean up the temporary file 34 | os.remove(temp_file_path) 35 | 36 | return data 37 | -------------------------------------------------------------------------------- /python/dendro/sdk/_make_spec_file.py: -------------------------------------------------------------------------------- 1 | import os 2 | import subprocess 3 | from pathlib import Path 4 | 5 | 6 | def make_app_spec_file_function(app_dir: str, spec_output_file: str): 7 | # Ensure the directory path is an absolute path 8 | app_dir_path = Path(app_dir).resolve() 9 | 10 | if spec_output_file is None: 11 | spec_output_file = str(app_dir_path / 'spec.json') 12 | 13 | script_path = str(app_dir_path / 'main.py') 14 | 15 | # check whether script path is executable 16 | if not os.access(script_path, os.X_OK): 17 | raise Exception(f"Script {script_path} is not executable") 18 | 19 | env = os.environ.copy() 20 | env['SPEC_OUTPUT_FILE'] = spec_output_file 21 | subprocess.run([script_path], env=env) 22 | 23 | # When we do it the following way, the inspection of type hints in the processor context does not work 24 | 25 | # # Construct the absolute path to main.py in the specified directory 26 | # main_module_path = app_dir_path / 'main.py' 27 | 28 | # # Check if main.py exists 29 | # if not main_module_path.exists(): 30 | # raise FileNotFoundError(f"main.py not found in {app_dir_path}") 31 | 32 | # # Create a module name from the directory path 33 | # module_name = app_dir_path.name 34 | 35 | # # Use importlib to load the module 36 | # spec = importlib.util.spec_from_file_location(module_name, str(main_module_path)) 37 | # if spec is None: 38 | # raise ImportError(f"Unable to get spec for module {module_name} from {main_module_path}") 39 | # module = importlib.util.module_from_spec(spec) 40 | # if spec.loader is None: 41 | # raise ImportError(f"Unable to get loader for module {module_name} from {main_module_path}") 42 | # spec.loader.exec_module(module) 43 | 44 | # # Check if the App class exists in the loaded module 45 | # if hasattr(module, 'app') and isinstance(getattr(module, 'app'), pr.App): 46 | # # Create an instance of the App class 47 | # app_instance = module.app 48 | 49 | # # Call the make_spec_file method 50 | # app_instance.make_spec_file(spec_output_file) 51 | # else: 52 | # raise AttributeError("App class not found in main.py") 53 | -------------------------------------------------------------------------------- /python/dendro/sdk/_test_app_processor.py: -------------------------------------------------------------------------------- 1 | import os 2 | import subprocess 3 | from pathlib import Path 4 | 5 | 6 | def test_app_processor_function(app_dir: str, processor: str, context: str): 7 | # Ensure the directory path is an absolute path 8 | app_dir_path = Path(app_dir).resolve() 9 | 10 | executable_path = str(app_dir_path / 'main.py') 11 | 12 | env = os.environ.copy() 13 | env['TEST_APP_PROCESSOR'] = '1' 14 | env['PROCESSOR_NAME'] = processor 15 | env['CONTEXT_FILE'] = context 16 | subprocess.run([executable_path], env=env) 17 | -------------------------------------------------------------------------------- /python/dendro/sdk/get_project_file_from_uri.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | 4 | def get_project_file_from_uri(uri: str): 5 | from .Job import _parse_dendro_uri 6 | from .InputFile import InputFile 7 | 8 | job_id = os.environ.get('JOB_ID') 9 | if not job_id: 10 | raise Exception('Cannot get project file when JOB_ID is not set') 11 | job_private_key = os.environ.get('JOB_PRIVATE_KEY') 12 | if not job_private_key: 13 | raise Exception('Cannot get project file when JOB_PRIVATE_KEY is not set') 14 | 15 | file_id, is_folder, label = _parse_dendro_uri(uri) 16 | if is_folder: 17 | raise Exception('Cannot get project file for a folder') 18 | ret = InputFile( 19 | name=None, 20 | url=None, 21 | local_file_name=None, 22 | project_file_uri=uri, # use this instead of name 23 | project_file_name=label, 24 | job_id=job_id, 25 | job_private_key=job_private_key 26 | ) 27 | ret._check_file_cache() 28 | return ret 29 | -------------------------------------------------------------------------------- /python/dendro/version.txt: -------------------------------------------------------------------------------- 1 | 0.2.17 -------------------------------------------------------------------------------- /python/mock-output-file.txt: -------------------------------------------------------------------------------- 1 | mock output -------------------------------------------------------------------------------- /python/pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | markers = 3 | api: tests for api that require the api packages to be installed -------------------------------------------------------------------------------- /python/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | # read version from dendro/version.txt 4 | with open('dendro/version.txt') as f: 5 | __version__ = f.read().strip() 6 | 7 | setup( 8 | name='dendro', 9 | version=__version__, 10 | author="Jeremy Magland, Luiz Tauffer", 11 | author_email="jmagland@flatironinstitute.org", 12 | url="https://github.com/flatironinstitute/dendro", 13 | description="Web framework for neurophysiology data analysis", 14 | packages=find_packages(), 15 | include_package_data=True, 16 | package_data={'dendro': ['version.txt']}, 17 | install_requires=[ 18 | 'click', 19 | 'simplejson', 20 | 'numpy', 21 | 'PyYAML', 22 | 'remfile', 23 | 'pydantic', # intentionally do not specify version 1 or 2 since we support both 24 | 'cryptography', 25 | 'h5py>=3.10.0', 26 | 'psutil' 27 | ], 28 | extras_require={ 29 | 'compute_resource': [ 30 | 'pubnub>=7.2.0', 31 | 'boto3' 32 | ], 33 | 'api': [ 34 | 'fastapi', 35 | 'motor', 36 | 'simplejson', 37 | 'pydantic', 38 | 'aiohttp', 39 | 'boto3' 40 | ] 41 | }, 42 | entry_points={ 43 | "console_scripts": [ 44 | "dendro=dendro.cli:main", 45 | ], 46 | } 47 | ) 48 | -------------------------------------------------------------------------------- /python/tests/mock_app/main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from typing import List, Optional 4 | import time 5 | import os 6 | from dendro import BaseModel, Field 7 | from dendro.sdk import App, ProcessorBase, InputFile, OutputFile 8 | 9 | 10 | class MockParameterGroup(BaseModel): 11 | num: int = Field(description='Number', default=1) 12 | secret_param: str = Field(description='Secret param', default='123', json_schema_extra={'secret': True}) 13 | 14 | class MockProcessor1Context(BaseModel): 15 | input_file: InputFile = Field(description='Input file') 16 | input_list: List[InputFile] = Field(description='Input file list') 17 | output_file: OutputFile = Field(description='Output file') 18 | text1: str = Field(description='Text 1', default='abc') 19 | text2: str = Field(description='Text 2') 20 | text3: str = Field(description='Text 3', default='xyz', json_schema_extra={'options': ['abc', 'xyz']}) 21 | val1: float = Field(description='Value 1', default=1.0) 22 | val2: Optional[float] = Field(description='Value 2 could be a float or it could be None') 23 | group: MockParameterGroup = Field(description='Group', default=MockParameterGroup()) 24 | intentional_error: bool = Field(description='Intentional error', default=False) 25 | 26 | class MockProcessor1(ProcessorBase): 27 | name = 'mock-processor1' 28 | description = 'This is mock processor 1' 29 | label = 'Mock Processor 1' 30 | tags = ['mock-processor1', 'test'] 31 | attributes = {'test': True} 32 | 33 | @staticmethod 34 | def run(context: MockProcessor1Context): 35 | if context.intentional_error: 36 | raise Exception('Received intentional error parameter') 37 | 38 | print('Start mock processor1 in mock app') 39 | assert context.text1 != '' 40 | assert context.input_file.get_url() 41 | assert len(context.input_list) > 0 42 | for input_file in context.input_list: 43 | assert input_file.get_url() 44 | print(f'text1: {context.text1}') 45 | print(f'text2: {context.text2}') 46 | print(f'text3: {context.text3}') 47 | print(f'val1: {context.val1}') 48 | print(f'group.num: {context.group.num}') 49 | time.sleep(0.001) # important not to wait too long because we are calling this synchronously during testing 50 | with open('mock-output-file.txt', 'w') as f: 51 | f.write('mock output') 52 | context.output_file.upload('mock-output-file.txt') 53 | print('End mock processor1 in mock app') 54 | 55 | app = App( 56 | name='test-app', 57 | description='This is a test app', 58 | app_image=None, 59 | app_executable=os.path.abspath(__file__) 60 | ) 61 | 62 | app.add_processor(MockProcessor1) 63 | 64 | if __name__ == '__main__': 65 | app.run() 66 | -------------------------------------------------------------------------------- /python/tests/mock_app_2/main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os 4 | from dendro import BaseModel, Field 5 | from dendro.sdk import App, ProcessorBase 6 | 7 | 8 | class MockProcessor2Context(BaseModel): 9 | text1: str = Field(description='Text 1') 10 | 11 | class MockProcessor2(ProcessorBase): 12 | name = 'mock-processor2' 13 | description = 'This is mock processor 2' 14 | label = 'Mock Processor 2' 15 | tags = ['mock-processor2', 'test'] 16 | attributes = {'test': True} 17 | 18 | @staticmethod 19 | def run(context: MockProcessor2Context): 20 | print('Start mock processor2 in mock app') 21 | assert context.text1 != '' 22 | print('End mock processor2 in mock app') 23 | 24 | app = App( 25 | name='test-app-2', 26 | description='This is a test app 2', 27 | app_image=None, 28 | app_executable=os.path.abspath(__file__) 29 | ) 30 | 31 | app.add_processor(MockProcessor2) 32 | 33 | if __name__ == '__main__': 34 | app.run() 35 | -------------------------------------------------------------------------------- /python/tests/test_api_request_failures.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from dendro.common._api_request import _processor_get_api_request, _processor_put_api_request 3 | from dendro.common._api_request import _client_get_api_request 4 | from dendro.common._api_request import _gui_get_api_request, _gui_put_api_request, _gui_post_api_request, _gui_delete_api_request 5 | 6 | @pytest.mark.api 7 | def test_api_request_failures(): 8 | from dendro.common._api_request import _use_api_test_client 9 | from dendro.mock import set_use_mock 10 | from test_integration import _get_fastapi_app 11 | from dendro.api_helpers.clients._get_mongo_client import _clear_mock_mongo_databases 12 | 13 | from fastapi.testclient import TestClient 14 | app = _get_fastapi_app() 15 | test_client = TestClient(app) 16 | _use_api_test_client(test_client) 17 | set_use_mock(True) 18 | 19 | try: 20 | # from requests import exceptions 21 | with pytest.raises(Exception): 22 | _processor_get_api_request(url_path='/api/incorrect', headers={}) 23 | with pytest.raises(Exception): 24 | _processor_put_api_request(url_path='/api/incorrect', headers={}, data={}) 25 | with pytest.raises(Exception): 26 | _client_get_api_request(url_path='/api/incorrect') 27 | with pytest.raises(Exception): 28 | _gui_get_api_request(url_path='/api/incorrect', github_access_token='incorrect') 29 | with pytest.raises(Exception): 30 | _gui_put_api_request(url_path='/api/incorrect', github_access_token='incorrect', data={}) 31 | with pytest.raises(Exception): 32 | _gui_post_api_request(url_path='/api/incorrect', github_access_token='incorrect', data={}) 33 | with pytest.raises(Exception): 34 | _gui_delete_api_request(url_path='/api/incorrect', github_access_token='incorrect') 35 | finally: 36 | _use_api_test_client(None) 37 | set_use_mock(False) 38 | _clear_mock_mongo_databases() 39 | -------------------------------------------------------------------------------- /python/tests/test_client.py: -------------------------------------------------------------------------------- 1 | import os 2 | from dendro.client import load_project 3 | 4 | 5 | def test_load_project(): 6 | if os.getenv('NO_INTERNET') == '1': 7 | print('Skipping test_load_project because NO_INTERNET is set to 1') 8 | return 9 | 10 | # if this project or file disappears then we'll need to update that here 11 | project_id = 'eb87e88a' 12 | file_name = 'imported/000618/sub-paired-english/sub-paired-english_ses-paired-english-m108-191125-163508_ecephys.nwb' 13 | project = load_project(project_id) 14 | file = project.get_file(file_name) 15 | assert file is not None 16 | assert file.get_url().startswith('https://') 17 | 18 | folder = project.get_folder('imported') 19 | assert len(folder.get_folders()) >= 1 20 | -------------------------------------------------------------------------------- /python/tests/test_compute_resource.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | import tempfile 3 | from dendro.compute_resource.register_compute_resource import register_compute_resource 4 | 5 | 6 | def test_register_compute_resource(): 7 | with TemporaryDirectory() as tmpdir: 8 | register_compute_resource( 9 | dir=tmpdir 10 | ) 11 | 12 | class TemporaryDirectory: 13 | """A context manager for temporary directories""" 14 | def __init__(self): 15 | self._dir = None 16 | def __enter__(self): 17 | self._dir = tempfile.mkdtemp() 18 | return self._dir 19 | def __exit__(self, exc_type, exc_value, traceback): 20 | if self._dir: 21 | shutil.rmtree(self._dir) 22 | -------------------------------------------------------------------------------- /python/tests/test_crypto_keys.py: -------------------------------------------------------------------------------- 1 | from dendro.api_helpers.services._crypto_keys import generate_keypair 2 | 3 | 4 | def test_crypto_keys(): 5 | public_key_hex, private_key_hex = generate_keypair() # this does checks internally 6 | assert public_key_hex 7 | assert private_key_hex 8 | -------------------------------------------------------------------------------- /python/tests/test_github_auth_route.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import os 3 | 4 | 5 | @pytest.mark.asyncio 6 | @pytest.mark.api 7 | async def test_github_auth(): 8 | from dendro.api_helpers.routers.gui.github_auth_routes import github_auth, GithubAuthError 9 | old_env = os.environ.copy() 10 | try: 11 | os.environ['VITE_GITHUB_CLIENT_ID'] = 'test-client-id' 12 | os.environ['GITHUB_CLIENT_SECRET'] = 'test-client-secret' 13 | with pytest.raises(GithubAuthError): 14 | await github_auth(code='test-code') 15 | finally: 16 | os.environ = old_env 17 | -------------------------------------------------------------------------------- /python/tests/test_sdk.py: -------------------------------------------------------------------------------- 1 | import shutil 2 | import tempfile 3 | from dendro import BaseModel, Field 4 | from dendro.sdk import App, ProcessorBase, InputFile, OutputFile 5 | from dendro.mock import set_use_mock 6 | 7 | 8 | def test_app(): 9 | set_use_mock(True) 10 | 11 | try: 12 | class Processor1Context(BaseModel): 13 | input_file: InputFile = Field(description='Input file') 14 | output_file: OutputFile = Field(description='Output file') 15 | text1: str = Field(description='Text 1', default='abc') 16 | text2: str = Field(description='Text 2') 17 | text3: str = Field(description='Text 3', default='xyz', json_schema_extra={'options': ['abc', 'xyz']}) 18 | val1: float = Field(description='Value 1', default=1.0) 19 | 20 | class Processor1(ProcessorBase): 21 | name = 'processor1' 22 | description = 'This is processor 1' 23 | label = 'Processor 1' 24 | tags = ['processor1', 'test'] 25 | attributes = {'test': True} 26 | 27 | @staticmethod 28 | def run(context: Processor1Context): 29 | assert context.text1 != '' 30 | 31 | app = App( 32 | name='test-app', 33 | description='This is a test app', 34 | app_image='fake-image' 35 | ) 36 | 37 | app.add_processor(Processor1) 38 | 39 | spec = app.get_spec() 40 | assert spec['name'] == 'test-app' 41 | finally: 42 | set_use_mock(False) 43 | 44 | class TemporaryDirectory: 45 | """A context manager for temporary directories""" 46 | def __init__(self): 47 | self._dir = None 48 | def __enter__(self): 49 | self._dir = tempfile.mkdtemp() 50 | return self._dir 51 | def __exit__(self, exc_type, exc_value, traceback): 52 | if self._dir: 53 | shutil.rmtree(self._dir) 54 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # NOTE: these are the requirements for the api, picked up by vercel 2 | fastapi 3 | motor 4 | simplejson 5 | # was having some trouble with the latest version of pydantic, so pinning it for now 6 | # error was: "No module named 'pydantic_core._pydantic_core'" 7 | pydantic==2.4 8 | aiohttp 9 | cryptography 10 | boto3 -------------------------------------------------------------------------------- /src/App.css: -------------------------------------------------------------------------------- 1 | /* #root { 2 | max-width: 1280px; 3 | margin: 0 auto; 4 | padding: 2rem; 5 | text-align: center; 6 | } 7 | 8 | .logo { 9 | height: 6em; 10 | padding: 1.5em; 11 | will-change: filter; 12 | transition: filter 300ms; 13 | } 14 | .logo:hover { 15 | filter: drop-shadow(0 0 2em #646cffaa); 16 | } 17 | .logo.react:hover { 18 | filter: drop-shadow(0 0 2em #61dafbaa); 19 | } 20 | 21 | @keyframes logo-spin { 22 | from { 23 | transform: rotate(0deg); 24 | } 25 | to { 26 | transform: rotate(360deg); 27 | } 28 | } 29 | 30 | @media (prefers-reduced-motion: no-preference) { 31 | a:nth-of-type(2) .logo { 32 | animation: logo-spin infinite 20s linear; 33 | } 34 | } 35 | 36 | .card { 37 | padding: 2em; 38 | } 39 | 40 | .read-the-docs { 41 | color: #888; 42 | } */ 43 | -------------------------------------------------------------------------------- /src/App.tsx: -------------------------------------------------------------------------------- 1 | import { BrowserRouter } from 'react-router-dom' 2 | import './App.css' 3 | import MainWindow from './MainWindow' 4 | import GithubAuthSetup from './GithubAuth/GithubAuthSetup' 5 | import { SetupDendro } from './DendroContext/DendroContext' 6 | 7 | function App() { 8 | return ( 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | ) 17 | } 18 | 19 | export default App 20 | -------------------------------------------------------------------------------- /src/ComputeResourceNameDisplay.tsx: -------------------------------------------------------------------------------- 1 | import { Hyperlink } from "@fi-sci/misc"; 2 | import { FunctionComponent, useMemo } from "react"; 3 | import { useComputeResources } from "./pages/ComputeResourcesPage/ComputeResourcesContext"; 4 | import useRoute from "./useRoute"; 5 | 6 | type Props = { 7 | computeResourceId: string | undefined 8 | link?: boolean 9 | } 10 | 11 | const ComputeResourceNameDisplay: FunctionComponent = ({ computeResourceId, link }) => { 12 | const {computeResources} = useComputeResources() 13 | const displayString = useMemo(() => { 14 | if (!computeResourceId) return 'DEFAULT' 15 | const cr = computeResources.find(cr => cr.computeResourceId === computeResourceId) 16 | return (cr ? cr.name + ` (${abbreviate(computeResourceId, 9)})` : undefined) || abbreviate(computeResourceId, 16) 17 | }, [computeResources, computeResourceId]) 18 | const {setRoute} = useRoute() 19 | const a = {displayString || ''} 20 | const crId = computeResourceId || import.meta.env.VITE_DEFAULT_COMPUTE_RESOURCE_ID 21 | if (link) { 22 | return setRoute({page: 'compute-resource', computeResourceId: crId})}>{a} 23 | } 24 | else { 25 | return a 26 | } 27 | } 28 | 29 | function abbreviate(s: string, maxLength: number) { 30 | if (s.length <= maxLength) return s 31 | return s.slice(0, maxLength - 3) + '...' 32 | } 33 | 34 | export default ComputeResourceNameDisplay -------------------------------------------------------------------------------- /src/DendroContext/DendroContext.tsx: -------------------------------------------------------------------------------- 1 | import React, { FunctionComponent, PropsWithChildren, useCallback, useMemo } from 'react'; 2 | import { DendroProject } from '../types/dendro-types'; 3 | 4 | type Props = { 5 | // none 6 | } 7 | 8 | type DendroState = { 9 | loadedProjects: DendroProject[] 10 | } 11 | 12 | type DendroAction = { 13 | type: 'reportLoadedProject' 14 | project: DendroProject 15 | } 16 | 17 | const dendroReducer = (state: DendroState, action: DendroAction) => { 18 | switch (action.type) { 19 | case 'reportLoadedProject': 20 | return { 21 | ...state, 22 | loadedProjects: [...state.loadedProjects.filter(x => x.projectId !== action.project.projectId), action.project] 23 | } 24 | } 25 | } 26 | 27 | type DendroContextType = { 28 | loadedProjects: DendroProject[] 29 | reportLoadedProject: (project: DendroProject) => void 30 | } 31 | 32 | const DendroContext = React.createContext({ 33 | loadedProjects: [], 34 | reportLoadedProject: () => {} 35 | }) 36 | 37 | export const SetupDendro: FunctionComponent> = ({children}) => { 38 | const [state, dispatch] = React.useReducer(dendroReducer, { 39 | loadedProjects: [] 40 | }) 41 | 42 | const value = { 43 | loadedProjects: state.loadedProjects, 44 | reportLoadedProject: useCallback((project: DendroProject) => { 45 | dispatch({ 46 | type: 'reportLoadedProject', 47 | project 48 | }) 49 | }, []) 50 | } 51 | 52 | return ( 53 | 54 | {children} 55 | 56 | ) 57 | } 58 | 59 | export const useDendro = () => { 60 | return React.useContext(DendroContext) 61 | } -------------------------------------------------------------------------------- /src/GitHub/GitHubAuthPage.tsx: -------------------------------------------------------------------------------- 1 | import { FunctionComponent, useEffect, useState } from "react"; 2 | import { setGitHubTokenInfoToLocalStorage } from "../GithubAuth/getGithubAuthFromLocalStorage"; 3 | import { apiBase } from "../dbInterface/dbInterface"; 4 | 5 | type Props = any 6 | 7 | function parseQuery(queryString: string) { 8 | const ind = queryString.indexOf('?') 9 | if (ind <0) return {} 10 | const query: {[k: string]: string} = {}; 11 | const pairs = queryString.slice(ind + 1).split('&'); 12 | for (let i = 0; i < pairs.length; i++) { 13 | const pair = pairs[i].split('='); 14 | query[decodeURIComponent(pair[0])] = decodeURIComponent(pair[1] || ''); 15 | } 16 | return query; 17 | } 18 | 19 | // Important to do it this way because it is difficult to handle special characters (especially #) by using URLSearchParams or window.location.search 20 | const queryParams = parseQuery(window.location.href) 21 | 22 | const GitHubAuthPage: FunctionComponent = () => { 23 | const [status, setStatus] = useState<'checking' | 'okay' | 'error'>('checking') 24 | const [error, setError] = useState('') 25 | const code = queryParams.code 26 | useEffect(() => { 27 | (async () => { 28 | const rr = await fetch(`${apiBase}/api/gui/github_auth/${code}`) 29 | const r = await rr.json() 30 | if ((!r.access_token) || (r.error)) { 31 | setStatus('error') 32 | setError(r.error) 33 | return 34 | } 35 | setGitHubTokenInfoToLocalStorage({ 36 | token: r.access_token, 37 | isPersonalAccessToken: false 38 | }) 39 | setStatus('okay') 40 | })() 41 | }, [code]) 42 | return ( 43 |
44 | { 45 | status === 'checking' ? ( 46 |
Checking authorization
47 | ) : status === 'okay' ? ( 48 |
Logged in. You may now close this window.
49 | ) : status === 'error' ? ( 50 |
Error: {error}
51 | ) : ( 52 |
Unexpected status: {status}
53 | ) 54 | } 55 |
56 | ) 57 | } 58 | 59 | export default GitHubAuthPage 60 | -------------------------------------------------------------------------------- /src/GitHub/PersonalAccessTokenWindow.tsx: -------------------------------------------------------------------------------- 1 | import { Button, Input } from "@mui/material"; 2 | import { FunctionComponent, useCallback, useState } from "react"; 3 | import { getGitHubTokenInfoFromLocalStorage, setGitHubTokenInfoToLocalStorage } from "../GithubAuth/getGithubAuthFromLocalStorage"; 4 | 5 | type Props ={ 6 | onChange: () => void 7 | } 8 | 9 | const PersonalAccessTokenWindow: FunctionComponent = ({onChange}) => { 10 | const [newToken, setNewToken] = useState('') 11 | const handleNewTokenChange: React.ChangeEventHandler = useCallback((e) => { 12 | setNewToken(e.target.value as string) 13 | }, []) 14 | const handleSubmit = useCallback(() => { 15 | setGitHubTokenInfoToLocalStorage({token: newToken, isPersonalAccessToken: true}) 16 | setNewToken('') 17 | onChange() 18 | }, [newToken, onChange]) 19 | 20 | const oldTokenInfo = getGitHubTokenInfoFromLocalStorage() 21 | 22 | return ( 23 |
24 |

25 | To write to public GitHub repositories and to read and write from private GitHub repositories 26 | you will need to set a GitHub access token. This token will be stored in the local storage of your browser. 27 | You should create a personal access token 28 |  with the least amount of permissions needed. 29 |

30 |

31 | Use the classic type personal access token with repo scope. 32 |

33 | 34 | 35 | { 36 | newToken && ( 37 |
38 | ) 39 | } 40 | { 41 | oldTokenInfo?.token ? ( 42 | 43 |

GitHub access token has been set.

44 | 45 | ) : ( 46 | 47 |

GitHub access token not set.

48 | 49 | ) 50 | } 51 |
52 | ) 53 | } 54 | 55 | export default PersonalAccessTokenWindow 56 | -------------------------------------------------------------------------------- /src/GithubAuth/GithubAuthContext.ts: -------------------------------------------------------------------------------- 1 | import React from 'react' 2 | 3 | export type GithubAuthData = { 4 | signedIn: boolean 5 | userId?: string, 6 | accessToken?: string 7 | isPersonalAccessToken?: boolean 8 | clearAccessToken: () => void 9 | loginStatus?: 'not-logged-in' | 'checking' | 'logged-in' 10 | } 11 | 12 | const dummyGithubAuthData: GithubAuthData = {signedIn: false, clearAccessToken: () => {}} 13 | 14 | const GithubAuthContext = React.createContext(dummyGithubAuthData) 15 | 16 | export default GithubAuthContext -------------------------------------------------------------------------------- /src/GithubAuth/GithubAuthSetup.tsx: -------------------------------------------------------------------------------- 1 | import React, { FunctionComponent, PropsWithChildren } from 'react'; 2 | import GithubAuthContext from './GithubAuthContext'; 3 | import useSetupGithubAuth from './useSetupGithubAuth'; 4 | 5 | const GithubAuthSetup: FunctionComponent = (props) => { 6 | const githubAuthData = useSetupGithubAuth() 7 | return ( 8 | 9 | {props.children} 10 | 11 | ) 12 | } 13 | 14 | export default GithubAuthSetup -------------------------------------------------------------------------------- /src/GithubAuth/getGithubAuthFromLocalStorage.ts: -------------------------------------------------------------------------------- 1 | import validateObject, { isBoolean, isNumber, isString, optional } from "../types/validateObject" 2 | 3 | export type GitHubTokenInfo = { 4 | token?: string 5 | userId?: string 6 | userIdTimestamp?: number 7 | isPersonalAccessToken?: boolean 8 | } 9 | 10 | export const isGithubTokenInfo = (x: any): x is GitHubTokenInfo => { 11 | return validateObject(x, { 12 | token: optional(isString), 13 | userId: optional(isString), 14 | userIdTimestamp: optional(isNumber), 15 | isPersonalAccessToken: optional(isBoolean) 16 | }) 17 | } 18 | 19 | export const setGitHubTokenInfoToLocalStorage = (tokenInfo: GitHubTokenInfo) => { 20 | localStorage.setItem('githubToken', JSON.stringify(tokenInfo)) 21 | } 22 | 23 | export const getGitHubTokenInfoFromLocalStorage = (): GitHubTokenInfo | undefined => { 24 | const a = localStorage.getItem('githubToken') 25 | if (!a) return undefined 26 | try { 27 | const b = JSON.parse(a) 28 | if (isGithubTokenInfo(b)) { 29 | return b 30 | } 31 | else { 32 | console.warn(b) 33 | console.warn('Invalid GitHub token info.') 34 | localStorage.removeItem('githubToken') 35 | return undefined 36 | } 37 | } 38 | catch { 39 | console.warn(a) 40 | console.warn('Error with github token info.') 41 | return undefined 42 | } 43 | } -------------------------------------------------------------------------------- /src/GithubAuth/useGithubAuth.ts: -------------------------------------------------------------------------------- 1 | import { useContext } from "react" 2 | import GithubAuthContext from "./GithubAuthContext" 3 | 4 | export const useGithubAuth = () => { 5 | return useContext(GithubAuthContext) 6 | } -------------------------------------------------------------------------------- /src/RemoteH5File/h5wasm/file_handlers.js: -------------------------------------------------------------------------------- 1 | import { ready } from "./hdf5_hl.js"; 2 | export const UPLOADED_FILES = []; 3 | export async function uploader(event) { 4 | const { FS } = await ready; 5 | const target = event.target; 6 | let file = target.files?.[0]; // only one file allowed 7 | if (file) { 8 | let datafilename = file.name; 9 | let ab = await file.arrayBuffer(); 10 | FS.writeFile(datafilename, new Uint8Array(ab)); 11 | if (!UPLOADED_FILES.includes(datafilename)) { 12 | UPLOADED_FILES.push(datafilename); 13 | console.log("file loaded:", datafilename); 14 | } 15 | else { 16 | console.log("file updated: ", datafilename); 17 | } 18 | target.value = ""; 19 | } 20 | } 21 | function create_downloader() { 22 | let a = document.createElement("a"); 23 | document.body.appendChild(a); 24 | a.style.display = "none"; 25 | a.id = "savedata"; 26 | return function (data, fileName) { 27 | let blob = (data instanceof Blob) ? data : new Blob([data], { type: 'application/x-hdf5' }); 28 | // IE 10 / 11 29 | const nav = window.navigator; 30 | if (nav.msSaveOrOpenBlob) { 31 | nav.msSaveOrOpenBlob(blob, fileName); 32 | } 33 | else { 34 | let url = window.URL.createObjectURL(blob); 35 | a.href = url; 36 | a.download = fileName; 37 | a.target = "_blank"; 38 | //window.open(url, '_blank', fileName); 39 | a.click(); 40 | setTimeout(function () { window.URL.revokeObjectURL(url); }, 1000); 41 | } 42 | // cleanup: this seems to break things! 43 | //document.body.removeChild(a); 44 | }; 45 | } 46 | ; 47 | export const downloader = create_downloader(); 48 | export async function to_blob(hdf5_file) { 49 | const { FS } = await ready; 50 | hdf5_file.flush(); 51 | return new Blob([FS.readFile(hdf5_file.filename)], { type: 'application/x-hdf5' }); 52 | } 53 | export async function download(hdf5_file) { 54 | let b = await to_blob(hdf5_file); 55 | downloader(b, hdf5_file.filename); 56 | } 57 | export function dirlisting(path, FS) { 58 | let node = FS.analyzePath(path).object; 59 | if (node && node.isFolder) { 60 | let files = Object.values(node.contents).filter(v => !(v.isFolder)).map(v => v.name); 61 | let subfolders = Object.values(node.contents).filter(v => (v.isFolder)).map(v => v.name); 62 | return { files, subfolders }; 63 | } 64 | else { 65 | return {}; 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/RemoteH5File/h5wasm/readme.txt: -------------------------------------------------------------------------------- 1 | The files in this directory were taken from a transpiled snapshot of h5wasm 2 | https://github.com/usnistgov/h5wasm 3 | 4 | Various modifications were made in hdf5_util_jfm.js to support specification 5 | of the chunk size, which is important for performance reasons. 6 | 7 | Another modification was to extract out the wasm binary program into a separate 8 | file (wasmBinaryFile.js). This was done to avoid including multiple modified 9 | versions of a large file (3.2M) in the git repo. The idea being that 10 | wasmBinaryFile.js should not change over a relatively long period of time, 11 | whereas hdf5_util_jfm.js may be tweaked more often. 12 | 13 | I am sure there's a better way to deal with this situation, but this is what 14 | I'm doing for now. 15 | 16 | - Jeremy 7/23/2023 -------------------------------------------------------------------------------- /src/TabWidget/TabWidgetTabBar.tsx: -------------------------------------------------------------------------------- 1 | import { Tab, Tabs } from '@mui/material'; 2 | import { FunctionComponent, useEffect } from 'react'; 3 | 4 | type Props = { 5 | tabs: { 6 | id: string 7 | label: string 8 | closeable: boolean 9 | icon?: any 10 | title?: string 11 | }[] 12 | currentTabIndex: number | undefined 13 | onCurrentTabIndexChanged: (i: number) => void 14 | onCloseTab: (id: string) => void 15 | } 16 | 17 | const TabWidgetTabBar: FunctionComponent = ({ tabs, currentTabIndex, onCurrentTabIndexChanged, onCloseTab }) => { 18 | useEffect(() => { 19 | if (currentTabIndex === undefined) { 20 | if (tabs.length > 0) { 21 | onCurrentTabIndexChanged(0) 22 | } 23 | } 24 | }, [currentTabIndex, onCurrentTabIndexChanged, tabs.length]) 25 | return ( 26 | {onCurrentTabIndexChanged(value)}} 31 | > 32 | {tabs.map((tab, i) => ( 33 | 36 | { 37 | tab.icon ? ( 38 | {tab.icon} 39 | ) : 40 | } 41 | {tab.label} 42 |    {onCloseTab(tab.id)}}>✕ 43 | 44 | ) : ( 45 | tab.label 46 | ) 47 | } sx={{minHeight: 0, height: 0, fontSize: 12}} /> 48 | ))} 49 | 50 | ) 51 | } 52 | 53 | export default TabWidgetTabBar -------------------------------------------------------------------------------- /src/UserIdComponent.tsx: -------------------------------------------------------------------------------- 1 | import { FunctionComponent } from "react"; 2 | 3 | type Props = { 4 | userId: string | undefined 5 | } 6 | 7 | const UserIdComponent: FunctionComponent = ({ userId }) => { 8 | const x = userId ? (userId.startsWith('github|') ? userId.slice('github|'.length) : userId) : '' 9 | return {x} 10 | } 11 | 12 | export default UserIdComponent -------------------------------------------------------------------------------- /src/confirm_prompt_alert.ts: -------------------------------------------------------------------------------- 1 | export const confirm = async (message: string): Promise => { 2 | return window.confirm(message) 3 | } 4 | 5 | export const prompt = async (message: string, defaultValue: string): Promise => { 6 | return window.prompt(message, defaultValue) 7 | } 8 | 9 | export const alert = async (message: string): Promise => { 10 | return window.alert(message) 11 | } -------------------------------------------------------------------------------- /src/dbInterface/getAuthorizationHeaderForUrl.ts: -------------------------------------------------------------------------------- 1 | const getAuthorizationHeaderForUrl = (url?: string) => { 2 | if (!url) return '' 3 | let key = '' 4 | if (url.startsWith('https://api-staging.dandiarchive.org/')) { 5 | key = localStorage.getItem('dandiStagingApiKey') || '' 6 | } 7 | else if (url.startsWith('https://api.dandiarchive.org/')) { 8 | key = localStorage.getItem('dandiApiKey') || '' 9 | } 10 | if (key) return 'token ' + key 11 | else return '' 12 | } 13 | 14 | export default getAuthorizationHeaderForUrl -------------------------------------------------------------------------------- /src/index.css: -------------------------------------------------------------------------------- 1 | body { 2 | margin: 0; 3 | font-family: 'Arial', sans-serif; 4 | } 5 | 6 | /* :root { 7 | font-family: Inter, system-ui, Avenir, Helvetica, Arial, sans-serif; 8 | line-height: 1.5; 9 | font-weight: 400; 10 | 11 | color-scheme: light dark; 12 | color: rgba(255, 255, 255, 0.87); 13 | background-color: #242424; 14 | 15 | font-synthesis: none; 16 | text-rendering: optimizeLegibility; 17 | -webkit-font-smoothing: antialiased; 18 | -moz-osx-font-smoothing: grayscale; 19 | -webkit-text-size-adjust: 100%; 20 | } 21 | 22 | a { 23 | font-weight: 500; 24 | color: #646cff; 25 | text-decoration: inherit; 26 | } 27 | a:hover { 28 | color: #535bf2; 29 | } 30 | 31 | body { 32 | margin: 0; 33 | display: flex; 34 | place-items: center; 35 | min-width: 320px; 36 | min-height: 100vh; 37 | } 38 | 39 | h1 { 40 | font-size: 3.2em; 41 | line-height: 1.1; 42 | } 43 | 44 | button { 45 | border-radius: 8px; 46 | border: 1px solid transparent; 47 | padding: 0.6em 1.2em; 48 | font-size: 1em; 49 | font-weight: 500; 50 | font-family: inherit; 51 | background-color: #1a1a1a; 52 | cursor: pointer; 53 | transition: border-color 0.25s; 54 | } 55 | button:hover { 56 | border-color: #646cff; 57 | } 58 | button:focus, 59 | button:focus-visible { 60 | outline: 4px auto -webkit-focus-ring-color; 61 | } 62 | 63 | @media (prefers-color-scheme: light) { 64 | :root { 65 | color: #213547; 66 | background-color: #ffffff; 67 | } 68 | a:hover { 69 | color: #747bff; 70 | } 71 | button { 72 | background-color: #f9f9f9; 73 | } 74 | } */ 75 | -------------------------------------------------------------------------------- /src/main.tsx: -------------------------------------------------------------------------------- 1 | import ReactDOM from 'react-dom/client' 2 | import App from './App' 3 | import './index.css' 4 | import './table1.css' 5 | import './scientific-table.css' 6 | 7 | ReactDOM.createRoot(document.getElementById('root') as HTMLElement).render( 8 | // 9 | 10 | // , 11 | ) 12 | -------------------------------------------------------------------------------- /src/pages/AboutPage/AboutPage.tsx: -------------------------------------------------------------------------------- 1 | import { FunctionComponent } from "react"; 2 | 3 | type Props = { 4 | width: number 5 | height: number 6 | } 7 | 8 | const AboutPage: FunctionComponent = ({width, height}) => { 9 | return ( 10 |
11 | This is dendro. 12 |
13 | ) 14 | } 15 | 16 | export default AboutPage -------------------------------------------------------------------------------- /src/pages/AdminPage/AdminPage.tsx: -------------------------------------------------------------------------------- 1 | import { FunctionComponent } from "react" 2 | import ProjectsTable from "../ProjectsPage/ProjectsTable" 3 | 4 | type AdminPageProps = { 5 | width: number 6 | height: number 7 | } 8 | 9 | const AdminPage: FunctionComponent = ({width, height}) => { 10 | return ( 11 | 14 | ) 15 | } 16 | 17 | export default AdminPage -------------------------------------------------------------------------------- /src/pages/ComputeResourcePage/ComputeResourceAppsTableMenuBar.tsx: -------------------------------------------------------------------------------- 1 | import { Add, Delete, Edit } from "@mui/icons-material" 2 | import { FunctionComponent, useCallback } from "react" 3 | import { SmallIconButton } from "@fi-sci/misc"; 4 | import { confirm } from "../../confirm_prompt_alert" 5 | 6 | type ComputeResourceAppsTableMenuBarProps = { 7 | width: number 8 | height: number 9 | selectedAppNames: string[] 10 | onDeleteApps: (appNames: string[]) => void 11 | onAddApp: () => void 12 | onEditApp: () => void 13 | } 14 | 15 | const ComputeResourceAppsTableMenuBar: FunctionComponent = ({width, height, selectedAppNames, onAddApp, onDeleteApps, onEditApp}) => { 16 | const handleDelete = useCallback(async () => { 17 | const okay = await confirm(`Are you sure you want to delete these ${selectedAppNames.length} apps?`) 18 | if (!okay) return 19 | onDeleteApps(selectedAppNames) 20 | }, [selectedAppNames, onDeleteApps]) 21 | 22 | return ( 23 |
24 | 25 | } 27 | title={"Add an app"} 28 | label="Add app" 29 | onClick={onAddApp} 30 | /> 31 |        32 | 33 | } 35 | disabled={(selectedAppNames.length === 0)} 36 | title={selectedAppNames.length > 0 ? `Delete these ${selectedAppNames.length} apps` : ''} 37 | onClick={handleDelete} 38 | /> 39 |     40 | } 42 | disabled={(selectedAppNames.length !== 1)} 43 | title={selectedAppNames.length === 1 ? `Edit app ${selectedAppNames[0]}` : ''} 44 | onClick={onEditApp} 45 | /> 46 |
47 | ) 48 | } 49 | 50 | export default ComputeResourceAppsTableMenuBar -------------------------------------------------------------------------------- /src/pages/ComputeResourcesPage/ComputeResourcesContext.tsx: -------------------------------------------------------------------------------- 1 | import React, { FunctionComponent, PropsWithChildren, useCallback, useEffect, useMemo, useState } from 'react'; 2 | import { deleteComputeResource, fetchComputeResources } from '../../dbInterface/dbInterface'; 3 | import { useGithubAuth } from '../../GithubAuth/useGithubAuth'; 4 | import { DendroComputeResource } from '../../types/dendro-types'; 5 | 6 | type Props = { 7 | // none 8 | } 9 | 10 | type ComputeResourcesContextType = { 11 | computeResources: DendroComputeResource[] 12 | refreshComputeResources: () => void 13 | deleteComputeResource: (computeResourceId: string) => void 14 | } 15 | 16 | const ComputeResourcesContext = React.createContext({ 17 | computeResources: [], 18 | refreshComputeResources: () => {}, 19 | deleteComputeResource: () => {} 20 | }) 21 | 22 | export const SetupComputeResources: FunctionComponent> = ({children}) => { 23 | const [computeResources, setComputeResources] = useState([]) 24 | const [refreshComputeResourcesCode, setRefreshComputeResourcesCode] = useState(0) 25 | const refreshComputeResources = useCallback(() => setRefreshComputeResourcesCode(rc => rc + 1), []) 26 | 27 | const auth = useGithubAuth() 28 | 29 | useEffect(() => { 30 | (async () => { 31 | setComputeResources([]) 32 | if (!auth) return 33 | const cr = await fetchComputeResources(auth) 34 | setComputeResources(cr) 35 | })() 36 | }, [auth, refreshComputeResourcesCode]) 37 | 38 | const deleteComputeResourceHandler = useCallback(async (computeResourceId: string) => { 39 | if (!auth) return 40 | await deleteComputeResource(computeResourceId, auth) 41 | refreshComputeResources() 42 | }, [auth, refreshComputeResources]) 43 | 44 | const value = React.useMemo(() => ({ 45 | computeResources, 46 | refreshComputeResources, 47 | deleteComputeResource: deleteComputeResourceHandler 48 | }), [computeResources, refreshComputeResources, deleteComputeResourceHandler]) 49 | 50 | return ( 51 | 52 | {children} 53 | 54 | ) 55 | } 56 | 57 | export const useComputeResources = () => { 58 | const context = React.useContext(ComputeResourcesContext) 59 | return context 60 | } -------------------------------------------------------------------------------- /src/pages/ComputeResourcesPage/ComputeResourcesPage.css: -------------------------------------------------------------------------------- 1 | .compute-resources-page { 2 | padding: 20px; 3 | font-family: Arial, sans-serif; 4 | max-width: 800px; 5 | margin: auto; 6 | } 7 | 8 | .compute-resources-page h1, .compute-resources-page h2 { 9 | color: #333; 10 | text-align: center; 11 | } 12 | 13 | .compute-resources-page p { 14 | margin-bottom: 20px; 15 | text-align: justify; 16 | } -------------------------------------------------------------------------------- /src/pages/ComputeResourcesPage/ComputeResourcesPage.tsx: -------------------------------------------------------------------------------- 1 | import { FunctionComponent } from "react"; 2 | import { SetupComputeResources } from "./ComputeResourcesContext"; 3 | import ComputeResourcesTable from "./ComputeResourcesTable"; 4 | import './ComputeResourcesPage.css' 5 | 6 | type Props = { 7 | width: number 8 | height: number 9 | } 10 | 11 | const ComputeResourcesPage: FunctionComponent = ({width, height}) => { 12 | return ( 13 | 14 |
15 |

Your compute resources

16 | 17 |
18 |

19 | 20 | Add a compute resource 21 | 22 |

23 |
24 | 25 | ) 26 | } 27 | 28 | export default ComputeResourcesPage -------------------------------------------------------------------------------- /src/pages/ComputeResourcesPage/ComputeResourcesTable.tsx: -------------------------------------------------------------------------------- 1 | import { Delete } from "@mui/icons-material" 2 | import { FunctionComponent, useCallback } from "react" 3 | import { Hyperlink } from "@fi-sci/misc"; 4 | import ComputeResourceNameDisplay from "../../ComputeResourceNameDisplay" 5 | import { confirm } from "../../confirm_prompt_alert" 6 | import { timeAgoString } from "../../timeStrings" 7 | import UserIdComponent from "../../UserIdComponent" 8 | import useRoute from "../../useRoute" 9 | import { useComputeResources } from "./ComputeResourcesContext" 10 | 11 | type Props = { 12 | // none 13 | } 14 | 15 | const ComputeResourcesTable: FunctionComponent = () => { 16 | const {computeResources, deleteComputeResource} = useComputeResources() 17 | 18 | const handleDeleteComputeResource = useCallback(async (computeResourceId: string) => { 19 | const okay = await confirm('Are you sure you want to delete this compute resource?') 20 | if (!okay) return 21 | deleteComputeResource(computeResourceId) 22 | }, [deleteComputeResource]) 23 | 24 | const { setRoute } = useRoute() 25 | 26 | return ( 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | { 39 | computeResources.map((cr) => ( 40 | 41 | 42 | 47 | 52 | 53 | 54 | 55 | )) 56 | } 57 | 58 |
Compute resourceIDOwnerCreated
handleDeleteComputeResource(cr.computeResourceId)} /> 43 | setRoute({page: 'compute-resource', computeResourceId: cr.computeResourceId})}> 44 | {cr.name} 45 | 46 | 48 | setRoute({page: 'compute-resource', computeResourceId: cr.computeResourceId})}> 49 | 50 | 51 | {timeAgoString(cr.timestampCreated)}
59 | ) 60 | } 61 | 62 | export default ComputeResourcesTable -------------------------------------------------------------------------------- /src/pages/DandiBrowser/formatByteCount.ts: -------------------------------------------------------------------------------- 1 | const formatByteCount = (a: number) => { 2 | if (a < 10000) { 3 | return `${a} bytes` 4 | } 5 | else if (a < 100 * 1024) { 6 | return `${formatNum(a / 1024)} KB` 7 | } 8 | else if (a < 100 * 1024 * 1024) { 9 | return `${formatNum(a / (1024 * 1024))} MB` 10 | } 11 | else if (a < 600 * 1024 * 1024 * 1024) { 12 | return `${formatNum(a / (1024 * 1024 * 1025))} GB` 13 | } 14 | else { 15 | return `${formatNum(a / (1024 * 1024 * 1024 * 1024))} TB` 16 | } 17 | } 18 | 19 | export const formatGiBCount = (a: number) => { 20 | return formatByteCount(a * 1000 * 1000 * 1000) 21 | } 22 | 23 | const formatNum = (a: number) => { 24 | const b = a.toFixed(2) 25 | if (Number(b) - Math.floor(Number(b)) === 0) { 26 | return a.toFixed(0) 27 | } 28 | else return b 29 | } 30 | 31 | export default formatByteCount -------------------------------------------------------------------------------- /src/pages/DandiBrowser/useProjectsForTag.ts: -------------------------------------------------------------------------------- 1 | import { useEffect, useState } from "react"; 2 | import { DendroProject } from "../../types/dendro-types"; 3 | import { useGithubAuth } from "../../GithubAuth/useGithubAuth"; 4 | import { fetchProjectsForTag } from "../../dbInterface/dbInterface"; 5 | 6 | const useProjectsForTag = (tag: string | undefined): DendroProject[] | undefined => { 7 | const [projects, setProjects] = useState(undefined) 8 | const auth = useGithubAuth() 9 | useEffect(() => { 10 | let canceled = false 11 | 12 | if (!tag) { 13 | setProjects([]) 14 | return 15 | } 16 | 17 | setProjects(undefined) 18 | 19 | ; (async () => { 20 | const projects = await fetchProjectsForTag(tag, auth) 21 | if (canceled) return 22 | setProjects(projects) 23 | })() 24 | return () => { 25 | canceled = true 26 | } 27 | }, [tag, auth]) 28 | return projects 29 | } 30 | 31 | export default useProjectsForTag -------------------------------------------------------------------------------- /src/pages/HomePage/HomePage.css: -------------------------------------------------------------------------------- 1 | .HomePage { 2 | display: flex; 3 | flex-direction: column; 4 | align-items: center; 5 | height: 100vh; 6 | font-size: 20px; 7 | } 8 | 9 | p { 10 | padding: 0px; 11 | margin: 7px; 12 | } -------------------------------------------------------------------------------- /src/pages/HomePage/HomePage.tsx: -------------------------------------------------------------------------------- 1 | import { FunctionComponent, useEffect } from "react"; 2 | import './HomePage.css' 3 | import { Hyperlink } from "@fi-sci/misc"; 4 | import useRoute from "../../useRoute"; 5 | 6 | type Props = { 7 | width: number 8 | height: number 9 | } 10 | 11 | const HomePage: FunctionComponent = ({width, height}) => { 12 | const {setRoute} = useRoute() 13 | return ( 14 |
15 |

Welcome to Dendro!

16 |

17 | setRoute({page: 'dandisets'})}> 18 | Import from DANDI 19 | 20 |

21 |

22 | setRoute({page: 'projects'})}> 23 | Manage projects 24 | 25 |

26 |

27 | setRoute({page: 'compute-resources'})}> 28 | Configure compute 29 | 30 |

31 |

32 | { 33 | window.open('https://github.com/flatironinstitute/dendro', '_blank') 34 | }}> 35 | Star us on GitHub 36 | 37 |

38 |

39 | { 40 | window.open('https://flatironinstitute.github.io/dendro-docs/docs/intro', '_blank') 41 | }}> 42 | View the documentation 43 | 44 |

45 |
46 | ) 47 | } 48 | 49 | export default HomePage -------------------------------------------------------------------------------- /src/pages/ProjectPage/ComputeResourceAppsTable2/ComputeResourceAppsTableMenuBar2.tsx: -------------------------------------------------------------------------------- 1 | import { Add, Delete, Edit } from "@mui/icons-material" 2 | import { FunctionComponent, useCallback } from "react" 3 | import { SmallIconButton } from "@fi-sci/misc"; 4 | 5 | type ComputeResourceAppsTableMenuBarProps = { 6 | selectedAppNames: string[] 7 | onDeleteApps: (appNames: string[]) => void 8 | onAddApp: () => void 9 | onEditApp: () => void 10 | } 11 | 12 | const ComputeResourceAppsTableMenuBar2: FunctionComponent = ({selectedAppNames, onAddApp, onDeleteApps, onEditApp}) => { 13 | const handleDelete = useCallback(async () => { 14 | const okay = await confirm(`Are you sure you want to delete these ${selectedAppNames.length} apps?`) 15 | if (!okay) return 16 | onDeleteApps(selectedAppNames) 17 | }, [selectedAppNames, onDeleteApps]) 18 | 19 | return ( 20 |
21 | 22 | } 24 | title={"Add an app"} 25 | label="Add app" 26 | onClick={onAddApp} 27 | /> 28 |        29 | 30 | } 32 | disabled={(selectedAppNames.length === 0)} 33 | title={selectedAppNames.length > 0 ? `Delete these ${selectedAppNames.length} apps` : ''} 34 | onClick={handleDelete} 35 | /> 36 |     37 | } 39 | disabled={(selectedAppNames.length !== 1)} 40 | title={selectedAppNames.length === 1 ? `Edit app ${selectedAppNames[0]}` : ''} 41 | onClick={onEditApp} 42 | /> 43 |
44 | ) 45 | } 46 | 47 | export default ComputeResourceAppsTableMenuBar2 -------------------------------------------------------------------------------- /src/pages/ProjectPage/ComputeResourceUsageComponent/ComputeResourceUsageComponent.tsx: -------------------------------------------------------------------------------- 1 | import { FunctionComponent, useEffect, useState } from "react" 2 | import { ComputeResourceUserUsage, isComputeResourceUserUsage } from "../../../types/dendro-types" 3 | import { apiBase, getRequest } from "../../../dbInterface/dbInterface" 4 | import { useGithubAuth } from "../../../GithubAuth/useGithubAuth" 5 | import { GithubAuthData } from "../../../GithubAuth/GithubAuthContext" 6 | 7 | type ComputeResourceUsageComponentProps = { 8 | computeResourceId: string 9 | } 10 | 11 | const useComputeResourceUserUsage = (computeResourceId: string, userId: string | undefined, auth: GithubAuthData) => { 12 | const [usage, setUsage] = useState(undefined) 13 | 14 | useEffect(() => { 15 | if (!userId) return 16 | let canceled = false 17 | const fetchUsage = async () => { 18 | const url = `${apiBase}/api/gui/usage/compute_resource/${computeResourceId}/user/${userId}` 19 | const response = await getRequest(url, auth) 20 | if (canceled) return 21 | if (!response.success) { 22 | console.error("Error fetching usage", response) 23 | return 24 | } 25 | const u = response.usage 26 | if (!isComputeResourceUserUsage(u)) { 27 | console.error("Invalid usage", u) 28 | return 29 | } 30 | setUsage(u) 31 | } 32 | fetchUsage() 33 | return () => { canceled = true } 34 | }, [computeResourceId, userId, auth]) 35 | 36 | return usage 37 | } 38 | 39 | const ComputeResourceUsageComponent: FunctionComponent = ({computeResourceId}) => { 40 | const auth = useGithubAuth() 41 | const usage = useComputeResourceUserUsage(computeResourceId, auth.userId, auth) 42 | if (!auth.userId) return
Not logged in
43 | if (!usage) return
Loading...
44 | const numJobsIncludingDeleted = usage.jobsIncludingDeleted.length 45 | const numJobs = usage.jobsIncludingDeleted.filter(j => (!j.deleted)).length 46 | return ( 47 |
48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 |
Num. jobs{numJobs}
Num. jobs including deleted{numJobsIncludingDeleted}
60 |
61 | ) 62 | } 63 | 64 | export default ComputeResourceUsageComponent -------------------------------------------------------------------------------- /src/pages/ProjectPage/DandiUpload/prepareDandiUploadTask.ts: -------------------------------------------------------------------------------- 1 | export type DandiUploadTask = { 2 | dandisetId: string 3 | dandiInstance: string 4 | fileNames: string[] 5 | names: string[] 6 | } 7 | 8 | const prepareDandiUploadTask = (fileNames: string[]): DandiUploadTask | undefined => { 9 | if (fileNames.length === 0) { 10 | return undefined 11 | } 12 | let topLevelDirectoryName: string | undefined = undefined 13 | let dandisetId: string | undefined = undefined 14 | let staging: boolean | undefined = undefined 15 | const names: string[] = [] 16 | for (const fileName of fileNames) { 17 | if (!fileName.endsWith('.nwb')) { 18 | return undefined 19 | } 20 | const a = fileName.split('/') 21 | if (a.length <3) { 22 | return undefined 23 | } 24 | if ((topLevelDirectoryName) && (topLevelDirectoryName !== a[0])) { 25 | return undefined 26 | } 27 | const dd = a[1].startsWith('staging-') ? a[1].substring('staging-'.length) : a[1] 28 | if ((dandisetId) && (dandisetId !== dd)) { 29 | return undefined 30 | } 31 | const ss = a[1].startsWith('staging-') 32 | if ((staging !== undefined) && (staging !== ss)) { 33 | return undefined 34 | } 35 | topLevelDirectoryName = a[0] 36 | dandisetId = dd 37 | staging = ss 38 | names.push(a.slice(2).join('/')) 39 | } 40 | if ((!topLevelDirectoryName) || (!dandisetId) || (staging === undefined)) { 41 | return undefined 42 | } 43 | if (!validDandisetId(dandisetId)) { 44 | return undefined 45 | } 46 | if (topLevelDirectoryName !== 'generated') { 47 | return undefined 48 | } 49 | return { 50 | dandisetId, 51 | dandiInstance: staging ? 'dandi-staging' : 'dandi', 52 | fileNames, 53 | names 54 | } 55 | } 56 | 57 | const validDandisetId = (dandisetId: string): boolean => { 58 | // must be an integer 59 | if (!dandisetId.match(/^\d+$/)) { 60 | return false 61 | } 62 | return true 63 | } 64 | 65 | export default prepareDandiUploadTask 66 | -------------------------------------------------------------------------------- /src/pages/ProjectPage/FileActions.tsx: -------------------------------------------------------------------------------- 1 | export type FileAction = { 2 | name: string 3 | label: string 4 | processorName: string 5 | outputFileBaseName: string 6 | icon: any 7 | } 8 | 9 | export const fileActions: FileAction[] = [ 10 | // { 11 | // name: 'spike_sorting_summary', 12 | // label: 'SS Summary', 13 | // processorName: 'dandi-vis-1.spike_sorting_summary', 14 | // outputFileBaseName: 'spike_sorting_summary.nh5', 15 | // icon: 16 | // }, 17 | // { 18 | // name: 'ecephys_summary', 19 | // label: 'Ecephys Summary', 20 | // processorName: 'dandi-vis-1.ecephys_summary', 21 | // outputFileBaseName: 'ecephys_summary.nh5', 22 | // icon: 23 | // } 24 | ] 25 | -------------------------------------------------------------------------------- /src/pages/ProjectPage/FileBrowser/DropdownMenu.css: -------------------------------------------------------------------------------- 1 | .DropdownMenu li.enabled:hover { 2 | /* background-color:rgb(212, 210, 210); */ 3 | color: rgb(131, 129, 129); 4 | } 5 | -------------------------------------------------------------------------------- /src/pages/ProjectPage/FileBrowser/DropdownMenu.tsx: -------------------------------------------------------------------------------- 1 | import { Menu } from '@mui/icons-material'; 2 | import React, { useState } from 'react'; 3 | import { SmallIconButton } from "@fi-sci/misc"; 4 | import './DropdownMenu.css'; 5 | 6 | type Option = { 7 | label: string; 8 | onClick?: () => void; 9 | }; 10 | 11 | type MinimalDropdownProps = { 12 | options: Option[]; 13 | }; 14 | 15 | const DropdownMenu: React.FC = ({ options }) => { 16 | const [isOpen, setIsOpen] = useState(false) 17 | 18 | return ( 19 |
20 | } onClick={() => setIsOpen(!isOpen)} /> 21 | {isOpen && ( 22 |
    34 | {options.map(option => ( 35 |
  • { 39 | option.onClick?.(); 40 | setIsOpen(false); 41 | }} 42 | style={{ 43 | padding: '5px 10px', 44 | cursor: 'pointer', 45 | textAlign: 'left', 46 | color: option.onClick ? '#000' : '#999' 47 | }} 48 | > 49 | {option.label} 50 |
    • 51 | ))} 52 |
53 | )} 54 |
55 | ); 56 | }; 57 | 58 | export default DropdownMenu; 59 | -------------------------------------------------------------------------------- /src/pages/ProjectPage/FileBrowser/file-browser-table.css: -------------------------------------------------------------------------------- 1 | .file-browser-table { 2 | width: 100%; 3 | border-collapse: collapse; 4 | font-size: 0.9em; 5 | font-family: sans-serif; 6 | text-align: left; 7 | margin: 0px 0; 8 | } 9 | 10 | .file-browser-table thead tr { 11 | background-color: #aaa; 12 | color: #ffffff; 13 | text-align: left; 14 | } 15 | 16 | .file-browser-table th, 17 | .file-browser-table td { 18 | padding: 5px 2px; 19 | } 20 | 21 | .file-browser-table tbody tr { 22 | border-bottom: thin solid #dddddd; 23 | } 24 | 25 | .file-browser-table tbody tr:nth-of-type(even) { 26 | background-color: #f3f3f3; 27 | } 28 | 29 | .file-browser-table tbody tr:last-of-type { 30 | border-bottom: 2px solid #009879; 31 | } 32 | 33 | .file-browser-table tbody tr:hover { 34 | background-color: #ededed; 35 | transition: background-color 0.3s ease; 36 | } 37 | 38 | .file-browser-context-menu { 39 | position: absolute; 40 | background-color: white; 41 | border: 1px solid #ddd; 42 | padding: 10px; 43 | box-shadow: 0px 0px 10px rgba(0, 0, 0, 0.1); 44 | z-index: 999; 45 | border-radius: 3px; 46 | min-width: 100px; 47 | } 48 | 49 | .file-browser-context-menu div { 50 | padding: 5px 0; 51 | cursor: pointer; 52 | } 53 | 54 | .file-browser-context-menu div:hover { 55 | background-color: #ddd; 56 | } -------------------------------------------------------------------------------- /src/pages/ProjectPage/FileBrowser/formatByteCount.ts: -------------------------------------------------------------------------------- 1 | const formatByteCount = (a: number) => { 2 | if (a < 10000) { 3 | return `${a} bytes` 4 | } 5 | else if (a < 100 * 1024) { 6 | return `${formatNum(a / 1024)} KB` 7 | } 8 | else if (a < 100 * 1024 * 1024) { 9 | return `${formatNum(a / (1024 * 1024))} MB` 10 | } 11 | else if (a < 600 * 1024 * 1024 * 1024) { 12 | return `${formatNum(a / (1024 * 1024 * 1025))} GB` 13 | } 14 | else { 15 | return `${formatNum(a / (1024 * 1024 * 1024 * 1024))} TB` 16 | } 17 | } 18 | 19 | export const formatGiBCount = (a: number) => { 20 | return formatByteCount(a * 1000 * 1000 * 1000) 21 | } 22 | 23 | const formatNum = (a: number) => { 24 | const b = a.toFixed(2) 25 | if (Number(b) - Math.floor(Number(b)) === 0) { 26 | return a.toFixed(0) 27 | } 28 | else return b 29 | } 30 | 31 | export default formatByteCount -------------------------------------------------------------------------------- /src/pages/ProjectPage/FileView/ElectricalSeriesSection/ElectricalSeriesSection.tsx: -------------------------------------------------------------------------------- 1 | import { FunctionComponent, useCallback, useState } from "react" 2 | import { useElectricalSeriesPaths, useNwbFile } from "../NwbFileView" 3 | import { Hyperlink } from "@fi-sci/misc"; 4 | import ModalWindow from "@fi-sci/modal-window"; 5 | import { useModalWindow } from "@fi-sci/modal-window" 6 | import { useProject } from "../../ProjectPageContext" 7 | import LoadElectricalSeriesScriptWindow from "./LoadElectricalSeriesScriptWindow" 8 | 9 | type ElectricalSeriesSectionProps = { 10 | fileName: string 11 | nwbUrl?: string 12 | } 13 | 14 | const ElectricalSeriesSection: FunctionComponent = ({fileName, nwbUrl}) => { 15 | const nwbFile = useNwbFile(nwbUrl) 16 | const electricalSeriesPaths = useElectricalSeriesPaths(nwbFile) 17 | const [selectedElectricalSeriesPath, setSelectedElectricalSeriesPath] = useState('') 18 | const {visible: loadInScriptVisible, handleOpen: openLoadInScriptWindow, handleClose: closeLoadInScriptWindow} = useModalWindow() 19 | const {project} = useProject() 20 | const loadInScript = useCallback((path: string) => { 21 | setSelectedElectricalSeriesPath(path) 22 | openLoadInScriptWindow() 23 | }, [openLoadInScriptWindow]) 24 | if ((!electricalSeriesPaths) || (electricalSeriesPaths.length === 0)) return ( 25 | <> 26 | ) 27 | return ( 28 |
29 |

30 | Electrical series: {fileName} 31 |

32 |
    33 | { 34 | electricalSeriesPaths.map((path, ii) => ( 35 |
  • 36 | {path} - loadInScript(path)}>load in script 37 |
    • 38 | )) 39 | } 40 |
41 | 45 | {selectedElectricalSeriesPath && project && fileName && ( 46 | 52 | )} 53 | 54 |
55 | ) 56 | } 57 | 58 | export default ElectricalSeriesSection -------------------------------------------------------------------------------- /src/pages/ProjectPage/FileView/ElectricalSeriesSection/LoadElectricalSeriesScriptWindow.tsx: -------------------------------------------------------------------------------- 1 | import { FunctionComponent, useCallback, useEffect, useMemo, useState } from "react" 2 | import { DendroProject } from "../../../../types/dendro-types" 3 | import Markdown from "../../../../Markdown/Markdown" 4 | import nunjucks from "nunjucks" 5 | 6 | type LoadElectricalSeriesScriptWindowProps = { 7 | onClose: () => void 8 | project: DendroProject 9 | fileName: string 10 | electricalSeriesPath: string 11 | } 12 | 13 | export const useScript = (url: string) => { 14 | const [script, setScript] = useState(`# Loading script from ${url}...`) 15 | useEffect(() => { 16 | let canceled = false 17 | ;(async () => { 18 | const resp = await fetch(url) 19 | const text = await resp.text() 20 | if (canceled) return 21 | setScript(text) 22 | })() 23 | return () => {canceled = true} 24 | }, [url]) 25 | return script 26 | } 27 | 28 | const LoadElectricalSeriesScriptWindow: FunctionComponent = ({project, fileName, electricalSeriesPath}) => { 29 | const [copied, setCopied] = useState(false) 30 | const script = useScript('/scripts/load_electrical_series.py') 31 | const processedScript = useMemo(() => { 32 | return nunjucks.renderString(script, {project, fileName, electricalSeriesPath}) 33 | }, [script, project, fileName, electricalSeriesPath]) 34 | const markdownSource = ` 35 | Use this script to load this electrical series into a SpikeInterface recording object. If this is an embargoed dandiset, then be sure to set the DANDI_API_KEY environment variable prior to running the script. 36 | 37 | You first need install dendro and spikeinterface via pip: \`pip install --upgrade dendro spikeinterface\` 38 | 39 | ${copied ? '*Copied to clipboard*' : '[Copy](#copy)'} 40 | 41 | \`\`\`python 42 | ${processedScript} 43 | \`\`\` 44 | ` 45 | const processedMarkdownSource = useMemo(() => { 46 | return nunjucks.renderString(markdownSource, {project, fileName, electricalSeriesPath}) 47 | }, [markdownSource, project, fileName, electricalSeriesPath]) 48 | const handleLinkClick = useCallback((href: string) => { 49 | if (href === '#copy') { 50 | navigator.clipboard.writeText(processedScript) 51 | setCopied(true) 52 | } 53 | }, [processedScript]) 54 | return ( 55 | 56 | ) 57 | } 58 | 59 | export default LoadElectricalSeriesScriptWindow -------------------------------------------------------------------------------- /src/pages/ProjectPage/FileView/FigurlFileView.tsx: -------------------------------------------------------------------------------- 1 | import { FunctionComponent, useEffect, useMemo, useState } from "react" 2 | import { useProject } from "../ProjectPageContext" 3 | 4 | type FigurlFileViewProps = { 5 | fileName: string 6 | width: number 7 | height: number 8 | } 9 | 10 | const FigurlFileView: FunctionComponent = ({fileName, width, height}) => { 11 | if (!fileName.endsWith('.figurl')) { 12 | throw Error('Unexpected file extension: ' + fileName) 13 | } 14 | const {files} = useProject() 15 | const file = useMemo(() => { 16 | if (!files) return undefined 17 | return files.find(f => (f.fileName === fileName)) 18 | }, [files, fileName]) 19 | 20 | const [url, setUrl] = useState(undefined) 21 | useEffect(() => { 22 | if (!file) return 23 | if (!file.content.startsWith('url:')) { 24 | console.warn('Unexpected file content: ' + file.content) 25 | return 26 | } 27 | let canceled = false 28 | ; (async () => { 29 | const u = await fetchTextFile(file.content.slice('url:'.length)) 30 | if (canceled) return 31 | setUrl(u) 32 | })() 33 | return () => {canceled = true} 34 | }, [file]) 35 | 36 | if (!files) return
Loading...
37 | if (!file) return
File not found: {fileName}
38 | if (!url) return
Loading file content...
39 | 40 | return ( 41 |
42 | {url} 43 |
44 | ) 45 | } 46 | 47 | const fetchTextFile = async (url: string) => { 48 | const response = await fetch(url) 49 | if (!response.ok) { 50 | throw Error(`Unexpected response for ${url}: ${response.status}`) 51 | } 52 | return await response.text() 53 | } 54 | 55 | export default FigurlFileView -------------------------------------------------------------------------------- /src/pages/ProjectPage/FileView/OtherFileView.tsx: -------------------------------------------------------------------------------- 1 | import { FunctionComponent } from "react"; 2 | import FileViewTable from "./FileViewTable"; 3 | 4 | 5 | type Props = { 6 | fileName: string 7 | width: number 8 | height: number 9 | } 10 | 11 | const OtherFileView: FunctionComponent = ({fileName, width, height}) => { 12 | return ( 13 |
14 |
15 | 16 |
 
17 |
...
18 |
 
19 |
20 | ) 21 | } 22 | 23 | export default OtherFileView -------------------------------------------------------------------------------- /src/pages/ProjectPage/FileView/SpikeSortingOutputSection/LoadSpikeSortingInScriptWindow.tsx: -------------------------------------------------------------------------------- 1 | import { FunctionComponent, useCallback, useEffect, useMemo, useState } from "react" 2 | import { DendroProject } from "../../../../types/dendro-types" 3 | import Markdown from "../../../../Markdown/Markdown" 4 | import nunjucks from "nunjucks" 5 | import { useScript } from "../ElectricalSeriesSection/LoadElectricalSeriesScriptWindow" 6 | 7 | type LoadSpikeSortingInScriptWindowProps = { 8 | onClose: () => void 9 | project: DendroProject 10 | fileName: string 11 | } 12 | 13 | const LoadSpikeSortingInScriptWindow: FunctionComponent = ({project, fileName}) => { 14 | const [copied, setCopied] = useState(false) 15 | const script = useScript('/scripts/load_spike_sorting.py') 16 | const processedScript = useMemo(() => { 17 | return nunjucks.renderString(script, {project, fileName}) 18 | }, [script, project, fileName]) 19 | const markdownSource = ` 20 | Use this script to load this spike sorting into a SpikeInterface sorting object. If this is an embargoed dandiset, then be sure to set the DANDI_API_KEY environment variable prior to running the script. 21 | 22 | You first need install dendro and spikeinterface via pip: \`pip install --upgrade dendro spikeinterface\` 23 | 24 | ${copied ? '*Copied to clipboard*' : '[Copy](#copy)'} 25 | 26 | \`\`\`python 27 | ${processedScript} 28 | \`\`\` 29 | ` 30 | const processedMarkdownSource = useMemo(() => { 31 | return nunjucks.renderString(markdownSource, {project, fileName}) 32 | }, [markdownSource, project, fileName]) 33 | const handleLinkClick = useCallback((href: string) => { 34 | if (href === '#copy') { 35 | navigator.clipboard.writeText(processedScript) 36 | setCopied(true) 37 | } 38 | }, [processedScript]) 39 | return ( 40 | 41 | ) 42 | } 43 | 44 | export default LoadSpikeSortingInScriptWindow -------------------------------------------------------------------------------- /src/pages/ProjectPage/JobView/OutputsTable.tsx: -------------------------------------------------------------------------------- 1 | import { FunctionComponent, useCallback } from "react" 2 | import { Hyperlink } from "@fi-sci/misc"; 3 | import { DendroJob } from "../../../types/dendro-types" 4 | import useRoute from "../../../useRoute" 5 | import { useProject } from "../ProjectPageContext" 6 | 7 | type OutputsTableProps = { 8 | job: DendroJob 9 | } 10 | 11 | const OutputsTable: FunctionComponent = ({ job }) => { 12 | const {openTab} = useProject() 13 | const {setRoute} = useRoute() 14 | const handleOpenFile = useCallback((fileName: string) => { 15 | openTab(`file:${fileName}`) 16 | setRoute({ 17 | page: 'project', 18 | projectId: job.projectId, 19 | tab: `project-files` 20 | }) 21 | }, [openTab, setRoute, job.projectId]) 22 | return ( 23 | 24 | 25 | { 26 | job.processorSpec.outputs.map((output, ii) => { 27 | const x = job.outputFiles.find(x => (x.name === output.name)) 28 | return ( 29 | 30 | 31 | 40 | 41 | 42 | ) 43 | }) 44 | } 45 | 46 |
{output.name} 32 | { 34 | x && handleOpenFile(x?.fileName) 35 | }} 36 | > 37 | {x?.fileName || 'unknown'} 38 | 39 | {output.description}
47 | ) 48 | } 49 | 50 | export default OutputsTable -------------------------------------------------------------------------------- /src/pages/ProjectPage/JobsWindow/JobsTableMenuBar.tsx: -------------------------------------------------------------------------------- 1 | import { SmallIconButton } from "@fi-sci/misc"; 2 | import { Check, Delete, Refresh } from "@mui/icons-material"; 3 | import { FunctionComponent, useCallback, useState } from "react"; 4 | import { confirm } from "../../../confirm_prompt_alert"; 5 | import { useProject } from "../ProjectPageContext"; 6 | 7 | type JobsTableMenuBarProps = { 8 | width: number 9 | height: number 10 | selectedJobIds: string[] 11 | onResetSelection: () => void 12 | createJobEnabled?: boolean 13 | createJobTitle?: string 14 | onApproveAll?: () => void 15 | } 16 | 17 | const JobsTableMenuBar: FunctionComponent = ({width, height, selectedJobIds, onResetSelection, createJobEnabled, createJobTitle, onApproveAll: onApproveAll}) => { 18 | const {deleteJob, refreshJobs, refreshFiles, projectRole} = useProject() 19 | const [operating, setOperating] = useState(false) 20 | const handleDelete = useCallback(async () => { 21 | if (!['admin', 'editor'].includes(projectRole || '')) { 22 | alert('You are not authorized to delete jobs in this project.') 23 | return 24 | } 25 | const okay = await confirm(`Are you sure you want to delete these ${selectedJobIds.length} jobs?`) 26 | if (!okay) return 27 | try { 28 | setOperating(true) 29 | for (const jobId of selectedJobIds) { 30 | await deleteJob(jobId) 31 | } 32 | } 33 | finally { 34 | setOperating(false) 35 | refreshJobs() 36 | refreshFiles() 37 | onResetSelection() 38 | } 39 | }, [selectedJobIds, deleteJob, refreshJobs, refreshFiles, onResetSelection, projectRole]) 40 | 41 | return ( 42 |
43 | } 45 | disabled={operating} 46 | title='Refresh' 47 | onClick={refreshJobs} 48 | /> 49 | } 51 | disabled={(selectedJobIds.length === 0) || operating} 52 | title={selectedJobIds.length > 0 ? `Delete these ${selectedJobIds.length} jobs` : ''} 53 | onClick={handleDelete} 54 | /> 55 | { 56 | onApproveAll && ( 57 | } 59 | disabled={operating} 60 | title='Approve all jobs' 61 | label="Approve all jobs" 62 | onClick={onApproveAll} 63 | /> 64 | ) 65 | } 66 |
67 | ) 68 | } 69 | 70 | export default JobsTableMenuBar -------------------------------------------------------------------------------- /src/pages/ProjectPage/JobsWindow/JobsWindow.tsx: -------------------------------------------------------------------------------- 1 | import { FunctionComponent, useMemo } from "react"; 2 | import { useProject } from "../ProjectPageContext"; 3 | import JobsTable from "./JobsTable"; 4 | import { useGithubAuth } from "../../../GithubAuth/useGithubAuth"; 5 | 6 | type Props = { 7 | width: number, 8 | height: number, 9 | fileName: string 10 | createJobEnabled?: boolean 11 | createJobTitle?: string 12 | } 13 | 14 | const JobsWindow: FunctionComponent = ({ width, height, fileName, createJobEnabled, createJobTitle }) => { 15 | const {jobs, openTab, computeResource} = useProject() 16 | 17 | const filteredJobs = useMemo(() => { 18 | if (!jobs) return undefined 19 | if (fileName) { 20 | return jobs.filter(jj => ( 21 | jj.inputFiles.map(x => (x.fileName)).includes(fileName) || 22 | jj.outputFiles.map(x => (x.fileName)).includes(fileName) 23 | )) 24 | } 25 | else return jobs 26 | }, [jobs, fileName]) 27 | 28 | // const iconFontSize = 20 29 | 30 | const auth = useGithubAuth() 31 | 32 | const userIsComputeResourceOwner = useMemo(() => { 33 | if (!computeResource) return false 34 | if (!auth.userId) return false 35 | return computeResource.ownerId === auth.userId 36 | }, [computeResource, auth.userId]) 37 | 38 | return ( 39 | openTab(`job:${jobId}`)} 45 | createJobEnabled={createJobEnabled} 46 | createJobTitle={createJobTitle} 47 | userCanApproveJobs={userIsComputeResourceOwner} 48 | /> 49 | // <> 50 | //
51 | // {fileName.endsWith('.py') && ( 52 | // } 54 | // onClick={handleCreateScriptJob} 55 | // disabled={!canCreateJob} 56 | // title={createJobTitle} 57 | // label="Run" 58 | // fontSize={iconFontSize} 59 | // /> 60 | // )} 61 | //      62 | // } 64 | // onClick={refreshJobs} 65 | // title="Refresh jobs" 66 | // label="Refresh" 67 | // fontSize={iconFontSize} 68 | // /> 69 | //
70 | // 71 | ) 72 | } 73 | 74 | export default JobsWindow -------------------------------------------------------------------------------- /src/pages/ProjectPage/LoadNwbInPythonWindow/LoadNwbInPythonWindow.tsx: -------------------------------------------------------------------------------- 1 | import { FunctionComponent, useMemo } from "react" 2 | import Markdown from "../../../Markdown/Markdown" 3 | import { DendroProject } from "../../../types/dendro-types" 4 | 5 | type LoadNwbInPythonWindowProps = { 6 | onClose: () => void 7 | project: DendroProject 8 | fileName: string 9 | } 10 | 11 | const getMdSource = (project: DendroProject, fileName: string) => { 12 | const source = ` 13 | \`\`\`python 14 | import h5py 15 | import pynwb 16 | import dendro.client as prc 17 | import remfile 18 | 19 | 20 | # Load project ${project.name} 21 | project = prc.load_project('${project.projectId}') 22 | 23 | # Lazy load ${fileName} 24 | nwb_file = project.get_file('${fileName}') 25 | nwb_remf = remfile.File(nwb_file) 26 | io = pynwb.NWBHDF5IO(file=h5py.File(nwb_remf, 'r'), mode='r') 27 | nwb = io.read() 28 | 29 | # Explore the NWB file 30 | print(nwb) 31 | \`\`\` 32 | ` 33 | return source 34 | } 35 | 36 | const LoadNwbInPythonWindow: FunctionComponent = ({project, fileName}) => { 37 | const source = useMemo(() => (getMdSource(project, fileName)), [project, fileName]) 38 | return ( 39 | 40 | ) 41 | } 42 | 43 | export default LoadNwbInPythonWindow -------------------------------------------------------------------------------- /src/pages/ProjectPage/Processor.css: -------------------------------------------------------------------------------- 1 | .Processor { 2 | } 3 | 4 | .Processor:hover { 5 | background: #eee; 6 | } 7 | 8 | .ProcessorImage { 9 | } 10 | 11 | .ProcessorTitle { 12 | font-size: 20px; 13 | font-weight: bold; 14 | } 15 | 16 | .ProcessorDescription { 17 | font-size: 16px; 18 | } 19 | 20 | .ProcessorParameters { 21 | font-size: 14px; 22 | color: #444; 23 | } 24 | 25 | .ProcessorTags { 26 | font-size: 14px; 27 | color: #444; 28 | } -------------------------------------------------------------------------------- /src/pages/ProjectPage/ProjectAnalysis/AnalysisSourceClient.ts: -------------------------------------------------------------------------------- 1 | import ClonedRepo, { joinPaths } from "./ClonedRepo"; 2 | 3 | class AnalysisSourceClient { 4 | constructor(private url: string, private path: string, private clonedRepo: ClonedRepo) { 5 | } 6 | static async create(url: string, setStatus: (status: string) => void) { 7 | const {repoUrl, branch, path} = parseAnalysisSourceUrl(url); 8 | setStatus(`Cloning repo ${repoUrl}`) 9 | const clonedRepo = new ClonedRepo({url: repoUrl, branch}) 10 | await clonedRepo.initialize(setStatus) 11 | return new AnalysisSourceClient(url, path, clonedRepo); 12 | } 13 | async readDirectory(path: string) { 14 | const fullPath = joinPaths(this.path, path); 15 | const {subdirectories, files} = await this.clonedRepo.readDirectory(fullPath); 16 | return {subdirectories, files}; 17 | } 18 | async readTextFile(path: string) { 19 | const fullPath = joinPaths(this.path, path); 20 | const txt = await this.clonedRepo.readTextFile(fullPath); 21 | return txt; 22 | } 23 | } 24 | 25 | export const parseAnalysisSourceUrl = (url: string) => { 26 | // for example, url = https://github.com/magland/dendro_analysis/tree/main/projects/eb87e88a 27 | if (!url.startsWith('https://github.com')) { 28 | throw new Error('Invalid url: ' + url); 29 | } 30 | const parts = url.split('/'); 31 | const owner = parts[3]; 32 | const repo = parts[4]; 33 | const a = parts[5]; 34 | if (a === 'tree') { 35 | const branch = parts[6]; 36 | const path = parts.slice(7).join('/'); 37 | if (!branch) throw new Error('Invalid url: ' + url); 38 | if (!path) throw new Error('Invalid url: ' + url); 39 | return { 40 | repoUrl: `https://github.com/${owner}/${repo}`, 41 | branch, 42 | path, 43 | repoName: repo 44 | }; 45 | } 46 | else if (!a) { 47 | return { 48 | repoUrl: url, 49 | branch: '', 50 | path: '', 51 | repoName: repo 52 | }; 53 | } 54 | else { 55 | throw new Error('Invalid url: ' + url); 56 | } 57 | } 58 | 59 | export default AnalysisSourceClient; -------------------------------------------------------------------------------- /src/pages/ProjectPage/ProjectAnalysis/AnalysisSourceFileBrowser.tsx: -------------------------------------------------------------------------------- 1 | import { FunctionComponent, useMemo, useReducer } from "react" 2 | import { selectedStringsReducer } from "../FileBrowser/FileBrowser2" 3 | import FileBrowserTable, { FileBrowserTableFile } from "../FileBrowser/FileBrowserTable" 4 | import AnalysisSourceClient from "./AnalysisSourceClient" 5 | 6 | type AnalysisSourceFileBrowserProps = { 7 | width: number 8 | height: number 9 | analysisSourceClient: AnalysisSourceClient 10 | onOpenFile: (path: string) => void 11 | allFiles?: AnalysisSourceFile[] 12 | } 13 | 14 | export type AnalysisSourceFile = { 15 | path: string 16 | } 17 | 18 | const AnalysisSourceFileBrowser: FunctionComponent = ({width, height, analysisSourceClient, onOpenFile, allFiles}) => { 19 | const [selectedFileNames, selectedFileNamesDispatch] = useReducer(selectedStringsReducer, new Set()) 20 | 21 | const files2: FileBrowserTableFile[] | undefined = useMemo(() => { 22 | if (!allFiles) return undefined 23 | return allFiles.map(f => ({ 24 | fileName: f.path, 25 | size: 0, 26 | timestampCreated: 0, 27 | content: undefined 28 | })) 29 | }, [allFiles]) 30 | 31 | if (!allFiles) { 32 | return
Loading files...
33 | } 34 | 35 | return ( 36 |
37 | 47 |
48 | ) 49 | } 50 | 51 | export default AnalysisSourceFileBrowser -------------------------------------------------------------------------------- /src/pages/ProjectPage/ProjectAnalysis/AnalysisSourceFileView.css: -------------------------------------------------------------------------------- 1 | .ipynb-renderer-root pre { 2 | font-size: 12px!important; 3 | } 4 | 5 | .ipynb-renderer-root div.output_html { 6 | font-size: 15px!important; 7 | } -------------------------------------------------------------------------------- /src/pages/ProjectPage/ProjectAnalysis/fs.ts: -------------------------------------------------------------------------------- 1 | import FS from "@isomorphic-git/lightning-fs" 2 | import {Buffer} from 'buffer' 3 | 4 | window.Buffer = Buffer 5 | 6 | const fs = new FS('default') 7 | 8 | export default fs -------------------------------------------------------------------------------- /src/pages/ProjectPage/ProjectJobs.tsx: -------------------------------------------------------------------------------- 1 | import { FunctionComponent, useMemo, useState } from "react"; 2 | import { Splitter } from "@fi-sci/splitter"; 3 | import JobsTable from "./JobsWindow/JobsTable"; 4 | import JobView from "./JobView/JobView"; 5 | import { useProject } from "./ProjectPageContext"; 6 | import { useGithubAuth } from "../../GithubAuth/useGithubAuth"; 7 | 8 | const ProjectJobs: FunctionComponent<{width: number, height: number}> = ({width, height}) => { 9 | const {jobs, computeResource} = useProject() 10 | const [selectedJobId, setSelectedJobId] = useState(undefined) 11 | 12 | const auth = useGithubAuth() 13 | 14 | const userIsComputeResourceOwner = useMemo(() => { 15 | if (!computeResource) return false 16 | if (!auth.userId) return false 17 | return computeResource.ownerId === auth.userId 18 | }, [computeResource, auth.userId]) 19 | 20 | return ( 21 | j.jobId).includes(selectedJobId))} 27 | > 28 | setSelectedJobId(jobId)} 34 | userCanApproveJobs={userIsComputeResourceOwner} 35 | /> 36 | 41 | 42 | ) 43 | } 44 | 45 | export default ProjectJobs -------------------------------------------------------------------------------- /src/pages/ProjectPage/ProjectScripts.tsx: -------------------------------------------------------------------------------- 1 | import { Splitter } from "@fi-sci/splitter"; 2 | import { FunctionComponent, useCallback, useMemo, useState } from "react"; 3 | import { useProject } from "./ProjectPageContext"; 4 | import ScriptsTable from "./scripts/ScriptsTable"; 5 | import ScriptView from "./scripts/ScriptView"; 6 | import { setScriptContent } from "../../dbInterface/dbInterface"; 7 | import { useGithubAuth } from "../../GithubAuth/useGithubAuth"; 8 | 9 | const ProjectScripts: FunctionComponent<{width: number, height: number}> = ({width, height}) => { 10 | const {scripts, projectRole, refreshScripts} = useProject() 11 | const [selectedScriptId, setSelectedScriptId] = useState(undefined) 12 | 13 | const createScriptEnabled = projectRole === 'admin' || projectRole === 'editor' 14 | 15 | const selectedScript = useMemo(() => { 16 | if (!selectedScriptId) return undefined 17 | return scripts?.find(s => s.scriptId === selectedScriptId) 18 | }, [selectedScriptId, scripts]) 19 | 20 | const auth = useGithubAuth() 21 | 22 | const handleSetScriptContent = useCallback(async (content: string) => { 23 | if (!selectedScriptId) return 24 | await setScriptContent(selectedScriptId, content, auth) 25 | refreshScripts() 26 | }, [selectedScriptId, auth, refreshScripts]) 27 | 28 | return ( 29 | s.scriptId).includes(selectedScriptId))} 35 | > 36 | {scripts ? setSelectedScriptId(scriptId)} 41 | createScriptEnabled={createScriptEnabled} 42 | createScriptTitle={createScriptEnabled ? "Create a new script" : ""} 43 | /> :
} 44 | 50 | 51 | ) 52 | } 53 | 54 | export default ProjectScripts -------------------------------------------------------------------------------- /src/pages/ProjectPage/ResourceUtilizationView/LogPlot.tsx: -------------------------------------------------------------------------------- 1 | import { FunctionComponent } from "react"; 2 | import Plot from 'react-plotly.js'; 3 | 4 | 5 | type LogPlotProps = { 6 | series: { 7 | label: string, 8 | data: {x: number, y: number}[] 9 | color: string 10 | }[], 11 | referenceTime: number 12 | yAxisLabel?: string 13 | } 14 | 15 | const LogPlot: FunctionComponent = ({series, referenceTime, yAxisLabel}) => { 16 | const height = 220 17 | const data = series.map(s => ({ 18 | x: s.data.map(d => d.x - referenceTime), 19 | y: s.data.map(d => d.y), 20 | name: s.label, 21 | mode: 'lines', 22 | line: { 23 | color: s.color 24 | } 25 | })) 26 | const xAxisLabel = 'Time (s)' 27 | return ( 28 | 49 | ) 50 | } 51 | 52 | export default LogPlot -------------------------------------------------------------------------------- /src/pages/ProjectPage/RunBatchSpikeSortingWindow/ElectrodeGeometryView.tsx: -------------------------------------------------------------------------------- 1 | import { FunctionComponent, useEffect, useState } from "react" 2 | import { RemoteH5File } from "../../../RemoteH5File/RemoteH5File" 3 | import {ElectrodeGeometryWidget, ElectrodeLocation} from "@fi-sci/electrode-geometry" 4 | 5 | type ElectrodeGeometryViewProps = { 6 | width: number 7 | height: number 8 | nwbFile: RemoteH5File 9 | } 10 | 11 | const ElectrodeGeometryView: FunctionComponent = ({width, height, nwbFile}) => { 12 | const [electrodeLocations, setElectrodeLocations] = useState(undefined) 13 | useEffect(() => { 14 | (async () => { 15 | setElectrodeLocations(undefined) 16 | const grp = await nwbFile.getGroup('/general/extracellular_ephys/electrodes') 17 | if (!grp) return 18 | if (!grp.datasets) return 19 | let xDatasetPath = '' 20 | let yDatasetPath = '' 21 | if (grp.datasets.find(ds => (ds.name === 'rel_x'))) { 22 | xDatasetPath = '/general/extracellular_ephys/electrodes/rel_x' 23 | yDatasetPath = '/general/extracellular_ephys/electrodes/rel_y' 24 | } 25 | else if (grp.datasets.find(ds => (ds.name === 'x'))) { 26 | xDatasetPath = '/general/extracellular_ephys/electrodes/x' 27 | yDatasetPath = '/general/extracellular_ephys/electrodes/y' 28 | } 29 | else { 30 | return 31 | } 32 | if ((!xDatasetPath) || (!yDatasetPath)) return 33 | const x = await nwbFile.getDatasetData(xDatasetPath, {}) 34 | const y = await nwbFile.getDatasetData(yDatasetPath, {}) 35 | const locations: ElectrodeLocation[] = [] 36 | for (let i = 0; i < x.length; i++) { 37 | locations.push({x: x[i], y: y[i]}) 38 | } 39 | setElectrodeLocations(locations) 40 | })() 41 | }, [nwbFile]) 42 | if (!electrodeLocations) return
43 | return ( 44 | 49 | ) 50 | } 51 | 52 | export default ElectrodeGeometryView -------------------------------------------------------------------------------- /src/pages/ProjectPage/RunBatchSpikeSortingWindow/SelectProcessorComponent.tsx: -------------------------------------------------------------------------------- 1 | import { FunctionComponent } from "react" 2 | import { Hyperlink } from "@fi-sci/misc"; 3 | import { ComputeResourceSpecApp, ComputeResourceSpecProcessor, DendroComputeResourceApp } from "../../../types/dendro-types" 4 | 5 | type SelectProcessorComponentProps = { 6 | processors: {appSpec: ComputeResourceSpecApp, app?: DendroComputeResourceApp, processor: ComputeResourceSpecProcessor}[] 7 | onSelected: (processorName: string) => void 8 | } 9 | 10 | const SelectProcessorComponent: FunctionComponent = ({processors, onSelected}) => { 11 | return ( 12 |
13 |

Select a spike sorter

14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | { 23 | processors.map((processor, i) => ( 24 | 25 | 30 | 33 | 34 | )) 35 | } 36 | 37 |
NameApp
26 | {onSelected(processor.processor.name)}}> 27 | {processor.processor.name} 28 | 29 | 31 | {processor.appSpec.name} 32 |
38 |
39 | ) 40 | } 41 | 42 | export default SelectProcessorComponent -------------------------------------------------------------------------------- /src/pages/ProjectPage/RunBatchSpikeSortingWindow/getDefaultRequiredResources.ts: -------------------------------------------------------------------------------- 1 | import { ComputeResourceSpecProcessor, DendroJobRequiredResources } from "../../../types/dendro-types" 2 | 3 | 4 | const getDefaultRequiredResources = (processor: ComputeResourceSpecProcessor | undefined): DendroJobRequiredResources | undefined => { 5 | if (!processor) return undefined 6 | const tags = processor.tags.map(t => t.tag) 7 | if ((tags.includes('kilosort2_5') || tags.includes('kilosort3'))) { 8 | return { 9 | numCpus: 4, 10 | numGpus: 1, 11 | memoryGb: 16, 12 | timeSec: 3600 * 3 // todo: determine this based on the size of the recording! 13 | } 14 | } 15 | else if (tags.includes('mountainsort5')) { 16 | return { 17 | numCpus: 8, 18 | numGpus: 0, 19 | memoryGb: 16, 20 | timeSec: 3600 * 3 // todo: determine this based on the size of the recording! 21 | } 22 | } 23 | else { 24 | return { 25 | numCpus: 8, 26 | numGpus: 0, 27 | memoryGb: 16, 28 | timeSec: 3600 * 3 // todo: determine this based on the size of the recording! 29 | } 30 | } 31 | } 32 | 33 | export default getDefaultRequiredResources -------------------------------------------------------------------------------- /src/pages/ProjectPage/UploadSmalFileWindow/UploadSmallFileWindow.tsx: -------------------------------------------------------------------------------- 1 | import { FunctionComponent, useCallback } from "react" 2 | import { createFileAndInitiateUpload } from "../../../dbInterface/dbInterface" 3 | import { useProject } from "../ProjectPageContext" 4 | import { useGithubAuth } from "../../../GithubAuth/useGithubAuth" 5 | 6 | type UploadSmallFileWindowProps = { 7 | onClose: () => void 8 | } 9 | 10 | const UploadSmallFileWindow: FunctionComponent = ({ onClose }) => { 11 | const {projectId, refreshFiles} = useProject() 12 | const auth = useGithubAuth() 13 | const handleSelectFile = useCallback(() => { 14 | if (!auth) return 15 | const fileInput = document.createElement("input") 16 | fileInput.type = "file" 17 | fileInput.addEventListener("change", () => { 18 | const file = fileInput.files?.[0] 19 | if (!file) return 20 | // get the binary content of the file 21 | const reader = new FileReader() 22 | reader.readAsBinaryString(file) 23 | reader.addEventListener("load", () => { 24 | const content = reader.result 25 | if (!content) return 26 | ; (async () => { 27 | const {uploadUrl} = await createFileAndInitiateUpload( 28 | projectId, 29 | `uploads/${file.name}`, 30 | contentSize(content), 31 | auth 32 | ) 33 | const uploadRequest = new XMLHttpRequest() 34 | uploadRequest.open("PUT", uploadUrl) 35 | uploadRequest.setRequestHeader("Content-Type", "application/octet-stream") 36 | uploadRequest.send(content) 37 | uploadRequest.addEventListener("load", () => { 38 | refreshFiles() 39 | onClose() 40 | }) 41 | uploadRequest.addEventListener("error", (e) => { 42 | console.error(e) 43 | alert("Upload failed") 44 | }, {once: true}) 45 | })() 46 | }) 47 | }) 48 | fileInput.click() 49 | }, [auth, projectId, onClose, refreshFiles]) 50 | if (!auth) { 51 | return
Please login first.
52 | } 53 | return ( 54 |
55 | 58 |
59 | ) 60 | } 61 | 62 | const contentSize = (content: string | ArrayBuffer) => { 63 | if (typeof content === "string") { 64 | return content.length 65 | } 66 | return content.byteLength 67 | } 68 | 69 | export default UploadSmallFileWindow 70 | -------------------------------------------------------------------------------- /src/pages/ProjectPage/openFilesInNeurosift.ts: -------------------------------------------------------------------------------- 1 | import { url } from "inspector"; 2 | import { DendroFile } from "../../types/dendro-types"; 3 | 4 | const openFilesInNeurosift = async (files: DendroFile[], dendroProjectId: string) => { 5 | const urls = files.map((file) => urlFromFileContent(file.content)); 6 | let urlQuery = urls.map((url) => `url=${url}`).join('&'); 7 | // const fileNameQuery = files.map((file) => `fileName=${file.fileName}`).join('&'); 8 | // const neurosiftUrl = `https://flatironinstitute.github.io/neurosift/?p=/nwb&${urlQuery}&dendroProjectId=${dendroProjectId}&${fileNameQuery}`; 9 | const dandisetId = files[0]?.metadata.dandisetId; 10 | const dandisetVersion = files[0]?.metadata.dandisetVersion; 11 | const dandiAssetId = files[0]?.metadata.dandiAssetId; 12 | if (dandisetId) urlQuery += `&dandisetId=${dandisetId}`; 13 | if (dandisetVersion) urlQuery += `&dandisetVersion=${dandisetVersion}`; 14 | if (dandiAssetId) urlQuery += `&dandiAssetId=${dandiAssetId}`; 15 | const neurosiftUrl = `https://flatironinstitute.github.io/neurosift/?p=/nwb&${urlQuery}`; 16 | window.open(neurosiftUrl, '_blank'); 17 | } 18 | 19 | const urlFromFileContent = (content: string) => { 20 | if (!content.startsWith('url:')) { 21 | throw new Error('Invalid file content: ' + content); 22 | } 23 | return content.substring('url:'.length); 24 | } 25 | 26 | export default openFilesInNeurosift; -------------------------------------------------------------------------------- /src/pages/ProjectPage/scripts/CodeEditor.tsx: -------------------------------------------------------------------------------- 1 | import { Editor } from "@monaco-editor/react" 2 | import { editor as editor0 } from 'monaco-editor'; 3 | import * as monaco from 'monaco-editor/esm/vs/editor/editor.api'; 4 | import { FunctionComponent, useCallback, useEffect, useState } from "react" 5 | 6 | type Monaco = typeof monaco 7 | 8 | type CodeEditorProps = { 9 | width: number 10 | height: number 11 | content: string 12 | readOnly: boolean 13 | onContentChanged: (content: string) => void 14 | onSave: () => void 15 | } 16 | 17 | const CodeEditor: FunctionComponent = ({width, height, content, readOnly, onContentChanged, onSave}) => { 18 | const handleChange = useCallback((value: string | undefined) => { 19 | if (value === undefined) return 20 | onContentChanged(value) 21 | }, [onContentChanged]) 22 | const wordWrap = false 23 | 24 | const [editor, setEditor] = useState(undefined) 25 | useEffect(() => { 26 | if (!editor) return 27 | if (editor.getValue() === content) return 28 | editor.setValue(content || '') 29 | }, [editor, content]) 30 | const handleEditorDidMount = useCallback((editor: editor0.IStandaloneCodeEditor, monaco: Monaco) => { 31 | (async () => { 32 | // fancy stuff would go here 33 | setEditor(editor) 34 | })() 35 | }, []) 36 | // Can't do this in the usual way with monaco editor: 37 | // See: https://github.com/microsoft/monaco-editor/issues/2947 38 | const handleKeyDown = useCallback((e: React.KeyboardEvent) => { 39 | if ((e.ctrlKey || e.metaKey) && e.key === 's') { 40 | e.preventDefault() 41 | if (readOnly) return 42 | if (!readOnly) { 43 | onSave() 44 | } 45 | } 46 | }, [onSave, readOnly]) 47 | return ( 48 |
49 | 62 |
63 | ) 64 | } 65 | 66 | export default CodeEditor -------------------------------------------------------------------------------- /src/pages/ProjectPage/scripts/RunScript/RunScriptWorkerTypes.ts: -------------------------------------------------------------------------------- 1 | export type RunScriptAddJobInputFile = { 2 | name: string 3 | fileName: string 4 | isFolder?: boolean 5 | } 6 | 7 | export type RunScriptAddJobOutputFile = { 8 | name: string 9 | fileName: string 10 | isFolder?: boolean 11 | skipCloudUpload?: boolean 12 | } 13 | 14 | export type RunScriptAddJobParameter = { 15 | name: string 16 | value: any 17 | } 18 | 19 | export type RunScriptAddJobRequiredResources = { 20 | numCpus: number 21 | numGpus: number 22 | memoryGb: number 23 | timeSec: number 24 | } 25 | 26 | export type RunScriptAddJob = { 27 | processorName: string 28 | inputFiles: RunScriptAddJobInputFile[] 29 | outputFiles: RunScriptAddJobOutputFile[] 30 | inputParameters: RunScriptAddJobParameter[] 31 | requiredResources: RunScriptAddJobRequiredResources 32 | runMethod: 'local' | 'aws_batch' | 'slurm' 33 | } 34 | 35 | export type RunScriptAddedFile = { 36 | fileName: string 37 | url: string 38 | } 39 | 40 | export type RunScriptResult = { 41 | jobs: RunScriptAddJob[] 42 | addedFiles: RunScriptAddedFile[] 43 | } 44 | 45 | export {} -------------------------------------------------------------------------------- /src/pages/ProjectsPage/ProjectsPage.css: -------------------------------------------------------------------------------- 1 | .projects-page { 2 | /* padding: 20px; */ 3 | font-family: Arial, sans-serif; 4 | max-width: 1400px; 5 | margin: auto; 6 | } 7 | 8 | .projects-page h3 { 9 | color: #333; 10 | } 11 | 12 | .projects-page p { 13 | margin-bottom: 20px; 14 | text-align: justify; 15 | } -------------------------------------------------------------------------------- /src/pages/ProjectsPage/ProjectsPage.tsx: -------------------------------------------------------------------------------- 1 | import { Hyperlink, SmallIconButton } from "@fi-sci/misc"; 2 | import { Add, ImportExport } from "@mui/icons-material"; 3 | import { FunctionComponent, useCallback } from "react"; 4 | import useRoute from "../../useRoute"; 5 | import './ProjectsPage.css'; 6 | import ProjectsTable from "./ProjectsTable"; 7 | import { createProject } from "../../dbInterface/dbInterface"; 8 | import { useGithubAuth } from "../../GithubAuth/useGithubAuth"; 9 | 10 | type Props = { 11 | width: number 12 | height: number 13 | } 14 | 15 | const ProjectsPage: FunctionComponent = ({width, height}) => { 16 | const {setRoute} = useRoute() 17 | 18 | const auth = useGithubAuth() 19 | 20 | const handleAdd = useCallback(async () => { 21 | if (!auth.signedIn) { 22 | alert('You must be signed in to create a project') 23 | return 24 | } 25 | const projectName = prompt('Enter a name for your project', 'untitled') 26 | if (!projectName) return 27 | const newProjectId = await createProject(projectName, auth) 28 | setRoute({page: 'project', projectId: newProjectId}) 29 | }, [setRoute, auth]) 30 | 31 | return ( 32 |
33 |

Your projects

34 |
35 | } onClick={handleAdd} label="Create a new project" /> 36 |        37 | } onClick={() => setRoute({page: 'dandisets'})} label="Import data from DANDI" /> 38 |
39 |
40 | 41 |
42 | ) 43 | } 44 | 45 | export default ProjectsPage -------------------------------------------------------------------------------- /src/pages/ProjectsPage/useProjectsForUser.ts: -------------------------------------------------------------------------------- 1 | import { useEffect, useState } from "react"; 2 | import { useGithubAuth } from "../../GithubAuth/useGithubAuth"; 3 | import { fetchAdminAllProjects, fetchProjectsForUser } from "../../dbInterface/dbInterface"; 4 | import { DendroProject } from "../../types/dendro-types"; 5 | 6 | const useProjectsForUser = (o: {admin?: boolean} = {}): DendroProject[] | undefined => { 7 | const [projects, setProjects] = useState(undefined) 8 | const auth = useGithubAuth() 9 | useEffect(() => { 10 | let canceled = false 11 | 12 | if (!auth) { 13 | setProjects([]) 14 | return 15 | } 16 | 17 | setProjects(undefined) 18 | 19 | ; (async () => { 20 | let projects 21 | if (!o.admin) { 22 | projects = await fetchProjectsForUser(auth) 23 | } 24 | else { 25 | projects = await fetchAdminAllProjects(auth) 26 | } 27 | if (canceled) return 28 | setProjects(projects) 29 | })() 30 | return () => { 31 | canceled = true 32 | } 33 | }, [auth, o.admin]) 34 | return projects 35 | } 36 | 37 | export default useProjectsForUser -------------------------------------------------------------------------------- /src/pages/RegisterComputeResourcePage/RegisterComputeResourcePage.tsx: -------------------------------------------------------------------------------- 1 | import { FunctionComponent, useCallback, useMemo, useState } from "react"; 2 | import { registerComputeResource } from "../../dbInterface/dbInterface"; 3 | import { useGithubAuth } from "../../GithubAuth/useGithubAuth"; 4 | import UserIdComponent from "../../UserIdComponent"; 5 | import useRoute from "../../useRoute"; 6 | 7 | type Props = { 8 | // none 9 | } 10 | 11 | const RegisterComputeResourcePage: FunctionComponent = () => { 12 | const {route, setRoute} = useRoute() 13 | const [name, setName] = useState('') 14 | 15 | if (route.page !== 'register-compute-resource') throw Error('Unexpected') 16 | 17 | const {computeResourceId, resourceCode} = route 18 | 19 | const auth = useGithubAuth() 20 | 21 | const handleRegister = useCallback(async () => { 22 | if (!auth) return 23 | try { 24 | await registerComputeResource(computeResourceId, resourceCode, name, auth) 25 | } 26 | catch(e) { 27 | alert(`Error registering compute resource: ${e}`) 28 | return 29 | } 30 | setRoute({page: 'compute-resources'}) 31 | }, [computeResourceId, resourceCode, name, auth, setRoute]) 32 | 33 | if (!auth.userId) { 34 | return

To register this compute resource, you must first log in. Click the "Log in" button in the upper right corner.

35 | } 36 | 37 | return ( 38 |
39 |

You are registering compute resource {abbreviate(computeResourceId, 12)} to user

40 |
41 | Choose a name for this resource: setName(e.target.value)} /> 42 |
43 |
44 |
45 | 46 |
47 |
48 | ) 49 | } 50 | 51 | const abbreviate = (s: string, maxLength: number) => { 52 | if (s.length <= maxLength) return s 53 | return s.slice(0, maxLength - 3) + '...' 54 | } 55 | 56 | export default RegisterComputeResourcePage; -------------------------------------------------------------------------------- /src/plugins/DendroFrontendPlugin.ts: -------------------------------------------------------------------------------- 1 | import { DendroJobRequiredResources } from "../types/dendro-types" 2 | 3 | export type PluginAction = { 4 | type: 'processor-that-generates-a-single-output', 5 | name: string, 6 | label: string, 7 | defaultOutputFileName: string, 8 | processorTag: string, 9 | defaultRequiredResources: DendroJobRequiredResources 10 | } 11 | 12 | export type PluginContext = { 13 | registerAction: (action: PluginAction) => void 14 | } 15 | 16 | export type DendroFrontendPlugin = { 17 | pluginName: string 18 | initialize: (context: PluginContext) => void 19 | } -------------------------------------------------------------------------------- /src/plugins/initializePlugins.ts: -------------------------------------------------------------------------------- 1 | import { PluginAction, PluginContext } from "./DendroFrontendPlugin" 2 | import mearecPlugin from "./mearec/mearecPlugin" 3 | 4 | const initializePlugins = () => { 5 | const actions: PluginAction[] = [] 6 | const pluginContext: PluginContext = { 7 | registerAction: (action) => { 8 | actions.push(action) 9 | } 10 | } 11 | mearecPlugin.initialize(pluginContext) 12 | 13 | return { 14 | actions 15 | } 16 | } 17 | 18 | export default initializePlugins -------------------------------------------------------------------------------- /src/plugins/mearec/mearecPlugin.ts: -------------------------------------------------------------------------------- 1 | import { DendroFrontendPlugin, PluginContext } from "../DendroFrontendPlugin" 2 | 3 | const mearecPlugin: DendroFrontendPlugin = { 4 | pluginName: 'mearec', 5 | initialize: (context: PluginContext) => { 6 | context.registerAction({ 7 | name: 'mearec-generate-templates', 8 | type: 'processor-that-generates-a-single-output', 9 | label: 'MEArec: generate templates', 10 | defaultOutputFileName: 'generated/mearec/default.templates.h5', 11 | processorTag: 'mearec_generate_templates', 12 | defaultRequiredResources: { 13 | numCpus: 4, 14 | numGpus: 0, 15 | memoryGb: 8, 16 | timeSec: 60 * 60 17 | } 18 | }) 19 | } 20 | } 21 | 22 | export default mearecPlugin 23 | -------------------------------------------------------------------------------- /src/pubnub/pubnub.ts: -------------------------------------------------------------------------------- 1 | import PubNub from 'pubnub' 2 | 3 | const PUBNUB_SUBSCRIBE_KEY = import.meta.env.VITE_PUBNUB_SUBSCRIBE_KEY 4 | 5 | let pnClient: PubNub | undefined = undefined 6 | if (PUBNUB_SUBSCRIBE_KEY) { 7 | pnClient = new PubNub({ 8 | subscribeKey: PUBNUB_SUBSCRIBE_KEY, 9 | userId: 'browser' 10 | }) 11 | } 12 | else { 13 | console.warn('PUBNUB_SUBSCRIBE_KEY not set. Not connecting to PubNub.') 14 | } 15 | 16 | export const onPubsubMessage = (callback: (message: any) => void) => { 17 | const listener = { 18 | message: (messageEvent: any) => { 19 | callback(messageEvent.message) 20 | } 21 | } 22 | if (pnClient) { 23 | pnClient.addListener(listener) 24 | } 25 | const cancel = () => { 26 | if (pnClient) { 27 | pnClient.removeListener(listener) 28 | } 29 | } 30 | return cancel 31 | } 32 | 33 | export const setPubNubListenChannel = (channel: string) => { 34 | if (pnClient) { 35 | pnClient.unsubscribeAll() 36 | pnClient.subscribe({ 37 | channels: [channel] 38 | }) 39 | } 40 | } -------------------------------------------------------------------------------- /src/scientific-table.css: -------------------------------------------------------------------------------- 1 | /* Table container */ 2 | .table-container { 3 | font-family: "Courier New", monospace; 4 | width: 100%; 5 | overflow-x: auto; 6 | white-space: nowrap; 7 | cursor: default; 8 | } 9 | 10 | /* Table */ 11 | .scientific-table { 12 | border-collapse: collapse; 13 | width: 100%; 14 | } 15 | 16 | /* Table header */ 17 | .scientific-table th { 18 | background-color: #c5af96; 19 | color: #ffffff; 20 | text-align: left; 21 | padding: 4px 8px; 22 | border: 1px solid #aaa7a7; 23 | } 24 | 25 | /* Table rows */ 26 | .scientific-table tr:nth-child(even) { 27 | background-color: #f2f2f2; 28 | } 29 | 30 | .scientific-table tr:nth-child(odd) { 31 | background-color: #ffffff; 32 | } 33 | 34 | /* Table data */ 35 | .scientific-table td { 36 | padding: 4px 8px; 37 | border: 1px solid #ccc; 38 | } 39 | 40 | /* Table hover */ 41 | .scientific-table tbody tr:hover { 42 | background-color: #ddd; 43 | } 44 | 45 | .scientific-table tr.selected { 46 | background-color: #aad; 47 | } 48 | 49 | .scientific-table tr.selected:hover { 50 | background-color: #ccf; 51 | } -------------------------------------------------------------------------------- /src/table1.css: -------------------------------------------------------------------------------- 1 | .table1 { 2 | border-collapse: collapse; 3 | width: 100%; 4 | text-align: left; 5 | color: #333; 6 | font-family: 'Roboto', sans-serif; 7 | } 8 | 9 | .table1 td { 10 | border: 1px solid #dfcece; 11 | padding: 3px; 12 | } 13 | 14 | .table1 tr:nth-child(even) { 15 | background-color: #f2f2f2; 16 | } 17 | 18 | .table1 tr:hover { 19 | background-color: #ddd; 20 | } 21 | 22 | .table1 td:first-child { 23 | font-weight: bold; 24 | } 25 | 26 | .table2 { 27 | border-collapse: collapse; 28 | width: 100%; 29 | text-align: left; 30 | color: #333; 31 | font-family: 'Roboto', sans-serif; 32 | } -------------------------------------------------------------------------------- /src/timeStrings.ts: -------------------------------------------------------------------------------- 1 | // thanks https://stackoverflow.com/a/6109105/160863 and gh copilot! 2 | export function timeAgoString(timestampSeconds?: number) { 3 | if (timestampSeconds === undefined) return '' 4 | const now = Date.now() 5 | const diff = now - timestampSeconds * 1000 6 | const diffSeconds = Math.floor(diff / 1000) 7 | const diffMinutes = Math.floor(diffSeconds / 60) 8 | const diffHours = Math.floor(diffMinutes / 60) 9 | const diffDays = Math.floor(diffHours / 24) 10 | const diffWeeks = Math.floor(diffDays / 7) 11 | const diffMonths = Math.floor(diffWeeks / 4) 12 | const diffYears = Math.floor(diffMonths / 12) 13 | if (diffYears > 0) { 14 | return `${diffYears} yr${diffYears === 1 ? '' : 's'} ago` 15 | } 16 | else if (diffWeeks > 0) { 17 | return `${diffWeeks} wk${diffWeeks === 1 ? '' : 's'} ago` 18 | } 19 | else if (diffDays > 0) { 20 | return `${diffDays} day${diffDays === 1 ? '' : 's'} ago` 21 | } 22 | else if (diffHours > 0) { 23 | return `${diffHours} hr${diffHours === 1 ? '' : 's'} ago` 24 | } 25 | else if (diffMinutes > 0) { 26 | return `${diffMinutes} min ago` 27 | } 28 | else { 29 | return `${diffSeconds} sec ago` 30 | } 31 | } 32 | 33 | export function elapsedTimeString(numSeconds?: number) { 34 | if (numSeconds === undefined) return '' 35 | numSeconds = Math.floor(numSeconds) 36 | const numMinutes = Math.floor(numSeconds / 60) 37 | const numHours = Math.floor(numMinutes / 60) 38 | const numDays = Math.floor(numHours / 24) 39 | const numWeeks = Math.floor(numDays / 7) 40 | const numMonths = Math.floor(numWeeks / 4) 41 | const numYears = Math.floor(numMonths / 12) 42 | if (numYears > 0) { 43 | return `${numYears} yr${numYears === 1 ? '' : 's'}` 44 | } 45 | else if (numWeeks > 5) { 46 | return `${numWeeks} wk${numWeeks === 1 ? '' : 's'}` 47 | } 48 | else if (numDays > 5) { 49 | return `${numDays} day${numDays === 1 ? '' : 's'}` 50 | } 51 | else if (numHours > 5) { 52 | return `${numHours} hr${numHours === 1 ? '' : 's'}` 53 | } 54 | else if (numMinutes > 5) { 55 | return `${numMinutes} min` 56 | } 57 | else { 58 | return `${numSeconds} sec` 59 | } 60 | } -------------------------------------------------------------------------------- /src/useWindowDimensions.ts: -------------------------------------------------------------------------------- 1 | import { useEffect, useState } from 'react' 2 | 3 | // Thanks: https://stackoverflow.com/questions/36862334/get-viewport-window-height-in-reactjs 4 | function getWindowDimensions() { 5 | const { innerWidth: width, innerHeight: height } = window; 6 | return { 7 | width, 8 | height 9 | }; 10 | } 11 | 12 | const useWindowDimensions = () => { 13 | const [windowDimensions, setWindowDimensions] = useState(getWindowDimensions()); 14 | 15 | useEffect(() => { 16 | function handleResize() { 17 | setWindowDimensions(getWindowDimensions()); 18 | } 19 | 20 | window.addEventListener('resize', handleResize); 21 | return () => window.removeEventListener('resize', handleResize); 22 | }, []); 23 | 24 | return windowDimensions; 25 | } 26 | ////////////////////////////////////////////////////////////////////////////////////////////////// 27 | 28 | export default useWindowDimensions -------------------------------------------------------------------------------- /src/vite-env.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ESNext", 4 | "useDefineForClassFields": true, 5 | "lib": ["DOM", "DOM.Iterable", "ESNext"], 6 | "allowJs": false, 7 | "skipLibCheck": true, 8 | "esModuleInterop": false, 9 | "allowSyntheticDefaultImports": true, 10 | "strict": true, 11 | "forceConsistentCasingInFileNames": true, 12 | "module": "ESNext", 13 | "moduleResolution": "Node", 14 | "resolveJsonModule": true, 15 | "isolatedModules": true, 16 | "noEmit": true, 17 | "jsx": "react-jsx" 18 | }, 19 | "include": ["src"], 20 | "references": [{ "path": "./tsconfig.node.json" }] 21 | } 22 | -------------------------------------------------------------------------------- /tsconfig.node.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "composite": true, 4 | // "skipLibCheck": true, 5 | "module": "ESNext", 6 | "moduleResolution": "Node", 7 | "allowSyntheticDefaultImports": true 8 | }, 9 | "include": ["vite.config.ts"] 10 | } 11 | -------------------------------------------------------------------------------- /vercel.json: -------------------------------------------------------------------------------- 1 | { 2 | "rewrites": [ 3 | { 4 | "source": "/api/(.*)", 5 | "destination": "/api/index.py" 6 | }, 7 | { 8 | "source": "/docs", 9 | "destination": "/api/index.py" 10 | }, 11 | { 12 | "source": "/redoc", 13 | "destination": "/api/index.py" 14 | }, 15 | { 16 | "source": "/openapi.json", 17 | "destination": "/api/index.py" 18 | } 19 | ] 20 | } -------------------------------------------------------------------------------- /vercel.json.readme.md: -------------------------------------------------------------------------------- 1 | Here are some important notes for developers regarding the `vercel.json` 2 | See: https://vercel.com/docs/projects/project-configuration 3 | 4 | Note that there is a `_vercel_dev.json` and a `_vercel_prod.json`. One is required for local dev and the other is required for prod. In package.json, notice in the deploy script, the `_vercel_prod.json` gets copied onto `vercel.json` and then after the build, `_vercel_dev.json` gets copied onto the `vercel.json`. I'd like the `_vercel_dev.json` to be the one in the source repo. -------------------------------------------------------------------------------- /vite.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from 'vite' 2 | import react from '@vitejs/plugin-react' 3 | 4 | // https://vitejs.dev/config/ 5 | export default defineConfig({ 6 | plugins: [react()], 7 | assetsInclude: ['**/*.md'] 8 | }) 9 | --------------------------------------------------------------------------------