├── .codecov.yml ├── .docker ├── app_dockerfile ├── app_entrypoint.sh ├── mongo_dockerfile └── server_dockerfile ├── .dockerignore ├── .github ├── CODEOWNERS ├── dependabot.yml └── workflows │ ├── ci.yml │ ├── dependable-bot.yml │ ├── license-check.yml │ ├── release.yml │ └── yarn-upgrade-bot.yml ├── .gitignore ├── .pre-commit-config.yaml ├── .prettierrc ├── .readthedocs.yaml ├── CHANGELOG.md ├── CITATION.cff ├── INSTALL.md ├── LICENSE ├── README.md ├── docker-compose.yml ├── pydatalab ├── README.md ├── docs │ ├── .pages │ ├── CHANGELOG.md │ ├── INSTALL.md │ ├── LICENSE │ ├── blocks │ │ ├── .pages │ │ ├── apps │ │ │ ├── .pages │ │ │ ├── chat.md │ │ │ ├── echem.md │ │ │ ├── eis.md │ │ │ ├── ftir.md │ │ │ ├── nmr.md │ │ │ ├── raman.md │ │ │ ├── tga.md │ │ │ ├── uvvis.md │ │ │ └── xrd.md │ │ ├── base.md │ │ └── common.md │ ├── config.md │ ├── css │ │ └── reference.css │ ├── deployment.md │ ├── index.md │ └── schemas │ │ ├── .pages │ │ ├── cells.md │ │ ├── collections.md │ │ ├── equipment.md │ │ ├── items.md │ │ ├── samples.md │ │ └── starting_materials.md ├── example_data │ ├── FTIR │ │ └── 2024-10-10_FeSO4_ref.asp │ ├── NMR │ │ ├── 1.zip │ │ ├── 13c.jdx │ │ ├── 1h.dx │ │ ├── 71.zip │ │ ├── 72.zip │ │ └── README │ ├── TGA-MS │ │ ├── 20221128 134958 TGA MS Megan.asc │ │ └── mp2028_281122_LNO+C.txt │ ├── UV-Vis │ │ ├── 1908047U1_0000.Raw8.TXT │ │ ├── 1908047U1_0001.Raw8.txt │ │ └── 1908047U1_0060.Raw8.txt │ ├── XRD │ │ ├── CG20396_jdb12-1.xrdml │ │ ├── HL1-2_5-90_60min.xrdml │ │ ├── JO_KL_16_ZF_3_60deg_0.02step_12degpermin_001.rasx │ │ ├── Scan_C1.xrdml │ │ ├── Scan_C10.xrdml │ │ ├── Scan_C2.xrdml │ │ ├── Scan_C3.xrdml │ │ ├── Scan_C4.xrdml │ │ ├── Scan_C5.xrdml │ │ ├── Scan_C6.xrdml │ │ ├── Scan_C7.xrdml │ │ ├── Scan_C8.xrdml │ │ ├── Scan_C9.xrdml │ │ ├── cod_9004112.cif │ │ ├── example_bmb.xye │ │ ├── example_evadiffract.xy │ │ ├── example_mac.xye │ │ ├── example_mythen.dat │ │ ├── example_mythen.xye │ │ └── example_ocx.xy │ ├── csv │ │ ├── simple.csv │ │ └── simple.xlsx │ ├── echem │ │ ├── README │ │ ├── jdb11-1_c3_gcpl_5cycles_2V-3p8V_C-24_data.mps │ │ ├── jdb11-1_c3_gcpl_5cycles_2V-3p8V_C-24_data_C09.mpr │ │ ├── jdb11-1_c3_gcpl_5cycles_2V-3p8V_C-24_data_D1_C09.sta │ │ └── jdb11-1_e1_s3_squidTest_data_C15.mpr │ └── raman │ │ ├── labspec_raman_example.txt │ │ ├── raman_example.txt │ │ └── raman_example.wdf ├── mkdocs.yml ├── pyproject.toml ├── schemas │ ├── cell.json │ ├── equipment.json │ ├── sample.json │ └── startingmaterial.json ├── scripts │ ├── add_test_cell_to_db.py │ ├── create_mongo_indices.py │ ├── generate_cy_links_json.py │ ├── generate_cy_links_json_typedRelationship.py │ ├── migrate_add_fields_to_files.py │ ├── migrate_add_item_ids_to_all_blocks.py │ ├── migrate_copy_data_collection_to_items.py │ ├── migrate_file_last_modified_remote_timestamp_to_last_modified_remote.py │ ├── migrate_file_sample_ids_to_item_ids.py │ ├── migrate_files_to_files_ObjectId_v2.py │ ├── migrate_image_blocks_to_media_blocks.py │ ├── migrate_rename_data_collection_to_items.py │ ├── migrate_rename_item_kind_to_type.py │ ├── migrate_rename_starting_material_field.py │ ├── migrate_sample_id_to_item_id.py │ ├── migrate_set_all_constituents_as_parents.py │ ├── migrate_set_all_constituents_as_parents_TypedRelationship.py │ └── migrate_set_all_samples_to_have_type_samples.py ├── src │ └── pydatalab │ │ ├── __init__.py │ │ ├── apps │ │ ├── __init__.py │ │ ├── _canary │ │ │ └── __init__.py │ │ ├── chat │ │ │ ├── __init__.py │ │ │ └── blocks.py │ │ ├── echem │ │ │ ├── __init__.py │ │ │ ├── blocks.py │ │ │ └── utils.py │ │ ├── eis │ │ │ └── __init__.py │ │ ├── ftir │ │ │ └── __init__.py │ │ ├── nmr │ │ │ ├── __init__.py │ │ │ ├── blocks.py │ │ │ └── utils.py │ │ ├── raman │ │ │ ├── __init__.py │ │ │ └── blocks.py │ │ ├── tga │ │ │ ├── __init__.py │ │ │ ├── blocks.py │ │ │ └── parsers.py │ │ ├── uvvis │ │ │ └── __init__.py │ │ └── xrd │ │ │ ├── __init__.py │ │ │ ├── blocks.py │ │ │ ├── models.py │ │ │ └── utils.py │ │ ├── backups.py │ │ ├── blocks │ │ ├── __init__.py │ │ ├── base.py │ │ └── common.py │ │ ├── bokeh_plots.py │ │ ├── config.py │ │ ├── errors.py │ │ ├── file_utils.py │ │ ├── logger.py │ │ ├── login.py │ │ ├── main.py │ │ ├── models │ │ ├── __init__.py │ │ ├── cells.py │ │ ├── collections.py │ │ ├── entries.py │ │ ├── equipment.py │ │ ├── files.py │ │ ├── items.py │ │ ├── people.py │ │ ├── relationships.py │ │ ├── samples.py │ │ ├── starting_materials.py │ │ ├── traits.py │ │ └── utils.py │ │ ├── mongo.py │ │ ├── permissions.py │ │ ├── remote_filesystems.py │ │ ├── routes │ │ ├── __init__.py │ │ └── v0_1 │ │ │ ├── __init__.py │ │ │ ├── _version.py │ │ │ ├── admin.py │ │ │ ├── auth.py │ │ │ ├── blocks.py │ │ │ ├── collections.py │ │ │ ├── files.py │ │ │ ├── graphs.py │ │ │ ├── healthcheck.py │ │ │ ├── info.py │ │ │ ├── items.py │ │ │ ├── remotes.py │ │ │ └── users.py │ │ ├── send_email.py │ │ └── utils.py ├── tasks.py ├── tests │ ├── __init__.py │ ├── apps │ │ ├── test_echem_block.py │ │ ├── test_ftir_block.py │ │ ├── test_nmr.py │ │ ├── test_plugins.py │ │ ├── test_raman.py │ │ ├── test_tabular.py │ │ ├── test_tga.py │ │ ├── test_uvvis.py │ │ └── test_xrd_block.py │ ├── conftest.py │ ├── server │ │ ├── conftest.py │ │ ├── test_auth.py │ │ ├── test_backup.py │ │ ├── test_equipment.py │ │ ├── test_files.py │ │ ├── test_graph.py │ │ ├── test_info_and_health.py │ │ ├── test_item_graph.py │ │ ├── test_items.py │ │ ├── test_permissions.py │ │ ├── test_remote_filesystems.py │ │ ├── test_remotes.py │ │ ├── test_samples.py │ │ ├── test_starting_materials.py │ │ └── test_users.py │ ├── test_config.py │ ├── test_models.py │ └── test_version.py └── uv.lock └── webapp ├── .browserslistrc ├── .env.test_e2e ├── .eslintrc.js ├── .gitignore ├── .prettierrc ├── README.md ├── babel.config.js ├── cypress.config.ts ├── cypress ├── component │ ├── ChemicalFormulaTest.cy.jsx │ ├── CollectionTableTest.cy.jsx │ ├── CreatorsTest.cy.jsx │ ├── EquipmentTableTest.cy.jsx │ ├── FormattedItemNameTest.cy.jsx │ ├── GHSInformationTest.cy.jsx │ ├── NavbarTest.cy.jsx │ ├── NotificationDotTest.cy.jsx │ ├── SampleTableTest.cy.jsx │ ├── StartingMaterialTableTest.cy.jsx │ ├── StyledBlockHelpTest.cy.jsx │ ├── StyledBlockInfoTest.cy.jsx │ ├── UserBubbleLoginTest.cy.jsx │ └── UserBubbleTest.cy.jsx ├── e2e │ ├── batchSampleFeature.cy.js │ ├── editPage.cy.js │ ├── equipment.cy.js │ ├── sampleTablePage.cy.js │ └── startingMaterial.cy.js ├── fixtures │ └── example_data ├── plugins │ └── index.js └── support │ ├── commands.js │ ├── component-index.html │ ├── component.js │ └── e2e.js ├── package.json ├── public ├── favicon.ico └── index.html ├── scripts └── get-version.js ├── src ├── App.vue ├── components │ ├── AddToCollectionModal.vue │ ├── AdminDisplay.vue │ ├── AdminNavbar.vue │ ├── BarcodeModal.vue │ ├── BaseIconCounter.vue │ ├── BatchCreateItemModal.vue │ ├── BlocksIconCounter.vue │ ├── BokehPlot.vue │ ├── CellInformation.vue │ ├── CellPreparationInformation.vue │ ├── ChatWindow.vue │ ├── ChemFormulaInput.vue │ ├── ChemicalFormula.vue │ ├── CollectionInformation.vue │ ├── CollectionList.vue │ ├── CollectionRelationshipVisualization.vue │ ├── CollectionSelect.vue │ ├── CollectionTable.vue │ ├── CompactConstituentTable.vue │ ├── CreateCollectionModal.vue │ ├── CreateEquipmentModal.vue │ ├── CreateItemModal.vue │ ├── Creators.vue │ ├── DynamicDataTable.vue │ ├── DynamicDataTableButtons.vue │ ├── EditAccountSettingsModal.vue │ ├── EquipmentInformation.vue │ ├── EquipmentTable.vue │ ├── FileList.vue │ ├── FileMultiSelectDropdown.vue │ ├── FileSelectDropdown.vue │ ├── FileSelectModal.vue │ ├── FilesIconCounter.vue │ ├── FormField.vue │ ├── FormattedBarcode.vue │ ├── FormattedCollectionName.vue │ ├── FormattedItemName.vue │ ├── FormattedRefcode.vue │ ├── GHSHazardInformation.vue │ ├── GHSHazardPictograms.vue │ ├── GetEmailModal.vue │ ├── Isotope.vue │ ├── ItemGraph.vue │ ├── ItemRelationshipVisualization.vue │ ├── ItemSelect.vue │ ├── LoginDetails.vue │ ├── LoginDropdown.vue │ ├── MessageBubble.vue │ ├── Modal.vue │ ├── Navbar.vue │ ├── NotificationDot.vue │ ├── QRCode.vue │ ├── QRCodeModal.vue │ ├── QRScannerModal.vue │ ├── RelationshipVisualization.vue │ ├── SampleInformation.vue │ ├── SampleTable.vue │ ├── SelectableFileTree.vue │ ├── StartingMaterialInformation.vue │ ├── StartingMaterialTable.vue │ ├── StatisticsTable.vue │ ├── StyledBlockHelp.vue │ ├── StyledBlockInfo.vue │ ├── StyledInput.vue │ ├── SynthesisInformation.vue │ ├── TableOfContents.vue │ ├── TinyMceInline.vue │ ├── ToggleableCollectionFormGroup.vue │ ├── ToggleableCreatorsFormGroup.vue │ ├── TreeMenu.vue │ ├── UserBubble.vue │ ├── UserBubbleLogin.vue │ ├── UserDropdown.vue │ ├── UserSelect.vue │ ├── UserTable.vue │ ├── __mocks__ │ │ └── bokeh.js │ ├── datablocks │ │ ├── BokehBlock.vue │ │ ├── ChatBlock.vue │ │ ├── CycleBlock.vue │ │ ├── DataBlockBase.vue │ │ ├── GenericBlock.vue │ │ ├── MediaBlock.vue │ │ ├── NMRBlock.vue │ │ ├── NMRInsituBlock.vue │ │ ├── NotImplementedBlock.vue │ │ ├── UVVisBlock.vue │ │ └── XRDBlock.vue │ └── itemCreateModalAddons │ │ ├── CellCreateModalAddon.vue │ │ ├── SampleCreateModalAddon.vue │ │ └── StartingMaterialCreateModalAddon.vue ├── field_utils.js ├── file_upload.js ├── main.js ├── primevue-theme-preset.js ├── resources.js ├── router │ └── index.js ├── server_fetch_utils.js ├── store │ └── index.js └── views │ ├── About.vue │ ├── Admin.vue │ ├── CollectionPage.vue │ ├── Collections.vue │ ├── EditPage.vue │ ├── Equipment.vue │ ├── ExampleGraph.vue │ ├── ItemGraphPage.vue │ ├── Login.vue │ ├── Login2.vue │ ├── Login3.vue │ ├── NotFound.vue │ ├── Samples.vue │ └── StartingMaterials.vue ├── vue.config.js └── yarn.lock /.codecov.yml: -------------------------------------------------------------------------------- 1 | comment: 2 | layout: "diff, files" 3 | behavior: default 4 | require_changes: false 5 | require_base: false 6 | require_head: true 7 | coverage: 8 | status: 9 | project: 10 | default: 11 | informational: true 12 | patch: 13 | default: 14 | informational: true 15 | github_checks: 16 | annotations: false 17 | -------------------------------------------------------------------------------- /.docker/app_dockerfile: -------------------------------------------------------------------------------- 1 | # syntax=docker/dockerfile:1 2 | FROM node:20-bullseye AS build 3 | SHELL ["/bin/bash", "--login", "-c"] 4 | 5 | WORKDIR /app 6 | 7 | EXPOSE 8081 8 | 9 | COPY webapp/package.json webapp/yarn.lock ./ 10 | 11 | # Using a custom node_modules location to avoid mounting it outside of docker 12 | RUN --mount=type=cache,target=/root/.cache/yarn yarn install --frozen-lockfile --modules-folder /node_modules 13 | 14 | # These get replaced by the entrypoint script for production builds. 15 | # Set the real values in `.env` files or an external docker-compose. 16 | ENV NODE_ENV=production 17 | ARG VUE_APP_API_URL=magic-api-url 18 | ARG VUE_APP_LOGO_URL=magic-logo-url 19 | ARG VUE_APP_HOMEPAGE_URL=magic-homepage-url 20 | ARG VUE_APP_EDITABLE_INVENTORY=magic-setting 21 | ARG VUE_APP_WEBSITE_TITLE=magic-title 22 | ARG VUE_APP_QR_CODE_RESOLVER_URL=magic-qr-code-resolver-url 23 | ARG VUE_APP_AUTOMATICALLY_GENERATE_ID_DEFAULT=magic-generate-id-setting 24 | 25 | COPY webapp ./ 26 | RUN --mount=type=bind,target=/.git,src=./.git VUE_APP_GIT_VERSION=$(node scripts/get-version.js) /node_modules/.bin/vue-cli-service build 27 | 28 | FROM node:20-bullseye AS production 29 | 30 | ENV PATH=$PATH:/node_modules/.bin 31 | COPY ./.docker/app_entrypoint.sh /app/ 32 | 33 | COPY webapp/package.json webapp/yarn.lock ./ 34 | 35 | COPY --from=build /app/dist /app/dist 36 | RUN --mount=type=cache,target=/root/.cache/yarn yarn install --frozen-lockfile --modules-folder /node_modules --production 37 | 38 | CMD [ "/bin/bash", "-c", "/app/app_entrypoint.sh" ] 39 | 40 | HEALTHCHECK --interval=30s --timeout=5s --start-period=5s --retries=3 CMD curl --fail http://localhost:8081 || exit 1 41 | -------------------------------------------------------------------------------- /.docker/app_entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Patch the built app to use the specified environment variables 4 | # at runtime, then serve the app 5 | # 6 | # See https://stackoverflow.com/questions/53010064/pass-environment-variable-into-a-vue-app-at-runtime for inspiration. 7 | # 8 | set -e 9 | ROOT_DIR=/app/dist 10 | 11 | if [ -z "$VUE_APP_API_URL" ]; then 12 | echo "VUE_APP_API_URL is unset and we are in production mode. Exiting." 13 | echo "" 14 | echo "Found settings:" 15 | echo "" 16 | env 17 | echo "" 18 | exit 1 19 | fi 20 | 21 | # If the VUE_APP_GIT_VERSION has not been overridden, set it to the default 22 | # from package.json; the real `.git` version, if available, should still 23 | # take precedence. 24 | if [ -z "$VUE_APP_GIT_VERSION" ]; then 25 | VUE_APP_GIT_VERSION="0.0.0-git" 26 | fi 27 | 28 | echo "Replacing env vars in Javascript files" 29 | echo "Settings:" 30 | echo "" 31 | echo " APP_VERSION: ${VUE_APP_GIT_VERSION}" 32 | echo " API_URL: ${VUE_APP_API_URL}" 33 | echo " LOGO_URL: ${VUE_APP_LOGO_URL}" 34 | echo " HOMEPAGE_URL: ${VUE_APP_HOMPAGE_URL}" 35 | echo " EDITABLE_INVENTORY: ${VUE_APP_EDITABLE_INVENTORY}" 36 | echo " WEBSITE_TITLE: ${VUE_APP_WEBSITE_TITLE}" 37 | echo " QR_CODE_RESOLVER_URL: ${VUE_APP_QR_CODE_RESOLVER_URL}" 38 | echo " AUTOMATICALLY_GENERATE_ID_DEFAULT: ${VUE_APP_AUTOMATICALLY_GENERATE_ID_DEFAULT}" 39 | echo "" 40 | echo "Patching..." 41 | 42 | for file in $ROOT_DIR/js/app.*.js* $ROOT_DIR/*html; do 43 | echo "$file" 44 | sed -i "s|0.0.0-git|${VUE_APP_GIT_VERSION}|g" $file 45 | sed -i "s|magic-api-url|${VUE_APP_API_URL}|g" $file 46 | sed -i "s|magic-logo-url|${VUE_APP_LOGO_URL}|g" $file 47 | sed -i "s|magic-homepage-url|${VUE_APP_HOMEPAGE_URL}|g" $file 48 | sed -i "s|magic-setting|${VUE_APP_EDITABLE_INVENTORY}|g" $file 49 | sed -i "s|magic-title|${VUE_APP_WEBSITE_TITLE}|g" $file 50 | sed -i "s|magic-qr-code-resolver-url|${VUE_APP_QR_CODE_RESOLVER_URL}|g" $file 51 | sed -i "s|magic-generate-id-setting|${VUE_APP_AUTOMATICALLY_GENERATE_ID_DEFAULT}|g" $file 52 | done 53 | 54 | echo "Done!" 55 | 56 | serve -s ${ROOT_DIR} -p 8081 57 | -------------------------------------------------------------------------------- /.docker/mongo_dockerfile: -------------------------------------------------------------------------------- 1 | FROM mongo:8-noble 2 | EXPOSE 27017 3 | -------------------------------------------------------------------------------- /.docker/server_dockerfile: -------------------------------------------------------------------------------- 1 | # syntax=docker/dockerfile:1 2 | FROM python:3.10 AS api 3 | SHELL ["/bin/bash", "--login", "-c"] 4 | 5 | # Useful for installing deps from git, preventing big downloads and bandwith quotas 6 | ENV GIT_LFS_SKIP_SMUDGE=1 7 | 8 | # Install system dependencies 9 | RUN apt update && apt install -y gnupg curl tree mdbtools && apt clean 10 | 11 | # Install MongoDB tools in the official way 12 | WORKDIR /opt 13 | RUN wget https://fastdl.mongodb.org/tools/db/mongodb-database-tools-ubuntu2204-x86_64-100.9.0.deb && apt install ./mongodb-database-tools-*-100.9.0.deb 14 | 15 | COPY --from=ghcr.io/astral-sh/uv:0.6.4 /uv /usr/local/bin/uv 16 | ENV UV_LINK_MODE=copy \ 17 | UV_COMPILE_BYTECODE=1 \ 18 | UV_PYTHON_DOWNLOADS=never \ 19 | UV_PROJECT_ENVIRONMENT=/opt/.venv \ 20 | UV_PYTHON=python3.10 21 | 22 | # Used to fake the version of the package in cases where datalab is only 23 | # available as a git submodule or worktree 24 | ARG SETUPTOOLS_SCM_PRETEND_VERSION 25 | ENV SETUPTOOLS_SCM_PRETEND_VERSION=${SETUPTOOLS_SCM_PRETEND_VERSION} 26 | 27 | WORKDIR /app 28 | COPY ./pydatalab/pyproject.toml . 29 | COPY ./pydatalab/uv.lock . 30 | RUN uv sync --locked --no-dev --all-extras 31 | 32 | WORKDIR /app 33 | 34 | # Install the local version of the package and mount the repository data to get version info 35 | COPY ./pydatalab/ ./ 36 | RUN git config --global --add safe.directory / 37 | 38 | # Install editable mode so that the server runs from a sensible place where we can stuff .env files 39 | RUN --mount=type=bind,target=/.git,source=./.git uv pip install --python /opt/.venv/bin/python --no-deps --editable . 40 | 41 | # This will define the number of gunicorn workers 42 | ARG WEB_CONCURRENCY=4 43 | ENV WEB_CONCURRENCY=${WEB_CONCURRENCY} 44 | 45 | ARG PORT=5001 46 | EXPOSE ${PORT} 47 | ENV PORT=${PORT} 48 | 49 | CMD ["/bin/bash", "-c", "/opt/.venv/bin/python -m gunicorn --preload -w ${WEB_CONCURRENCY} --error-logfile /logs/pydatalab_error.log --access-logfile - -b 0.0.0.0:${PORT} 'pydatalab.main:create_app()'"] 50 | 51 | HEALTHCHECK --interval=30s --timeout=30s --start-interval=15s --start-period=30s --retries=3 CMD curl --fail http://localhost:${PORT}/healthcheck/is_ready || exit 1 52 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | **/node_modules 2 | **/cypress 3 | **/.venv 4 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @jdbocarsly @ml-evs @BenjaminCharmes 2 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: pip 4 | directory: "./pydatalab" 5 | schedule: 6 | interval: monthly 7 | day: monday 8 | time: "05:43" 9 | target-branch: main 10 | labels: 11 | - dependency_updates 12 | versioning-strategy: "increase-if-necessary" 13 | ignore: 14 | - dependency-name: "pydantic" 15 | versions: [ ">=2" ] 16 | - dependency-name: "bokeh" 17 | versions: [ ">=3" ] 18 | - dependency-name: "langchain" 19 | versions: [ ">=0.3" ] 20 | # Updates GH actions versions as often as needed 21 | - package-ecosystem: github-actions 22 | directory: "/" 23 | schedule: 24 | day: monday 25 | interval: monthly 26 | time: "05:33" 27 | target-branch: main 28 | labels: 29 | - CI 30 | - dependency_updates 31 | groups: 32 | github-actions: 33 | applies-to: version-updates 34 | dependency-type: production 35 | -------------------------------------------------------------------------------- /.github/workflows/dependable-bot.yml: -------------------------------------------------------------------------------- 1 | name: Automatic `uv` dependency upgrades 2 | on: 3 | schedule: 4 | - cron: "4 3 * * 3" # Wednesdays in the early morning 5 | workflow_dispatch: 6 | 7 | permissions: 8 | contents: write 9 | pull-requests: write 10 | 11 | jobs: 12 | uv-update-pins: 13 | name: Update `uv.lock` pins 14 | runs-on: ubuntu-latest 15 | 16 | steps: 17 | - uses: actions/checkout@v4 18 | with: 19 | # Make sure to pull all tags so the version is set correctly 20 | fetch-tags: true 21 | fetch-depth: 0 22 | 23 | - name: Set up Python 3.10 24 | uses: actions/setup-python@v5 25 | with: 26 | python-version: "3.10" 27 | 28 | - name: Set up uv 29 | uses: astral-sh/setup-uv@v5 30 | with: 31 | version: "0.5.22" 32 | enable-cache: true 33 | 34 | - name: Sync latest compatible dependencies and commit 35 | working-directory: ./pydatalab 36 | run: | 37 | uv lock -U 2> output.txt 38 | 39 | - name: Create PR with changes 40 | uses: peter-evans/create-pull-request@v7 41 | with: 42 | base: main 43 | add-paths: pydatalab/uv.lock 44 | sign-commits: true 45 | branch: ci/update-uv-lock-main-deps 46 | delete-branch: true 47 | commit-message: "ci: update uv lock file" 48 | title: "Update `uv.lock` with latest dependencies" 49 | body-path: ./pydatalab/output.txt 50 | labels: dependency_updates,Python 51 | draft: always-true 52 | -------------------------------------------------------------------------------- /.github/workflows/license-check.yml: -------------------------------------------------------------------------------- 1 | name: Check dependency licensing 2 | on: 3 | workflow_dispatch: 4 | workflow_call: 5 | 6 | jobs: 7 | licensing-checks: 8 | name: "Check dependency licensing" 9 | runs-on: ubuntu-latest 10 | 11 | steps: 12 | - name: Checkout repository 13 | uses: actions/checkout@v4 14 | with: 15 | submodules: true 16 | fetch-depth: 0 17 | ref: ${{ env.PUBLISH_UPDATE_BRANCH }} 18 | 19 | - name: Set up Python 3.11 20 | uses: actions/setup-python@v5 21 | with: 22 | python-version: '3.11' 23 | 24 | - name: Set up uv 25 | uses: astral-sh/setup-uv@v5 26 | with: 27 | version: "0.6.4" 28 | enable-cache: true 29 | 30 | - name: Run liccheck 31 | working-directory: ./pydatalab 32 | run: | 33 | uv venv 34 | uv sync --all-extras --dev 35 | uv export --locked --all-extras --no-hashes --no-dev > requirements.txt 36 | uv pip install liccheck==0.9.2 pip 37 | uv run liccheck -r requirements.txt 38 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Publish and release on PyPI 2 | on: 3 | release: 4 | types: 5 | - published 6 | 7 | env: 8 | PUBLISH_UPDATE_BRANCH: main 9 | GIT_USER_NAME: datalab developers 10 | GIT_USER_EMAIL: "dev@datalab-org.io" 11 | 12 | jobs: 13 | 14 | publish-to-pypi: 15 | name: "Publish on PyPI" 16 | runs-on: ubuntu-latest 17 | permissions: 18 | id-token: write 19 | if: github.repository == 'datalab-org/datalab' && startsWith(github.ref, 'refs/tags/v') 20 | environment: 21 | name: pypi-release 22 | url: https://pypi.org/project/datalab-server 23 | 24 | steps: 25 | 26 | - name: Checkout repository 27 | uses: actions/checkout@v4 28 | with: 29 | submodules: true 30 | fetch-depth: 0 31 | ref: ${{ env.PUBLISH_UPDATE_BRANCH }} 32 | 33 | - name: Set up Python 3.11 34 | uses: actions/setup-python@v5 35 | with: 36 | python-version: '3.11' 37 | 38 | - name: Build source distribution 39 | working-directory: "./pydatalab" 40 | run: | 41 | pip install -U build 42 | python -m build 43 | 44 | - name: Publish package to PyPI 45 | uses: pypa/gh-action-pypi-publish@release/v1 46 | with: 47 | packages-dir: "./pydatalab/dist" 48 | 49 | 50 | publish-container-build: 51 | name: "Build and publish container" 52 | runs-on: ubuntu-latest 53 | if: github.repository == 'datalab-org/datalab' && startsWith(github.ref, 'refs/tags/v') 54 | needs: [publish-to-pypi] 55 | permissions: 56 | packages: write 57 | environment: 58 | name: docker-release 59 | url: https://github.com/orgs/datalab-org/packages 60 | 61 | steps: 62 | 63 | - name: Checkout repository 64 | uses: actions/checkout@v4 65 | with: 66 | submodules: true 67 | fetch-depth: 0 68 | fetch-tags: true 69 | ref: ${{ env.PUBLISH_UPDATE_BRANCH }} 70 | 71 | - name: Set up Docker Buildx 72 | uses: docker/setup-buildx-action@v3 73 | 74 | - name: Build Docker images 75 | uses: docker/bake-action@v6 76 | with: 77 | files: docker-compose.yml 78 | load: true 79 | push: false 80 | source: . 81 | targets: 'app,api' 82 | set: | 83 | app.tags=ghcr.io/datalab-org/datalab-app:latest 84 | api.tags=ghcr.io/datalab-org/datalab-server-api:latest 85 | -------------------------------------------------------------------------------- /.github/workflows/yarn-upgrade-bot.yml: -------------------------------------------------------------------------------- 1 | name: Automatic `yarn` dependency upgrades 2 | on: 3 | schedule: 4 | - cron: "5 3 * * 3" # Wednesdays in the early morning 5 | workflow_dispatch: 6 | 7 | permissions: 8 | contents: write 9 | pull-requests: write 10 | 11 | jobs: 12 | yarn-update-pins: 13 | name: Update `yarn.lock` pins 14 | runs-on: ubuntu-latest 15 | 16 | steps: 17 | - uses: actions/checkout@v4 18 | 19 | - name: Set up Node 20 | uses: actions/setup-node@v4 21 | env: 22 | FORCE_COLOR: 0 23 | with: 24 | node-version: "20" 25 | cache: "yarn" 26 | cache-dependency-path: ./webapp/yarn.lock 27 | 28 | - name: Sync latest compatible dependencies and commit 29 | working-directory: ./webapp 30 | run: yarn upgrade 31 | 32 | - name: Create PR with changes 33 | uses: peter-evans/create-pull-request@v7 34 | with: 35 | base: main 36 | add-paths: ./webapp/yarn.lock 37 | sign-commits: true 38 | branch: ci/update-yarn-lock-main-deps 39 | delete-branch: true 40 | commit-message: "ci: update yarn lock file" 41 | title: "Update `yarn.lock` with latest dependencies" 42 | body: "This PR updates the `yarn.lock` file with the latest compatible dependencies. The changes to the lockfile must be reviewed; to run the tests, mark the PR as ready-for-review." 43 | labels: dependency_updates,javascript 44 | draft: always-true 45 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Datalab database dirs - these can be removed once we have a better place for them 2 | files/ 3 | uploads/ 4 | ssh_config 5 | logs/ 6 | site/ 7 | docker-compose.override.yml 8 | 9 | # App testing files 10 | webapp/cypress/screenshots 11 | webapp/cypress/videos 12 | 13 | #starting material files 14 | greyGroup_chemInventory* 15 | 16 | pydatalab/Pipfile 17 | 18 | 19 | *.sublime-workspace 20 | *.sublime-project 21 | *.tmlanguage.cache 22 | *.tmPreferences.cache 23 | *.stTheme.cache 24 | .vscode 25 | vetur.config.js 26 | .DS_Store 27 | vetur.config.js 28 | 29 | *~ 30 | \#.*\# 31 | 32 | node_modules 33 | 34 | ## Python files 35 | __pycache__/ 36 | *.py[cod] 37 | *$py.class 38 | .mypy_cache 39 | 40 | # Ignore any *additional* pipfile/pyprojects that get updated 41 | pydatalab/pyproject.toml 42 | Pipfile 43 | Pipfile.lock 44 | 45 | ## Packaging 46 | build/ 47 | dist/ 48 | eggs/ 49 | .eggs/ 50 | sdist/ 51 | wheels/ 52 | *.egg-info/ 53 | *.egg 54 | 55 | # Pip logs 56 | pip-log.txt 57 | pip-delete-this-directory.txt 58 | 59 | # Testing 60 | htmlcov/ 61 | .coverage 62 | .coverage.* 63 | .cache 64 | coverage.xml 65 | .pytest-cache/ 66 | 67 | # Envs 68 | .python-version 69 | .env 70 | .venv 71 | .env.local 72 | env/ 73 | venv/ 74 | ENV/ 75 | .matplotlibrc 76 | matplotlibrc 77 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | default_language_version: 2 | python: python3.10 3 | node: "20.17.0" 4 | 5 | ci: 6 | skip: 7 | # skip generate-schemas as pre-commit-ci will not have correct environment 8 | - generate-schemas 9 | # skip eslint as it is too large to run in pre-commit-ci for free 10 | - eslint 11 | autoupdate_schedule: "monthly" 12 | autofix_prs: false 13 | 14 | repos: 15 | - repo: https://github.com/pre-commit/pre-commit-hooks 16 | rev: v5.0.0 17 | hooks: 18 | - id: trailing-whitespace 19 | exclude: (pydatalab/example_data/)|(.*.snap) 20 | args: [--markdown-linebreak-ext=md] 21 | - id: check-yaml 22 | args: [--unsafe] 23 | - id: check-json 24 | - id: end-of-file-fixer 25 | exclude: ^(pydatalab/example_data/|pydatalab/schemas) 26 | - id: check-added-large-files 27 | args: [--maxkb=1024] 28 | - id: check-symlinks 29 | - id: mixed-line-ending 30 | 31 | - repo: https://github.com/astral-sh/ruff-pre-commit 32 | rev: v0.7.2 33 | hooks: 34 | - id: ruff 35 | args: [--fix] 36 | - id: ruff-format 37 | 38 | - repo: https://github.com/pre-commit/mirrors-eslint 39 | rev: v8.56.0 40 | hooks: 41 | - id: eslint 42 | additional_dependencies: 43 | - prettier@3.1.0 44 | - eslint@8.57.1 45 | - eslint-config-prettier@8.10.0 46 | - eslint-plugin-prettier@5.1.3 47 | - eslint-plugin-vue@8.0.3 48 | - "@vue/cli-plugin-babel@5.0.8" 49 | - "@vue/eslint-config-prettier@8.0.0" 50 | - eslint-plugin-cypress@3.3.0 51 | - "@babel/core@7.24.8" 52 | - "@babel/eslint-parser@7.24.8" 53 | - "@babel/plugin-transform-export-namespace-from@7.24.7" 54 | types: [file] 55 | files: \.(js|jsx|ts|tsx|vue)$ 56 | args: [--fix] 57 | 58 | - repo: https://github.com/asottile/pyupgrade 59 | rev: v3.19.0 60 | hooks: 61 | - id: pyupgrade 62 | args: [--py310-plus] 63 | 64 | - repo: https://github.com/pre-commit/mirrors-mypy 65 | rev: v1.13.0 66 | hooks: 67 | - id: mypy 68 | additional_dependencies: 69 | ["types-python-dateutil", "types-requests", "types-paramiko", "pydantic~=1.10"] 70 | args: ["--config-file", "pydatalab/pyproject.toml"] 71 | 72 | - repo: local 73 | hooks: 74 | - id: generate-schemas 75 | name: Regenerate item model JSONSchemas 76 | files: "^pydatalab/src/pydatalab/models/.*.$" 77 | description: Check if the current code changes have enacted changes to the resulting JSONSchemas 78 | entry: invoke -r pydatalab dev.generate-schemas 79 | pass_filenames: false 80 | language: system 81 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | printWidth: 100 2 | trailingComma: all 3 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | formats: [] 4 | 5 | build: 6 | os: ubuntu-22.04 7 | tools: 8 | python: "3.10" 9 | commands: 10 | - asdf plugin add uv 11 | - asdf install uv latest 12 | - asdf global uv latest 13 | - cd pydatalab && uv sync --all-extras --dev 14 | - cd pydatalab && uv pip install . 15 | - cd pydatalab && .venv/bin/mkdocs build --site-dir $READTHEDOCS_OUTPUT/html 16 | -------------------------------------------------------------------------------- /CITATION.cff: -------------------------------------------------------------------------------- 1 | # YAML 1.2 2 | # Metadata for citation of this software according to the CFF format (https://citation-file-format.github.io/) 3 | cff-version: 1.2.0 4 | message: If you use this software in a publication, please cite the source code archive on Zenodo. 5 | title: datalab 6 | abstract: datalab is a place to store experimental data and the connections between them. 7 | repository-code: https://github.com/datalab-org/datalab 8 | url: https://github.com/datalab-org/datalab 9 | type: software 10 | license: MIT 11 | identifiers: 12 | - type: doi 13 | value: 10.5281/zenodo.14719467 14 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020-2024 Joshua Bocarsly, Matthew Evans & The Datalab Development Team 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | name: datalab 2 | services: 3 | app: 4 | profiles: ["prod"] 5 | build: 6 | context: . 7 | dockerfile: .docker/app_dockerfile 8 | volumes: 9 | - ./logs:/logs 10 | restart: unless-stopped 11 | environment: 12 | - VUE_APP_GIT_VERSION 13 | - VUE_APP_API_URL 14 | - VUE_APP_LOGO_URL 15 | - VUE_APP_HOMEPAGE_URL 16 | - VUE_APP_EDITABLE_INVENTORY 17 | - VUE_APP_WEBSITE_TITLE 18 | - VUE_APP_QR_CODE_RESOLVER_URL 19 | - VUE_APP_AUTOMATICALLY_GENERATE_ID_DEFAULT 20 | ports: 21 | - "8081:8081" 22 | 23 | api: 24 | profiles: ["prod"] 25 | build: 26 | context: . 27 | dockerfile: .docker/server_dockerfile 28 | args: 29 | WEB_CONCURRENCY: ${WEB_CONCURRENCY:-4} 30 | SETUPTOOLS_SCM_PRETEND_VERSION: ${SETUPTOOLS_SCM_PRETEND_VERSION} 31 | depends_on: 32 | - database 33 | restart: unless-stopped 34 | volumes: 35 | - ./logs:/logs 36 | - /data/files:/app/files 37 | - /data/backups:/tmp/datalab-backups 38 | ports: 39 | - "5001:5001" 40 | networks: 41 | - backend 42 | environment: 43 | - PYDATALAB_MONGO_URI=mongodb://database:27017/datalabvue 44 | 45 | database: 46 | profiles: ["prod"] 47 | build: 48 | context: . 49 | dockerfile: .docker/mongo_dockerfile 50 | volumes: 51 | - ./logs:/var/logs/mongod 52 | - /data/db:/data/db 53 | restart: unless-stopped 54 | networks: 55 | - backend 56 | 57 | networks: 58 | backend: 59 | driver: bridge 60 | -------------------------------------------------------------------------------- /pydatalab/README.md: -------------------------------------------------------------------------------- 1 | ../README.md -------------------------------------------------------------------------------- /pydatalab/docs/.pages: -------------------------------------------------------------------------------- 1 | nav: 2 | - index.md 3 | - INSTALL.md 4 | - deployment.md 5 | - config.md 6 | - schemas 7 | - blocks 8 | - CHANGELOG.md 9 | -------------------------------------------------------------------------------- /pydatalab/docs/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ../../CHANGELOG.md -------------------------------------------------------------------------------- /pydatalab/docs/INSTALL.md: -------------------------------------------------------------------------------- 1 | ../../INSTALL.md -------------------------------------------------------------------------------- /pydatalab/docs/LICENSE: -------------------------------------------------------------------------------- 1 | ../../LICENSE -------------------------------------------------------------------------------- /pydatalab/docs/blocks/.pages: -------------------------------------------------------------------------------- 1 | title: Data blocks 2 | -------------------------------------------------------------------------------- /pydatalab/docs/blocks/apps/.pages: -------------------------------------------------------------------------------- 1 | title: Applications 2 | -------------------------------------------------------------------------------- /pydatalab/docs/blocks/apps/chat.md: -------------------------------------------------------------------------------- 1 | title: Whinchat (LLM assistant) 2 | ::: pydatalab.apps.chat 3 | -------------------------------------------------------------------------------- /pydatalab/docs/blocks/apps/echem.md: -------------------------------------------------------------------------------- 1 | title: Electrochemistry 2 | ::: pydatalab.apps.echem 3 | -------------------------------------------------------------------------------- /pydatalab/docs/blocks/apps/eis.md: -------------------------------------------------------------------------------- 1 | title: EIS 2 | ::: pydatalab.apps.eis 3 | -------------------------------------------------------------------------------- /pydatalab/docs/blocks/apps/ftir.md: -------------------------------------------------------------------------------- 1 | title: FTIR 2 | ::: pydatalab.apps.ftir 3 | -------------------------------------------------------------------------------- /pydatalab/docs/blocks/apps/nmr.md: -------------------------------------------------------------------------------- 1 | title: NMR 2 | ::: pydatalab.apps.nmr 3 | -------------------------------------------------------------------------------- /pydatalab/docs/blocks/apps/raman.md: -------------------------------------------------------------------------------- 1 | ::: pydatalab.apps.raman 2 | -------------------------------------------------------------------------------- /pydatalab/docs/blocks/apps/tga.md: -------------------------------------------------------------------------------- 1 | title: TGA 2 | ::: pydatalab.apps.tga 3 | -------------------------------------------------------------------------------- /pydatalab/docs/blocks/apps/uvvis.md: -------------------------------------------------------------------------------- 1 | title: UV-Vis 2 | ::: pydatalab.apps.uvvis 3 | -------------------------------------------------------------------------------- /pydatalab/docs/blocks/apps/xrd.md: -------------------------------------------------------------------------------- 1 | title: XRD 2 | ::: pydatalab.apps.xrd 3 | -------------------------------------------------------------------------------- /pydatalab/docs/blocks/base.md: -------------------------------------------------------------------------------- 1 | ::: pydatalab.blocks.base 2 | -------------------------------------------------------------------------------- /pydatalab/docs/blocks/common.md: -------------------------------------------------------------------------------- 1 | ::: pydatalab.blocks.common 2 | -------------------------------------------------------------------------------- /pydatalab/docs/css/reference.css: -------------------------------------------------------------------------------- 1 | @import url("https://cdnjs.cloudflare.com/ajax/libs/Iosevka/6.0.1/iosevka/iosevka.min.css"); 2 | 3 | .md-grid { 4 | max-width: 60rem; 5 | } 6 | -------------------------------------------------------------------------------- /pydatalab/docs/index.md: -------------------------------------------------------------------------------- 1 | ../../README.md -------------------------------------------------------------------------------- /pydatalab/docs/schemas/.pages: -------------------------------------------------------------------------------- 1 | title: "Data models" 2 | -------------------------------------------------------------------------------- /pydatalab/docs/schemas/cells.md: -------------------------------------------------------------------------------- 1 | # Cells 2 | 3 | ::: pydatalab.models.cells 4 | options: 5 | inherited_members: true 6 | summary: 7 | attributes: true 8 | functions: false 9 | modules: false 10 | -------------------------------------------------------------------------------- /pydatalab/docs/schemas/collections.md: -------------------------------------------------------------------------------- 1 | # Collections 2 | 3 | ::: pydatalab.models.collections 4 | options: 5 | inherited_members: true 6 | summary: 7 | attributes: true 8 | functions: false 9 | modules: false 10 | -------------------------------------------------------------------------------- /pydatalab/docs/schemas/equipment.md: -------------------------------------------------------------------------------- 1 | # Equipment 2 | 3 | ::: pydatalab.models.equipment 4 | options: 5 | inherited_members: true 6 | summary: 7 | attributes: true 8 | functions: false 9 | modules: false 10 | -------------------------------------------------------------------------------- /pydatalab/docs/schemas/items.md: -------------------------------------------------------------------------------- 1 | # Items 2 | 3 | ::: pydatalab.models.items 4 | options: 5 | inherited_members: true 6 | summary: 7 | attributes: true 8 | functions: false 9 | modules: false 10 | -------------------------------------------------------------------------------- /pydatalab/docs/schemas/samples.md: -------------------------------------------------------------------------------- 1 | # Samples 2 | 3 | ::: pydatalab.models.samples 4 | options: 5 | inherited_members: true 6 | summary: 7 | attributes: true 8 | functions: false 9 | modules: false 10 | -------------------------------------------------------------------------------- /pydatalab/docs/schemas/starting_materials.md: -------------------------------------------------------------------------------- 1 | # Starting Materials 2 | 3 | ::: pydatalab.models.starting_materials 4 | options: 5 | inherited_members: true 6 | summary: 7 | attributes: true 8 | functions: false 9 | modules: false 10 | -------------------------------------------------------------------------------- /pydatalab/example_data/NMR/1.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datalab-org/datalab/05f761881757985ac8b761a85163571434316d9c/pydatalab/example_data/NMR/1.zip -------------------------------------------------------------------------------- /pydatalab/example_data/NMR/71.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datalab-org/datalab/05f761881757985ac8b761a85163571434316d9c/pydatalab/example_data/NMR/71.zip -------------------------------------------------------------------------------- /pydatalab/example_data/NMR/72.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datalab-org/datalab/05f761881757985ac8b761a85163571434316d9c/pydatalab/example_data/NMR/72.zip -------------------------------------------------------------------------------- /pydatalab/example_data/NMR/README: -------------------------------------------------------------------------------- 1 | 1.zip: example of 1D solution-state NMR data (H3PO4 in H2O) 2 | 71.zip: example of 1D solid-state NMR data 3 | 72.zip: example of 2D (MATPASS) solid-state NMR data 4 | 1h.dx: example of 1D JCAMP-DX data, sourced from [nmRxiv](https://docs.nmrxiv.org/introduction/data/exemplary-data.html#jcamp-datasets) 5 | 13c.dx: example of 1D JCAMP-DX data, sourced from [nmRxiv](https://docs.nmrxiv.org/introduction/data/exemplary-data.html#jcamp-datasets) 6 | -------------------------------------------------------------------------------- /pydatalab/example_data/TGA-MS/mp2028_281122_LNO+C.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datalab-org/datalab/05f761881757985ac8b761a85163571434316d9c/pydatalab/example_data/TGA-MS/mp2028_281122_LNO+C.txt -------------------------------------------------------------------------------- /pydatalab/example_data/XRD/JO_KL_16_ZF_3_60deg_0.02step_12degpermin_001.rasx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datalab-org/datalab/05f761881757985ac8b761a85163571434316d9c/pydatalab/example_data/XRD/JO_KL_16_ZF_3_60deg_0.02step_12degpermin_001.rasx -------------------------------------------------------------------------------- /pydatalab/example_data/XRD/cod_9004112.cif: -------------------------------------------------------------------------------- 1 | #------------------------------------------------------------------------------ 2 | #$Date: 2024-04-25 17:59:02 +0300 (Thu, 25 Apr 2024) $ 3 | #$Revision: 291351 $ 4 | #$URL: file:///home/coder/svn-repositories/cod/cif/9/00/41/9004112.cif $ 5 | #------------------------------------------------------------------------------ 6 | # 7 | # This file is available in the Crystallography Open Database (COD), 8 | # http://www.crystallography.net/. The original data for this entry 9 | # were provided the American Mineralogist Crystal Structure Database, 10 | # http://rruff.geo.arizona.edu/AMS/amcsd.php 11 | # 12 | # The file may be used within the scientific community so long as 13 | # proper attribution is given to the journal article from which the 14 | # data were obtained. 15 | # 16 | data_9004112 17 | loop_ 18 | _publ_author_name 19 | 'Scott, J. D.' 20 | 'Nowacki, W.' 21 | _publ_section_title 22 | ; 23 | The crystal structure of alloclasite, CoAsS, and the 24 | alloclasite-cobaltite transformation 25 | ; 26 | _journal_name_full 'The Canadian Mineralogist' 27 | _journal_page_first 561 28 | _journal_page_last 566 29 | _journal_volume 14 30 | _journal_year 1976 31 | _chemical_formula_sum 'As Co S' 32 | _chemical_name_mineral Alloclasite 33 | _space_group_IT_number 4 34 | _symmetry_space_group_name_Hall 'P 2yb' 35 | _symmetry_space_group_name_H-M 'P 1 21 1' 36 | _cell_angle_alpha 90 37 | _cell_angle_beta 90.2 38 | _cell_angle_gamma 90 39 | _cell_length_a 4.661 40 | _cell_length_b 5.602 41 | _cell_length_c 3.411 42 | _cell_formula_units_Z 2 43 | _cell_volume 89.064 44 | _database_code_amcsd 0005134 45 | _exptl_crystal_density_diffrn 6.187 46 | _cod_original_formula_sum 'Co As S' 47 | _cod_database_code 9004112 48 | loop_ 49 | _space_group_symop_operation_xyz 50 | x,y,z 51 | -x,1/2+y,-z 52 | loop_ 53 | _atom_site_aniso_label 54 | _atom_site_aniso_U_11 55 | _atom_site_aniso_U_22 56 | _atom_site_aniso_U_33 57 | _atom_site_aniso_U_12 58 | _atom_site_aniso_U_13 59 | _atom_site_aniso_U_23 60 | Co 0.01244 0.01335 0.01309 0.00000 0.00000 -0.00019 61 | As 0.01695 0.01781 0.01527 -0.00053 -0.00040 -0.00019 62 | S 0.01750 0.01860 0.01886 0.00079 -0.00072 0.00039 63 | loop_ 64 | _atom_site_label 65 | _atom_site_fract_x 66 | _atom_site_fract_y 67 | _atom_site_fract_z 68 | Co 0.24824 0.00000 0.24480 69 | As 0.04766 0.37149 0.25547 70 | S 0.44104 0.62236 0.24239 71 | loop_ 72 | _cod_related_entry_id 73 | _cod_related_entry_database 74 | _cod_related_entry_code 75 | 1 AMCSD 0005134 76 | -------------------------------------------------------------------------------- /pydatalab/example_data/XRD/example_evadiffract.xy: -------------------------------------------------------------------------------- 1 | 'Id: "" Comment: "Example file from Bruker diffrac.eva export" Operator: "Emmy Noether" Anode: "Cu" Scantype: "Coupled TwoTheta/Theta" TimePerStep: "96" 2 | 5.0001 4.852 3 | 5.0103 4.097 4 | 5.0204 4.667 5 | 5.0306 4.410 6 | 5.0407 4.484 7 | 5.0509 5.542 8 | 5.0610 5.244 9 | 5.0712 4.579 10 | 5.0813 4.856 11 | 5.0915 5.335 12 | 5.1017 4.999 13 | 5.1118 4.624 14 | 5.1220 4.898 15 | 5.1321 4.363 16 | 5.1423 3.871 17 | 5.1524 4.346 18 | 5.1626 5.583 19 | 5.1728 5.091 20 | 5.1829 5.001 21 | 5.1931 5.310 22 | 5.2032 4.702 23 | 5.2134 4.534 24 | 5.2235 4.486 25 | 5.2337 4.715 26 | 5.2438 5.022 27 | 5.2540 5.130 28 | 5.2642 4.412 29 | 5.2743 4.324 30 | -------------------------------------------------------------------------------- /pydatalab/example_data/XRD/example_mythen.xye: -------------------------------------------------------------------------------- 1 | 2.038000 76131.000000 275.918466 2 | 2.042000 66343.000000 257.571349 3 | 2.046000 60502.000000 245.971543 4 | 2.050000 58150.000000 241.143111 5 | 2.054000 56412.000000 237.512105 6 | 2.058000 55123.000000 234.782878 7 | 2.062000 53971.000000 232.316594 8 | 2.066000 53188.000000 230.625237 9 | 2.070000 54024.000000 232.430635 10 | 2.074000 53292.000000 230.850601 11 | 2.078000 53469.000000 231.233648 12 | 2.082000 52889.000000 229.976086 13 | 2.086000 53168.000000 230.581873 14 | 2.090000 53371.000000 231.021644 15 | 2.094000 52916.000000 230.034780 16 | 2.098000 53496.000000 231.292023 17 | 2.102000 53343.000000 230.961036 18 | 2.106000 53620.000000 231.559927 19 | 2.110000 53646.000000 231.616062 20 | 2.114000 53524.000000 231.352545 21 | 2.118000 53705.000000 231.743393 22 | 2.122000 53505.000000 231.311478 23 | 2.126000 53688.000000 231.706711 24 | 2.130000 53922.000000 232.211111 25 | 2.134000 54236.000000 232.886238 26 | 2.138000 54310.000000 233.045060 27 | 2.142000 53962.000000 232.297223 28 | 2.146000 54222.000000 232.856179 29 | 2.150000 55096.000000 234.725371 30 | 2.154000 54517.000000 233.488758 31 | 2.158000 54560.000000 233.580821 32 | 2.162000 55165.000000 234.872306 33 | 2.166000 54692.000000 233.863208 34 | 2.170000 55456.000000 235.490976 35 | 2.174000 55524.000000 235.635311 36 | 2.178000 55197.000000 234.940418 37 | 2.182000 55153.000000 234.846759 38 | 2.186000 55525.000000 235.637433 39 | 2.190000 55760.000000 236.135554 40 | 2.194000 55877.000000 236.383164 41 | 2.198000 56447.000000 237.585774 42 | 2.202000 55403.000000 235.378419 43 | 2.206000 56343.000000 237.366805 44 | 2.210000 56813.000000 238.354778 45 | 2.214000 57553.000000 239.902063 46 | 2.218000 57033.000000 238.815829 47 | 2.222000 57970.000000 240.769599 48 | 2.226000 57339.000000 239.455633 49 | 2.230000 58282.000000 241.416652 50 | 2.234000 58364.000000 241.586423 51 | 2.238000 59331.000000 243.579556 52 | 2.242000 60224.000000 245.405786 53 | 2.246000 59706.000000 244.348112 54 | 2.250000 60354.000000 245.670511 55 | 2.254000 61154.000000 247.293348 56 | 2.258000 61661.000000 248.316331 57 | 2.262000 62631.000000 250.261863 58 | 2.266000 63572.000000 252.134885 59 | 2.270000 64879.000000 254.713565 60 | 2.274000 65929.000000 256.766431 61 | 2.278000 67451.000000 259.713303 62 | 2.282000 68661.000000 262.032441 63 | 2.286000 69910.000000 264.404992 64 | 2.290000 71842.000000 268.033580 65 | 2.294000 73201.000000 270.556833 66 | 2.298000 76228.000000 276.094187 67 | 2.302000 77395.000000 278.199569 68 | 2.306000 80773.000000 284.205911 69 | 2.310000 83256.000000 288.541158 70 | 2.314000 86421.000000 293.974489 71 | 2.318000 89546.000000 299.242377 72 | 2.322000 94388.000000 307.226301 73 | 2.326000 97704.000000 312.576391 74 | 2.330000 105132.000000 324.240651 75 | -------------------------------------------------------------------------------- /pydatalab/example_data/csv/simple.csv: -------------------------------------------------------------------------------- 1 | test,test2,test3 2 | 1,2,3 3 | 4,5,6 4 | -------------------------------------------------------------------------------- /pydatalab/example_data/csv/simple.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datalab-org/datalab/05f761881757985ac8b761a85163571434316d9c/pydatalab/example_data/csv/simple.xlsx -------------------------------------------------------------------------------- /pydatalab/example_data/echem/README: -------------------------------------------------------------------------------- 1 | jdb11-1_c3_gcpl_5cycles_2V-3p8V_C-24_data_C09.* : coin cell GCPL data, biologic 2 | -> mpr : results file (binary) 3 | -> mps : settings file (information about the sample program) 4 | -> sta : I'm not sure what this one does 5 | 6 | jdb11-1_e1_s3_squidTest_data_C15.mpr : in situ SQUID cell GCPL data, biologic file 7 | 8 | -------------------------------------------------------------------------------- /pydatalab/example_data/echem/jdb11-1_c3_gcpl_5cycles_2V-3p8V_C-24_data.mps: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datalab-org/datalab/05f761881757985ac8b761a85163571434316d9c/pydatalab/example_data/echem/jdb11-1_c3_gcpl_5cycles_2V-3p8V_C-24_data.mps -------------------------------------------------------------------------------- /pydatalab/example_data/echem/jdb11-1_c3_gcpl_5cycles_2V-3p8V_C-24_data_C09.mpr: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datalab-org/datalab/05f761881757985ac8b761a85163571434316d9c/pydatalab/example_data/echem/jdb11-1_c3_gcpl_5cycles_2V-3p8V_C-24_data_C09.mpr -------------------------------------------------------------------------------- /pydatalab/example_data/echem/jdb11-1_c3_gcpl_5cycles_2V-3p8V_C-24_data_D1_C09.sta: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datalab-org/datalab/05f761881757985ac8b761a85163571434316d9c/pydatalab/example_data/echem/jdb11-1_c3_gcpl_5cycles_2V-3p8V_C-24_data_D1_C09.sta -------------------------------------------------------------------------------- /pydatalab/example_data/echem/jdb11-1_e1_s3_squidTest_data_C15.mpr: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datalab-org/datalab/05f761881757985ac8b761a85163571434316d9c/pydatalab/example_data/echem/jdb11-1_e1_s3_squidTest_data_C15.mpr -------------------------------------------------------------------------------- /pydatalab/example_data/raman/labspec_raman_example.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datalab-org/datalab/05f761881757985ac8b761a85163571434316d9c/pydatalab/example_data/raman/labspec_raman_example.txt -------------------------------------------------------------------------------- /pydatalab/example_data/raman/raman_example.wdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datalab-org/datalab/05f761881757985ac8b761a85163571434316d9c/pydatalab/example_data/raman/raman_example.wdf -------------------------------------------------------------------------------- /pydatalab/mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: datalab 2 | site_description: Documentation for datalab 3 | site_url: https://docs.datalab-org.io 4 | 5 | theme: 6 | name: material 7 | icon: 8 | repo: fontawesome/brands/github 9 | language: en 10 | palette: 11 | # Palette toggle for light mode 12 | - media: "(prefers-color-scheme: light)" 13 | scheme: default 14 | toggle: 15 | icon: material/brightness-7 16 | name: Switch to dark mode 17 | primary: amber 18 | accent: brown 19 | 20 | # Palette toggle for dark mode 21 | - media: "(prefers-color-scheme: dark)" 22 | scheme: slate 23 | toggle: 24 | icon: material/brightness-4 25 | name: Switch to light mode 26 | primary: amber 27 | accent: brown 28 | 29 | font: 30 | text: Figtree 31 | code: Iosevka Web 32 | features: 33 | - content.code.copy 34 | 35 | features: 36 | - content.tabs.link 37 | 38 | repo_name: datalab-org/datalab 39 | repo_url: https://github.com/datalab-org/datalab 40 | 41 | docs_dir: "docs" 42 | 43 | extra: 44 | social: 45 | - icon: fontawesome/brands/github 46 | link: https://github.com/datalab-org 47 | 48 | markdown_extensions: 49 | - admonition 50 | - pymdownx.details 51 | - pymdownx.highlight 52 | - pymdownx.superfences: 53 | # Allows mermaid code blocks to be rendered via mermaid.js 54 | custom_fences: 55 | - name: mermaid 56 | class: mermaid 57 | format: !!python/name:pymdownx.superfences.fence_code_format 58 | 59 | - pymdownx.inlinehilite 60 | - pymdownx.tabbed: 61 | alternate_style: true 62 | - pymdownx.tasklist 63 | - pymdownx.snippets 64 | - toc: 65 | permalink: true 66 | 67 | extra_css: 68 | - css/reference.css 69 | 70 | plugins: 71 | - mkdocstrings: 72 | default_handler: python 73 | handlers: 74 | python: 75 | options: 76 | show_root_heading: true 77 | show_root_toc_entry: true 78 | show_root_full_path: true 79 | show_object_full_path: false 80 | show_category_heading: false 81 | show_if_no_docstring: true 82 | show_signature_annotations: true 83 | show_source: false 84 | show_labels: false 85 | show_bases: true 86 | group_by_category: true 87 | heading_level: 2 88 | summary: 89 | attributes: true 90 | functions: false 91 | modules: false 92 | inherited_members: false 93 | docstring_style: google 94 | filters: 95 | - "!^_[^_]" 96 | - "!__json_encoder__$" 97 | - "!__all__$" 98 | - "!__config__$" 99 | - "!^Config$" 100 | - awesome-pages 101 | - autorefs 102 | - search: 103 | lang: en 104 | 105 | watch: 106 | - src 107 | -------------------------------------------------------------------------------- /pydatalab/scripts/add_test_cell_to_db.py: -------------------------------------------------------------------------------- 1 | from pymongo import MongoClient, uri_parser 2 | 3 | from pydatalab.config import CONFIG 4 | from pydatalab.models import Cell 5 | 6 | client = MongoClient(CONFIG.MONGO_URI) 7 | database = uri_parser.parse_uri(CONFIG.MONGO_URI).get("database") 8 | db = client.datalabvue 9 | 10 | new_cell = Cell( 11 | **{ 12 | "item_id": "test_cell", 13 | "name": "test cell", 14 | "date": "1970-02-01", 15 | "anode": [ 16 | { 17 | "item": {"item_id": "test", "chemform": "Li15Si4", "type": "samples"}, 18 | "quantity": 2.0, 19 | "unit": "mg", 20 | }, 21 | { 22 | "item": {"item_id": "test", "chemform": "C", "type": "samples"}, 23 | "quantity": 2.0, 24 | "unit": "mg", 25 | }, 26 | ], 27 | "cathode": [ 28 | { 29 | "item": {"item_id": "test_cathode", "chemform": "LiCoO2", "type": "samples"}, 30 | "quantity": 2000, 31 | "unit": "kg", 32 | } 33 | ], 34 | "cell_format": "swagelok", 35 | "type": "cells", 36 | } 37 | ) 38 | 39 | 40 | db.items.insert_one(new_cell.dict()) 41 | -------------------------------------------------------------------------------- /pydatalab/scripts/create_mongo_indices.py: -------------------------------------------------------------------------------- 1 | from pydatalab.mongo import create_default_indices 2 | 3 | create_default_indices() 4 | -------------------------------------------------------------------------------- /pydatalab/scripts/generate_cy_links_json.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from pymongo import MongoClient 4 | 5 | from pydatalab.config import CONFIG 6 | 7 | client = MongoClient(CONFIG.MONGO_URI) 8 | 9 | db = client.datalabvue 10 | 11 | all_documents = db.items.find() 12 | 13 | nodes = [] 14 | edges = [] 15 | for document in all_documents: 16 | if ("parent_items" not in document) and ("child_items" not in document): 17 | continue 18 | nodes.append( 19 | {"data": {"id": document["item_id"], "name": document["name"], "type": document["type"]}} 20 | ) 21 | if "parent_items" not in document: 22 | continue 23 | for parent_id in document["parent_items"]: 24 | target = document["item_id"] 25 | source = parent_id 26 | edges.append( 27 | { 28 | "data": { 29 | "id": f"{source}->{target}", 30 | "source": source, 31 | "target": target, 32 | "value": 1, 33 | } 34 | } 35 | ) 36 | 37 | 38 | with open("cy_links_production.json", "w") as f: 39 | json.dump({"nodes": nodes, "edges": edges}, f) 40 | -------------------------------------------------------------------------------- /pydatalab/scripts/generate_cy_links_json_typedRelationship.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from pymongo import MongoClient 4 | 5 | from pydatalab.config import CONFIG 6 | 7 | client = MongoClient(CONFIG.MONGO_URI) 8 | 9 | db = client.datalabvue 10 | 11 | all_documents = db.items.find() 12 | 13 | nodes = [] 14 | edges = [] 15 | for document in all_documents: 16 | nodes.append( 17 | {"data": {"id": document["item_id"], "name": document["name"], "type": document["type"]}} 18 | ) 19 | 20 | if "relationships" not in document: 21 | continue 22 | 23 | for relationship in document["relationships"]: 24 | # only considering child-parent relationships: 25 | if relationship["relation"] != "parent": 26 | continue 27 | 28 | target = document["item_id"] 29 | source = relationship["item_id"] 30 | edges.append( 31 | { 32 | "data": { 33 | "id": f"{source}->{target}", 34 | "source": source, 35 | "target": target, 36 | "value": 1, 37 | } 38 | } 39 | ) 40 | 41 | 42 | # We want to filter out all the starting materials that don't have relationships since there are so many of them: 43 | whitelist = {edge["data"]["source"] for edge in edges} 44 | 45 | nodes = [ 46 | node 47 | for node in nodes 48 | if ((node["data"]["type"] == "samples") or (node["data"]["id"] in whitelist)) 49 | ] 50 | 51 | 52 | with open("cy_links_production_v2.json", "w") as f: 53 | json.dump({"nodes": nodes, "edges": edges}, f) 54 | -------------------------------------------------------------------------------- /pydatalab/scripts/migrate_add_fields_to_files.py: -------------------------------------------------------------------------------- 1 | from pymongo import MongoClient, uri_parser 2 | 3 | from pydatalab.config import CONFIG 4 | 5 | client = MongoClient(uri_parser.parse_host(CONFIG.MONGO_URI)) 6 | database = uri_parser.parse_uri(CONFIG.MONGO_URI).get("database") 7 | db = client.datalabvue 8 | file_collection = db.files 9 | 10 | 11 | all_files = file_collection.find({}) 12 | 13 | 14 | for file in all_files: 15 | file_collection.update_one( 16 | {"_id": file["_id"]}, 17 | { 18 | "$set": { 19 | "time_added": file["last_modified"], 20 | "version": 1, 21 | } 22 | }, 23 | ) 24 | -------------------------------------------------------------------------------- /pydatalab/scripts/migrate_add_item_ids_to_all_blocks.py: -------------------------------------------------------------------------------- 1 | from pymongo import MongoClient 2 | 3 | from pydatalab.config import CONFIG 4 | 5 | client = MongoClient(CONFIG.MONGO_URI) 6 | 7 | db = client.datalabvue 8 | 9 | all_items = db.items.find({}) 10 | 11 | for item in all_items: 12 | print(f"processing item: {item['_id']}") 13 | for block_id in item["blocks_obj"]: 14 | print(f"\tadding item_id field to block with id: {block_id}") 15 | res = db.items.update_one( 16 | {"item_id": item["item_id"]}, 17 | {"$set": {f"blocks_obj.{block_id}.item_id": item["item_id"]}}, 18 | ) 19 | -------------------------------------------------------------------------------- /pydatalab/scripts/migrate_copy_data_collection_to_items.py: -------------------------------------------------------------------------------- 1 | from pymongo import MongoClient 2 | 3 | from pydatalab.config import CONFIG 4 | 5 | client = MongoClient(CONFIG.MONGO_URI) 6 | 7 | db = client.datalabvue 8 | 9 | pipeline = [ 10 | {"$match": {}}, 11 | {"$out": "items"}, 12 | ] 13 | 14 | db.data.aggregate(pipeline) 15 | -------------------------------------------------------------------------------- /pydatalab/scripts/migrate_file_last_modified_remote_timestamp_to_last_modified_remote.py: -------------------------------------------------------------------------------- 1 | from pymongo import MongoClient 2 | 3 | from pydatalab.config import CONFIG 4 | 5 | client = MongoClient(CONFIG.MONGO_URI) 6 | 7 | db = client.datalabvue 8 | 9 | db.files.update_many({}, {"$rename": {"last_modified_remote_timestamp": "last_modified_remote"}}) 10 | -------------------------------------------------------------------------------- /pydatalab/scripts/migrate_file_sample_ids_to_item_ids.py: -------------------------------------------------------------------------------- 1 | from pymongo import MongoClient 2 | 3 | from pydatalab.config import CONFIG 4 | 5 | client = MongoClient(CONFIG.MONGO_URI) 6 | 7 | db = client.datalabvue 8 | 9 | db.files.update_many({}, {"$rename": {"sample_ids": "item_ids"}}) 10 | -------------------------------------------------------------------------------- /pydatalab/scripts/migrate_image_blocks_to_media_blocks.py: -------------------------------------------------------------------------------- 1 | from pydatalab.mongo import get_database 2 | 3 | db = get_database() 4 | 5 | for item in db.items.find({"blocks_obj": {"$ne": {}}}): 6 | print(f"processing item {item['item_id']}") 7 | for key in item["blocks_obj"]: 8 | if item["blocks_obj"][key]["blocktype"] == "image": 9 | print(f"need to update block: {key}") 10 | db.items.update_one( 11 | {"item_id": item["item_id"]}, {"$set": {f"blocks_obj.{key}.blocktype": "media"}} 12 | ) 13 | -------------------------------------------------------------------------------- /pydatalab/scripts/migrate_rename_data_collection_to_items.py: -------------------------------------------------------------------------------- 1 | from pymongo import MongoClient 2 | 3 | from pydatalab.config import CONFIG 4 | 5 | client = MongoClient(CONFIG.MONGO_URI) 6 | 7 | db = client.datalabvue 8 | 9 | db.data.copyTo("items") 10 | -------------------------------------------------------------------------------- /pydatalab/scripts/migrate_rename_item_kind_to_type.py: -------------------------------------------------------------------------------- 1 | from pymongo import MongoClient 2 | 3 | from pydatalab.config import CONFIG 4 | 5 | client = MongoClient(CONFIG.MONGO_URI) 6 | 7 | db = client.datalabvue 8 | 9 | data_collection = db.data 10 | 11 | data_collection.update_many({}, {"$rename": {"item_kind": "type"}}) 12 | 13 | 14 | data_collection.update_many({"type": "sample"}, {"$set": {"type": "samples"}}) 15 | -------------------------------------------------------------------------------- /pydatalab/scripts/migrate_rename_starting_material_field.py: -------------------------------------------------------------------------------- 1 | from pymongo import MongoClient 2 | 3 | from pydatalab.config import CONFIG 4 | 5 | client = MongoClient(CONFIG.MONGO_URI) 6 | 7 | db = client.datalabvue 8 | 9 | # because spelling is hard 10 | db.items.update_many( 11 | {"date_aquired": {"$exists": True}}, {"$rename": {"date_aquired": "date_acquired"}} 12 | ) 13 | -------------------------------------------------------------------------------- /pydatalab/scripts/migrate_sample_id_to_item_id.py: -------------------------------------------------------------------------------- 1 | from pymongo import MongoClient 2 | 3 | from pydatalab.config import CONFIG 4 | 5 | client = MongoClient(CONFIG.MONGO_URI) 6 | 7 | db = client.datalabvue 8 | 9 | db.items.update_many({"type": "samples"}, [{"$set": {"item_id": "$sample_id"}}]) 10 | -------------------------------------------------------------------------------- /pydatalab/scripts/migrate_set_all_constituents_as_parents.py: -------------------------------------------------------------------------------- 1 | from pymongo import MongoClient 2 | 3 | from pydatalab.config import CONFIG 4 | 5 | client = MongoClient(CONFIG.MONGO_URI) 6 | 7 | db = client.datalabvue 8 | 9 | 10 | all_documents = db.items.find() 11 | 12 | for document in all_documents: 13 | if "synthesis_constituents" not in document: 14 | continue 15 | constituent_ids = [entry["item"]["item_id"] for entry in document["synthesis_constituents"]] 16 | 17 | print( 18 | f"Item {document['item_id']} has constituents: {constituent_ids}. Creating relationships from these." 19 | ) 20 | 21 | # add all constituents as parents to this item (addToSet only adds if its not already there) 22 | db.items.update_one( 23 | {"item_id": document["item_id"]}, 24 | {"$addToSet": {"parent_items": {"$each": constituent_ids}}}, 25 | ) 26 | 27 | # add this item as children in each constituent 28 | for constituent_id in constituent_ids: 29 | db.items.update_one( 30 | {"item_id": constituent_id}, {"$addToSet": {"child_items": document["item_id"]}} 31 | ) 32 | -------------------------------------------------------------------------------- /pydatalab/scripts/migrate_set_all_constituents_as_parents_TypedRelationship.py: -------------------------------------------------------------------------------- 1 | from pymongo import MongoClient 2 | 3 | from pydatalab.config import CONFIG 4 | from pydatalab.models.relationships import RelationshipType, TypedRelationship 5 | 6 | client = MongoClient(CONFIG.MONGO_URI) 7 | 8 | db = client.datalabvue 9 | 10 | 11 | all_documents = db.items.find() 12 | 13 | for document in all_documents: 14 | if "synthesis_constituents" not in document: 15 | continue 16 | constituent_items = [entry["item"] for entry in document["synthesis_constituents"]] 17 | 18 | print( 19 | f"Item {document['item_id']} has constituents: {constituent_items}. Creating relationships from these." 20 | ) 21 | 22 | relationships = [ 23 | TypedRelationship( 24 | description="Is a constituent of", 25 | relation=RelationshipType.PARENT, 26 | type=item["type"], 27 | item_id=item["item_id"], 28 | ).dict() 29 | for item in constituent_items 30 | ] 31 | 32 | db.items.update_one( 33 | {"item_id": document["item_id"]}, 34 | {"$addToSet": {"relationships": {"$each": relationships}}}, 35 | upsert=True, 36 | ) 37 | 38 | # # # add all constituents as parents to this item (addToSet only adds if its not already there) 39 | # for constituent_id, item_type in zip(constituent_ids, types): 40 | # print(constituent_id, item_type) 41 | # relationship = TypedRelationship( 42 | # description = "Is a constituent of", 43 | # relation = RelationshipType.PARENT, 44 | # type = item_type, 45 | # item_id = constituent_id, 46 | # ) 47 | # db.items.update_one( 48 | # {"item_id": document["item_id"]}, 49 | # {"$addToSet": {"parent_items": {"$each": constituent_ids}}}, 50 | # ) 51 | 52 | # # add this item as children in each constituent 53 | # for constituent_id in constituent_ids: 54 | # db.items.update_one( 55 | # {"item_id": constituent_id}, {"$addToSet": {"child_items": document["item_id"]}} 56 | # ) 57 | -------------------------------------------------------------------------------- /pydatalab/scripts/migrate_set_all_samples_to_have_type_samples.py: -------------------------------------------------------------------------------- 1 | from pymongo import MongoClient 2 | 3 | from pydatalab.config import CONFIG 4 | 5 | client = MongoClient(CONFIG.MONGO_URI) 6 | 7 | db = client.datalabvue 8 | 9 | db.items.update_many({"sample_id": {"$exists": True}}, {"$set": {"type": "samples"}}) 10 | -------------------------------------------------------------------------------- /pydatalab/src/pydatalab/__init__.py: -------------------------------------------------------------------------------- 1 | from importlib.metadata import PackageNotFoundError, version 2 | 3 | try: 4 | __version__ = version("datalab-server") 5 | except PackageNotFoundError: 6 | __version__ = "develop" 7 | -------------------------------------------------------------------------------- /pydatalab/src/pydatalab/apps/_canary/__init__.py: -------------------------------------------------------------------------------- 1 | from pydatalab.blocks.base import DataBlock 2 | 3 | raise ImportError("This canary block is only used for internal testing of dynamic block loading.") 4 | 5 | 6 | class CanaryBlock(DataBlock): ... 7 | -------------------------------------------------------------------------------- /pydatalab/src/pydatalab/apps/chat/__init__.py: -------------------------------------------------------------------------------- 1 | from .blocks import ChatBlock 2 | 3 | __all__ = ("ChatBlock",) 4 | -------------------------------------------------------------------------------- /pydatalab/src/pydatalab/apps/echem/__init__.py: -------------------------------------------------------------------------------- 1 | from .blocks import CycleBlock 2 | 3 | __all__ = ("CycleBlock",) 4 | -------------------------------------------------------------------------------- /pydatalab/src/pydatalab/apps/eis/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | from pathlib import Path 3 | 4 | import bokeh.embed 5 | import pandas as pd 6 | from bokeh.models import HoverTool, LogColorMapper 7 | 8 | from pydatalab.blocks.base import DataBlock 9 | from pydatalab.bokeh_plots import DATALAB_BOKEH_THEME, selectable_axes_plot 10 | from pydatalab.file_utils import get_file_info_by_id 11 | from pydatalab.logger import LOGGER 12 | 13 | 14 | def parse_ivium_eis_txt(filename: Path): 15 | eis = pd.read_csv(filename, sep="\t") 16 | eis["Z2 /ohm"] *= -1 17 | eis.rename( 18 | {"Z1 /ohm": "Re(Z) [Ω]", "Z2 /ohm": "-Im(Z) [Ω]", "freq. /Hz": "Frequency [Hz]"}, 19 | inplace=True, 20 | axis="columns", 21 | ) 22 | return eis 23 | 24 | 25 | class EISBlock(DataBlock): 26 | accepted_file_extensions = (".txt",) 27 | blocktype = "eis" 28 | name = "EIS" 29 | description = "This block can plot electrochemical impedance spectroscopy (EIS) data from Ivium .txt files" 30 | 31 | @property 32 | def plot_functions(self): 33 | return (self.generate_eis_plot,) 34 | 35 | def generate_eis_plot(self): 36 | file_info = None 37 | # all_files = None 38 | eis_data = None 39 | 40 | if "file_id" not in self.data: 41 | LOGGER.warning("No file set in the DataBlock") 42 | return 43 | else: 44 | file_info = get_file_info_by_id(self.data["file_id"], update_if_live=True) 45 | ext = os.path.splitext(file_info["location"].split("/")[-1])[-1].lower() 46 | if ext not in self.accepted_file_extensions: 47 | LOGGER.warning( 48 | "Unsupported file extension (must be one of %s, not %s)", 49 | self.accepted_file_extensions, 50 | ext, 51 | ) 52 | return 53 | 54 | eis_data = parse_ivium_eis_txt(Path(file_info["location"])) 55 | 56 | if eis_data is not None: 57 | plot = selectable_axes_plot( 58 | eis_data, 59 | x_options=["Re(Z) [Ω]"], 60 | y_options=["-Im(Z) [Ω]"], 61 | color_options=["Frequency [Hz]"], 62 | color_mapper=LogColorMapper("Cividis256"), 63 | plot_points=True, 64 | plot_line=False, 65 | tools=HoverTool(tooltips=[("Frequency [Hz]", "@{Frequency [Hz]}")]), 66 | ) 67 | 68 | self.data["bokeh_plot_data"] = bokeh.embed.json_item(plot, theme=DATALAB_BOKEH_THEME) 69 | -------------------------------------------------------------------------------- /pydatalab/src/pydatalab/apps/nmr/__init__.py: -------------------------------------------------------------------------------- 1 | from .blocks import NMRBlock 2 | 3 | __all__ = ("NMRBlock",) 4 | -------------------------------------------------------------------------------- /pydatalab/src/pydatalab/apps/raman/__init__.py: -------------------------------------------------------------------------------- 1 | from .blocks import RamanBlock 2 | 3 | __all__ = ("RamanBlock",) 4 | -------------------------------------------------------------------------------- /pydatalab/src/pydatalab/apps/tga/__init__.py: -------------------------------------------------------------------------------- 1 | from .blocks import MassSpecBlock 2 | from .parsers import parse_mt_mass_spec_ascii 3 | 4 | __all__ = ("MassSpecBlock", "parse_mt_mass_spec_ascii") 5 | -------------------------------------------------------------------------------- /pydatalab/src/pydatalab/apps/xrd/__init__.py: -------------------------------------------------------------------------------- 1 | from .blocks import XRDBlock 2 | 3 | __all__ = ("XRDBlock",) 4 | -------------------------------------------------------------------------------- /pydatalab/src/pydatalab/apps/xrd/models.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel 2 | 3 | __all__ = ("PeakInformation",) 4 | 5 | 6 | class PeakInformation(BaseModel): 7 | positions: list[float] | None = None 8 | intensities: list[float] | None = None 9 | widths: list[float] | None = None 10 | hkls: list[tuple[int, int, int]] | None = None 11 | theoretical: bool = False 12 | -------------------------------------------------------------------------------- /pydatalab/src/pydatalab/blocks/__init__.py: -------------------------------------------------------------------------------- 1 | # Base block import has to go first to avoid circular deps 2 | from pydatalab.blocks.base import DataBlock 3 | from pydatalab.blocks.common import CommentBlock, MediaBlock, NotSupportedBlock, TabularDataBlock 4 | 5 | COMMON_BLOCKS: list[type[DataBlock]] = [ 6 | CommentBlock, 7 | MediaBlock, 8 | NotSupportedBlock, 9 | TabularDataBlock, 10 | ] 11 | 12 | __all__ = ("COMMON_BLOCKS", "DataBlock") 13 | -------------------------------------------------------------------------------- /pydatalab/src/pydatalab/models/__init__.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel 2 | 3 | from pydatalab.models.cells import Cell 4 | from pydatalab.models.collections import Collection 5 | from pydatalab.models.equipment import Equipment 6 | from pydatalab.models.files import File 7 | from pydatalab.models.people import Person 8 | from pydatalab.models.samples import Sample 9 | from pydatalab.models.starting_materials import StartingMaterial 10 | 11 | ITEM_MODELS: dict[str, type[BaseModel]] = { 12 | "samples": Sample, 13 | "starting_materials": StartingMaterial, 14 | "cells": Cell, 15 | "equipment": Equipment, 16 | } 17 | 18 | __all__ = ( 19 | "File", 20 | "Sample", 21 | "StartingMaterial", 22 | "Person", 23 | "Cell", 24 | "Collection", 25 | "Equipment", 26 | "ITEM_MODELS", 27 | ) 28 | -------------------------------------------------------------------------------- /pydatalab/src/pydatalab/models/collections.py: -------------------------------------------------------------------------------- 1 | from pydantic import Field, root_validator 2 | 3 | from pydatalab.models.entries import Entry 4 | from pydatalab.models.traits import HasBlocks, HasOwner 5 | from pydatalab.models.utils import HumanReadableIdentifier 6 | 7 | 8 | class Collection(Entry, HasOwner, HasBlocks): 9 | type: str = Field("collections", const="collections", pattern="^collections$") 10 | 11 | collection_id: HumanReadableIdentifier = Field(None) 12 | """A short human-readable/usable name for the collection.""" 13 | 14 | title: str | None 15 | """A descriptive title for the collection.""" 16 | 17 | description: str | None 18 | """A description of the collection, either in plain-text or a markup language.""" 19 | 20 | num_items: int | None = Field(None) 21 | """Inlined number of items associated with this collection.""" 22 | 23 | @root_validator 24 | def check_ids(cls, values): 25 | if not any(values.get(k) is not None for k in ("collection_id", "immutable_id")): 26 | raise ValueError("Collection must have at least collection_id or immutable_id") 27 | 28 | return values 29 | -------------------------------------------------------------------------------- /pydatalab/src/pydatalab/models/entries.py: -------------------------------------------------------------------------------- 1 | import abc 2 | 3 | from pydantic import BaseModel, Field, root_validator 4 | 5 | from pydatalab.models.relationships import TypedRelationship 6 | from pydatalab.models.utils import ( 7 | JSON_ENCODERS, 8 | EntryReference, 9 | IsoformatDateTime, 10 | PyObjectId, 11 | ) 12 | 13 | 14 | class Entry(BaseModel, abc.ABC): 15 | """An Entry is an abstract base class for any model that can be 16 | deserialized and stored in the database. 17 | 18 | """ 19 | 20 | type: str 21 | """The resource type of the entry.""" 22 | 23 | immutable_id: PyObjectId = Field( 24 | None, 25 | title="Immutable ID", 26 | alias="_id", 27 | format="uuid", 28 | ) 29 | """The immutable database ID of the entry.""" 30 | 31 | last_modified: IsoformatDateTime | None = None 32 | """The timestamp at which the entry was last modified.""" 33 | 34 | relationships: list[TypedRelationship] | None = None 35 | """A list of related entries and their types.""" 36 | 37 | @root_validator(pre=True) 38 | def check_id_names(cls, values): 39 | """Slightly upsetting hack: this case *should* be covered by the pydantic setting for 40 | populating fields by alias names. 41 | """ 42 | if "_id" in values: 43 | values["immutable_id"] = values.pop("_id") 44 | 45 | return values 46 | 47 | def to_reference(self, additional_fields: list[str] | None = None) -> "EntryReference": 48 | """Populate an EntryReference model from this entry, selecting additional fields to inline. 49 | 50 | Parameters: 51 | additional_fields: A list of fields to inline in the reference. 52 | 53 | """ 54 | if additional_fields is None: 55 | additional_fields = [] 56 | 57 | data = { 58 | "type": self.type, 59 | "item_id": getattr(self, "item_id", None), 60 | "immutable_id": getattr(self, "immutable_id", None), 61 | } 62 | data.update({field: getattr(self, field, None) for field in additional_fields}) 63 | 64 | return EntryReference(**data) 65 | 66 | class Config: 67 | allow_population_by_field_name = True 68 | json_encoders = JSON_ENCODERS 69 | extra = "ignore" 70 | -------------------------------------------------------------------------------- /pydatalab/src/pydatalab/models/equipment.py: -------------------------------------------------------------------------------- 1 | from pydantic import Field 2 | 3 | from pydatalab.models.items import Item 4 | 5 | 6 | class Equipment(Item): 7 | """A model for representing an experimental sample.""" 8 | 9 | type: str = Field("equipment", const="equipment", pattern="^equipment$") 10 | 11 | serial_numbers: str | None 12 | """A string describing one or more serial numbers for the instrument.""" 13 | 14 | manufacturer: str | None 15 | """The manufacturer of this piece of equipment""" 16 | 17 | location: str | None 18 | """Place where the equipment is located""" 19 | 20 | contact: str | None 21 | """Contact information for equipment (e.g., email address or phone number).""" 22 | -------------------------------------------------------------------------------- /pydatalab/src/pydatalab/models/files.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | 3 | from pydantic import Field 4 | 5 | from pydatalab.models.entries import Entry 6 | from pydatalab.models.traits import HasOwner, HasRevisionControl 7 | from pydatalab.models.utils import IsoformatDateTime 8 | 9 | 10 | class File(Entry, HasOwner, HasRevisionControl): 11 | """A model for representing a file that has been tracked or uploaded to datalab.""" 12 | 13 | type: str = Field("files", const="files", pattern="^files$") 14 | 15 | size: int | None 16 | """The size of the file on disk in bytes.""" 17 | 18 | last_modified_remote: IsoformatDateTime | None 19 | """The last date/time at which the remote file was modified.""" 20 | 21 | item_ids: list[str] 22 | """A list of item IDs associated with this file.""" 23 | 24 | blocks: list[str] 25 | """A list of block IDs associated with this file.""" 26 | 27 | name: str 28 | """The filename on disk.""" 29 | 30 | extension: str 31 | """The file extension that the file was uploaded with.""" 32 | 33 | original_name: str | None 34 | """The raw filename as uploaded.""" 35 | 36 | location: str | None 37 | """The location of the file on disk.""" 38 | 39 | url_path: str | None 40 | """The path to a remote file.""" 41 | 42 | source: str | None 43 | """The source of the file, e.g. 'remote' or 'uploaded'.""" 44 | 45 | time_added: IsoformatDateTime 46 | """The timestamp for the original file upload.""" 47 | 48 | metadata: dict[Any, Any] | None 49 | """Any additional metadata.""" 50 | 51 | representation: Any | None 52 | 53 | source_server_name: str | None 54 | """The server name at which the file is stored.""" 55 | 56 | source_path: str | None 57 | """The path to the file on the remote resource.""" 58 | 59 | is_live: bool 60 | """Whether or not the file should be watched for future updates.""" 61 | -------------------------------------------------------------------------------- /pydatalab/src/pydatalab/models/items.py: -------------------------------------------------------------------------------- 1 | import abc 2 | 3 | from pydantic import Field, validator 4 | 5 | from pydatalab.models.entries import Entry 6 | from pydatalab.models.files import File 7 | from pydatalab.models.traits import ( 8 | HasBlocks, 9 | HasOwner, 10 | HasRevisionControl, 11 | IsCollectable, 12 | ) 13 | from pydatalab.models.utils import ( 14 | HumanReadableIdentifier, 15 | IsoformatDateTime, 16 | PyObjectId, 17 | Refcode, 18 | ) 19 | 20 | 21 | class Item(Entry, HasOwner, HasRevisionControl, IsCollectable, HasBlocks, abc.ABC): 22 | """The generic model for data types that will be exposed with their own named endpoints.""" 23 | 24 | refcode: Refcode = None # type: ignore 25 | """A globally unique immutable ID comprised of the deployment prefix (e.g., `grey`) 26 | and a locally unique string, ideally created with some consistent scheme. 27 | """ 28 | 29 | item_id: HumanReadableIdentifier 30 | """A locally unique, human-readable identifier for the entry. This ID is mutable.""" 31 | 32 | description: str | None 33 | """A description of the item, either in plain-text or a markup language.""" 34 | 35 | date: IsoformatDateTime | None 36 | """A relevant 'creation' timestamp for the entry (e.g., purchase date, synthesis date).""" 37 | 38 | name: str | None 39 | """An optional human-readable/usable name for the entry.""" 40 | 41 | files: list[File] | None 42 | """Any files attached to this sample.""" 43 | 44 | file_ObjectIds: list[PyObjectId] = Field([]) 45 | """Links to object IDs of files stored within the database.""" 46 | 47 | @validator("refcode", pre=True, always=True) 48 | def refcode_validator(cls, v): 49 | """Generate a refcode if not provided.""" 50 | 51 | if v: 52 | prefix = None 53 | id = None 54 | prefix, id = v.split(":") 55 | if prefix is None or id is None: 56 | raise ValueError(f"refcode missing prefix or ID {id=}, {prefix=} from {v=}") 57 | 58 | return v 59 | -------------------------------------------------------------------------------- /pydatalab/src/pydatalab/models/relationships.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | 3 | from pydantic import BaseModel, root_validator, validator 4 | 5 | from pydatalab.models.utils import ( 6 | HumanReadableIdentifier, 7 | KnownType, 8 | PyObjectId, 9 | Refcode, 10 | ) 11 | 12 | 13 | class RelationshipType(str, Enum): 14 | """An enumeration of the possible types of relationship between two entries. 15 | 16 | ```mermaid 17 | classDiagram 18 | class entryC 19 | entryC --|> entryA: parent 20 | entryC ..|> entryD 21 | entryA <..> entryD: sibling 22 | entryA --|> entryB : child 23 | ``` 24 | 25 | """ 26 | 27 | PARENT = "parent" 28 | CHILD = "child" 29 | SIBLING = "sibling" 30 | PARTHOOD = "is_part_of" 31 | OTHER = "other" 32 | 33 | 34 | class TypedRelationship(BaseModel): 35 | description: str | None 36 | """A description of the relationship.""" 37 | 38 | relation: RelationshipType | None 39 | """The type of relationship between the two items. If the type is 'other', then a human-readable description should be provided.""" 40 | 41 | type: KnownType 42 | """The type of the related resource.""" 43 | 44 | immutable_id: PyObjectId | None 45 | """The immutable ID of the entry that is related to this entry.""" 46 | 47 | item_id: HumanReadableIdentifier | None 48 | """The ID of the entry that is related to this entry.""" 49 | 50 | refcode: Refcode | None 51 | """The refcode of the entry that is related to this entry.""" 52 | 53 | @validator("relation") 54 | def check_for_description(cls, v, values): 55 | if v == RelationshipType.OTHER and values.get("description") is None: 56 | raise ValueError( 57 | f"A description must be provided if the relationship type is {RelationshipType.OTHER.value!r}." 58 | ) 59 | 60 | return v 61 | 62 | @root_validator 63 | def check_id_fields(cls, values): 64 | """Check that only one of the possible identifier fields is provided.""" 65 | id_fields = ("immutable_id", "item_id", "refcode") 66 | if all(values[f] is None for f in id_fields): 67 | raise ValueError(f"Must provide at least one of {id_fields!r}") 68 | if sum(1 for f in id_fields if values[f] is not None) > 1: 69 | raise ValueError("Must provide only one of {id_fields!r}") 70 | 71 | return values 72 | -------------------------------------------------------------------------------- /pydatalab/src/pydatalab/models/samples.py: -------------------------------------------------------------------------------- 1 | from pydantic import Field 2 | 3 | from pydatalab.models.items import Item 4 | from pydatalab.models.traits import HasSynthesisInfo 5 | 6 | 7 | class Sample(Item, HasSynthesisInfo): 8 | """A model for representing an experimental sample.""" 9 | 10 | type: str = Field("samples", const="samples", pattern="^samples$") 11 | 12 | chemform: str | None = Field(example=["Na3P", "LiNiO2@C"]) 13 | """A string representation of the chemical formula or composition associated with this sample.""" 14 | -------------------------------------------------------------------------------- /pydatalab/src/pydatalab/routes/__init__.py: -------------------------------------------------------------------------------- 1 | from pydatalab.routes.v0_1 import BLUEPRINTS, OAUTH, OAUTH_PROXIES, __api_version__ 2 | 3 | __all__ = ("ENDPOINTS", "__api_version__", "OAUTH", "OAUTH_PROXIES", "BLUEPRINTS") 4 | -------------------------------------------------------------------------------- /pydatalab/src/pydatalab/routes/v0_1/__init__.py: -------------------------------------------------------------------------------- 1 | from flask import Blueprint 2 | 3 | from ._version import __api_version__ 4 | from .admin import ADMIN 5 | from .auth import AUTH, OAUTH, OAUTH_PROXIES 6 | from .blocks import BLOCKS 7 | from .collections import COLLECTIONS 8 | from .files import FILES 9 | from .graphs import GRAPHS 10 | from .healthcheck import HEALTHCHECK 11 | from .info import INFO 12 | from .items import ITEMS 13 | from .remotes import REMOTES 14 | from .users import USERS 15 | 16 | BLUEPRINTS: tuple[Blueprint, ...] = ( 17 | AUTH, 18 | COLLECTIONS, 19 | REMOTES, 20 | USERS, 21 | ADMIN, 22 | ITEMS, 23 | BLOCKS, 24 | FILES, 25 | HEALTHCHECK, 26 | INFO, 27 | GRAPHS, 28 | ) 29 | 30 | __all__ = ("BLUEPRINTS", "OAUTH", "__api_version__", "OAUTH_PROXIES") 31 | -------------------------------------------------------------------------------- /pydatalab/src/pydatalab/routes/v0_1/_version.py: -------------------------------------------------------------------------------- 1 | __api_version__ = "0.1.0" 2 | -------------------------------------------------------------------------------- /pydatalab/src/pydatalab/routes/v0_1/healthcheck.py: -------------------------------------------------------------------------------- 1 | from flask import Blueprint, jsonify 2 | 3 | HEALTHCHECK = Blueprint("healthcheck", __name__) 4 | 5 | 6 | @HEALTHCHECK.route("/healthcheck/is_ready", methods=["GET"]) 7 | def is_ready(): 8 | from pydatalab.mongo import check_mongo_connection 9 | 10 | try: 11 | check_mongo_connection() 12 | except RuntimeError: 13 | return ( 14 | jsonify(status="error", message="Unable to connect to MongoDB at specified URI."), 15 | 500, 16 | ) 17 | return (jsonify(status="success", message="Server and database are ready"), 200) 18 | 19 | 20 | @HEALTHCHECK.route("/healthcheck/is_alive", methods=["GET"]) 21 | def is_alive(): 22 | return (jsonify(status="success", message="Server is alive"), 200) 23 | -------------------------------------------------------------------------------- /pydatalab/src/pydatalab/routes/v0_1/users.py: -------------------------------------------------------------------------------- 1 | from bson import ObjectId 2 | from flask import Blueprint, jsonify, request 3 | from flask_login import current_user 4 | 5 | from pydatalab.config import CONFIG 6 | from pydatalab.models.people import DisplayName, EmailStr 7 | from pydatalab.mongo import flask_mongo 8 | from pydatalab.permissions import active_users_or_get_only 9 | 10 | USERS = Blueprint("users", __name__) 11 | 12 | 13 | @USERS.before_request 14 | @active_users_or_get_only 15 | def _(): ... 16 | 17 | 18 | @USERS.route("/users/", methods=["PATCH"]) 19 | def save_user(user_id): 20 | request_json = request.get_json() 21 | 22 | display_name: str | None = None 23 | contact_email: str | None = None 24 | account_status: str | None = None 25 | 26 | if request_json is not None: 27 | display_name = request_json.get("display_name", False) 28 | contact_email = request_json.get("contact_email", False) 29 | account_status = request_json.get("account_status", None) 30 | 31 | if not current_user.is_authenticated and not CONFIG.TESTING: 32 | return (jsonify({"status": "error", "message": "No user authenticated."}), 401) 33 | 34 | if not CONFIG.TESTING and current_user.id != user_id and current_user.role != "admin": 35 | return ( 36 | jsonify({"status": "error", "message": "User not allowed to edit this profile."}), 37 | 403, 38 | ) 39 | 40 | update = {} 41 | 42 | try: 43 | if display_name: 44 | update["display_name"] = DisplayName(display_name) 45 | 46 | if contact_email or contact_email in (None, ""): 47 | if contact_email in ("", None): 48 | update["contact_email"] = None 49 | else: 50 | update["contact_email"] = EmailStr(contact_email) 51 | 52 | if account_status: 53 | update["account_status"] = account_status 54 | 55 | except ValueError as e: 56 | return jsonify( 57 | {"status": "error", "message": f"Invalid display name or email was passed: {str(e)}"} 58 | ), 400 59 | 60 | if not update: 61 | return jsonify({"status": "success", "message": "No update was performed."}), 200 62 | 63 | update_result = flask_mongo.db.users.update_one({"_id": ObjectId(user_id)}, {"$set": update}) 64 | 65 | if update_result.matched_count != 1: 66 | return (jsonify({"status": "error", "message": "Unable to update user."}), 400) 67 | 68 | if update_result.modified_count != 1: 69 | return ( 70 | jsonify( 71 | { 72 | "status": "success", 73 | "message": "No update was performed", 74 | } 75 | ), 76 | 200, 77 | ) 78 | 79 | return (jsonify({"status": "success"}), 200) 80 | -------------------------------------------------------------------------------- /pydatalab/src/pydatalab/send_email.py: -------------------------------------------------------------------------------- 1 | """Send an email using Flask-Mail with the configured SMTP server.""" 2 | 3 | from flask_mail import Mail, Message 4 | 5 | from pydatalab.config import CONFIG 6 | from pydatalab.logger import LOGGER 7 | 8 | MAIL = Mail() 9 | 10 | 11 | def send_mail(recipient: str, subject: str, body: str): 12 | """Send an email via the configured SMTP server. 13 | 14 | Mail will be sent from the configured `MAIL_DEFAULT_SENDER` address, 15 | via the configured `MAIL_SERVER` using `MAIL_USERNAME` and `MAIL_PASSWORD` credentials 16 | on `MAIL_PORT` using `MAIL_USE_TLS` encryption. 17 | 18 | Parameters: 19 | recipient (str): The email address of the recipient. 20 | subject (str): The subject of the email. 21 | body (str): The body of the email. 22 | 23 | """ 24 | LOGGER.debug("Sending email to %s", recipient) 25 | 26 | sender = None 27 | if CONFIG.EMAIL_AUTH_SMTP_SETTINGS is not None: 28 | sender = CONFIG.EMAIL_AUTH_SMTP_SETTINGS.MAIL_DEFAULT_SENDER 29 | 30 | message = Message( 31 | sender=sender, 32 | recipients=[recipient], 33 | body=body, 34 | subject=subject, 35 | ) 36 | MAIL.connect() 37 | MAIL.send(message) 38 | LOGGER.debug("Email sent to %s", recipient) 39 | -------------------------------------------------------------------------------- /pydatalab/src/pydatalab/utils.py: -------------------------------------------------------------------------------- 1 | """This module contains utility functions that can be used 2 | anywhere in the package. 3 | 4 | """ 5 | 6 | import datetime 7 | from json import JSONEncoder 8 | from math import ceil 9 | 10 | import pandas as pd 11 | from bson import json_util 12 | from flask.json.provider import DefaultJSONProvider 13 | 14 | 15 | def reduce_df_size(df: pd.DataFrame, target_nrows: int, endpoint: bool = True) -> pd.DataFrame: 16 | """Reduce the dataframe to the number of target rows by applying a stride. 17 | 18 | Parameters: 19 | df: The dataframe to reduce. 20 | target_nrows: The target number of rows to reduce each column to. 21 | endpoint: Whether to include the endpoint of the dataframe. 22 | 23 | Returns: 24 | A copy of the input dataframe with the applied stride. 25 | 26 | """ 27 | num_rows = len(df) 28 | stride = ceil(num_rows / target_nrows) 29 | if endpoint: 30 | indices = [0] + list(range(stride, num_rows - 1, stride)) + [num_rows - 1] 31 | else: 32 | indices = list(range(0, num_rows, stride)) 33 | 34 | return df.iloc[indices].copy() 35 | 36 | 37 | class CustomJSONEncoder(JSONEncoder): 38 | """A custom JSON encoder that uses isoformat datetime strings and 39 | BSON for other serialization.""" 40 | 41 | @staticmethod 42 | def default(o): 43 | if isinstance(o, (datetime.date, datetime.datetime)): 44 | return o.isoformat() 45 | 46 | return json_util.default(o) 47 | 48 | 49 | class BSONProvider(DefaultJSONProvider): 50 | """A custom JSON provider that uses isoformat datetime strings and 51 | BSON for other serialization.""" 52 | 53 | @staticmethod 54 | def default(o): 55 | return CustomJSONEncoder.default(o) 56 | -------------------------------------------------------------------------------- /pydatalab/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datalab-org/datalab/05f761881757985ac8b761a85163571434316d9c/pydatalab/tests/__init__.py -------------------------------------------------------------------------------- /pydatalab/tests/apps/test_ftir_block.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import pytest 4 | 5 | from pydatalab.apps.ftir import FTIRBlock 6 | 7 | 8 | @pytest.fixture 9 | def data_files(): 10 | return (Path(__file__).parent.parent.parent / "example_data" / "FTIR").glob("*") 11 | 12 | 13 | def test_load(data_files): 14 | for f in data_files: 15 | df = FTIRBlock.parse_ftir_asp(f) 16 | assert len(df) == 1932 17 | assert all(x in df.columns for x in ["Wavenumber", "Absorbance"]) 18 | assert df["Wavenumber"].min() == pytest.approx(400.688817501068, 1e-5) 19 | assert df["Wavenumber"].max() == pytest.approx(3999.43349933624, 1e-5) 20 | assert df["Absorbance"].argmin() == 987 21 | assert df["Absorbance"].argmax() == 1928 22 | # Checking height of peak at 1079 cm-1 has correct height 23 | mask = (df["Wavenumber"] > 800) & (df["Wavenumber"] < 1500) 24 | assert max(df["Absorbance"][mask]) == pytest.approx(0.0536771319493808, 1e-5) 25 | 26 | 27 | def test_plot(data_files): 28 | f = next(data_files) 29 | ftir_data = FTIRBlock.parse_ftir_asp(f) 30 | layout = FTIRBlock._format_ftir_plot(ftir_data) 31 | assert layout 32 | -------------------------------------------------------------------------------- /pydatalab/tests/apps/test_plugins.py: -------------------------------------------------------------------------------- 1 | # Load block types first to avoid circular dependency issues 2 | from pydatalab.apps import BLOCK_TYPES, BLOCKS, load_block_plugins 3 | 4 | 5 | def test_load_plugins(): 6 | from datalab_app_plugin_insitu import InsituBlock 7 | 8 | plugins = load_block_plugins() 9 | 10 | assert plugins["insitu-nmr"] == InsituBlock 11 | assert isinstance(BLOCK_TYPES["insitu-nmr"], type) 12 | assert BLOCK_TYPES["insitu-nmr"] == InsituBlock 13 | assert InsituBlock in BLOCKS 14 | 15 | 16 | def test_load_app_blocks(): 17 | from pydatalab.apps import load_app_blocks 18 | from pydatalab.apps.echem import CycleBlock 19 | from pydatalab.apps.xrd import XRDBlock 20 | 21 | app_blocks = load_app_blocks() 22 | 23 | assert CycleBlock in app_blocks 24 | assert XRDBlock in app_blocks 25 | -------------------------------------------------------------------------------- /pydatalab/tests/apps/test_raman.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import numpy as np 4 | import pytest 5 | 6 | from pydatalab.apps.raman.blocks import RamanBlock 7 | 8 | 9 | @pytest.fixture 10 | def wdf_example(): 11 | return Path(__file__).parent.parent.parent / "example_data" / "raman" / "raman_example.wdf" 12 | 13 | 14 | @pytest.fixture 15 | def labspec_txt_example(): 16 | return ( 17 | Path(__file__).parent.parent.parent / "example_data" / "raman" / "labspec_raman_example.txt" 18 | ) 19 | 20 | 21 | @pytest.fixture 22 | def renishaw_txt_example(): 23 | return Path(__file__).parent.parent.parent / "example_data" / "raman" / "raman_example.txt" 24 | 25 | 26 | def test_load_wdf(wdf_example): 27 | df, metadata, y_options = RamanBlock.load(wdf_example) 28 | assert all(y in df.columns for y in y_options) 29 | assert df.shape == (1011, 11) 30 | np.testing.assert_almost_equal(df["intensity"].mean(), 364.29358, decimal=5) 31 | np.testing.assert_almost_equal(df["normalized intensity"].max(), 1.0, decimal=5) 32 | 33 | # TODO It is likely this is the "real" value we should be reading, but after switching to the older 34 | # package, we now longer have access to the offset. 35 | # np.testing.assert_almost_equal( 36 | # df["wavenumber"][np.argmax(df["intensity"])], 1587.546335309901, decimal=5 37 | # ) 38 | # np.testing.assert_almost_equal(df["wavenumber"].min(), 44.812868, decimal=5) 39 | # np.testing.assert_almost_equal(df["wavenumber"].max(), 1919.855951, decimal=5) 40 | np.testing.assert_almost_equal( 41 | df["wavenumber"][np.argmax(df["intensity"])], 1581.730469, decimal=5 42 | ) 43 | np.testing.assert_almost_equal(df["wavenumber"].min(), 0.380859, decimal=5) 44 | np.testing.assert_almost_equal(df["wavenumber"].max(), 1879.449219, decimal=5) 45 | 46 | 47 | def test_load_renishaw_txt(renishaw_txt_example): 48 | with pytest.warns(UserWarning, match="Unable to find wavenumber unit"): 49 | with pytest.warns(UserWarning, match="Unable to find wavenumber offset"): 50 | df, metadata, y_options = RamanBlock.load(renishaw_txt_example) 51 | assert all(y in df.columns for y in y_options) 52 | assert df.shape == (1011, 11) 53 | np.testing.assert_almost_equal(df["intensity"].mean(), 364.293613265, decimal=5) 54 | np.testing.assert_almost_equal(df["normalized intensity"].max(), 1.0, decimal=5) 55 | np.testing.assert_almost_equal( 56 | df["wavenumber"][np.argmax(df["intensity"])], 1581.730469, decimal=5 57 | ) 58 | 59 | np.testing.assert_almost_equal(df["wavenumber"].min(), 0.380859, decimal=5) 60 | np.testing.assert_almost_equal(df["wavenumber"].max(), 1879.449219, decimal=5) 61 | 62 | 63 | def test_load_labspec_txt(labspec_txt_example): 64 | df, metadata, y_options = RamanBlock.load(labspec_txt_example) 65 | assert all(y in df.columns for y in y_options) 66 | assert df.shape == (341, 11) 67 | np.testing.assert_almost_equal(df["normalized intensity"].max(), 1.0, decimal=5) 68 | -------------------------------------------------------------------------------- /pydatalab/tests/apps/test_tabular.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from pydatalab.blocks.common import TabularDataBlock 4 | 5 | 6 | def test_load_raman_txt(example_data_dir): 7 | with pytest.warns(UserWarning): 8 | df = TabularDataBlock.load(example_data_dir / "raman" / "raman_example.txt") 9 | assert df.shape == (1011, 2) 10 | 11 | 12 | def test_load_labspec_raman_txt(example_data_dir): 13 | with pytest.warns(UserWarning): 14 | df = TabularDataBlock.load(example_data_dir / "raman" / "labspec_raman_example.txt") 15 | assert df.shape == (341, 2) 16 | 17 | 18 | def test_simple_csv(example_data_dir): 19 | df = TabularDataBlock.load(example_data_dir / "csv" / "simple.csv") 20 | assert df.shape == (2, 3) 21 | assert df.columns.tolist() == ["test", "test2", "test3"] 22 | 23 | 24 | def test_simple_xlsx(example_data_dir): 25 | df = TabularDataBlock.load(example_data_dir / "csv" / "simple.xlsx") 26 | assert df.shape == (4, 4) 27 | -------------------------------------------------------------------------------- /pydatalab/tests/apps/test_tga.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import pytest 4 | 5 | 6 | def test_ms_parse_and_plot(): 7 | from pydatalab.apps.tga.blocks import MassSpecBlock 8 | from pydatalab.apps.tga.parsers import parse_mt_mass_spec_ascii 9 | 10 | ms = parse_mt_mass_spec_ascii( 11 | Path(__file__).parent.parent.parent 12 | / "example_data" 13 | / "TGA-MS" 14 | / "20221128 134958 TGA MS Megan.asc" 15 | ) 16 | 17 | expected_species = ( 18 | "14", 19 | "16", 20 | "Water", 21 | "Nitrogen/Carbonmonoxide", 22 | "Oxygen", 23 | "Argon", 24 | "Carbondioxide", 25 | ) 26 | 27 | # the final 3 columns are missing in the example file 28 | expected_shapes = [ 29 | (1366, 2), 30 | (1366, 2), 31 | (1366, 2), 32 | (1365, 2), 33 | (1365, 2), 34 | (1365, 2), 35 | (1365, 2), 36 | ] 37 | 38 | assert all(k in ms["data"] for k in expected_species) 39 | 40 | for ind, k in enumerate(expected_species): 41 | assert ms["data"][k].shape == expected_shapes[ind] 42 | 43 | assert all(k in ms["meta"] for k in ("Start Time", "End Time", "Sourcefile", "Exporttime")) 44 | 45 | assert MassSpecBlock._plot_ms_data(ms) 46 | 47 | 48 | @pytest.mark.parametrize( 49 | "filename", (Path(__file__).parent.parent.parent / "example_data" / "TGA-MS").glob("*.asc") 50 | ) 51 | def test_ms_parse_no_validation(filename): 52 | from pydatalab.apps.tga.parsers import parse_mt_mass_spec_ascii 53 | 54 | ms = parse_mt_mass_spec_ascii(filename) 55 | assert ms 56 | 57 | assert all(k in ms["meta"] for k in ("Start Time", "End Time", "Sourcefile", "Exporttime")) 58 | 59 | for species in ms["data"]: 60 | assert ( 61 | "Ion Current [A]" in ms["data"][species] 62 | or "Partial Pressure [mbar]" in ms["data"][species] 63 | ) 64 | assert "Time Relative [s]" in ms["data"][species] 65 | assert "Time" not in ms["data"][species] 66 | -------------------------------------------------------------------------------- /pydatalab/tests/apps/test_xrd_block.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import pytest 4 | 5 | from pydatalab.apps.xrd.blocks import XRDBlock 6 | from pydatalab.bokeh_plots import selectable_axes_plot 7 | 8 | XRD_DATA_FILES = list((Path(__file__).parent.parent.parent / "example_data" / "XRD").glob("*")) 9 | 10 | 11 | @pytest.mark.parametrize("f", XRD_DATA_FILES) 12 | def test_load(f): 13 | if f.suffix in XRDBlock.accepted_file_extensions: 14 | df, y_options, _ = XRDBlock.load_pattern(f) 15 | assert all(y in df.columns for y in y_options) 16 | 17 | 18 | @pytest.mark.parametrize("f", XRD_DATA_FILES) 19 | def test_plot(f): 20 | if f.suffix in XRDBlock.accepted_file_extensions: 21 | df, y_options, _ = XRDBlock.load_pattern(f) 22 | p = selectable_axes_plot( 23 | [df], 24 | x_options=["2θ (°)", "Q (Å⁻¹)", "d (Å)"], 25 | y_options=y_options, 26 | plot_line=True, 27 | plot_points=True, 28 | point_size=3, 29 | ) 30 | assert p 31 | -------------------------------------------------------------------------------- /pydatalab/tests/conftest.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import pytest 4 | 5 | 6 | @pytest.fixture(scope="session") 7 | def example_data_dir(): 8 | return Path(__file__).parent.parent / "example_data" 9 | 10 | 11 | @pytest.fixture(scope="session", name="default_filepath") 12 | def fixture_default_filepath(example_data_dir): 13 | return example_data_dir / "echem" / "jdb11-1_c3_gcpl_5cycles_2V-3p8V_C-24_data_C09.mpr" 14 | -------------------------------------------------------------------------------- /pydatalab/tests/server/test_auth.py: -------------------------------------------------------------------------------- 1 | from pydatalab.routes.v0_1.auth import _check_email_domain 2 | 3 | 4 | def test_allow_emails(): 5 | # Test that a valid email is allowed 6 | assert _check_email_domain("test@example.org", ["example.org"]) 7 | assert _check_email_domain("test@example.org", ["example.org", "example2.org"]) 8 | assert _check_email_domain("test@subdomain.example.org", ["example.org", "example2.org"]) 9 | assert _check_email_domain("test@subdomain.example.org", ["subdomain.example.org"]) 10 | assert not _check_email_domain("test@example2.org", []) 11 | assert _check_email_domain("test@example2.org", None) 12 | assert not _check_email_domain("test@example.org", ["subdomain.example.org"]) 13 | assert not _check_email_domain("test@example2.org", ["example.org"]) 14 | 15 | 16 | def test_magic_link_account_creation(unauthenticated_client, app, database): 17 | with app.extensions["mail"].record_messages() as outbox: 18 | response = unauthenticated_client.post( 19 | "/login/magic-link", 20 | json={"email": "test@ml-evs.science", "referrer": "datalab.example.org"}, 21 | ) 22 | assert response.json["detail"] == "Email sent successfully." 23 | assert response.status_code == 200 24 | assert len(outbox) == 1 25 | 26 | doc = database.magic_links.find_one() 27 | assert "jwt" in doc 28 | 29 | response = unauthenticated_client.get(f"/login/email?token={doc['jwt']}") 30 | assert response.status_code == 307 31 | assert database.users.find_one({"contact_email": "test@ml-evs.science"}) 32 | 33 | 34 | def test_magic_links_expected_failures(unauthenticated_client, app): 35 | with app.extensions["mail"].record_messages() as outbox: 36 | response = unauthenticated_client.post( 37 | "/login/magic-link", 38 | json={"email": "test@ml-evs.science"}, 39 | ) 40 | assert response.status_code == 400 41 | assert len(outbox) == 0 42 | 43 | response = unauthenticated_client.post( 44 | "/login/magic-link", 45 | json={"email": "not_an_email", "referrer": "datalab.example.org"}, 46 | ) 47 | assert response.status_code == 400 48 | assert len(outbox) == 0 49 | assert response.json["detail"] == "Invalid email provided." 50 | 51 | response = unauthenticated_client.post( 52 | "/login/magic-link", 53 | json={"email": "banned_email@gmail.com", "referrer": "datalab.example.org"}, 54 | ) 55 | assert response.status_code == 403 56 | assert len(outbox) == 0 57 | -------------------------------------------------------------------------------- /pydatalab/tests/server/test_backup.py: -------------------------------------------------------------------------------- 1 | """Tests for backup creation and restoration. 2 | Does not test backup scheduling. 3 | 4 | """ 5 | 6 | import shutil 7 | import tarfile 8 | import time 9 | 10 | import pytest 11 | 12 | from pydatalab.backups import create_backup 13 | from pydatalab.config import BackupStrategy 14 | 15 | # Check whether mongodump (and mongorestore by extension) is present; skip backup tests if not 16 | mongodump_present = pytest.mark.skipif( 17 | shutil.which("mongodump") is None, reason="mongodump not installed" 18 | ) 19 | 20 | 21 | @mongodump_present 22 | def test_backup_creation( 23 | client, database, default_filepath, insert_default_sample, default_sample, tmp_path 24 | ): 25 | """Test whether a simple local backup can be created.""" 26 | assert database.items.count_documents({}) == 1 27 | with open(default_filepath, "rb") as f: 28 | response = client.post( 29 | "/upload-file/", 30 | buffered=True, 31 | content_type="multipart/form-data", 32 | data={ 33 | "item_id": default_sample.item_id, 34 | "file": [(f, default_filepath.name)], 35 | "type": "application/octet-stream", 36 | "replace_file": "null", 37 | "relativePath": "null", 38 | }, 39 | ) 40 | assert response.status_code == 201 41 | 42 | strategy = BackupStrategy( 43 | hostname=None, 44 | location=tmp_path, 45 | frequency="5 4 * * *", # 4:05 every day 46 | retention=2, 47 | ) 48 | create_backup(strategy) 49 | assert len(list(tmp_path.glob("*"))) == 1 50 | 51 | # make sure the next ones have a different timestamp 52 | time.sleep(1) 53 | create_backup(strategy) 54 | assert len(list(tmp_path.glob("*"))) == 2 55 | 56 | time.sleep(1) 57 | create_backup(strategy) 58 | assert len(list(tmp_path.glob("*"))) == 2 59 | 60 | # remove the first one and check contents of the second 61 | backups = list(tmp_path.glob("*")) 62 | backups.pop().unlink() 63 | 64 | backup = backups.pop() 65 | 66 | with tarfile.open(backup, mode="r:gz") as tar: 67 | members = {m.name for m in tar.getmembers()} 68 | 69 | assert any(m.startswith("mongodb/") for m in members) 70 | # Only one file is backed up but the tar file reports a "member" for the containing directory too 71 | assert sum(1 for m in members if m.startswith("files/")) == 2 72 | assert sum(1 for m in members if m.startswith("config/")) == 1 73 | -------------------------------------------------------------------------------- /pydatalab/tests/server/test_item_graph.py: -------------------------------------------------------------------------------- 1 | """More item graph tests (in addition to `test_graph.py`) that 2 | are written to be more isolated from one another. 3 | 4 | """ 5 | 6 | import json 7 | 8 | from pydatalab.models import Sample, StartingMaterial 9 | 10 | 11 | def test_single_starting_material(admin_client): 12 | item_id = "material" 13 | 14 | material = StartingMaterial(item_id=item_id) 15 | 16 | creation = admin_client.post( 17 | "/new-sample/", 18 | json={"new_sample_data": json.loads(material.json())}, 19 | ) 20 | 21 | assert creation.status_code == 201 22 | 23 | # A single material without connections should be ignored 24 | graph = admin_client.get("/item-graph").json 25 | assert len(graph["nodes"]) == 0 26 | 27 | # Unless it is asked for directly 28 | graph = admin_client.get(f"/item-graph/{item_id}").json 29 | assert len(graph["nodes"]) == 1 30 | 31 | # Now make a sample and connect it to the starting material; check that the 32 | # starting material is now shown by default 33 | parent = Sample( 34 | item_id="parent", 35 | synthesis_constituents=[ 36 | {"item": {"item_id": item_id, "type": "starting_materials"}, "quantity": None} 37 | ], 38 | ) 39 | 40 | creation = admin_client.post( 41 | "/new-sample/", 42 | json={"new_sample_data": json.loads(parent.json())}, 43 | ) 44 | 45 | assert creation.status_code == 201 46 | 47 | graph = admin_client.get("/item-graph").json 48 | assert len(graph["nodes"]) == 2 49 | assert len(graph["edges"]) == 1 50 | -------------------------------------------------------------------------------- /pydatalab/tests/server/test_items.py: -------------------------------------------------------------------------------- 1 | def test_single_item_endpoints(client, inserted_default_items): 2 | for item in inserted_default_items: 3 | response = client.get(f"/items/{item.refcode}") 4 | assert response.status_code == 200, response.json 5 | assert response.json["item_id"] == item.item_id 6 | assert response.json["item_data"]["item_id"] == item.item_id 7 | assert response.json["status"] == "success" 8 | 9 | test_ref = item.refcode.split(":")[1] 10 | response = client.get(f"/items/{test_ref}") 11 | assert response.status_code == 200, response.json 12 | assert response.json["item_id"] == item.item_id 13 | assert response.json["item_data"]["item_id"] == item.item_id 14 | assert response.json["status"] == "success" 15 | 16 | response = client.get(f"/get-item-data/{item.item_id}") 17 | assert response.status_code == 200, response.json 18 | assert response.json["status"] == "success" 19 | assert response.json["item_id"] == item.item_id 20 | assert response.json["item_data"]["item_id"] == item.item_id 21 | 22 | 23 | def test_fts_fields(): 24 | """Test non-exhaustively that certain fields make it into the fts index.""" 25 | from pydatalab.mongo import ITEMS_FTS_FIELDS 26 | 27 | fields = ("item_id", "name", "description", "refcode", "synthesis_description", "supplier") 28 | assert all(field in ITEMS_FTS_FIELDS for field in fields) 29 | -------------------------------------------------------------------------------- /pydatalab/tests/server/test_remotes.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | 4 | @pytest.mark.dependency() 5 | def test_directories_list(client, unauthenticated_client, unverified_client, deactivated_client): 6 | response = client.get("/list-remote-directories") 7 | assert response.json 8 | toplevel = response.json["data"][0] 9 | assert toplevel["type"] == "toplevel" 10 | assert toplevel["status"] == "updated" 11 | 12 | response = unauthenticated_client.get("/list-remote-directories") 13 | assert response.status_code == 401 14 | response = unverified_client.get("/list-remote-directories") 15 | assert response.status_code == 401 16 | response = deactivated_client.get("/list-remote-directories") 17 | assert response.status_code == 401 18 | 19 | response = client.get("/remotes") 20 | assert response.json 21 | toplevel = response.json["data"][0] 22 | assert toplevel["type"] == "toplevel" 23 | assert toplevel["status"] == "cached" 24 | 25 | response = unauthenticated_client.get("/remotes") 26 | assert response.status_code == 401 27 | response = unverified_client.get("/remotes") 28 | assert response.status_code == 401 29 | response = deactivated_client.get("/remotes") 30 | assert response.status_code == 401 31 | 32 | 33 | @pytest.mark.dependency(depends=["test_directories_list"]) 34 | def test_single_directory(client): 35 | response = client.get("/remotes/example_data?invalidate_cache=1") 36 | assert response.json 37 | toplevel = response.json["data"] 38 | assert toplevel["type"] == "toplevel" 39 | # even though `invalidate_cache` is set to 1, the directory is cached 40 | # because it was just updated in the previous test and the min age is 1 41 | assert toplevel["status"] == "cached" 42 | 43 | response = client.get("/remotes/example/data?invalidate_cache=0") 44 | assert response.json 45 | toplevel = response.json["data"] 46 | assert toplevel["type"] == "toplevel" 47 | assert toplevel["status"] == "cached" 48 | -------------------------------------------------------------------------------- /pydatalab/tests/test_config.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | 3 | import pytest 4 | 5 | from pydatalab.config import ServerConfig, SMTPSettings 6 | from pydatalab.main import create_app 7 | 8 | 9 | def test_default_settings(): 10 | config = ServerConfig() 11 | assert config.MONGO_URI == "mongodb://localhost:27017/datalabvue" 12 | assert config.SECRET_KEY 13 | assert Path(config.FILE_DIRECTORY).name == "files" 14 | 15 | 16 | def test_update_settings(): 17 | config = ServerConfig() 18 | new_settings = { 19 | "mongo_uri": "mongodb://test", 20 | "new_key": "some new data", 21 | } 22 | config.update(new_settings) 23 | 24 | assert new_settings["mongo_uri"] == config.MONGO_URI 25 | assert new_settings["new_key"] == config.NEW_KEY 26 | assert config.SECRET_KEY 27 | assert Path(config.FILE_DIRECTORY).name == "files" 28 | 29 | 30 | def test_config_override(): 31 | app = create_app( 32 | config_override={"REMOTE_FILESYSTEMS": [{"hostname": None, "path": "/", "name": "local"}]} 33 | ) 34 | assert app.config["REMOTE_FILESYSTEMS"][0]["hostname"] is None 35 | assert app.config["REMOTE_FILESYSTEMS"][0]["path"] == Path("/") 36 | 37 | from pydatalab.config import CONFIG 38 | 39 | assert CONFIG.REMOTE_FILESYSTEMS[0].hostname is None 40 | assert CONFIG.REMOTE_FILESYSTEMS[0].path == Path("/") 41 | 42 | 43 | def test_validators(): 44 | # check bad prefix 45 | with pytest.raises( 46 | RuntimeError, match="Identifier prefix must be less than 12 characters long," 47 | ): 48 | _ = ServerConfig(IDENTIFIER_PREFIX="this prefix is way way too long", TESTING=False) 49 | 50 | 51 | def test_mail_settings_combinations(tmpdir): 52 | """Tests that the config file mail settings get passed 53 | correctly to the flask settings, and that additional 54 | overrides can be provided as environment variables. 55 | """ 56 | 57 | from pydatalab.config import CONFIG 58 | 59 | CONFIG.update( 60 | { 61 | "EMAIL_AUTH_SMTP_SETTINGS": SMTPSettings( 62 | MAIL_SERVER="example.com", 63 | MAIL_DEFAULT_SENDER="test@example.com", 64 | MAIL_PORT=587, 65 | MAIL_USE_TLS=True, 66 | MAIL_USERNAME="user", 67 | ) 68 | } 69 | ) 70 | 71 | app = create_app() 72 | assert app.config["MAIL_SERVER"] == "example.com" 73 | assert app.config["MAIL_DEFAULT_SENDER"] == "test@example.com" 74 | assert app.config["MAIL_PORT"] == 587 75 | assert app.config["MAIL_USE_TLS"] is True 76 | assert app.config["MAIL_USERNAME"] == "user" 77 | 78 | # write temporary .env file and check that it overrides the config 79 | env_file = Path(tmpdir.join(".env")) 80 | env_file.write_text("MAIL_PASSWORD=password\nMAIL_DEFAULT_SENDER=test2@example.com") 81 | 82 | app = create_app(env_file=env_file) 83 | assert app.config["MAIL_PASSWORD"] == "password" # noqa: S105 84 | assert app.config["MAIL_DEFAULT_SENDER"] == "test2@example.com" 85 | -------------------------------------------------------------------------------- /pydatalab/tests/test_version.py: -------------------------------------------------------------------------------- 1 | def test_version(): 2 | from pydatalab import __version__ 3 | 4 | assert isinstance(__version__, str) 5 | assert int(__version__.split(".")[0]) == 0 6 | assert int(__version__.split(".")[1]) >= 4 7 | -------------------------------------------------------------------------------- /webapp/.browserslistrc: -------------------------------------------------------------------------------- 1 | > 1% 2 | last 2 versions 3 | not dead 4 | -------------------------------------------------------------------------------- /webapp/.env.test_e2e: -------------------------------------------------------------------------------- 1 | NODE_ENV=production 2 | VUE_APP_API_URL="http://localhost:5001" 3 | -------------------------------------------------------------------------------- /webapp/.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | root: true, 3 | env: { 4 | node: true, 5 | "cypress/globals": true, 6 | }, 7 | extends: [ 8 | "plugin:vue/vue3-recommended", 9 | "eslint:recommended", 10 | "plugin:prettier/recommended", 11 | "@vue/prettier", 12 | "plugin:cypress/recommended", 13 | ], 14 | parserOptions: { 15 | parser: "@babel/eslint-parser", 16 | requireConfigFile: false, 17 | babelOptions: { 18 | babelrc: false, 19 | configFile: false, 20 | presets: ["@vue/cli-plugin-babel/preset"], 21 | plugins: ["@babel/plugin-transform-export-namespace-from"], 22 | }, 23 | }, 24 | rules: { 25 | //"no-console": process.env.NODE_ENV === "production" ? "warn" : "off", 26 | "no-debugger": process.env.NODE_ENV === "production" ? "warn" : "off", 27 | "vue/multi-word-component-names": "off", 28 | // Rule disable for item_id, block_id and collection_id 29 | "vue/prop-name-casing": "off", 30 | "vue/no-unused-components": process.env.NODE_ENV === "production" ? "error" : "warn", 31 | "vue/no-unused-vars": process.env.NODE_ENV === "production" ? "error" : "warn", 32 | "cypress/no-assigning-return-values": "warn", 33 | "cypress/no-unnecessary-waiting": "warn", 34 | "cypress/unsafe-to-chain-command": "warn", 35 | "prettier/prettier": process.env.NODE_ENV === "production" ? "error" : "warn", 36 | }, 37 | }; 38 | -------------------------------------------------------------------------------- /webapp/.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | node_modules 3 | /dist 4 | 5 | /tests/e2e/videos/ 6 | /tests/e2e/screenshots/ 7 | 8 | 9 | # local env files 10 | .env.local 11 | .env.*.local 12 | 13 | tests/unit/__snapshots__/*.snap.html 14 | 15 | # Log files 16 | npm-debug.log* 17 | yarn-debug.log* 18 | yarn-error.log* 19 | pnpm-debug.log* 20 | 21 | # Editor directories and files 22 | .idea 23 | .vscode 24 | *.suo 25 | *.ntvs* 26 | *.njsproj 27 | *.sln 28 | *.sw? 29 | 30 | cypress_examples/ 31 | -------------------------------------------------------------------------------- /webapp/.prettierrc: -------------------------------------------------------------------------------- 1 | printWidth: 100 2 | trailingComma: all 3 | -------------------------------------------------------------------------------- /webapp/README.md: -------------------------------------------------------------------------------- 1 | # datalab webapp 2 | 3 | More detailed instructions can be found in the top-level [README](../README.md). 4 | 5 | ## Project setup 6 | ``` 7 | yarn install 8 | ``` 9 | 10 | ### Compiles and hot-reloads for development 11 | ``` 12 | yarn serve 13 | ``` 14 | 15 | ### Compiles and minifies for production 16 | ``` 17 | yarn build 18 | ``` 19 | 20 | ### Run your unit tests 21 | ``` 22 | yarn test:unit 23 | ``` 24 | 25 | ### Run your end-to-end tests 26 | ``` 27 | yarn test:e2e 28 | ``` 29 | 30 | ### Lints and fixes files 31 | ``` 32 | yarn lint 33 | ``` 34 | 35 | ### Customize configuration 36 | See [Configuration Reference](https://cli.vuejs.org/config/). 37 | -------------------------------------------------------------------------------- /webapp/babel.config.js: -------------------------------------------------------------------------------- 1 | module.exports = function override(api) { 2 | var env = api.cache(() => process.env.NODE_ENV); 3 | var isProd = api.cache(() => env === "production"); 4 | let config = {}; 5 | 6 | if (isProd) { 7 | config["plugins"] = [ 8 | "@babel/plugin-transform-export-namespace-from", 9 | "transform-remove-console", 10 | ]; 11 | } else { 12 | config["plugins"] = ["@babel/plugin-transform-export-namespace-from"]; 13 | } 14 | config["presets"] = ["@vue/cli-plugin-babel/preset"]; 15 | return config; 16 | }; 17 | -------------------------------------------------------------------------------- /webapp/cypress.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from "cypress"; 2 | 3 | export default defineConfig({ 4 | projectId: "4kqx5i", 5 | e2e: { 6 | baseUrl: "http://localhost:8080", 7 | apiUrl: "http://localhost:5001", 8 | defaultCommandTimeout: 10000, 9 | }, 10 | component: { 11 | devServer: { 12 | framework: "vue-cli", 13 | bundler: "webpack", 14 | }, 15 | }, 16 | }); 17 | -------------------------------------------------------------------------------- /webapp/cypress/component/ChemicalFormulaTest.cy.jsx: -------------------------------------------------------------------------------- 1 | import ChemFormulaInput from "@/components/ChemFormulaInput.vue"; 2 | 3 | describe("ChemFormulaInput", () => { 4 | beforeEach(() => { 5 | cy.mount(ChemFormulaInput); 6 | cy.get("span").click({ force: true }); 7 | }); 8 | 9 | it("renders single element formula correctly", () => { 10 | cy.get("input").type("Na"); 11 | cy.get("input").should("have.value", "Na"); 12 | }); 13 | 14 | // it("renders single element with subscript correctly", () => { 15 | // cy.get("input").type("Na3"); 16 | // cy.get("input").should("have.value", "Na3P"); 17 | // }); 18 | 19 | // it("renders formula with parentheses correctly", () => { 20 | // cy.get("input").type("Na3P"); 21 | // cy.get("input").should("have.value", "Na3P"); 22 | // }); 23 | 24 | // it("renders formula with multiple elements in parentheses correctly", () => { 25 | // cy.get("input").type("(NaLi)3P"); 26 | // cy.get("input").should("have.value", "(NaLi)3P"); 27 | // }); 28 | 29 | // it("renders formula with multiple elements and subscripts correctly", () => { 30 | // cy.get("input").type("Na3P4"); 31 | // cy.get("input").should("have.value", "Na3P4"); 32 | // }); 33 | 34 | // it("handles invalid input gracefully", () => { 35 | // cy.get("input").type("Invalid@Formula"); 36 | // cy.get("input").should("have.value", "InFo"); 37 | // }); 38 | }); 39 | -------------------------------------------------------------------------------- /webapp/cypress/component/NotificationDotTest.cy.jsx: -------------------------------------------------------------------------------- 1 | import { createStore } from "vuex"; 2 | import NotificationDot from "@/components/NotificationDot.vue"; 3 | 4 | describe("NotificationDot", () => { 5 | const testCases = [ 6 | { 7 | isUnverified: true, 8 | hasUnverifiedUser: true, 9 | tooltipText: "There is an unverified user in the database", 10 | }, 11 | { 12 | isUnverified: true, 13 | hasUnverifiedUser: false, 14 | tooltipText: "Your account is currently unverified, please contact an administrator.", 15 | }, 16 | { 17 | isUnverified: false, 18 | hasUnverifiedUser: true, 19 | tooltipText: "There is an unverified user in the database", 20 | }, 21 | { isUnverified: false, hasUnverifiedUser: false, tooltipText: "" }, 22 | ]; 23 | 24 | testCases.forEach(({ isUnverified, hasUnverifiedUser, tooltipText }) => { 25 | describe(`when isUnverified is ${isUnverified} and hasUnverifiedUser is ${hasUnverifiedUser}`, () => { 26 | let store; 27 | 28 | beforeEach(() => { 29 | store = createStore({ 30 | getters: { 31 | getCurrentUserIsUnverified: () => isUnverified, 32 | getHasUnverifiedUser: () => hasUnverifiedUser, 33 | }, 34 | }); 35 | 36 | cy.mount(NotificationDot, { 37 | global: { 38 | plugins: [store], 39 | }, 40 | }); 41 | }); 42 | 43 | it(`shows the correct tooltip message`, () => { 44 | if (tooltipText) { 45 | cy.get(".notification-dot").trigger("mouseenter"); 46 | cy.get("#tooltip").should("have.attr", "data-show"); 47 | cy.get("#tooltip p").contains(tooltipText); 48 | } else { 49 | cy.get(".notification-dot").should("not.exist"); 50 | cy.get("#tooltip").should("not.have.attr", "data-show"); 51 | } 52 | }); 53 | 54 | it(`shows and hides tooltip on mouseenter and mouseleave`, () => { 55 | if (tooltipText) { 56 | cy.get(".notification-dot").trigger("mouseenter"); 57 | cy.get("#tooltip").should("have.attr", "data-show"); 58 | 59 | cy.get(".notification-dot").trigger("mouseleave"); 60 | cy.get("#tooltip").should("not.have.attr", "data-show"); 61 | } 62 | }); 63 | 64 | it(`shows and hides tooltip on focus and blur`, () => { 65 | if (tooltipText) { 66 | cy.get(".notification-dot").focus(); 67 | cy.get("#tooltip").should("have.attr", "data-show"); 68 | 69 | cy.get(".notification-dot").blur(); 70 | cy.get("#tooltip").should("not.have.attr", "data-show"); 71 | } 72 | }); 73 | }); 74 | }); 75 | }); 76 | -------------------------------------------------------------------------------- /webapp/cypress/component/StyledBlockHelpTest.cy.jsx: -------------------------------------------------------------------------------- 1 | import StyledBlockHelp from "@/components/StyledBlockHelp.vue"; 2 | 3 | describe("StyledBlockHelp", () => { 4 | const blockInfo = { 5 | name: "Sample Block", 6 | description: "This is a sample block description.", 7 | accepted_file_extensions: [".jpg", ".png", ".gif"], 8 | }; 9 | 10 | it("renders the tooltip with correct content on hover", () => { 11 | cy.mount(StyledBlockHelp, { 12 | propsData: { blockInfo }, 13 | }); 14 | 15 | cy.get("a.dropdown-item").trigger("mouseenter"); 16 | 17 | cy.get("#tooltip") 18 | .should("be.visible") 19 | .within(() => { 20 | cy.contains(blockInfo.description).should("be.visible"); 21 | cy.contains("Accepted file extensions:").should("be.visible"); 22 | cy.contains(".jpg, .png, .gif").should("be.visible"); 23 | }); 24 | }); 25 | 26 | it("hides the tooltip on mouseleave", () => { 27 | cy.mount(StyledBlockHelp, { 28 | propsData: { blockInfo }, 29 | }); 30 | 31 | cy.get("a.dropdown-item").trigger("mouseenter"); 32 | cy.get("a.dropdown-item").trigger("mouseleave"); 33 | 34 | cy.get("#tooltip").should("not.have.attr", "data-show"); 35 | }); 36 | 37 | it("shows and hides tooltip on focus and blur", () => { 38 | cy.mount(StyledBlockHelp, { 39 | propsData: { blockInfo }, 40 | }); 41 | 42 | cy.get("a.dropdown-item").focus(); 43 | cy.get("#tooltip").should("be.visible"); 44 | 45 | cy.get("a.dropdown-item").blur(); 46 | cy.get("#tooltip").should("not.have.attr", "data-show"); 47 | }); 48 | }); 49 | -------------------------------------------------------------------------------- /webapp/cypress/component/StyledBlockInfoTest.cy.jsx: -------------------------------------------------------------------------------- 1 | import StyledBlockInfo from "@/components/StyledBlockInfo.vue"; 2 | 3 | describe("StyledBlockInfo", () => { 4 | const blockInfo = { 5 | attributes: { 6 | name: "Example Block", 7 | description: "This is an example block description.", 8 | accepted_file_extensions: [".pdf", ".docx", ".xlsx"], 9 | }, 10 | }; 11 | 12 | it("renders the tooltip with correct content on hover", () => { 13 | cy.mount(StyledBlockInfo, { 14 | propsData: { blockInfo }, 15 | }); 16 | 17 | cy.get("a").trigger("mouseenter", { force: true }); 18 | 19 | cy.get("#tooltip") 20 | .should("be.visible") 21 | .within(() => { 22 | cy.contains(blockInfo.attributes.name).should("be.visible"); 23 | cy.contains(blockInfo.attributes.description).should("be.visible"); 24 | cy.contains(".pdf").should("be.visible"); 25 | cy.contains(".docx").should("be.visible"); 26 | cy.contains(".xlsx").should("be.visible"); 27 | }); 28 | }); 29 | 30 | it("hides the tooltip on mouseleave", () => { 31 | cy.mount(StyledBlockInfo, { 32 | propsData: { blockInfo }, 33 | }); 34 | 35 | cy.get("a").trigger("mouseenter", { force: true }); 36 | cy.get("a").trigger("mouseleave", { force: true }); 37 | 38 | cy.get("#tooltip").should("not.have.attr", "data-show"); 39 | }); 40 | 41 | it("shows and hides tooltip on focus and blur", () => { 42 | cy.mount(StyledBlockInfo, { 43 | propsData: { blockInfo }, 44 | }); 45 | 46 | cy.get("a").focus(); 47 | cy.get("#tooltip").should("be.visible"); 48 | 49 | cy.get("a").blur(); 50 | cy.get("#tooltip").should("not.have.attr", "data-show"); 51 | }); 52 | }); 53 | -------------------------------------------------------------------------------- /webapp/cypress/component/UserBubbleTest.cy.jsx: -------------------------------------------------------------------------------- 1 | import UserBubble from "@/components/UserBubble.vue"; 2 | import crypto from "crypto"; 3 | 4 | describe("UserBubble", () => { 5 | const md5 = (value) => crypto.createHash("md5").update(value).digest("hex"); 6 | 7 | const creator = { 8 | contact_email: "test@contact.email", 9 | display_name: "Test User", 10 | }; 11 | 12 | beforeEach(() => { 13 | cy.mount(); 14 | }); 15 | 16 | it("renders the avatar image with correct gravatar URL based on contact_email", () => { 17 | const expectedHash = md5(creator.contact_email); 18 | cy.get("img.avatar") 19 | .should("have.attr", "src") 20 | .and("include", `https://www.gravatar.com/avatar/${expectedHash}`); 21 | }); 22 | 23 | it("renders the avatar image with correct gravatar URL based on display_name when contact_email is not provided", () => { 24 | const creatorWithoutEmail = { 25 | display_name: "Test User", 26 | }; 27 | 28 | cy.mount(); 29 | 30 | const expectedHash = md5(creatorWithoutEmail.display_name); 31 | cy.get("img.avatar") 32 | .should("have.attr", "src") 33 | .and("include", `https://www.gravatar.com/avatar/${expectedHash}`); 34 | }); 35 | 36 | it("uses the default size if not provided", () => { 37 | cy.get("img.avatar").should("have.attr", "width", "32").and("have.attr", "height", "32"); 38 | }); 39 | 40 | it("allows overriding the size via props", () => { 41 | const size = 64; 42 | cy.mount(); 43 | 44 | cy.get("img.avatar") 45 | .should("have.attr", "width", size.toString()) 46 | .and("have.attr", "height", size.toString()); 47 | }); 48 | 49 | it("sets the correct title attribute to display_name", () => { 50 | cy.get("img.avatar").should("have.attr", "title", creator.display_name); 51 | }); 52 | 53 | it("applies the correct styles to the avatar image", () => { 54 | cy.get("img.avatar").should("have.css", "border", "2px solid rgb(128, 128, 128)"); 55 | cy.get("img.avatar").should("have.css", "border-radius", "50%"); 56 | }); 57 | }); 58 | -------------------------------------------------------------------------------- /webapp/cypress/e2e/startingMaterial.cy.js: -------------------------------------------------------------------------------- 1 | const API_URL = Cypress.config("apiUrl"); 2 | console.log(API_URL); 3 | 4 | describe("Starting material table page - editable_inventory FALSE", () => { 5 | beforeEach(() => { 6 | cy.visit("/starting-materials"); 7 | }); 8 | 9 | it("Loads the Starting material page without any errors", () => { 10 | cy.findByText("About").should("exist"); 11 | cy.findByText("Inventory").should("exist"); 12 | }); 13 | 14 | it("Add a starting material button isn't displayed", () => { 15 | cy.get('[data-testid="add-starting-material-button"]').should("not.exist"); 16 | }); 17 | }); 18 | -------------------------------------------------------------------------------- /webapp/cypress/fixtures/example_data: -------------------------------------------------------------------------------- 1 | ../../../pydatalab/example_data -------------------------------------------------------------------------------- /webapp/cypress/plugins/index.js: -------------------------------------------------------------------------------- 1 | // *********************************************************** 2 | // This example plugins/index.js can be used to load plugins 3 | // 4 | // You can change the location of this file or turn off loading 5 | // the plugins file with the 'pluginsFile' configuration option. 6 | // 7 | // You can read more here: 8 | // https://on.cypress.io/plugins-guide 9 | // *********************************************************** 10 | 11 | // This function is called when a project is opened or re-opened (e.g. due to 12 | // the project's config changing) 13 | 14 | // eslint-disable-next-line no-unused-vars 15 | module.exports = (on, config) => { 16 | // `on` is used to hook into various events Cypress emits 17 | // `config` is the resolved Cypress config 18 | }; 19 | -------------------------------------------------------------------------------- /webapp/cypress/support/component-index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | Components App 8 | 9 | 10 | 11 |
12 | 13 | 14 | -------------------------------------------------------------------------------- /webapp/cypress/support/component.js: -------------------------------------------------------------------------------- 1 | // *********************************************************** 2 | // This example support/component.ts is processed and 3 | // loaded automatically before your test files. 4 | // 5 | // This is a great place to put global configuration and 6 | // behavior that modifies Cypress. 7 | // 8 | // You can change the location of this file or turn off 9 | // automatically serving support files with the 10 | // 'supportFile' configuration option. 11 | // 12 | // You can read more here: 13 | // https://on.cypress.io/configuration 14 | // *********************************************************** 15 | 16 | // Import commands.js using ES2015 syntax: 17 | import "./commands"; 18 | 19 | // Alternatively you can use CommonJS syntax: 20 | // require('./commands') 21 | 22 | import { mount } from "cypress/vue"; 23 | 24 | Cypress.Commands.add("mount", mount); 25 | 26 | // Example use: 27 | // cy.mount(MyComponent) 28 | -------------------------------------------------------------------------------- /webapp/cypress/support/e2e.js: -------------------------------------------------------------------------------- 1 | // *********************************************************** 2 | // This example support/index.js is processed and 3 | // loaded automatically before your test files. 4 | // 5 | // This is a great place to put global configuration and 6 | // behavior that modifies Cypress. 7 | // 8 | // You can change the location of this file or turn off 9 | // automatically serving support files with the 10 | // 'supportFile' configuration option. 11 | // 12 | // You can read more here: 13 | // https://on.cypress.io/configuration 14 | // *********************************************************** 15 | 16 | // Import commands.js using ES2015 syntax: 17 | import "./commands"; 18 | 19 | // eslint-disable-next-line no-unused-vars 20 | Cypress.on("uncaught:exception", (err, runnable) => { 21 | // returning false here prevents Cypress from 22 | // failing the test 23 | return false; 24 | }); 25 | -------------------------------------------------------------------------------- /webapp/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "datalab-vue", 3 | "version": "0.0.0-git", 4 | "private": true, 5 | "scripts": { 6 | "serve": "VUE_APP_GIT_VERSION=$(node scripts/get-version.js) vue-cli-service serve", 7 | "build": "VUE_APP_GIT_VERSION=$(node scripts/get-version.js) vue-cli-service build", 8 | "test:unit": "vue-cli-service test:unit", 9 | "test:e2e": "vue-cli-service test:e2e --mode test_e2e", 10 | "lint": "vue-cli-service lint" 11 | }, 12 | "dependencies": { 13 | "@fortawesome/fontawesome-svg-core": "^1.2.26-2", 14 | "@fortawesome/free-brands-svg-icons": "^5.15.4", 15 | "@fortawesome/free-regular-svg-icons": "^5.15.2", 16 | "@fortawesome/free-solid-svg-icons": "^5.12.0-2", 17 | "@fortawesome/vue-fontawesome": "^3.0.0-3", 18 | "@popperjs/core": "^2.11.8", 19 | "@primevue/themes": "^4.0.0", 20 | "@tinymce/tinymce-vue": "^4.0.0", 21 | "@uppy/core": "^1.16.0", 22 | "@uppy/dashboard": "^1.16.0", 23 | "@uppy/webcam": "^1.8.4", 24 | "@uppy/xhr-upload": "^1.6.10", 25 | "@vueuse/components": "^10.7.2", 26 | "bootstrap": "^4.5.3", 27 | "buffer": "^6.0.3", 28 | "crypto-browserify": "^3.12.0", 29 | "cytoscape": "^3.23.0", 30 | "cytoscape-cola": "^2.5.1", 31 | "cytoscape-elk": "^2.1.0", 32 | "cytoscape-euler": "^1.2.3", 33 | "cytoscape-fcose": "^2.2.0", 34 | "date-fns": "^2.29.3", 35 | "highlight.js": "^11.7.0", 36 | "markdown-it": "^13.0.1", 37 | "mermaid": "^10.1.0", 38 | "primeicons": "^7.0.0", 39 | "primevue": "^4.0.0", 40 | "process": "^0.11.10", 41 | "qrcode-vue3": "^1.6.8", 42 | "serve": "^14.2.1", 43 | "stream-browserify": "^3.0.0", 44 | "tinymce": "^5.10.9", 45 | "vue": "^3.2.4", 46 | "vue-qrcode-reader": "^5.5.7", 47 | "vue-router": "^4.0.0-0", 48 | "vue-select": "^4.0.0-beta.6", 49 | "vue3-easy-data-table": "^1.5.45", 50 | "vuex": "^4.0.0-0" 51 | }, 52 | "devDependencies": { 53 | "@babel/core": "^7.24.8", 54 | "@babel/eslint-parser": "^7.24.8", 55 | "@babel/plugin-transform-export-namespace-from": "^7.24.7", 56 | "@testing-library/cypress": "^10.0.0", 57 | "@testing-library/vue": "^8.0.3", 58 | "@vue/cli-plugin-babel": "~5.0.8", 59 | "@vue/cli-plugin-e2e-cypress": "5.0.8", 60 | "@vue/cli-service": "~5.0.8", 61 | "@vue/compiler-sfc": "^3.2.4", 62 | "@vue/eslint-config-prettier": "^8.0.0", 63 | "@vue/test-utils": "^2.4.5", 64 | "babel-plugin-transform-remove-console": "^6.9.4", 65 | "cypress": "^13.7.2", 66 | "eslint": "^8.56.0", 67 | "eslint-config-prettier": "^8.10.0", 68 | "eslint-plugin-cypress": "^3.3.0", 69 | "eslint-plugin-prettier": "^5.1.3", 70 | "eslint-plugin-vue": "^8.0.3", 71 | "prettier": "^3.1.0", 72 | "typescript": "~3.9.3", 73 | "web-worker": "^1.2.0" 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /webapp/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/datalab-org/datalab/05f761881757985ac8b761a85163571434316d9c/webapp/public/favicon.ico -------------------------------------------------------------------------------- /webapp/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | <%= htmlWebpackPlugin.options.title %> 9 | 14 | 19 | 24 | 25 | 26 | 32 |
33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /webapp/scripts/get-version.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | // This script tries to get the version of the app from three sources: 3 | // 4 | // 1. The VUE_APP_GIT_VERSION environment variable 5 | // 2. The latest git tag 6 | // 3. The version field in package.JSON 7 | // 8 | // It can be used as part of the build system to set the 9 | // `VUE_APP_GIT_VERSION` environment variable itself. 10 | 11 | const { execSync } = require("child_process"); 12 | const fs = require("fs"); 13 | 14 | function getPackageJsonVersion() { 15 | try { 16 | const packageJson = JSON.parse(fs.readFileSync("package.json", "utf8")); 17 | return packageJson.version || "unknown"; 18 | } catch (error) { 19 | console.error("Error reading package.json:", error); 20 | return "unknown"; 21 | } 22 | } 23 | 24 | function getGitVersion() { 25 | try { 26 | const tag = execSync("git describe --tags").toString().trim(); 27 | return tag; 28 | } catch (error) { 29 | console.error("Error getting git version:", error); 30 | return null; 31 | } 32 | } 33 | 34 | function getVersion() { 35 | // Check if VUE_APP_GIT_VERSION is already set (e.g., by build system) 36 | if (process.env.VUE_APP_GIT_VERSION) { 37 | return process.env.VUE_APP_GIT_VERSION; 38 | } 39 | 40 | // Try to get git version 41 | const gitVersion = getGitVersion(); 42 | if (gitVersion) { 43 | return gitVersion; 44 | } 45 | 46 | // Fall back to package.json version 47 | return getPackageJsonVersion(); 48 | } 49 | 50 | // Get and log the version 51 | const version = getVersion(); 52 | console.log(version); 53 | -------------------------------------------------------------------------------- /webapp/src/App.vue: -------------------------------------------------------------------------------- 1 | 4 | 5 | 106 | -------------------------------------------------------------------------------- /webapp/src/components/AdminDisplay.vue: -------------------------------------------------------------------------------- 1 | 6 | 7 | 21 | 22 | 30 | -------------------------------------------------------------------------------- /webapp/src/components/AdminNavbar.vue: -------------------------------------------------------------------------------- 1 | 22 | 23 | 44 | 45 | 72 | -------------------------------------------------------------------------------- /webapp/src/components/BarcodeModal.vue: -------------------------------------------------------------------------------- 1 | 19 | 20 | 34 | 35 | 41 | -------------------------------------------------------------------------------- /webapp/src/components/BaseIconCounter.vue: -------------------------------------------------------------------------------- 1 | 9 | 10 | 42 | 43 | 85 | -------------------------------------------------------------------------------- /webapp/src/components/BlocksIconCounter.vue: -------------------------------------------------------------------------------- 1 | 4 | 5 | 40 | -------------------------------------------------------------------------------- /webapp/src/components/ChatWindow.vue: -------------------------------------------------------------------------------- 1 | 13 | 14 | 42 | 43 | 62 | -------------------------------------------------------------------------------- /webapp/src/components/ChemFormulaInput.vue: -------------------------------------------------------------------------------- 1 | 19 | 20 | 69 | -------------------------------------------------------------------------------- /webapp/src/components/ChemicalFormula.vue: -------------------------------------------------------------------------------- 1 | 11 | 12 | 33 | -------------------------------------------------------------------------------- /webapp/src/components/CollectionList.vue: -------------------------------------------------------------------------------- 1 | 12 | 13 | 31 | -------------------------------------------------------------------------------- /webapp/src/components/CollectionRelationshipVisualization.vue: -------------------------------------------------------------------------------- 1 | 12 | 13 | 38 | -------------------------------------------------------------------------------- /webapp/src/components/CollectionTable.vue: -------------------------------------------------------------------------------- 1 | 10 | 11 | 60 | -------------------------------------------------------------------------------- /webapp/src/components/Creators.vue: -------------------------------------------------------------------------------- 1 | 14 | 15 | 48 | 49 | 59 | -------------------------------------------------------------------------------- /webapp/src/components/EquipmentTable.vue: -------------------------------------------------------------------------------- 1 | 9 | 10 | 49 | -------------------------------------------------------------------------------- /webapp/src/components/FilesIconCounter.vue: -------------------------------------------------------------------------------- 1 | 4 | 5 | 30 | -------------------------------------------------------------------------------- /webapp/src/components/FormField.vue: -------------------------------------------------------------------------------- 1 | 5 | 6 | 13 | -------------------------------------------------------------------------------- /webapp/src/components/FormattedBarcode.vue: -------------------------------------------------------------------------------- 1 | 24 | 25 | 54 | 55 | 62 | -------------------------------------------------------------------------------- /webapp/src/components/FormattedCollectionName.vue: -------------------------------------------------------------------------------- 1 | 14 | 15 | 67 | 68 | 80 | -------------------------------------------------------------------------------- /webapp/src/components/FormattedItemName.vue: -------------------------------------------------------------------------------- 1 | 21 | 22 | 79 | 80 | 90 | -------------------------------------------------------------------------------- /webapp/src/components/FormattedRefcode.vue: -------------------------------------------------------------------------------- 1 | 23 | 24 | 72 | 73 | 78 | -------------------------------------------------------------------------------- /webapp/src/components/GHSHazardInformation.vue: -------------------------------------------------------------------------------- 1 | 16 | 17 | 56 | 57 | 70 | -------------------------------------------------------------------------------- /webapp/src/components/GHSHazardPictograms.vue: -------------------------------------------------------------------------------- 1 | 18 | 19 | 36 | 37 | 55 | -------------------------------------------------------------------------------- /webapp/src/components/Isotope.vue: -------------------------------------------------------------------------------- 1 | 8 | 9 | 26 | -------------------------------------------------------------------------------- /webapp/src/components/ItemRelationshipVisualization.vue: -------------------------------------------------------------------------------- 1 | 12 | 13 | 35 | -------------------------------------------------------------------------------- /webapp/src/components/LoginDropdown.vue: -------------------------------------------------------------------------------- 1 | 29 | 30 | 60 | 61 | 74 | -------------------------------------------------------------------------------- /webapp/src/components/Navbar.vue: -------------------------------------------------------------------------------- 1 | 41 | 42 | 66 | 67 | 82 | -------------------------------------------------------------------------------- /webapp/src/components/NotificationDot.vue: -------------------------------------------------------------------------------- 1 | 23 | 24 | 73 | 74 | 117 | -------------------------------------------------------------------------------- /webapp/src/components/QRCode.vue: -------------------------------------------------------------------------------- 1 | 35 | 36 | 75 | 76 | 82 | -------------------------------------------------------------------------------- /webapp/src/components/QRCodeModal.vue: -------------------------------------------------------------------------------- 1 | 26 | 27 | 55 | -------------------------------------------------------------------------------- /webapp/src/components/SampleTable.vue: -------------------------------------------------------------------------------- 1 | 16 | 17 | 100 | -------------------------------------------------------------------------------- /webapp/src/components/StartingMaterialTable.vue: -------------------------------------------------------------------------------- 1 | 18 | 19 | 77 | -------------------------------------------------------------------------------- /webapp/src/components/StatisticsTable.vue: -------------------------------------------------------------------------------- 1 | 19 | 20 | 38 | 39 | 70 | -------------------------------------------------------------------------------- /webapp/src/components/StyledBlockHelp.vue: -------------------------------------------------------------------------------- 1 | 27 | 28 | 80 | 81 | 116 | -------------------------------------------------------------------------------- /webapp/src/components/TinyMceInline.vue: -------------------------------------------------------------------------------- 1 | 47 | 48 | 73 | 74 | 95 | -------------------------------------------------------------------------------- /webapp/src/components/ToggleableCollectionFormGroup.vue: -------------------------------------------------------------------------------- 1 | 33 | 34 | 74 | 75 | 88 | -------------------------------------------------------------------------------- /webapp/src/components/UserBubble.vue: -------------------------------------------------------------------------------- 1 | 17 | 18 | 46 | 58 | -------------------------------------------------------------------------------- /webapp/src/components/UserBubbleLogin.vue: -------------------------------------------------------------------------------- 1 | 18 | 19 | 65 | 66 | 84 | -------------------------------------------------------------------------------- /webapp/src/components/UserDropdown.vue: -------------------------------------------------------------------------------- 1 | 25 | 26 | 64 | 65 | 74 | -------------------------------------------------------------------------------- /webapp/src/components/UserSelect.vue: -------------------------------------------------------------------------------- 1 | 24 | 25 | 91 | -------------------------------------------------------------------------------- /webapp/src/components/__mocks__/bokeh.js: -------------------------------------------------------------------------------- 1 | // dummy mock for bokeh so that jest doesn't complain that 2 | // bokeh doesn't exist (bokeh is loaded globaly in index.html 3 | // from CDN ) 4 | export default {}; 5 | -------------------------------------------------------------------------------- /webapp/src/components/datablocks/BokehBlock.vue: -------------------------------------------------------------------------------- 1 | 20 | 21 | 72 | 73 | 74 | -------------------------------------------------------------------------------- /webapp/src/components/datablocks/GenericBlock.vue: -------------------------------------------------------------------------------- 1 | 4 | 5 | 8 | -------------------------------------------------------------------------------- /webapp/src/components/datablocks/NotImplementedBlock.vue: -------------------------------------------------------------------------------- 1 | 9 | 10 | 19 | -------------------------------------------------------------------------------- /webapp/src/components/datablocks/UVVisBlock.vue: -------------------------------------------------------------------------------- 1 | 20 | 21 | 64 | 65 | 66 | -------------------------------------------------------------------------------- /webapp/src/components/itemCreateModalAddons/SampleCreateModalAddon.vue: -------------------------------------------------------------------------------- 1 | 15 | 16 | 53 | -------------------------------------------------------------------------------- /webapp/src/components/itemCreateModalAddons/StartingMaterialCreateModalAddon.vue: -------------------------------------------------------------------------------- 1 | 15 | 16 | 53 | -------------------------------------------------------------------------------- /webapp/src/file_upload.js: -------------------------------------------------------------------------------- 1 | import "@uppy/core/dist/style.css"; 2 | import "@uppy/dashboard/dist/style.css"; 3 | import Uppy from "@uppy/core"; 4 | import Dashboard from "@uppy/dashboard"; 5 | import XHRUpload from "@uppy/xhr-upload"; 6 | import Webcam from "@uppy/webcam"; 7 | 8 | import store from "@/store/index.js"; 9 | import { construct_headers } from "@/server_fetch_utils.js"; 10 | 11 | import { API_URL, UPPY_MAX_NUMBER_OF_FILES, UPPY_MAX_TOTAL_FILE_SIZE } from "@/resources.js"; 12 | // file-upload loaded 13 | 14 | export default function setupUppy(item_id, trigger_selector, reactive_file_list) { 15 | var uppy = new Uppy({ 16 | restrictions: { 17 | // Somewhat arbitrary restrictions that prevent numbers that would break the server in one go -- the API should also refuse files when 'full' 18 | maxTotalFileSize: UPPY_MAX_TOTAL_FILE_SIZE, // Set this UI restriction arbitrarily high at 100 GB for now --- this is the point at which I would be unsure if the upload could even complete 19 | maxNumberOfFiles: UPPY_MAX_NUMBER_OF_FILES, // Similarly, a max of 10000 files in one upload as a single "File" entry feels reasonable, once we move to uploading folders etc. 20 | }, 21 | }); 22 | let headers = construct_headers(); 23 | uppy 24 | .use(Dashboard, { 25 | inline: false, 26 | trigger: trigger_selector, 27 | close_after_finish: true, 28 | }) 29 | .use(Webcam, { target: Dashboard }) 30 | .use(XHRUpload, { 31 | //someday, try to upgrade this to Tus for resumable uploads 32 | headers: headers, 33 | endpoint: `${API_URL}/upload-file/`, 34 | FormData: true, // send as form 35 | fieldName: "files[]", // default, think about whether or not to change this 36 | showProgressDetails: true, 37 | withCredentials: true, 38 | }); 39 | 40 | uppy.on("file-added", (file) => { 41 | var matching_file_id = null; 42 | for (const file_id in reactive_file_list) { 43 | if (reactive_file_list[file_id].name == file.name) { 44 | alert( 45 | "A file with this name already exists in the sample. If you upload this, it will be duplicated on the current item.", 46 | ); 47 | } 48 | } 49 | 50 | uppy.setFileMeta(file.id, { 51 | size: file.size, 52 | item_id: item_id, 53 | replace_file: matching_file_id, 54 | }); 55 | }); 56 | 57 | uppy.on("complete", function (result) { 58 | result.successful.forEach(function (f) { 59 | var response_body = f.response.body; 60 | if (!response_body.is_update) { 61 | store.commit("addFileToSample", { 62 | item_id: item_id, 63 | file_id: response_body.file_id, 64 | file_info: { 65 | ...response_body.file_information, 66 | immutable_id: response_body.file_information.immutable_id.$oid, 67 | }, 68 | }); 69 | } 70 | }); 71 | }); 72 | } 73 | -------------------------------------------------------------------------------- /webapp/src/views/Admin.vue: -------------------------------------------------------------------------------- 1 | 11 | 12 | 50 | 51 | 70 | -------------------------------------------------------------------------------- /webapp/src/views/Collections.vue: -------------------------------------------------------------------------------- 1 | 12 | 13 | 25 | 26 | 39 | -------------------------------------------------------------------------------- /webapp/src/views/Equipment.vue: -------------------------------------------------------------------------------- 1 | 12 | 13 | 25 | 26 | 39 | -------------------------------------------------------------------------------- /webapp/src/views/ItemGraphPage.vue: -------------------------------------------------------------------------------- 1 | 5 | 6 | 31 | -------------------------------------------------------------------------------- /webapp/src/views/NotFound.vue: -------------------------------------------------------------------------------- 1 | 4 | -------------------------------------------------------------------------------- /webapp/src/views/Samples.vue: -------------------------------------------------------------------------------- 1 | 12 | 13 | 25 | 26 | 39 | -------------------------------------------------------------------------------- /webapp/src/views/StartingMaterials.vue: -------------------------------------------------------------------------------- 1 | 12 | 13 | 25 | 26 | 39 | -------------------------------------------------------------------------------- /webapp/vue.config.js: -------------------------------------------------------------------------------- 1 | const { ProvidePlugin } = require("webpack"); 2 | 3 | module.exports = { 4 | transpileDependencies: ["mermaid"], 5 | configureWebpack: (config) => { 6 | config.resolve.fallback = { 7 | crypto: require.resolve("crypto-browserify"), 8 | stream: require.resolve("stream-browserify"), 9 | process: require.resolve("process/browser"), 10 | buffer: require.resolve("buffer/"), 11 | vm: false, 12 | }; 13 | // disable stats output in production due to bad `wrap-ansi` ESM/CommonJS interop 14 | if (process.env.NODE_ENV === "production") { 15 | config.stats = "none"; 16 | } 17 | 18 | config.externals = { 19 | ...config.externals, 20 | bokeh: "Bokeh", 21 | }; 22 | config.module.rules.push({ 23 | test: /\.mjs$/, 24 | include: /node_modules/, 25 | type: "javascript/auto", 26 | }); 27 | config.plugins = [ 28 | ...(config.plugins || []), 29 | new ProvidePlugin({ 30 | process: "process/browser", 31 | }), 32 | ]; 33 | }, 34 | chainWebpack: (config) => { 35 | config.plugin("html").tap((args) => { 36 | args[0].title = process.env.VUE_APP_WEBSITE_TITLE || "datalab"; 37 | args[0].meta = { 38 | x_datalab_api_url: process.env.VUE_APP_API_URL, 39 | }; 40 | return args; 41 | }); 42 | }, 43 | }; 44 | --------------------------------------------------------------------------------