├── ui-tests ├── yarn.lock ├── playwright.config.js ├── jupyter_server_test_config.py ├── package.json ├── tests │ └── jupydrive-s3.spec.ts └── README.md ├── style ├── index.js ├── index.css ├── base.css ├── driveIconFileBrowser.svg └── newDriveIcon.svg ├── setup.py ├── babel.config.js ├── .yarnrc.yml ├── src ├── svg.d.ts ├── __tests__ │ └── jupydrive-s3.spec.ts ├── icons.ts ├── index.ts ├── s3contents.ts └── s3.ts ├── tsconfig.test.json ├── dev.webpack.config.js ├── .prettierignore ├── install.json ├── .env.example ├── .github └── workflows │ ├── enforce-label.yml │ ├── check-release.yml │ ├── prep-release.yml │ ├── update-integration-tests.yml │ ├── publish-release.yml │ └── build.yml ├── tsconfig.json ├── .copier-answers.yml ├── jupydrive_s3 └── __init__.py ├── jest.config.js ├── schema ├── auth-file-browser.json └── file-browser-toolbar.json ├── LICENSE ├── .gitignore ├── pyproject.toml ├── RELEASE.md ├── README.md ├── package.json └── CHANGELOG.md /ui-tests/yarn.lock: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /style/index.js: -------------------------------------------------------------------------------- 1 | import './base.css'; 2 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | __import__("setuptools").setup() 2 | -------------------------------------------------------------------------------- /style/index.css: -------------------------------------------------------------------------------- 1 | @import url('base.css'); 2 | -------------------------------------------------------------------------------- /babel.config.js: -------------------------------------------------------------------------------- 1 | module.exports = require('@jupyterlab/testutils/lib/babel.config'); 2 | -------------------------------------------------------------------------------- /.yarnrc.yml: -------------------------------------------------------------------------------- 1 | compressionLevel: mixed 2 | 3 | enableGlobalCache: false 4 | 5 | nodeLinker: node-modules 6 | -------------------------------------------------------------------------------- /src/svg.d.ts: -------------------------------------------------------------------------------- 1 | declare module '*.svg' { 2 | const value: string; // @ts-ignore 3 | export default value; 4 | } 5 | -------------------------------------------------------------------------------- /tsconfig.test.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig", 3 | "compilerOptions": { 4 | "types": ["jest"] 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /dev.webpack.config.js: -------------------------------------------------------------------------------- 1 | const Dotenv = require('dotenv-webpack'); 2 | 3 | module.exports = { 4 | plugins: [new Dotenv()] 5 | }; 6 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | **/node_modules 3 | **/lib 4 | **/package.json 5 | !/package.json 6 | jupydrive_s3 7 | CHANGELOG.md 8 | -------------------------------------------------------------------------------- /style/base.css: -------------------------------------------------------------------------------- 1 | /* 2 | See the JupyterLab Developer Guide for useful CSS Patterns: 3 | 4 | https://jupyterlab.readthedocs.io/en/stable/developer/css.html 5 | */ 6 | -------------------------------------------------------------------------------- /install.json: -------------------------------------------------------------------------------- 1 | { 2 | "packageManager": "python", 3 | "packageName": "jupydrive_s3", 4 | "uninstallInstructions": "Use your Python package manager (pip, conda, etc.) to uninstall the package jupydrive_s3" 5 | } 6 | -------------------------------------------------------------------------------- /src/__tests__/jupydrive-s3.spec.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Example of [Jest](https://jestjs.io/docs/getting-started) unit tests 3 | */ 4 | 5 | describe('jupydrive-s3', () => { 6 | it('should be tested', () => { 7 | expect(1 + 1).toEqual(2); 8 | }); 9 | }); 10 | -------------------------------------------------------------------------------- /.env.example: -------------------------------------------------------------------------------- 1 | JP_S3_BUCKET= 2 | JP_S3_ROOT= 3 | JP_S3_REGION= 4 | JP_S3_ENDPOINT= 5 | JP_S3_ACCESS_KEY_ID= 6 | JP_S3_SECRET_ACCESS_KEY= 7 | -------------------------------------------------------------------------------- /style/driveIconFileBrowser.svg: -------------------------------------------------------------------------------- 1 | 6 | 9 | -------------------------------------------------------------------------------- /.github/workflows/enforce-label.yml: -------------------------------------------------------------------------------- 1 | name: Enforce PR label 2 | 3 | on: 4 | pull_request: 5 | types: [labeled, unlabeled, opened, edited, synchronize] 6 | jobs: 7 | enforce-label: 8 | runs-on: ubuntu-latest 9 | permissions: 10 | pull-requests: write 11 | steps: 12 | - name: enforce-triage-label 13 | uses: jupyterlab/maintainer-tools/.github/actions/enforce-label@v1 14 | -------------------------------------------------------------------------------- /src/icons.ts: -------------------------------------------------------------------------------- 1 | import { LabIcon } from '@jupyterlab/ui-components'; 2 | import driveSvg from '../style/driveIconFileBrowser.svg'; 3 | import newDriveSvg from '../style/newDriveIcon.svg'; 4 | 5 | export const DriveIcon = new LabIcon({ 6 | name: 'jupydrive-s3:drive', 7 | svgstr: driveSvg 8 | }); 9 | 10 | export const NewDriveIcon = new LabIcon({ 11 | name: 'jupydrive-s3:new-drive', 12 | svgstr: newDriveSvg 13 | }); 14 | -------------------------------------------------------------------------------- /ui-tests/playwright.config.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Configuration for Playwright using default from @jupyterlab/galata 3 | */ 4 | const baseConfig = require('@jupyterlab/galata/lib/playwright-config'); 5 | 6 | module.exports = { 7 | ...baseConfig, 8 | webServer: { 9 | command: 'jlpm start', 10 | url: 'http://localhost:8888/lab', 11 | timeout: 120 * 1000, 12 | reuseExistingServer: !process.env.CI 13 | } 14 | }; 15 | -------------------------------------------------------------------------------- /ui-tests/jupyter_server_test_config.py: -------------------------------------------------------------------------------- 1 | """Server configuration for integration tests. 2 | 3 | !! Never use this configuration in production because it 4 | opens the server to the world and provide access to JupyterLab 5 | JavaScript objects through the global window variable. 6 | """ 7 | from jupyterlab.galata import configure_jupyter_server 8 | 9 | configure_jupyter_server(c) 10 | 11 | # Uncomment to set server log level to debug level 12 | # c.ServerApp.log_level = "DEBUG" 13 | -------------------------------------------------------------------------------- /ui-tests/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "jupydrive-s3-ui-tests", 3 | "version": "1.0.0", 4 | "description": "JupyterLab jupydrive-s3 Integration Tests", 5 | "private": true, 6 | "scripts": { 7 | "start": "jupyter lab --config jupyter_server_test_config.py", 8 | "test": "jlpm playwright test", 9 | "test:update": "jlpm playwright test --update-snapshots" 10 | }, 11 | "devDependencies": { 12 | "@jupyterlab/galata": "^5.0.5", 13 | "@playwright/test": "^1.37.0" 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "allowSyntheticDefaultImports": true, 4 | "composite": true, 5 | "declaration": true, 6 | "esModuleInterop": true, 7 | "incremental": true, 8 | "jsx": "react", 9 | "module": "esnext", 10 | "moduleResolution": "node", 11 | "noEmitOnError": true, 12 | "noImplicitAny": false, 13 | "noUnusedLocals": true, 14 | "preserveWatchOutput": true, 15 | "resolveJsonModule": true, 16 | "outDir": "lib", 17 | "rootDir": "src", 18 | "strict": true, 19 | "target": "ES2018" 20 | }, 21 | "include": ["src/*"] 22 | } 23 | -------------------------------------------------------------------------------- /ui-tests/tests/jupydrive-s3.spec.ts: -------------------------------------------------------------------------------- 1 | import { expect, test } from '@jupyterlab/galata'; 2 | 3 | /** 4 | * Don't load JupyterLab webpage before running the tests. 5 | * This is required to ensure we capture all log messages. 6 | */ 7 | test.use({ autoGoto: false }); 8 | 9 | test.skip('should emit an activation console message', async ({ page }) => { 10 | const logs: string[] = []; 11 | 12 | page.on('console', message => { 13 | logs.push(message.text()); 14 | }); 15 | 16 | await page.goto(); 17 | 18 | expect( 19 | logs.filter(s => s === 'JupyterLab extension jupydrive-s3 is activated!') 20 | ).toHaveLength(1); 21 | }); 22 | -------------------------------------------------------------------------------- /.copier-answers.yml: -------------------------------------------------------------------------------- 1 | # Changes here will be overwritten by Copier; NEVER EDIT MANUALLY 2 | _commit: v4.2.4 3 | _src_path: https://github.com/jupyterlab/extension-template 4 | author_email: denisa.checiu@quantstack.net 5 | author_name: QuantStack 6 | data_format: string 7 | file_extension: '' 8 | has_binder: false 9 | has_settings: false 10 | kind: frontend 11 | labextension_name: jupydrive-s3 12 | mimetype: '' 13 | mimetype_name: '' 14 | project_short_description: A JupyterLab extension which enables client-side drives 15 | access. 16 | python_name: jupydrive_s3 17 | repository: https://github.com/QuantStack/jupydrive-s3 18 | test: true 19 | viewer_name: '' 20 | 21 | -------------------------------------------------------------------------------- /jupydrive_s3/__init__.py: -------------------------------------------------------------------------------- 1 | try: 2 | from ._version import __version__ 3 | except ImportError: 4 | # Fallback when using the package in dev mode without installing 5 | # in editable mode with pip. It is highly recommended to install 6 | # the package from a stable release or in editable mode: https://pip.pypa.io/en/stable/topics/local-project-installs/#editable-installs 7 | import warnings 8 | warnings.warn("Importing 'jupydrive_s3' outside a proper installation.") 9 | __version__ = "dev" 10 | 11 | 12 | def _jupyter_labextension_paths(): 13 | return [{ 14 | "src": "labextension", 15 | "dest": "jupydrive-s3" 16 | }] 17 | -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | const jestJupyterLab = require('@jupyterlab/testutils/lib/jest-config'); 2 | 3 | const esModules = [ 4 | '@codemirror', 5 | '@jupyter/ydoc', 6 | '@jupyterlab/', 7 | 'lib0', 8 | 'nanoid', 9 | 'vscode-ws-jsonrpc', 10 | 'y-protocols', 11 | 'y-websocket', 12 | 'yjs' 13 | ].join('|'); 14 | 15 | const baseConfig = jestJupyterLab(__dirname); 16 | 17 | module.exports = { 18 | ...baseConfig, 19 | automock: false, 20 | collectCoverageFrom: [ 21 | 'src/**/*.{ts,tsx}', 22 | '!src/**/*.d.ts', 23 | '!src/**/.ipynb_checkpoints/*' 24 | ], 25 | coverageReporters: ['lcov', 'text'], 26 | testRegex: 'src/.*/.*.spec.ts[x]?$', 27 | transformIgnorePatterns: [`/node_modules/(?!${esModules}).+`] 28 | }; 29 | -------------------------------------------------------------------------------- /.github/workflows/check-release.yml: -------------------------------------------------------------------------------- 1 | name: Check Release 2 | on: 3 | push: 4 | branches: ["main"] 5 | pull_request: 6 | branches: ["*"] 7 | 8 | jobs: 9 | check_release: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Checkout 13 | uses: actions/checkout@v3 14 | - name: Base Setup 15 | uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 16 | - name: Check Release 17 | uses: jupyter-server/jupyter_releaser/.github/actions/check-release@v2 18 | with: 19 | 20 | token: ${{ secrets.GITHUB_TOKEN }} 21 | 22 | - name: Upload Distributions 23 | uses: actions/upload-artifact@v4 24 | with: 25 | name: jupydrive_s3-releaser-dist-${{ github.run_number }} 26 | path: .jupyter_releaser_checkout/dist 27 | -------------------------------------------------------------------------------- /schema/auth-file-browser.json: -------------------------------------------------------------------------------- 1 | { 2 | "jupyter.lab.setting-icon": "jupydrive-s3:drive", 3 | "jupyter.lab.setting-icon-label": "Drive Browser", 4 | "title": "Credentials Provider", 5 | "description": "jupydrive-s3 credentials provider.", 6 | "type": "object", 7 | "properties": { 8 | "bucket": { 9 | "type": "string", 10 | "title": "Bucket", 11 | "description": "The S3 bucket name.", 12 | "default": "jupyter-drives-test-bucket-1" 13 | }, 14 | "root": { 15 | "type": "string", 16 | "title": "Custom root path", 17 | "description": "Path to folder within bucket, that should act as root. Defaults to bucket's top-level directory.", 18 | "default": "" 19 | }, 20 | "endpoint": { 21 | "type": "string", 22 | "title": "S3 endpoint", 23 | "description": "The custom S3 endpoint (e.g. : https://s3.eu-north-1.amazonaws.com).", 24 | "default": "https://example.com/s3" 25 | }, 26 | "region": { 27 | "type": "string", 28 | "title": "Bucket region", 29 | "description": "The S3 bucket region.", 30 | "default": "eu-north-1" 31 | }, 32 | "accessKeyId": { 33 | "type": "string", 34 | "title": "Access key ID", 35 | "description": "The access key id used to access S3 bucket.", 36 | "default": "" 37 | }, 38 | "secretAccessKey": { 39 | "type": "string", 40 | "title": "Secret access key", 41 | "default": "" 42 | } 43 | }, 44 | "additionalProperties": false 45 | } 46 | -------------------------------------------------------------------------------- /.github/workflows/prep-release.yml: -------------------------------------------------------------------------------- 1 | name: "Step 1: Prep Release" 2 | on: 3 | workflow_dispatch: 4 | inputs: 5 | version_spec: 6 | description: "New Version Specifier" 7 | default: "next" 8 | required: false 9 | branch: 10 | description: "The branch to target" 11 | required: false 12 | post_version_spec: 13 | description: "Post Version Specifier" 14 | required: false 15 | since: 16 | description: "Use PRs with activity since this date or git reference" 17 | required: false 18 | since_last_stable: 19 | description: "Use PRs with activity since the last stable git tag" 20 | required: false 21 | type: boolean 22 | jobs: 23 | prep_release: 24 | runs-on: ubuntu-latest 25 | steps: 26 | - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 27 | 28 | - name: Prep Release 29 | id: prep-release 30 | uses: jupyter-server/jupyter_releaser/.github/actions/prep-release@v2 31 | with: 32 | token: ${{ secrets.ADMIN_GITHUB_TOKEN }} 33 | version_spec: ${{ github.event.inputs.version_spec }} 34 | post_version_spec: ${{ github.event.inputs.post_version_spec }} 35 | branch: ${{ github.event.inputs.branch }} 36 | since: ${{ github.event.inputs.since }} 37 | since_last_stable: ${{ github.event.inputs.since_last_stable }} 38 | 39 | - name: "** Next Step **" 40 | run: | 41 | echo "Optional): Review Draft Release: ${{ steps.prep-release.outputs.release_url }}" 42 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2024, QuantStack 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | 1. Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | 2. Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | 3. Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /.github/workflows/update-integration-tests.yml: -------------------------------------------------------------------------------- 1 | name: Update Playwright Snapshots 2 | 3 | on: 4 | issue_comment: 5 | types: [created, edited] 6 | 7 | permissions: 8 | contents: write 9 | pull-requests: write 10 | 11 | jobs: 12 | update-snapshots: 13 | if: ${{ github.event.issue.pull_request && contains(github.event.comment.body, 'please update snapshots') }} 14 | runs-on: ubuntu-latest 15 | 16 | steps: 17 | - name: React to the triggering comment 18 | run: | 19 | gh api repos/${{ github.repository }}/issues/comments/${{ github.event.comment.id }}/reactions --raw-field 'content=+1' 20 | env: 21 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 22 | 23 | - name: Checkout 24 | uses: actions/checkout@v4 25 | with: 26 | token: ${{ secrets.GITHUB_TOKEN }} 27 | 28 | - name: Checkout the branch from the PR that triggered the job 29 | run: gh pr checkout ${{ github.event.issue.number }} 30 | env: 31 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 32 | 33 | - name: Base Setup 34 | uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 35 | 36 | - name: Install dependencies 37 | run: python -m pip install -U "jupyterlab>=4.0.0,<5" 38 | 39 | - name: Install extension 40 | run: | 41 | set -eux 42 | jlpm 43 | python -m pip install . 44 | 45 | - uses: jupyterlab/maintainer-tools/.github/actions/update-snapshots@v1 46 | with: 47 | github_token: ${{ secrets.GITHUB_TOKEN }} 48 | # Playwright knows how to start JupyterLab server 49 | start_server_script: 'null' 50 | test_folder: ui-tests 51 | -------------------------------------------------------------------------------- /.github/workflows/publish-release.yml: -------------------------------------------------------------------------------- 1 | name: "Step 2: Publish Release" 2 | on: 3 | workflow_dispatch: 4 | inputs: 5 | branch: 6 | description: "The target branch" 7 | required: false 8 | release_url: 9 | description: "The URL of the draft GitHub release" 10 | required: false 11 | steps_to_skip: 12 | description: "Comma separated list of steps to skip" 13 | required: false 14 | 15 | jobs: 16 | publish_release: 17 | runs-on: ubuntu-latest 18 | permissions: 19 | # This is useful if you want to use PyPI trusted publisher 20 | # and NPM provenance 21 | id-token: write 22 | steps: 23 | - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 24 | 25 | - name: Populate Release 26 | id: populate-release 27 | uses: jupyter-server/jupyter_releaser/.github/actions/populate-release@v2 28 | with: 29 | token: ${{ secrets.ADMIN_GITHUB_TOKEN }} 30 | branch: ${{ github.event.inputs.branch }} 31 | release_url: ${{ github.event.inputs.release_url }} 32 | steps_to_skip: ${{ github.event.inputs.steps_to_skip }} 33 | 34 | - name: Finalize Release 35 | id: finalize-release 36 | env: 37 | # The following are needed if you use legacy PyPI set up 38 | # PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} 39 | # PYPI_TOKEN_MAP: ${{ secrets.PYPI_TOKEN_MAP }} 40 | # TWINE_USERNAME: __token__ 41 | NPM_TOKEN: ${{ secrets.NPM_TOKEN }} 42 | uses: jupyter-server/jupyter_releaser/.github/actions/finalize-release@v2 43 | with: 44 | token: ${{ secrets.ADMIN_GITHUB_TOKEN }} 45 | release_url: ${{ steps.populate-release.outputs.release_url }} 46 | 47 | - name: "** Next Step **" 48 | if: ${{ success() }} 49 | run: | 50 | echo "Verify the final release" 51 | echo ${{ steps.finalize-release.outputs.release_url }} 52 | 53 | - name: "** Failure Message **" 54 | if: ${{ failure() }} 55 | run: | 56 | echo "Failed to Publish the Draft Release Url:" 57 | echo ${{ steps.populate-release.outputs.release_url }} 58 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.bundle.* 2 | lib/ 3 | node_modules/ 4 | *.log 5 | .eslintcache 6 | .stylelintcache 7 | *.egg-info/ 8 | .ipynb_checkpoints 9 | *.tsbuildinfo 10 | jupydrive_s3/labextension 11 | # Version file is handled by hatchling 12 | jupydrive_s3/_version.py 13 | 14 | # Integration tests 15 | ui-tests/test-results/ 16 | ui-tests/playwright-report/ 17 | 18 | # Created by https://www.gitignore.io/api/python 19 | # Edit at https://www.gitignore.io/?templates=python 20 | 21 | ### Python ### 22 | # Byte-compiled / optimized / DLL files 23 | __pycache__/ 24 | *.py[cod] 25 | *$py.class 26 | 27 | # C extensions 28 | *.so 29 | 30 | # Distribution / packaging 31 | .Python 32 | build/ 33 | develop-eggs/ 34 | dist/ 35 | downloads/ 36 | eggs/ 37 | .eggs/ 38 | lib/ 39 | lib64/ 40 | parts/ 41 | sdist/ 42 | var/ 43 | wheels/ 44 | pip-wheel-metadata/ 45 | share/python-wheels/ 46 | .installed.cfg 47 | *.egg 48 | MANIFEST 49 | 50 | # PyInstaller 51 | # Usually these files are written by a python script from a template 52 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 53 | *.manifest 54 | *.spec 55 | 56 | # Installer logs 57 | pip-log.txt 58 | pip-delete-this-directory.txt 59 | 60 | # Unit test / coverage reports 61 | htmlcov/ 62 | .tox/ 63 | .nox/ 64 | .coverage 65 | .coverage.* 66 | .cache 67 | nosetests.xml 68 | coverage/ 69 | coverage.xml 70 | *.cover 71 | .hypothesis/ 72 | .pytest_cache/ 73 | 74 | # Translations 75 | *.mo 76 | *.pot 77 | 78 | # Scrapy stuff: 79 | .scrapy 80 | 81 | # Sphinx documentation 82 | docs/_build/ 83 | 84 | # PyBuilder 85 | target/ 86 | 87 | # pyenv 88 | .python-version 89 | 90 | # celery beat schedule file 91 | celerybeat-schedule 92 | 93 | # SageMath parsed files 94 | *.sage.py 95 | 96 | # Spyder project settings 97 | .spyderproject 98 | .spyproject 99 | 100 | # Rope project settings 101 | .ropeproject 102 | 103 | # Mr Developer 104 | .mr.developer.cfg 105 | .project 106 | .pydevproject 107 | 108 | # mkdocs documentation 109 | /site 110 | 111 | # mypy 112 | .mypy_cache/ 113 | .dmypy.json 114 | dmypy.json 115 | 116 | # Pyre type checker 117 | .pyre/ 118 | 119 | # End of https://www.gitignore.io/api/python 120 | 121 | # OSX files 122 | .DS_Store 123 | 124 | # Yarn cache 125 | .yarn/ 126 | 127 | # Env file 128 | .env 129 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["hatchling>=1.5.0", "jupyterlab>=4.2.5,<5", "hatch-nodejs-version>=0.3.2"] 3 | build-backend = "hatchling.build" 4 | 5 | [project] 6 | name = "jupydrive_s3" 7 | readme = "README.md" 8 | license = { file = "LICENSE" } 9 | requires-python = ">=3.8" 10 | classifiers = [ 11 | "Framework :: Jupyter", 12 | "Framework :: Jupyter :: JupyterLab", 13 | "Framework :: Jupyter :: JupyterLab :: 4", 14 | "Framework :: Jupyter :: JupyterLab :: Extensions", 15 | "Framework :: Jupyter :: JupyterLab :: Extensions :: Prebuilt", 16 | "License :: OSI Approved :: BSD License", 17 | "Programming Language :: Python", 18 | "Programming Language :: Python :: 3", 19 | "Programming Language :: Python :: 3.8", 20 | "Programming Language :: Python :: 3.9", 21 | "Programming Language :: Python :: 3.10", 22 | "Programming Language :: Python :: 3.11", 23 | "Programming Language :: Python :: 3.12", 24 | ] 25 | dependencies = [ 26 | "jupyter-secrets-manager >=0.4,<0.5" 27 | ] 28 | dynamic = ["version", "description", "authors", "urls", "keywords"] 29 | 30 | [tool.hatch.version] 31 | source = "nodejs" 32 | 33 | [tool.hatch.metadata.hooks.nodejs] 34 | fields = ["description", "authors", "urls"] 35 | 36 | [tool.hatch.build.targets.sdist] 37 | artifacts = ["jupydrive_s3/labextension"] 38 | exclude = [".github", "binder"] 39 | 40 | [tool.hatch.build.targets.wheel.shared-data] 41 | "jupydrive_s3/labextension" = "share/jupyter/labextensions/jupydrive-s3" 42 | "install.json" = "share/jupyter/labextensions/jupydrive-s3/install.json" 43 | 44 | [tool.hatch.build.hooks.version] 45 | path = "jupydrive_s3/_version.py" 46 | 47 | [tool.hatch.build.hooks.jupyter-builder] 48 | dependencies = ["hatch-jupyter-builder>=0.5"] 49 | build-function = "hatch_jupyter_builder.npm_builder" 50 | ensured-targets = [ 51 | "jupydrive_s3/labextension/static/style.js", 52 | "jupydrive_s3/labextension/package.json", 53 | ] 54 | skip-if-exists = ["jupydrive_s3/labextension/static/style.js"] 55 | 56 | [tool.hatch.build.hooks.jupyter-builder.build-kwargs] 57 | build_cmd = "build:prod" 58 | npm = ["jlpm"] 59 | 60 | [tool.hatch.build.hooks.jupyter-builder.editable-build-kwargs] 61 | build_cmd = "install:extension" 62 | npm = ["jlpm"] 63 | source_dir = "src" 64 | build_dir = "jupydrive_s3/labextension" 65 | 66 | [tool.jupyter-releaser.options] 67 | version_cmd = "hatch version" 68 | 69 | [tool.jupyter-releaser.hooks] 70 | before-build-npm = [ 71 | "python -m pip install 'jupyterlab>=4.2.5,<5'", 72 | "jlpm", 73 | "jlpm build:prod" 74 | ] 75 | before-build-python = ["jlpm clean:all"] 76 | 77 | [tool.check-wheel-contents] 78 | ignore = ["W002"] 79 | -------------------------------------------------------------------------------- /schema/file-browser-toolbar.json: -------------------------------------------------------------------------------- 1 | { 2 | "jupyter.lab.toolbars": { 3 | "DriveBrowser": [ 4 | { 5 | "name": "switch-drive", 6 | "command": "drives:open-switch-drive-dialog", 7 | "rank": 35 8 | } 9 | ] 10 | }, 11 | "jupyter.lab.menus": { 12 | "context": [ 13 | { 14 | "command": "filebrowser:rename", 15 | "selector": ".jp-DirListing-item[data-isdir]", 16 | "rank": 5, 17 | "disabled": true 18 | }, 19 | { 20 | "command": "drives:rename", 21 | "selector": ".jp-DirListing-item[data-isdir]", 22 | "rank": 5 23 | } 24 | ] 25 | }, 26 | "jupyter.lab.setting-icon": "jupydrive-s3:drive", 27 | "jupyter.lab.setting-icon-label": "Drive Browser", 28 | "title": "Jupydrive-s3 Settings", 29 | "description": "jupydrive-s3 settings.", 30 | "type": "object", 31 | "jupyter.lab.transform": true, 32 | "properties": { 33 | "bucketSwitching": { 34 | "type": "boolean", 35 | "title": "Enable Bucket switching", 36 | "description": "This flag enables or disables the bucket switching UI.", 37 | "default": false 38 | }, 39 | "toolbar": { 40 | "title": "Drive browser toolbar items", 41 | "items": { 42 | "$ref": "#/definitions/toolbarItem" 43 | }, 44 | "type": "array", 45 | "default": [] 46 | } 47 | }, 48 | "additionalProperties": false, 49 | "definitions": { 50 | "toolbarItem": { 51 | "properties": { 52 | "name": { 53 | "title": "Unique name", 54 | "type": "string" 55 | }, 56 | "args": { 57 | "title": "Command arguments", 58 | "type": "object" 59 | }, 60 | "command": { 61 | "title": "Command id", 62 | "type": "string", 63 | "default": "" 64 | }, 65 | "disabled": { 66 | "title": "Whether the item is ignored or not", 67 | "type": "boolean", 68 | "default": false 69 | }, 70 | "icon": { 71 | "title": "Item icon id", 72 | "description": "If defined, it will override the command icon", 73 | "type": "string" 74 | }, 75 | "label": { 76 | "title": "Item label", 77 | "description": "If defined, it will override the command label", 78 | "type": "string" 79 | }, 80 | "caption": { 81 | "title": "Item caption", 82 | "description": "If defined, it will override the command caption", 83 | "type": "string" 84 | }, 85 | "type": { 86 | "title": "Item type", 87 | "type": "string", 88 | "enum": ["command", "spacer"] 89 | }, 90 | "rank": { 91 | "title": "Item rank", 92 | "type": "number", 93 | "minimum": 0, 94 | "default": 50 95 | } 96 | }, 97 | "required": ["name"], 98 | "additionalProperties": false, 99 | "type": "object" 100 | } 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build 2 | 3 | on: 4 | push: 5 | branches: main 6 | pull_request: 7 | branches: '*' 8 | 9 | jobs: 10 | build: 11 | runs-on: ubuntu-latest 12 | 13 | steps: 14 | - name: Checkout 15 | uses: actions/checkout@v3 16 | 17 | - name: Base Setup 18 | uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 19 | 20 | - name: Install dependencies 21 | run: python -m pip install -U "jupyterlab>=4.0.0,<5" 22 | 23 | - name: Lint the extension 24 | run: | 25 | set -eux 26 | jlpm 27 | jlpm run lint:check 28 | 29 | - name: Test the extension 30 | run: | 31 | set -eux 32 | jlpm run test 33 | 34 | - name: Build the extension 35 | run: | 36 | set -eux 37 | python -m pip install .[test] 38 | 39 | jupyter labextension list 40 | jupyter labextension list 2>&1 | grep -ie "jupydrive-s3.*OK" 41 | python -m jupyterlab.browser_check 42 | 43 | - name: Package the extension 44 | run: | 45 | set -eux 46 | 47 | pip install build 48 | python -m build 49 | pip uninstall -y "jupydrive_s3" jupyterlab 50 | 51 | - name: Upload extension packages 52 | uses: actions/upload-artifact@v4 53 | with: 54 | name: extension-artifacts 55 | path: dist/jupydrive_s3* 56 | if-no-files-found: error 57 | 58 | test_isolated: 59 | needs: build 60 | runs-on: ubuntu-latest 61 | 62 | steps: 63 | - name: Install Python 64 | uses: actions/setup-python@v4 65 | with: 66 | python-version: '3.9' 67 | architecture: 'x64' 68 | - uses: actions/download-artifact@v4 69 | with: 70 | name: extension-artifacts 71 | - name: Install and Test 72 | run: | 73 | set -eux 74 | # Remove NodeJS, twice to take care of system and locally installed node versions. 75 | sudo rm -rf $(which node) 76 | sudo rm -rf $(which node) 77 | 78 | pip install "jupyterlab>=4.0.0,<5" jupydrive_s3*.whl 79 | 80 | 81 | jupyter labextension list 82 | jupyter labextension list 2>&1 | grep -ie "jupydrive-s3.*OK" 83 | python -m jupyterlab.browser_check --no-browser-test 84 | 85 | integration-tests: 86 | name: Integration tests 87 | needs: build 88 | runs-on: ubuntu-latest 89 | 90 | env: 91 | PLAYWRIGHT_BROWSERS_PATH: ${{ github.workspace }}/pw-browsers 92 | 93 | steps: 94 | - name: Checkout 95 | uses: actions/checkout@v3 96 | 97 | - name: Base Setup 98 | uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 99 | 100 | - name: Download extension package 101 | uses: actions/download-artifact@v4 102 | with: 103 | name: extension-artifacts 104 | 105 | - name: Install the extension 106 | run: | 107 | set -eux 108 | python -m pip install "jupyterlab>=4.0.0,<5" jupydrive_s3*.whl 109 | 110 | - name: Install dependencies 111 | working-directory: ui-tests 112 | env: 113 | YARN_ENABLE_IMMUTABLE_INSTALLS: 0 114 | PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 115 | run: jlpm install 116 | 117 | - name: Set up browser cache 118 | uses: actions/cache@v3 119 | with: 120 | path: | 121 | ${{ github.workspace }}/pw-browsers 122 | key: ${{ runner.os }}-${{ hashFiles('ui-tests/yarn.lock') }} 123 | 124 | - name: Install browser 125 | run: jlpm playwright install chromium 126 | working-directory: ui-tests 127 | 128 | - name: Execute integration tests 129 | working-directory: ui-tests 130 | run: | 131 | jlpm playwright test 132 | 133 | - name: Upload Playwright Test report 134 | if: always() 135 | uses: actions/upload-artifact@v4 136 | with: 137 | name: jupydrive_s3-playwright-tests 138 | path: | 139 | ui-tests/test-results 140 | ui-tests/playwright-report 141 | 142 | check_links: 143 | name: Check Links 144 | runs-on: ubuntu-latest 145 | timeout-minutes: 15 146 | steps: 147 | - uses: actions/checkout@v3 148 | - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 149 | - uses: jupyterlab/maintainer-tools/.github/actions/check-links@v1 150 | -------------------------------------------------------------------------------- /RELEASE.md: -------------------------------------------------------------------------------- 1 | # Making a new release of jupydrive_s3 2 | 3 | The extension can be published to `PyPI` and `npm` manually or using the [Jupyter Releaser](https://github.com/jupyter-server/jupyter_releaser). 4 | 5 | ## Manual release 6 | 7 | ### Python package 8 | 9 | This extension can be distributed as Python packages. All of the Python 10 | packaging instructions are in the `pyproject.toml` file to wrap your extension in a 11 | Python package. Before generating a package, you first need to install some tools: 12 | 13 | ```bash 14 | pip install build twine hatch 15 | ``` 16 | 17 | Bump the version using `hatch`. By default this will create a tag. 18 | See the docs on [hatch-nodejs-version](https://github.com/agoose77/hatch-nodejs-version#semver) for details. 19 | 20 | ```bash 21 | hatch version 22 | ``` 23 | 24 | Make sure to clean up all the development files before building the package: 25 | 26 | ```bash 27 | jlpm clean:all 28 | ``` 29 | 30 | You could also clean up the local git repository: 31 | 32 | ```bash 33 | git clean -dfX 34 | ``` 35 | 36 | To create a Python source package (`.tar.gz`) and the binary package (`.whl`) in the `dist/` directory, do: 37 | 38 | ```bash 39 | python -m build 40 | ``` 41 | 42 | > `python setup.py sdist bdist_wheel` is deprecated and will not work for this package. 43 | 44 | Then to upload the package to PyPI, do: 45 | 46 | ```bash 47 | twine upload dist/* 48 | ``` 49 | 50 | ### NPM package 51 | 52 | To publish the frontend part of the extension as a NPM package, do: 53 | 54 | ```bash 55 | npm login 56 | npm publish --access public 57 | ``` 58 | 59 | ## Automated releases with the Jupyter Releaser 60 | 61 | The extension repository should already be compatible with the Jupyter Releaser. 62 | 63 | Check out the [workflow documentation](https://jupyter-releaser.readthedocs.io/en/latest/get_started/making_release_from_repo.html) for more information. 64 | 65 | Here is a summary of the steps to cut a new release: 66 | 67 | - Add tokens to the [Github Secrets](https://docs.github.com/en/actions/security-guides/encrypted-secrets) in the repository: 68 | - `ADMIN_GITHUB_TOKEN` (with "public_repo" and "repo:status" permissions); see the [documentation](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) 69 | - `NPM_TOKEN` (with "automation" permission); see the [documentation](https://docs.npmjs.com/creating-and-viewing-access-tokens) 70 | - Set up PyPI 71 | 72 |
Using PyPI trusted publisher (modern way) 73 | 74 | - Set up your PyPI project by [adding a trusted publisher](https://docs.pypi.org/trusted-publishers/adding-a-publisher/) 75 | - The _workflow name_ is `publish-release.yml` and the _environment_ should be left blank. 76 | - Ensure the publish release job as `permissions`: `id-token : write` (see the [documentation](https://docs.pypi.org/trusted-publishers/using-a-publisher/)) 77 | 78 |
79 | 80 |
Using PyPI token (legacy way) 81 | 82 | - If the repo generates PyPI release(s), create a scoped PyPI [token](https://packaging.python.org/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/#saving-credentials-on-github). We recommend using a scoped token for security reasons. 83 | 84 | - You can store the token as `PYPI_TOKEN` in your fork's `Secrets`. 85 | 86 | - Advanced usage: if you are releasing multiple repos, you can create a secret named `PYPI_TOKEN_MAP` instead of `PYPI_TOKEN` that is formatted as follows: 87 | 88 | ```text 89 | owner1/repo1,token1 90 | owner2/repo2,token2 91 | ``` 92 | 93 | If you have multiple Python packages in the same repository, you can point to them as follows: 94 | 95 | ```text 96 | owner1/repo1/path/to/package1,token1 97 | owner1/repo1/path/to/package2,token2 98 | ``` 99 | 100 |
101 | 102 | - Go to the Actions panel 103 | - Run the "Step 1: Prep Release" workflow 104 | - Check the draft changelog 105 | - Run the "Step 2: Publish Release" workflow 106 | 107 | ## Publishing to `conda-forge` 108 | 109 | If the package is not on conda forge yet, check the documentation to learn how to add it: https://conda-forge.org/docs/maintainer/adding_pkgs.html 110 | 111 | Otherwise a bot should pick up the new version publish to PyPI, and open a new PR on the feedstock repository automatically. 112 | -------------------------------------------------------------------------------- /ui-tests/README.md: -------------------------------------------------------------------------------- 1 | # Integration Testing 2 | 3 | This folder contains the integration tests of the extension. 4 | 5 | They are defined using [Playwright](https://playwright.dev/docs/intro) test runner 6 | and [Galata](https://github.com/jupyterlab/jupyterlab/tree/main/galata) helper. 7 | 8 | The Playwright configuration is defined in [playwright.config.js](./playwright.config.js). 9 | 10 | The JupyterLab server configuration to use for the integration test is defined 11 | in [jupyter_server_test_config.py](./jupyter_server_test_config.py). 12 | 13 | The default configuration will produce video for failing tests and an HTML report. 14 | 15 | > There is a new experimental UI mode that you may fall in love with; see [that video](https://www.youtube.com/watch?v=jF0yA-JLQW0). 16 | 17 | ## Run the tests 18 | 19 | > All commands are assumed to be executed from the root directory 20 | 21 | To run the tests, you need to: 22 | 23 | 1. Compile the extension: 24 | 25 | ```sh 26 | jlpm install 27 | jlpm build:prod 28 | ``` 29 | 30 | > Check the extension is installed in JupyterLab. 31 | 32 | 2. Install test dependencies (needed only once): 33 | 34 | ```sh 35 | cd ./ui-tests 36 | jlpm install 37 | jlpm playwright install 38 | cd .. 39 | ``` 40 | 41 | 3. Execute the [Playwright](https://playwright.dev/docs/intro) tests: 42 | 43 | ```sh 44 | cd ./ui-tests 45 | jlpm playwright test 46 | ``` 47 | 48 | Test results will be shown in the terminal. In case of any test failures, the test report 49 | will be opened in your browser at the end of the tests execution; see 50 | [Playwright documentation](https://playwright.dev/docs/test-reporters#html-reporter) 51 | for configuring that behavior. 52 | 53 | ## Update the tests snapshots 54 | 55 | > All commands are assumed to be executed from the root directory 56 | 57 | If you are comparing snapshots to validate your tests, you may need to update 58 | the reference snapshots stored in the repository. To do that, you need to: 59 | 60 | 1. Compile the extension: 61 | 62 | ```sh 63 | jlpm install 64 | jlpm build:prod 65 | ``` 66 | 67 | > Check the extension is installed in JupyterLab. 68 | 69 | 2. Install test dependencies (needed only once): 70 | 71 | ```sh 72 | cd ./ui-tests 73 | jlpm install 74 | jlpm playwright install 75 | cd .. 76 | ``` 77 | 78 | 3. Execute the [Playwright](https://playwright.dev/docs/intro) command: 79 | 80 | ```sh 81 | cd ./ui-tests 82 | jlpm playwright test -u 83 | ``` 84 | 85 | > Some discrepancy may occurs between the snapshots generated on your computer and 86 | > the one generated on the CI. To ease updating the snapshots on a PR, you can 87 | > type `please update playwright snapshots` to trigger the update by a bot on the CI. 88 | > Once the bot has computed new snapshots, it will commit them to the PR branch. 89 | 90 | ## Create tests 91 | 92 | > All commands are assumed to be executed from the root directory 93 | 94 | To create tests, the easiest way is to use the code generator tool of playwright: 95 | 96 | 1. Compile the extension: 97 | 98 | ```sh 99 | jlpm install 100 | jlpm build:prod 101 | ``` 102 | 103 | > Check the extension is installed in JupyterLab. 104 | 105 | 2. Install test dependencies (needed only once): 106 | 107 | ```sh 108 | cd ./ui-tests 109 | jlpm install 110 | jlpm playwright install 111 | cd .. 112 | ``` 113 | 114 | 3. Start the server: 115 | 116 | ```sh 117 | cd ./ui-tests 118 | jlpm start 119 | ``` 120 | 121 | 4. Execute the [Playwright code generator](https://playwright.dev/docs/codegen) in **another terminal**: 122 | 123 | ```sh 124 | cd ./ui-tests 125 | jlpm playwright codegen localhost:8888 126 | ``` 127 | 128 | ## Debug tests 129 | 130 | > All commands are assumed to be executed from the root directory 131 | 132 | To debug tests, a good way is to use the inspector tool of playwright: 133 | 134 | 1. Compile the extension: 135 | 136 | ```sh 137 | jlpm install 138 | jlpm build:prod 139 | ``` 140 | 141 | > Check the extension is installed in JupyterLab. 142 | 143 | 2. Install test dependencies (needed only once): 144 | 145 | ```sh 146 | cd ./ui-tests 147 | jlpm install 148 | jlpm playwright install 149 | cd .. 150 | ``` 151 | 152 | 3. Execute the Playwright tests in [debug mode](https://playwright.dev/docs/debug): 153 | 154 | ```sh 155 | cd ./ui-tests 156 | jlpm playwright test --debug 157 | ``` 158 | 159 | ## Upgrade Playwright and the browsers 160 | 161 | To update the web browser versions, you must update the package `@playwright/test`: 162 | 163 | ```sh 164 | cd ./ui-tests 165 | jlpm up "@playwright/test" 166 | jlpm playwright install 167 | ``` 168 | -------------------------------------------------------------------------------- /style/newDriveIcon.svg: -------------------------------------------------------------------------------- 1 | 12 | 17 | 18 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # jupydrive-s3 2 | 3 | [![Github Actions Status](https://github.com/QuantStack/jupydrive-s3/workflows/Build/badge.svg)](https://github.com/QuantStack/jupydrive-s3/actions/workflows/build.yml) 4 | A JupyterLab extension which enables client-side drives access. 5 | 6 | ![Screenshot from 2024-05-06 15-22-59](https://github.com/DenisaCG/jupydrive-s3/assets/91504950/c6912105-cc0b-4a95-9234-57faebe75b90) 7 | 8 | The drives are used as a filesystem, having support for all basic functionalities (file tree-view, editing contents, copying, renaming, deleting, downloading etc). 9 | 10 | The extension was built using the official JavaScript [`AWS SDK`](https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/). 11 | 12 | ## Drives Compatibility 13 | 14 | Currently, the extension offers support only for [`S3`](https://aws.amazon.com/s3/) drives. 15 | 16 | ## Configuration 17 | 18 | ### Set `CORS` Rules 19 | 20 | As the extension works in the browser, the `S3` buckets need to have certain `CORS` (Cross-Origin-Resource-Sharing) rules set: 21 | 22 | - `http://localhost:*` needs to be added to the `AllowedOrigins` section, 23 | - `GET`, `PUT`, `DELETE`, `HEAD` need to be added to the `AllowedMethods` section. 24 | 25 | More information about `CORS` [here](https://docs.aws.amazon.com/AmazonS3/latest/userguide/cors.html) and the various ways to configure it [here](https://docs.aws.amazon.com/AmazonS3/latest/userguide/enabling-cors-examples.html). 26 | 27 | ### Set up credentials 28 | 29 | User credentials can be set by accessing `Settings` -> `Settings Editor` -> `Credentials Provider`. Users need to offer a bucket name, region, endpoint, access key ID and secret access key, as well as optionally a path to the folder within the bucket that should act as root. 30 | 31 | The extension uses [`jupyter-secrets-manager`](https://github.com/jupyterlab-contrib/jupyter-secrets-manager) to deal with the secret input fields. 32 | 33 | > [!NOTE] 34 | > The extension uses the Settings Registry to allow users to set their credentials when not in development mode. As a result, secret values may still be accessible to other extensions. Be aware of the associated security risks. 35 | 36 | ## Requirements 37 | 38 | - JupyterLab >= 4.2.5 39 | 40 | ## Install 41 | 42 | To install the extension, execute: 43 | 44 | ```bash 45 | pip install jupydrive_s3 46 | ``` 47 | 48 | ## Uninstall 49 | 50 | To remove the extension, execute: 51 | 52 | ```bash 53 | pip uninstall jupydrive_s3 54 | ``` 55 | 56 | ## Contributing 57 | 58 | ### Development install 59 | 60 | Note: You will need NodeJS to build the extension package. 61 | 62 | The `jlpm` command is JupyterLab's pinned version of 63 | [yarn](https://yarnpkg.com/) that is installed with JupyterLab. You may use 64 | `yarn` or `npm` in lieu of `jlpm` below. 65 | 66 | ```bash 67 | # Clone the repo to your local environment 68 | # Change directory to the jupydrive_s3 directory 69 | # Install package in development mode 70 | pip install -e "." 71 | # Link your development version of the extension with JupyterLab 72 | jupyter labextension develop . --overwrite 73 | # Rebuild extension Typescript source after making changes 74 | jlpm build 75 | ``` 76 | 77 | You can watch the source directory and run JupyterLab at the same time in different terminals to watch for changes in the extension's source and automatically rebuild the extension. 78 | 79 | ```bash 80 | # Watch the source directory in one terminal, automatically rebuilding when needed 81 | jlpm watch 82 | # Run JupyterLab in another terminal 83 | jupyter lab 84 | ``` 85 | 86 | With the watch command running, every saved change will immediately be built locally and available in your running JupyterLab. Refresh JupyterLab to load the change in your browser (you may need to wait several seconds for the extension to be rebuilt). 87 | 88 | By default, the `jlpm build` command generates the source maps for this extension to make it easier to debug using the browser dev tools. To also generate source maps for the JupyterLab core extensions, you can run the following command: 89 | 90 | ```bash 91 | jupyter lab build --minimize=False 92 | ``` 93 | 94 | ### Local enviroment variables 95 | 96 | For the local development of the extension, enviroment variables are used to define the required bucket name, region and endpoint, as well as the access key id and secret key, with the additional possibility of defining a different root folder. 97 | 98 | Rename the `.env.example` file to `.env` and update it with the values needed for your local configuration. 99 | 100 | Note: Unless configured differently, the `S3` bucket endpoint should follow the format: `https://s3..amazonaws.com`. 101 | 102 | ### Development uninstall 103 | 104 | ```bash 105 | pip uninstall jupydrive_s3 106 | ``` 107 | 108 | In development mode, you will also need to remove the symlink created by `jupyter labextension develop` 109 | command. To find its location, you can run `jupyter labextension list` to figure out where the `labextensions` 110 | folder is located. Then you can remove the symlink named `jupydrive-s3` within that folder. 111 | 112 | ### Testing the extension 113 | 114 | #### Frontend tests 115 | 116 | This extension is using [Jest](https://jestjs.io/) for JavaScript code testing. 117 | 118 | To execute them, execute: 119 | 120 | ```sh 121 | jlpm 122 | jlpm test 123 | ``` 124 | 125 | #### Integration tests 126 | 127 | This extension uses [Playwright](https://playwright.dev/docs/intro) for the integration tests (aka user level tests). 128 | More precisely, the JupyterLab helper [Galata](https://github.com/jupyterlab/jupyterlab/tree/master/galata) is used to handle testing the extension in JupyterLab. 129 | 130 | More information are provided within the [ui-tests](./ui-tests/README.md) README. 131 | 132 | ### Packaging the extension 133 | 134 | See [RELEASE](RELEASE.md) 135 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "jupydrive-s3", 3 | "version": "0.1.0", 4 | "description": "A JupyterLab extension which enables client-side drives access.", 5 | "keywords": [ 6 | "jupyter", 7 | "jupyterlab", 8 | "jupyterlab-extension" 9 | ], 10 | "homepage": "https://github.com/QuantStack/jupydrive-s3", 11 | "bugs": { 12 | "url": "https://github.com/QuantStack/jupydrive-s3/issues" 13 | }, 14 | "license": "BSD-3-Clause", 15 | "author": { 16 | "name": "QuantStack", 17 | "email": "denisa.checiu@quantstack.net" 18 | }, 19 | "files": [ 20 | "lib/**/*.{d.ts,eot,gif,html,jpg,js,js.map,json,png,svg,woff2,ttf}", 21 | "style/**/*.{css,js,eot,gif,html,jpg,json,png,svg,woff2,ttf}", 22 | "schema/*.json" 23 | ], 24 | "main": "lib/index.js", 25 | "types": "lib/index.d.ts", 26 | "style": "style/index.css", 27 | "repository": { 28 | "type": "git", 29 | "url": "https://github.com/QuantStack/jupydrive-s3.git" 30 | }, 31 | "scripts": { 32 | "build": "jlpm build:lib && jlpm build:labextension:dev", 33 | "build:prod": "jlpm clean && jlpm build:lib:prod && jlpm build:labextension", 34 | "build:labextension": "jupyter labextension build .", 35 | "build:labextension:dev": "jupyter labextension build --development True .", 36 | "build:lib": "tsc --sourceMap", 37 | "build:lib:prod": "tsc", 38 | "clean": "jlpm clean:lib", 39 | "clean:lib": "rimraf lib tsconfig.tsbuildinfo", 40 | "clean:lintcache": "rimraf .eslintcache .stylelintcache", 41 | "clean:labextension": "rimraf jupydrive_s3/labextension jupydrive_s3/_version.py", 42 | "clean:all": "jlpm clean:lib && jlpm clean:labextension && jlpm clean:lintcache", 43 | "eslint": "jlpm eslint:check --fix", 44 | "eslint:check": "eslint . --cache --ext .ts,.tsx", 45 | "install:extension": "jlpm build", 46 | "lint": "jlpm stylelint && jlpm prettier && jlpm eslint", 47 | "lint:check": "jlpm stylelint:check && jlpm prettier:check && jlpm eslint:check", 48 | "prettier": "jlpm prettier:base --write --list-different", 49 | "prettier:base": "prettier \"**/*{.ts,.tsx,.js,.jsx,.css,.json,.md}\"", 50 | "prettier:check": "jlpm prettier:base --check", 51 | "stylelint": "jlpm stylelint:check --fix", 52 | "stylelint:check": "stylelint --cache \"style/**/*.css\"", 53 | "test": "jest --coverage", 54 | "watch": "run-p watch:src watch:labextension", 55 | "watch:src": "tsc -w --sourceMap", 56 | "watch:labextension": "jupyter labextension watch ." 57 | }, 58 | "dependencies": { 59 | "@aws-sdk/client-s3": "^3.511.0", 60 | "@aws-sdk/s3-request-presigner": "^3.511.0", 61 | "@jupyterlab/application": "^4.4.0", 62 | "@jupyterlab/apputils": "^4.5.0", 63 | "@jupyterlab/coreutils": "^6.4.0", 64 | "@jupyterlab/filebrowser": "^4.4.0", 65 | "@jupyterlab/settingregistry": "^4.4.0", 66 | "@jupyterlab/statedb": "^4.4.0", 67 | "@jupyterlab/translation": "^4.4.0", 68 | "@jupyterlab/ui-components": "^4.4.0", 69 | "@lumino/commands": "^2.3.0", 70 | "@lumino/coreutils": "^2.2.0", 71 | "@lumino/widgets": "^2.7.0", 72 | "jupyter-secrets-manager": "^0.4.0" 73 | }, 74 | "devDependencies": { 75 | "@jupyterlab/builder": "^4.4.0", 76 | "@jupyterlab/testutils": "^4.4.0", 77 | "@types/jest": "^29.2.0", 78 | "@types/json-schema": "^7.0.11", 79 | "@types/react": "^18.0.26", 80 | "@types/react-addons-linked-state-mixin": "^0.14.22", 81 | "@typescript-eslint/eslint-plugin": "^6.1.0", 82 | "@typescript-eslint/parser": "^6.1.0", 83 | "css-loader": "^6.7.1", 84 | "dotenv-webpack": "^8.1.0", 85 | "eslint": "^8.36.0", 86 | "eslint-config-prettier": "^8.8.0", 87 | "eslint-plugin-prettier": "^5.0.0", 88 | "jest": "^29.2.0", 89 | "npm-run-all": "^4.1.5", 90 | "prettier": "^3.0.0", 91 | "rimraf": "^5.0.1", 92 | "source-map-loader": "^1.0.2", 93 | "style-loader": "^3.3.1", 94 | "stylelint": "^15.10.1", 95 | "stylelint-config-recommended": "^13.0.0", 96 | "stylelint-config-standard": "^34.0.0", 97 | "stylelint-csstree-validator": "^3.0.0", 98 | "stylelint-prettier": "^4.0.0", 99 | "typescript": "~5.0.2", 100 | "yjs": "^13.5.0" 101 | }, 102 | "sideEffects": [ 103 | "style/*.css", 104 | "style/index.js" 105 | ], 106 | "styleModule": "style/index.js", 107 | "publishConfig": { 108 | "access": "public" 109 | }, 110 | "jupyterlab": { 111 | "extension": true, 112 | "outputDir": "jupydrive_s3/labextension", 113 | "disabledExtensions": [ 114 | "@jupyterlab/filebrowser-extension:default-file-browser" 115 | ], 116 | "schemaDir": "schema", 117 | "webpackConfig": "./dev.webpack.config.js" 118 | }, 119 | "eslintIgnore": [ 120 | "node_modules", 121 | "dist", 122 | "coverage", 123 | "**/*.d.ts", 124 | "tests", 125 | "**/__tests__", 126 | "ui-tests", 127 | "CHANGELOG.md" 128 | ], 129 | "eslintConfig": { 130 | "extends": [ 131 | "eslint:recommended", 132 | "plugin:@typescript-eslint/eslint-recommended", 133 | "plugin:@typescript-eslint/recommended", 134 | "plugin:prettier/recommended" 135 | ], 136 | "parser": "@typescript-eslint/parser", 137 | "parserOptions": { 138 | "project": "tsconfig.json", 139 | "sourceType": "module" 140 | }, 141 | "plugins": [ 142 | "@typescript-eslint" 143 | ], 144 | "rules": { 145 | "@typescript-eslint/naming-convention": [ 146 | "error", 147 | { 148 | "selector": "interface", 149 | "format": [ 150 | "PascalCase" 151 | ], 152 | "custom": { 153 | "regex": "^I[A-Z]", 154 | "match": true 155 | } 156 | } 157 | ], 158 | "@typescript-eslint/no-unused-vars": [ 159 | "warn", 160 | { 161 | "args": "none" 162 | } 163 | ], 164 | "@typescript-eslint/no-explicit-any": "off", 165 | "@typescript-eslint/no-namespace": "off", 166 | "@typescript-eslint/no-use-before-define": "off", 167 | "@typescript-eslint/quotes": [ 168 | "error", 169 | "single", 170 | { 171 | "avoidEscape": true, 172 | "allowTemplateLiterals": false 173 | } 174 | ], 175 | "curly": [ 176 | "error", 177 | "all" 178 | ], 179 | "eqeqeq": "error", 180 | "prefer-arrow-callback": "error" 181 | } 182 | }, 183 | "prettier": { 184 | "singleQuote": true, 185 | "trailingComma": "none", 186 | "arrowParens": "avoid", 187 | "endOfLine": "auto", 188 | "overrides": [ 189 | { 190 | "files": "package.json", 191 | "options": { 192 | "tabWidth": 4 193 | } 194 | } 195 | ] 196 | }, 197 | "stylelint": { 198 | "extends": [ 199 | "stylelint-config-recommended", 200 | "stylelint-config-standard", 201 | "stylelint-prettier/recommended" 202 | ], 203 | "plugins": [ 204 | "stylelint-csstree-validator" 205 | ], 206 | "rules": { 207 | "csstree/validator": true, 208 | "property-no-vendor-prefix": null, 209 | "selector-class-pattern": "^([a-z][A-z\\d]*)(-[A-z\\d]+)*$", 210 | "selector-no-vendor-prefix": null, 211 | "value-no-vendor-prefix": null 212 | } 213 | } 214 | } 215 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | 4 | 5 | ## 0.1.0 6 | 7 | ([Full Changelog](https://github.com/QuantStack/jupydrive-s3/compare/340862a8dfb8c96354f5c076c4bfd5660a5809fe...75d45c3ebf8065d6c4747d0cfc246d6d2f39f2ab)) 8 | 9 | ### Enhancements made 10 | 11 | - Set up credentials provider [#44](https://github.com/QuantStack/jupydrive-s3/pull/44) ([@DenisaCG](https://github.com/DenisaCG)) 12 | - Disable directory rename [#41](https://github.com/QuantStack/jupydrive-s3/pull/41) ([@DenisaCG](https://github.com/DenisaCG)) 13 | - Add limit to object listings in helping functions [#39](https://github.com/QuantStack/jupydrive-s3/pull/39) ([@DenisaCG](https://github.com/DenisaCG)) 14 | - Update `getFileType` function [#35](https://github.com/QuantStack/jupydrive-s3/pull/35) ([@DenisaCG](https://github.com/DenisaCG)) 15 | - Update `eslintIgnore` and `.prettierignore` [#32](https://github.com/QuantStack/jupydrive-s3/pull/32) ([@DenisaCG](https://github.com/DenisaCG)) 16 | - Change `DriveBrowser` icon [#30](https://github.com/QuantStack/jupydrive-s3/pull/30) ([@DenisaCG](https://github.com/DenisaCG)) 17 | - Improve user experience reliability [#29](https://github.com/QuantStack/jupydrive-s3/pull/29) ([@DenisaCG](https://github.com/DenisaCG)) 18 | - Update Drive Browser [#28](https://github.com/QuantStack/jupydrive-s3/pull/28) ([@DenisaCG](https://github.com/DenisaCG)) 19 | - Upgrade dependencies [#27](https://github.com/QuantStack/jupydrive-s3/pull/27) ([@DenisaCG](https://github.com/DenisaCG)) 20 | - Add flag which toggles bucket switching UI [#23](https://github.com/QuantStack/jupydrive-s3/pull/23) ([@DenisaCG](https://github.com/DenisaCG)) 21 | - Refactor functionalities and files [#19](https://github.com/QuantStack/jupydrive-s3/pull/19) ([@DenisaCG](https://github.com/DenisaCG)) 22 | - Make an arbitrary directory root [#17](https://github.com/QuantStack/jupydrive-s3/pull/17) ([@DenisaCG](https://github.com/DenisaCG)) 23 | - Extract config from `.env` file for dev mode [#15](https://github.com/QuantStack/jupydrive-s3/pull/15) ([@DenisaCG](https://github.com/DenisaCG)) 24 | - Refactor contents listing [#14](https://github.com/QuantStack/jupydrive-s3/pull/14) ([@DenisaCG](https://github.com/DenisaCG)) 25 | - Skip getting region if it is provided [#13](https://github.com/QuantStack/jupydrive-s3/pull/13) ([@trungleduc](https://github.com/trungleduc)) 26 | - Refresh Filebrowser contents after certain operations [#10](https://github.com/QuantStack/jupydrive-s3/pull/10) ([@DenisaCG](https://github.com/DenisaCG)) 27 | - Auth [#9](https://github.com/QuantStack/jupydrive-s3/pull/9) ([@afshin](https://github.com/afshin)) 28 | - Clean Up [#8](https://github.com/QuantStack/jupydrive-s3/pull/8) ([@DenisaCG](https://github.com/DenisaCG)) 29 | - Add functionality to Copy content to another bucket [#6](https://github.com/QuantStack/jupydrive-s3/pull/6) ([@DenisaCG](https://github.com/DenisaCG)) 30 | - Add download functionality [#5](https://github.com/QuantStack/jupydrive-s3/pull/5) ([@DenisaCG](https://github.com/DenisaCG)) 31 | - Add S3 contents functionalities [#2](https://github.com/QuantStack/jupydrive-s3/pull/2) ([@DenisaCG](https://github.com/DenisaCG)) 32 | - Replace default FileBrowser with custom FileBrowser for drives [#1](https://github.com/QuantStack/jupydrive-s3/pull/1) ([@DenisaCG](https://github.com/DenisaCG)) 33 | 34 | ### Bugs fixed 35 | 36 | - Fix check object promise handling [#42](https://github.com/QuantStack/jupydrive-s3/pull/42) ([@DenisaCG](https://github.com/DenisaCG)) 37 | - Fix counting functionality for directory rename [#37](https://github.com/QuantStack/jupydrive-s3/pull/37) ([@DenisaCG](https://github.com/DenisaCG)) 38 | - Update logic for checking object type [#34](https://github.com/QuantStack/jupydrive-s3/pull/34) ([@DenisaCG](https://github.com/DenisaCG)) 39 | - Fix root formatting [#31](https://github.com/QuantStack/jupydrive-s3/pull/31) ([@DenisaCG](https://github.com/DenisaCG)) 40 | - Unpin `@lumino/coreutils` package [#25](https://github.com/QuantStack/jupydrive-s3/pull/25) ([@DenisaCG](https://github.com/DenisaCG)) 41 | - Fix formatting root error [#21](https://github.com/QuantStack/jupydrive-s3/pull/21) ([@DenisaCG](https://github.com/DenisaCG)) 42 | - Improve switching drives functionality [#20](https://github.com/QuantStack/jupydrive-s3/pull/20) ([@DenisaCG](https://github.com/DenisaCG)) 43 | - Change the name of this package/repo to jupydrive-s3 [#18](https://github.com/QuantStack/jupydrive-s3/pull/18) ([@afshin](https://github.com/afshin)) 44 | - Refactor awaiting S3 operations [#16](https://github.com/QuantStack/jupydrive-s3/pull/16) ([@DenisaCG](https://github.com/DenisaCG)) 45 | 46 | ### Maintenance and upkeep improvements 47 | 48 | - Update to `v4` of actions upload and download artifact [#40](https://github.com/QuantStack/jupydrive-s3/pull/40) ([@DenisaCG](https://github.com/DenisaCG)) 49 | - Make increment functions protected [#38](https://github.com/QuantStack/jupydrive-s3/pull/38) ([@DenisaCG](https://github.com/DenisaCG)) 50 | 51 | ### Documentation improvements 52 | 53 | - Update `README.md` [#45](https://github.com/QuantStack/jupydrive-s3/pull/45) ([@DenisaCG](https://github.com/DenisaCG)) 54 | - Update README file [#22](https://github.com/QuantStack/jupydrive-s3/pull/22) ([@DenisaCG](https://github.com/DenisaCG)) 55 | - Update README.md file [#7](https://github.com/QuantStack/jupydrive-s3/pull/7) ([@DenisaCG](https://github.com/DenisaCG)) 56 | 57 | ### Other merged PRs 58 | 59 | - v0.1.0-alpha [#12](https://github.com/QuantStack/jupydrive-s3/pull/12) ([@afshin](https://github.com/afshin)) 60 | 61 | ### Contributors to this release 62 | 63 | ([GitHub contributors page for this release](https://github.com/QuantStack/jupydrive-s3/graphs/contributors?from=2024-01-22&to=2025-06-04&type=c)) 64 | 65 | [@afshin](https://github.com/search?q=repo%3AQuantStack%2Fjupydrive-s3+involves%3Aafshin+updated%3A2024-01-22..2025-06-04&type=Issues) | [@DenisaCG](https://github.com/search?q=repo%3AQuantStack%2Fjupydrive-s3+involves%3ADenisaCG+updated%3A2024-01-22..2025-06-04&type=Issues) | [@trungleduc](https://github.com/search?q=repo%3AQuantStack%2Fjupydrive-s3+involves%3Atrungleduc+updated%3A2024-01-22..2025-06-04&type=Issues) 66 | 67 | 68 | 69 | ## 0.1.0a6 70 | 71 | ([Full Changelog](https://github.com/QuantStack/jupydrive-s3/compare/0e42a5371c0cd5fb0cc3d95ef8c89121289374db...c7994feb5d5cb38e10b1cd5fc573a12f693be6b6)) 72 | 73 | ### Enhancements made 74 | 75 | - Disable directory rename [#41](https://github.com/QuantStack/jupydrive-s3/pull/41) ([@DenisaCG](https://github.com/DenisaCG)) 76 | - Add limit to object listings in helping functions [#39](https://github.com/QuantStack/jupydrive-s3/pull/39) ([@DenisaCG](https://github.com/DenisaCG)) 77 | - Update `getFileType` function [#35](https://github.com/QuantStack/jupydrive-s3/pull/35) ([@DenisaCG](https://github.com/DenisaCG)) 78 | - Update `eslintIgnore` and `.prettierignore` [#32](https://github.com/QuantStack/jupydrive-s3/pull/32) ([@DenisaCG](https://github.com/DenisaCG)) 79 | 80 | ### Bugs fixed 81 | 82 | - Fix check object promise handling [#42](https://github.com/QuantStack/jupydrive-s3/pull/42) ([@DenisaCG](https://github.com/DenisaCG)) 83 | - Fix counting functionality for directory rename [#37](https://github.com/QuantStack/jupydrive-s3/pull/37) ([@DenisaCG](https://github.com/DenisaCG)) 84 | - Update logic for checking object type [#34](https://github.com/QuantStack/jupydrive-s3/pull/34) ([@DenisaCG](https://github.com/DenisaCG)) 85 | 86 | ### Maintenance and upkeep improvements 87 | 88 | - Update to `v4` of actions upload and download artifact [#40](https://github.com/QuantStack/jupydrive-s3/pull/40) ([@DenisaCG](https://github.com/DenisaCG)) 89 | - Make increment functions protected [#38](https://github.com/QuantStack/jupydrive-s3/pull/38) ([@DenisaCG](https://github.com/DenisaCG)) 90 | 91 | ### Contributors to this release 92 | 93 | ([GitHub contributors page for this release](https://github.com/QuantStack/jupydrive-s3/graphs/contributors?from=2024-10-25&to=2025-02-24&type=c)) 94 | 95 | [@DenisaCG](https://github.com/search?q=repo%3AQuantStack%2Fjupydrive-s3+involves%3ADenisaCG+updated%3A2024-10-25..2025-02-24&type=Issues) 96 | 97 | ## 0.1.0a5 98 | 99 | ([Full Changelog](https://github.com/QuantStack/jupydrive-s3/compare/846580a2eb7a0b8d738c3770e056cb0bf6cd8502...1a21e51771dd64c44c11f010dfd3565c31d744d3)) 100 | 101 | ### Bugs fixed 102 | 103 | - Fix root formatting [#31](https://github.com/QuantStack/jupydrive-s3/pull/31) ([@DenisaCG](https://github.com/DenisaCG)) 104 | 105 | ### Contributors to this release 106 | 107 | ([GitHub contributors page for this release](https://github.com/QuantStack/jupydrive-s3/graphs/contributors?from=2024-10-23&to=2024-10-25&type=c)) 108 | 109 | [@DenisaCG](https://github.com/search?q=repo%3AQuantStack%2Fjupydrive-s3+involves%3ADenisaCG+updated%3A2024-10-23..2024-10-25&type=Issues) 110 | 111 | ## 0.1.0a4 112 | 113 | ([Full Changelog](https://github.com/QuantStack/jupydrive-s3/compare/f14bef6badc6b3b9165bd7b74fe175317a01d8b1...a3e36001f789636258a39347e98e1d993f60aece)) 114 | 115 | ### Enhancements made 116 | 117 | - Change `DriveBrowser` icon [#30](https://github.com/QuantStack/jupydrive-s3/pull/30) ([@DenisaCG](https://github.com/DenisaCG)) 118 | - Improve user experience reliability [#29](https://github.com/QuantStack/jupydrive-s3/pull/29) ([@DenisaCG](https://github.com/DenisaCG)) 119 | - Update Drive Browser [#28](https://github.com/QuantStack/jupydrive-s3/pull/28) ([@DenisaCG](https://github.com/DenisaCG)) 120 | - Upgrade dependencies [#27](https://github.com/QuantStack/jupydrive-s3/pull/27) ([@DenisaCG](https://github.com/DenisaCG)) 121 | 122 | ### Bugs fixed 123 | 124 | - Unpin `@lumino/coreutils` package [#25](https://github.com/QuantStack/jupydrive-s3/pull/25) ([@DenisaCG](https://github.com/DenisaCG)) 125 | 126 | ### Contributors to this release 127 | 128 | ([GitHub contributors page for this release](https://github.com/QuantStack/jupydrive-s3/graphs/contributors?from=2024-08-02&to=2024-10-23&type=c)) 129 | 130 | [@DenisaCG](https://github.com/search?q=repo%3AQuantStack%2Fjupydrive-s3+involves%3ADenisaCG+updated%3A2024-08-02..2024-10-23&type=Issues) 131 | 132 | ## 0.1.0a3 133 | 134 | ([Full Changelog](https://github.com/QuantStack/jupydrive-s3/compare/1183afc...98aec3dfa539442e8c6dd1451c0d0fd94fa2d98d)) 135 | 136 | ### Enhancements made 137 | 138 | - Add flag which toggles bucket switching UI [#23](https://github.com/QuantStack/jupydrive-s3/pull/23) ([@DenisaCG](https://github.com/DenisaCG)) 139 | 140 | ### Bugs fixed 141 | 142 | - Fix formatting root error [#21](https://github.com/QuantStack/jupydrive-s3/pull/21) ([@DenisaCG](https://github.com/DenisaCG)) 143 | 144 | ### Documentation improvements 145 | 146 | - Update README file [#22](https://github.com/QuantStack/jupydrive-s3/pull/22) ([@DenisaCG](https://github.com/DenisaCG)) 147 | 148 | ### Contributors to this release 149 | 150 | ([GitHub contributors page for this release](https://github.com/QuantStack/jupydrive-s3/graphs/contributors?from=2024-07-05&to=2024-07-09&type=c)) 151 | 152 | [@DenisaCG](https://github.com/search?q=repo%3AQuantStack%2Fjupydrive-s3+involves%3ADenisaCG+updated%3A2024-07-05..2024-07-09&type=Issues) 153 | 154 | ## 0.1.0a2 155 | 156 | ([Full Changelog](https://github.com/QuantStack/jupydrive-s3/compare/558f874...855034fae68961937a85ed29c81097c8ab51914a)) 157 | 158 | ### Enhancements made 159 | 160 | - Refactor functionalities and files [#19](https://github.com/QuantStack/jupydrive-s3/pull/19) ([@DenisaCG](https://github.com/DenisaCG)) 161 | - Make an arbitrary directory root [#17](https://github.com/QuantStack/jupydrive-s3/pull/17) ([@DenisaCG](https://github.com/DenisaCG)) 162 | - Extract config from `.env` file for dev mode [#15](https://github.com/QuantStack/jupydrive-s3/pull/15) ([@DenisaCG](https://github.com/DenisaCG)) 163 | - Refactor contents listing [#14](https://github.com/QuantStack/jupydrive-s3/pull/14) ([@DenisaCG](https://github.com/DenisaCG)) 164 | - Skip getting region if it is provided [#13](https://github.com/QuantStack/jupydrive-s3/pull/13) ([@trungleduc](https://github.com/trungleduc)) 165 | 166 | ### Bugs fixed 167 | 168 | - Improve switching drives functionality [#20](https://github.com/QuantStack/jupydrive-s3/pull/20) ([@DenisaCG](https://github.com/DenisaCG)) 169 | - Change the name of this package/repo to jupydrive-s3 [#18](https://github.com/QuantStack/jupydrive-s3/pull/18) ([@afshin](https://github.com/afshin)) 170 | - Refactor awaiting S3 operations [#16](https://github.com/QuantStack/jupydrive-s3/pull/16) ([@DenisaCG](https://github.com/DenisaCG)) 171 | 172 | ### Other merged PRs 173 | 174 | - v0.1.0-alpha [#12](https://github.com/QuantStack/jupydrive-s3/pull/12) ([@afshin](https://github.com/afshin)) 175 | 176 | ### Contributors to this release 177 | 178 | ([GitHub contributors page for this release](https://github.com/QuantStack/jupydrive-s3/graphs/contributors?from=2024-05-21&to=2024-06-25&type=c)) 179 | 180 | [@afshin](https://github.com/search?q=repo%3AQuantStack%2Fjupydrive-s3+involves%3Aafshin+updated%3A2024-05-21..2024-06-25&type=Issues) | [@DenisaCG](https://github.com/search?q=repo%3AQuantStack%2Fjupydrive-s3+involves%3ADenisaCG+updated%3A2024-05-21..2024-06-25&type=Issues) | [@trungleduc](https://github.com/search?q=repo%3AQuantStack%2Fjupydrive-s3+involves%3Atrungleduc+updated%3A2024-05-21..2024-06-25&type=Issues) 181 | 182 | ## 0.1.0-alpha1 183 | 184 | ([Full Changelog](https://github.com/QuantStack/jupydrive-s3/compare/558f874...855034fae68961937a85ed29c81097c8ab51914a)) 185 | 186 | ### Enhancements made 187 | 188 | - Refactor functionalities and files [#19](https://github.com/QuantStack/jupydrive-s3/pull/19) ([@DenisaCG](https://github.com/DenisaCG)) 189 | - Make an arbitrary directory root [#17](https://github.com/QuantStack/jupydrive-s3/pull/17) ([@DenisaCG](https://github.com/DenisaCG)) 190 | - Extract config from `.env` file for dev mode [#15](https://github.com/QuantStack/jupydrive-s3/pull/15) ([@DenisaCG](https://github.com/DenisaCG)) 191 | - Refactor contents listing [#14](https://github.com/QuantStack/jupydrive-s3/pull/14) ([@DenisaCG](https://github.com/DenisaCG)) 192 | - Skip getting region if it is provided [#13](https://github.com/QuantStack/jupydrive-s3/pull/13) ([@trungleduc](https://github.com/trungleduc)) 193 | 194 | ### Bugs fixed 195 | 196 | - Improve switching drives functionality [#20](https://github.com/QuantStack/jupydrive-s3/pull/20) ([@DenisaCG](https://github.com/DenisaCG)) 197 | - Change the name of this package/repo to jupydrive-s3 [#18](https://github.com/QuantStack/jupydrive-s3/pull/18) ([@afshin](https://github.com/afshin)) 198 | - Refactor awaiting S3 operations [#16](https://github.com/QuantStack/jupydrive-s3/pull/16) ([@DenisaCG](https://github.com/DenisaCG)) 199 | 200 | ### Other merged PRs 201 | 202 | - v0.1.0-alpha [#12](https://github.com/QuantStack/jupydrive-s3/pull/12) ([@afshin](https://github.com/afshin)) 203 | 204 | ### Contributors to this release 205 | 206 | ([GitHub contributors page for this release](https://github.com/QuantStack/jupydrive-s3/graphs/contributors?from=2024-05-21&to=2024-06-24&type=c)) 207 | 208 | [@afshin](https://github.com/search?q=repo%3AQuantStack%2Fjupydrive-s3+involves%3Aafshin+updated%3A2024-05-21..2024-06-24&type=Issues) | [@DenisaCG](https://github.com/search?q=repo%3AQuantStack%2Fjupydrive-s3+involves%3ADenisaCG+updated%3A2024-05-21..2024-06-24&type=Issues) | [@trungleduc](https://github.com/search?q=repo%3AQuantStack%2Fjupydrive-s3+involves%3Atrungleduc+updated%3A2024-05-21..2024-06-24&type=Issues) 209 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | import { 2 | ILabShell, 3 | JupyterFrontEndPlugin, 4 | IRouter, 5 | JupyterFrontEnd 6 | } from '@jupyterlab/application'; 7 | import { 8 | createToolbarFactory, 9 | IToolbarWidgetRegistry, 10 | setToolbar, 11 | showDialog, 12 | Dialog 13 | } from '@jupyterlab/apputils'; 14 | import { 15 | IDefaultFileBrowser, 16 | IFileBrowserFactory, 17 | FileBrowser, 18 | Uploader 19 | } from '@jupyterlab/filebrowser'; 20 | import { IStateDB } from '@jupyterlab/statedb'; 21 | import { editIcon } from '@jupyterlab/ui-components'; 22 | import { ISettingRegistry } from '@jupyterlab/settingregistry'; 23 | import { ITranslator } from '@jupyterlab/translation'; 24 | import { CommandRegistry } from '@lumino/commands'; 25 | import { Widget } from '@lumino/widgets'; 26 | import { 27 | FilenameSearcher, 28 | IScore, 29 | folderIcon 30 | } from '@jupyterlab/ui-components'; 31 | import { ReadonlyPartialJSONObject, Token } from '@lumino/coreutils'; 32 | import { S3ClientConfig } from '@aws-sdk/client-s3'; 33 | import { Drive } from './s3contents'; 34 | import { DriveIcon } from './icons'; 35 | import { SecretsManager, ISecretsManager } from 'jupyter-secrets-manager'; 36 | 37 | /** 38 | * The command IDs used by the filebrowser plugin. 39 | */ 40 | namespace CommandIDs { 41 | export const openPath = 'filebrowser:open-path'; 42 | export const openSwitchDrive = 'drives:open-switch-drive-dialog'; 43 | export const copyToAnotherBucket = 'drives:copy-to-another-bucket'; 44 | export const toggleBucketSwitching = 'drives:toggle-bucket-switching-ui'; 45 | export const toggleBrowser = 'filebrowser:toggle-main'; 46 | export const rename = 'drives:rename'; 47 | } 48 | 49 | const FILE_BROWSER_FACTORY = 'DriveBrowser'; 50 | const FILE_BROWSER_PLUGIN_ID = 'jupydrive-s3:file-browser-toolbar'; 51 | 52 | /** 53 | * The class name added to the drive filebrowser filterbox node. 54 | */ 55 | const FILTERBOX_CLASS = 'jp-DriveBrowser-filterBox'; 56 | 57 | /** 58 | * The class name added to file dialogs. 59 | */ 60 | const FILE_DIALOG_CLASS = 'jp-FileDialog'; 61 | 62 | /** 63 | * The class name added for the new drive label in the switch drive dialog. 64 | */ 65 | const SWITCH_DRIVE_TITLE_CLASS = 'jp-new-drive-title'; 66 | 67 | /** 68 | * The ID used for saving the drive name to the persistent state databse. 69 | */ 70 | const DRIVE_STATE_ID = 'jupydrive-s3:drive-name-id'; 71 | 72 | /** 73 | * A promise that resolves to S3 authentication credentials. 74 | */ 75 | export interface IS3Auth { 76 | factory: () => Promise<{ 77 | name: string; 78 | root: string; 79 | config: S3ClientConfig; 80 | secretsManager?: ISecretsManager; 81 | token?: symbol; 82 | }>; 83 | } 84 | 85 | /** 86 | * A token for a plugin that provides S3 authentication. 87 | */ 88 | export const IS3Auth = new Token('jupydrive-s3:auth-file-browser'); 89 | 90 | /** 91 | * Authentification file browser ID. 92 | */ 93 | export const AUTH_FILEBROWSER_ID = 'jupydrive-s3:auth-file-browser'; 94 | 95 | /** 96 | * The auth/credentials provider for the file browser. 97 | */ 98 | const authFileBrowser: JupyterFrontEndPlugin = SecretsManager.sign( 99 | AUTH_FILEBROWSER_ID, 100 | token => ({ 101 | id: 'jupydrive-s3:auth-file-browser', 102 | description: 'The default file browser auth/credentials provider', 103 | requires: [ISettingRegistry, ISecretsManager], 104 | provides: IS3Auth, 105 | activate: ( 106 | app: JupyterFrontEnd, 107 | settings: ISettingRegistry, 108 | secretsManager: ISecretsManager 109 | ): IS3Auth => { 110 | const secretFields: string[] = ['accessKeyId', 'secretAccessKey']; 111 | 112 | function loadCredentials(setting: ISettingRegistry.ISettings) { 113 | const bucket = setting.get('bucket').composite as string; 114 | const root = setting.get('root').composite as string; 115 | const endpoint = setting.get('endpoint').composite as string; 116 | const region = setting.get('region').composite as string; 117 | 118 | secretFields.forEach((key: string) => { 119 | const secretValue = setting.get(key).composite as string; 120 | // Save secret to secret manager. 121 | secretsManager.set( 122 | token, 123 | AUTH_FILEBROWSER_ID, 124 | AUTH_FILEBROWSER_ID + '::' + key, 125 | { 126 | namespace: AUTH_FILEBROWSER_ID, 127 | id: AUTH_FILEBROWSER_ID + '::' + key, 128 | value: secretValue 129 | } 130 | ); 131 | }); 132 | 133 | return { 134 | name: bucket, 135 | root: root, 136 | config: { 137 | forcePathStyle: true, 138 | endpoint: endpoint, 139 | region: region, 140 | credentials: { 141 | accessKeyId: '***', 142 | secretAccessKey: '***' 143 | } 144 | }, 145 | secretsManager: secretsManager, 146 | token: token 147 | }; 148 | } 149 | 150 | const attachSecretInput = (key: string) => { 151 | const input = document.getElementById( 152 | 'jp-SettingsEditor-jupydrive-s3:auth-file-browser_' + key 153 | ) as HTMLInputElement; 154 | if (input) { 155 | input.type = 'password'; 156 | secretsManager.attach( 157 | token, 158 | AUTH_FILEBROWSER_ID, 159 | AUTH_FILEBROWSER_ID + '::' + key, 160 | input 161 | ); 162 | } else { 163 | setTimeout(() => attachSecretInput(key), 300); 164 | } 165 | }; 166 | 167 | // Watch for DOM changes to make sure secret inputs are attached. 168 | const observer = new MutationObserver(() => { 169 | secretFields.forEach((key: string) => attachSecretInput(key)); 170 | }); 171 | 172 | observer.observe(document.body, { 173 | childList: true, 174 | subtree: true 175 | }); 176 | 177 | const getInitalSettings = async () => { 178 | // Read the inital credentials settings. 179 | const setting = await settings.load(authFileBrowser.id); 180 | const initial = loadCredentials(setting); 181 | if (initial.name !== '') { 182 | return initial; 183 | } else { 184 | return null; 185 | } 186 | }; 187 | 188 | // Wait for the application to be restored and for the 189 | // settings of credentials to be loaded. 190 | Promise.all([app.restored, settings.load(authFileBrowser.id)]).then( 191 | ([_, settingCredentials]) => { 192 | // Listen for the plugin setting changes using Signal. 193 | settingCredentials.changed.connect(() => { 194 | const s3Config = loadCredentials(settingCredentials); 195 | 196 | // Connect to drive using new config. 197 | const S3Drive = new Drive({ ...s3Config, secretsManager, token }); 198 | app.serviceManager.contents.addDrive(S3Drive); 199 | }); 200 | } 201 | ); 202 | 203 | return { 204 | factory: async () => { 205 | return ( 206 | (await getInitalSettings()) ?? { 207 | name: process.env.JP_S3_BUCKET ?? 'jupyter-drives-test-bucket-1', 208 | root: process.env.JP_S3_ROOT ?? '', 209 | config: { 210 | forcePathStyle: true, 211 | endpoint: 212 | process.env.JP_S3_ENDPOINT ?? 'https://example.com/s3', 213 | region: process.env.JP_S3_REGION ?? 'eu-west-1', 214 | credentials: { 215 | accessKeyId: 216 | process.env.JP_S3_ACCESS_KEY_ID ?? 217 | 'abcdefghijklmnopqrstuvwxyz', 218 | secretAccessKey: 219 | process.env.JP_S3_SECRET_ACCESS_KEY ?? 220 | 'SECRET123456789abcdefghijklmnopqrstuvwxyz' 221 | } 222 | } 223 | } 224 | ); 225 | } 226 | }; 227 | } 228 | }) 229 | ); 230 | 231 | /** 232 | * The default file browser factory provider. 233 | */ 234 | const defaultFileBrowser: JupyterFrontEndPlugin = { 235 | id: 'jupydrive-s3:default-file-browser', 236 | description: 'The default file browser factory provider', 237 | provides: IDefaultFileBrowser, 238 | requires: [IFileBrowserFactory, IS3Auth, IStateDB, ISettingRegistry], 239 | optional: [IRouter, JupyterFrontEnd.ITreeResolver, ILabShell], 240 | activate: async ( 241 | app: JupyterFrontEnd, 242 | fileBrowserFactory: IFileBrowserFactory, 243 | s3auth: IS3Auth, 244 | state: IStateDB, 245 | settings: ISettingRegistry, 246 | router: IRouter | null, 247 | tree: JupyterFrontEnd.ITreeResolver | null, 248 | labShell: ILabShell | null 249 | ): Promise => { 250 | const { commands } = app; 251 | const auth = await s3auth.factory(); 252 | // create S3 drive 253 | const S3Drive = new Drive({ 254 | name: auth.name, 255 | root: auth.root, 256 | config: auth.config, 257 | secretsManager: auth.secretsManager, 258 | token: auth.token 259 | }); 260 | 261 | app.serviceManager.contents.addDrive(S3Drive); 262 | 263 | // get registered file types 264 | S3Drive.getRegisteredFileTypes(app); 265 | 266 | // Manually restore and load the default file browser. 267 | const defaultBrowser = fileBrowserFactory.createFileBrowser( 268 | 'drivebrowser', 269 | { 270 | auto: false, 271 | restore: false, 272 | driveName: S3Drive.name 273 | } 274 | ); 275 | 276 | function loadSetting(setting: ISettingRegistry.ISettings): boolean { 277 | // Read the settings and convert to the correct type 278 | const bucketSwitching = setting.get('bucketSwitching') 279 | .composite as boolean; 280 | return bucketSwitching; 281 | } 282 | 283 | // Set attributes when adding the browser to the UI 284 | defaultBrowser.node.setAttribute('role', 'region'); 285 | defaultBrowser.node.setAttribute('aria-label', 'Drive Browser Section'); 286 | defaultBrowser.title.icon = folderIcon; 287 | 288 | // Show the current file browser shortcut in its title. 289 | const updateBrowserTitle = () => { 290 | const binding = app.commands.keyBindings.find( 291 | b => b.command === CommandIDs.toggleBrowser 292 | ); 293 | if (binding) { 294 | const ks = binding.keys.map(CommandRegistry.formatKeystroke).join(', '); 295 | defaultBrowser.title.caption = 'Drive Browser (' + ks + ')'; 296 | } else { 297 | defaultBrowser.title.caption = 'Drive Browser'; 298 | } 299 | }; 300 | updateBrowserTitle(); 301 | app.commands.keyBindingChanged.connect(() => { 302 | updateBrowserTitle(); 303 | }); 304 | 305 | // Wait for the application to be restored and for the 306 | // settings and persistent state database to be loaded 307 | app.restored 308 | .then(() => 309 | Promise.all([ 310 | state.fetch(DRIVE_STATE_ID), 311 | settings.load(toolbarFileBrowser.id) 312 | ]) 313 | ) 314 | .then(([value, setting]) => { 315 | if (value) { 316 | const bucket = (value as ReadonlyPartialJSONObject)[ 317 | 'bucket' 318 | ] as string; 319 | const root = (value as ReadonlyPartialJSONObject)['root'] as string; 320 | 321 | // if values are stored, change bucket name and root 322 | S3Drive.name = bucket; 323 | S3Drive.root = root; 324 | app.serviceManager.contents.addDrive(S3Drive); 325 | } 326 | 327 | // Listen for the plugin setting changes using Signal. 328 | setting.changed.connect(loadSetting); 329 | 330 | // adding commands 331 | Private.addCommands(app, S3Drive, fileBrowserFactory, state, setting); 332 | }); 333 | 334 | void Private.restoreBrowser( 335 | defaultBrowser, 336 | commands, 337 | router, 338 | tree, 339 | labShell 340 | ); 341 | 342 | return defaultBrowser; 343 | } 344 | }; 345 | 346 | /** 347 | * File browser toolbar buttons. 348 | */ 349 | const toolbarFileBrowser: JupyterFrontEndPlugin = { 350 | id: 'jupydrive-s3:file-browser-toolbar', 351 | description: 'The toolbar for the drives file browser', 352 | requires: [ 353 | IDefaultFileBrowser, 354 | IToolbarWidgetRegistry, 355 | ISettingRegistry, 356 | ITranslator, 357 | IFileBrowserFactory 358 | ], 359 | autoStart: true, 360 | activate: async ( 361 | _: JupyterFrontEnd, 362 | fileBrowser: IDefaultFileBrowser, 363 | toolbarRegistry: IToolbarWidgetRegistry, 364 | settingsRegistry: ISettingRegistry, 365 | translator: ITranslator 366 | ): Promise => { 367 | toolbarRegistry.addFactory( 368 | FILE_BROWSER_FACTORY, 369 | 'uploaderTest', 370 | (fileBrowser: FileBrowser) => 371 | new Uploader({ model: fileBrowser.model, translator }) 372 | ); 373 | 374 | toolbarRegistry.addFactory( 375 | FILE_BROWSER_FACTORY, 376 | 'fileNameSearcherTest', 377 | (fileBrowser: FileBrowser) => { 378 | const searcher = FilenameSearcher({ 379 | updateFilter: ( 380 | filterFn: (item: string) => Partial | null, 381 | query?: string 382 | ) => { 383 | fileBrowser.model.setFilter(value => { 384 | return filterFn(value.name.toLowerCase()); 385 | }); 386 | }, 387 | useFuzzyFilter: true, 388 | placeholder: 'Filter files by names', 389 | forceRefresh: true 390 | }); 391 | searcher.addClass(FILTERBOX_CLASS); 392 | return searcher; 393 | } 394 | ); 395 | 396 | // connect the filebrowser toolbar to the settings registry for the plugin 397 | setToolbar( 398 | fileBrowser, 399 | createToolbarFactory( 400 | toolbarRegistry, 401 | settingsRegistry, 402 | FILE_BROWSER_FACTORY, 403 | FILE_BROWSER_PLUGIN_ID, 404 | translator 405 | ) 406 | ); 407 | } 408 | }; 409 | 410 | /** 411 | * Export the plugins as default. 412 | */ 413 | const plugins: JupyterFrontEndPlugin[] = [ 414 | authFileBrowser, 415 | defaultFileBrowser, 416 | toolbarFileBrowser 417 | ]; 418 | 419 | export default plugins; 420 | 421 | namespace Private { 422 | /** 423 | * Create the node for a switch drive handler. 424 | */ 425 | const createSwitchDriveNode = (oldDriveName: string): HTMLElement => { 426 | const body = document.createElement('div'); 427 | 428 | const existingLabel = document.createElement('label'); 429 | existingLabel.textContent = 'Current Drive: ' + oldDriveName; 430 | 431 | const bucket = document.createElement('label'); 432 | bucket.textContent = 'Switch to another Drive'; 433 | bucket.className = SWITCH_DRIVE_TITLE_CLASS; 434 | const bucketName = document.createElement('input'); 435 | 436 | const root = document.createElement('label'); 437 | root.textContent = 'with root'; 438 | root.className = SWITCH_DRIVE_TITLE_CLASS; 439 | const rootPath = document.createElement('input'); 440 | 441 | body.appendChild(existingLabel); 442 | body.appendChild(bucket); 443 | body.appendChild(bucketName); 444 | body.appendChild(root); 445 | body.appendChild(rootPath); 446 | return body; 447 | }; 448 | 449 | /** 450 | * Create the node for a copy to another bucket handler. 451 | */ 452 | const createCopyToAnotherBucketNode = (): HTMLElement => { 453 | const body = document.createElement('div'); 454 | 455 | const bucket = document.createElement('label'); 456 | bucket.textContent = 'Copy to another Bucket'; 457 | bucket.className = SWITCH_DRIVE_TITLE_CLASS; 458 | const bucketName = document.createElement('input'); 459 | 460 | const root = document.createElement('label'); 461 | root.textContent = 'Location within the Bucket'; 462 | root.className = SWITCH_DRIVE_TITLE_CLASS; 463 | const rootPath = document.createElement('input'); 464 | 465 | body.appendChild(bucket); 466 | body.appendChild(bucketName); 467 | body.appendChild(root); 468 | body.appendChild(rootPath); 469 | return body; 470 | }; 471 | 472 | /** 473 | * A widget used to copy files or directories to another bucket. 474 | */ 475 | export class CopyToAnotherBucket extends Widget { 476 | /** 477 | * Construct a new "copy-to-another-bucket" dialog. 478 | */ 479 | constructor() { 480 | super({ node: createCopyToAnotherBucketNode() }); 481 | this.onAfterAttach(); 482 | } 483 | 484 | /** 485 | * The text input node for bucket name. 486 | */ 487 | protected get bucketInput(): HTMLInputElement { 488 | return this.node.getElementsByTagName('input')[0] as HTMLInputElement; 489 | } 490 | /** 491 | * The text input node root directory. 492 | */ 493 | protected get rootInput(): HTMLInputElement { 494 | return this.node.getElementsByTagName('input')[1] as HTMLInputElement; 495 | } 496 | 497 | protected onAfterAttach(): void { 498 | this.addClass(FILE_DIALOG_CLASS); 499 | const [bucket, root] = this.getValue(); 500 | this.bucketInput.setSelectionRange(0, bucket.length); 501 | this.rootInput.setSelectionRange(0, root.length); 502 | } 503 | 504 | /** 505 | * Get the value of the widget. 506 | */ 507 | getValue(): [bucket: string, root: string] { 508 | return [this.bucketInput.value, this.rootInput.value]; 509 | } 510 | } 511 | 512 | /** 513 | * A widget used to switch to another drive. 514 | */ 515 | export class SwitchDriveHandler extends Widget { 516 | /** 517 | * Construct a new "switch-drive" dialog. 518 | */ 519 | constructor(oldDriveName: string) { 520 | super({ node: createSwitchDriveNode(oldDriveName) }); 521 | this.onAfterAttach(); 522 | } 523 | 524 | protected onAfterAttach(): void { 525 | this.addClass(FILE_DIALOG_CLASS); 526 | const bucket = this.bucketInput.value; 527 | this.bucketInput.setSelectionRange(0, bucket.length); 528 | const root = this.rootInput.value; 529 | this.rootInput.setSelectionRange(0, root.length); 530 | } 531 | 532 | /** 533 | * Get the input text node for bucket name. 534 | */ 535 | get bucketInput(): HTMLInputElement { 536 | return this.node.getElementsByTagName('input')[0] as HTMLInputElement; 537 | } 538 | 539 | /** 540 | * Get the input text node for path to root. 541 | */ 542 | get rootInput(): HTMLInputElement { 543 | return this.node.getElementsByTagName('input')[1] as HTMLInputElement; 544 | } 545 | 546 | /** 547 | * Get the value of the widget. 548 | */ 549 | getValue(): string[] { 550 | return [this.bucketInput.value, this.rootInput.value]; 551 | } 552 | } 553 | 554 | export function addCommands( 555 | app: JupyterFrontEnd, 556 | drive: Drive, 557 | factory: IFileBrowserFactory, 558 | state: IStateDB, 559 | settings: ISettingRegistry.ISettings 560 | ): void { 561 | const { tracker } = factory; 562 | 563 | app.commands.addCommand(CommandIDs.openSwitchDrive, { 564 | isVisible: () => { 565 | return (settings.get('bucketSwitching').composite as boolean) ?? false; 566 | }, 567 | execute: async () => { 568 | return showDialog({ 569 | body: new Private.SwitchDriveHandler(drive.name), 570 | focusNodeSelector: 'input', 571 | buttons: [ 572 | Dialog.okButton({ 573 | label: 'Switch Drive', 574 | ariaLabel: 'Switch to another Drive' 575 | }) 576 | ] 577 | }).then(result => { 578 | if (result.value) { 579 | drive.name = result.value[0]; 580 | drive.root = result.value[1]; 581 | app.serviceManager.contents.addDrive(drive); 582 | 583 | // saving the new drive name to the persistent state database 584 | state.save(DRIVE_STATE_ID, { 585 | bucket: result.value[0], 586 | root: result.value[1] 587 | }); 588 | } 589 | }); 590 | }, 591 | icon: DriveIcon.bindprops({ stylesheet: 'menuItem' }) 592 | }); 593 | 594 | app.commands.addCommand(CommandIDs.copyToAnotherBucket, { 595 | execute: async () => { 596 | return showDialog({ 597 | body: new Private.CopyToAnotherBucket(), 598 | focusNodeSelector: 'input', 599 | buttons: [ 600 | Dialog.okButton({ 601 | label: 'Copy', 602 | ariaLabel: 'Copy to another Bucket' 603 | }) 604 | ] 605 | }).then(result => { 606 | const widget = tracker.currentWidget; 607 | 608 | if (widget) { 609 | const path = widget 610 | .selectedItems() 611 | .next()! 612 | .value.path.split(':')[1]; 613 | 614 | if (result.value) { 615 | drive.copyToAnotherBucket(path, result.value[1], result.value[0]); 616 | } 617 | } 618 | }); 619 | }, 620 | icon: DriveIcon.bindprops({ stylesheet: 'menuItem' }), 621 | label: 'Copy to another Bucket' 622 | }); 623 | 624 | app.contextMenu.addItem({ 625 | command: CommandIDs.copyToAnotherBucket, 626 | selector: 627 | '.jp-SidePanel .jp-DirListing-content .jp-DirListing-item[data-isDir]', 628 | rank: 10 629 | }); 630 | 631 | app.commands.addCommand(CommandIDs.rename, { 632 | execute: args => { 633 | const widget = tracker.currentWidget; 634 | 635 | if (widget) { 636 | return widget.rename(); 637 | } 638 | }, 639 | isVisible: () => 640 | // So long as this command only handles one file at time, don't show it 641 | // if multiple files are selected. 642 | !!tracker.currentWidget && 643 | Array.from(tracker.currentWidget.selectedItems()).length === 1, 644 | isEnabled: () => 645 | // Disable directory rename for S3 folders. 646 | !!tracker.currentWidget && 647 | tracker.currentWidget?.selectedItems().next()!.value.type !== 648 | 'directory', 649 | icon: editIcon.bindprops({ stylesheet: 'menuItem' }), 650 | label: 'Rename', 651 | mnemonic: 0 652 | }); 653 | } 654 | 655 | /** 656 | * Restores file browser state and overrides state if tree resolver resolves. 657 | */ 658 | export async function restoreBrowser( 659 | browser: FileBrowser, 660 | commands: CommandRegistry, 661 | router: IRouter | null, 662 | tree: JupyterFrontEnd.ITreeResolver | null, 663 | labShell: ILabShell | null 664 | ): Promise { 665 | const restoring = 'jp-mod-restoring'; 666 | 667 | browser.addClass(restoring); 668 | 669 | if (!router) { 670 | await browser.model.restore(browser.id); 671 | await browser.model.refresh(); 672 | browser.removeClass(restoring); 673 | return; 674 | } 675 | 676 | const listener = async () => { 677 | router.routed.disconnect(listener); 678 | 679 | const paths = await tree?.paths; 680 | if (paths?.file || paths?.browser) { 681 | // Restore the model without populating it. 682 | await browser.model.restore(browser.id, false); 683 | if (paths.file) { 684 | await commands.execute(CommandIDs.openPath, { 685 | path: paths.file, 686 | dontShowBrowser: true 687 | }); 688 | } 689 | if (paths.browser) { 690 | await commands.execute(CommandIDs.openPath, { 691 | path: paths.browser, 692 | dontShowBrowser: true 693 | }); 694 | } 695 | } else { 696 | await browser.model.restore(browser.id); 697 | await browser.model.refresh(); 698 | } 699 | browser.removeClass(restoring); 700 | 701 | if (labShell?.isEmpty('main')) { 702 | void commands.execute('launcher:create'); 703 | } 704 | }; 705 | router.routed.connect(listener); 706 | } 707 | } 708 | -------------------------------------------------------------------------------- /src/s3contents.ts: -------------------------------------------------------------------------------- 1 | import { Signal, ISignal } from '@lumino/signaling'; 2 | import { Contents, ServerConnection } from '@jupyterlab/services'; 3 | import { URLExt, PathExt } from '@jupyterlab/coreutils'; 4 | import { JupyterFrontEnd } from '@jupyterlab/application'; 5 | 6 | import { 7 | S3Client, 8 | GetBucketLocationCommand, 9 | S3ClientConfig 10 | } from '@aws-sdk/client-s3'; 11 | 12 | import { 13 | checkS3Object, 14 | createS3Object, 15 | copyS3Objects, 16 | countS3ObjectNameAppearances, 17 | deleteS3Objects, 18 | presignedS3Url, 19 | renameS3Objects, 20 | listS3Contents, 21 | IRegisteredFileTypes, 22 | isDirectory 23 | } from './s3'; 24 | import { ISecretsManager } from 'jupyter-secrets-manager'; 25 | import { AUTH_FILEBROWSER_ID } from '.'; 26 | 27 | let data: Contents.IModel = { 28 | name: '', 29 | path: '', 30 | last_modified: '', 31 | created: '', 32 | content: null, 33 | format: null, 34 | mimetype: '', 35 | size: 0, 36 | writable: true, 37 | type: '' 38 | }; 39 | 40 | export class Drive implements Contents.IDrive { 41 | /** 42 | * Construct a new drive object. 43 | * 44 | * @param options - The options used to initialize the object. 45 | */ 46 | constructor(options: Drive.IOptions) { 47 | const { config, name, root, secretsManager, token } = options; 48 | this._serverSettings = ServerConnection.makeSettings(); 49 | 50 | if (secretsManager) { 51 | this._secretsManager = secretsManager; 52 | } 53 | if (token) { 54 | Private.setToken(token); 55 | // Delete token used by secrets manager. 56 | delete options.token; 57 | } 58 | const s3Config = { ...config }; 59 | if (secretsManager && token) { 60 | // Retrieve secrets and set up S3 client. 61 | Object.entries(s3Config!.credentials!).forEach(([key, _]: any) => { 62 | secretsManager! 63 | .get(token, AUTH_FILEBROWSER_ID, AUTH_FILEBROWSER_ID + '::' + key) 64 | .then((secret: any) => { 65 | s3Config!.credentials![key] = secret.value; 66 | }) 67 | .catch(() => 68 | console.error('Error occurred retrieving secret: ', key) 69 | ); 70 | }); 71 | } 72 | this._s3Client = new S3Client(s3Config ?? {}); 73 | 74 | this._name = name; 75 | this._baseUrl = URLExt.join( 76 | (config?.endpoint as string) ?? 'https://s3.amazonaws.com/', 77 | this._name 78 | ); 79 | this._provider = 'S3'; 80 | const region = config?.region; 81 | if (typeof region === 'string') { 82 | this._region = region; 83 | } else { 84 | const regionPromise = region ?? this.getRegion; 85 | regionPromise().then((region: string) => { 86 | this._region = region!; 87 | }); 88 | } 89 | this._root = root; 90 | this._registeredFileTypes = {}; 91 | } 92 | 93 | /** 94 | * The Drive S3 client 95 | */ 96 | get s3Client(): S3Client { 97 | return this._s3Client; 98 | } 99 | 100 | /** 101 | * The Drive S3 client 102 | */ 103 | set s3Client(s3Client: S3Client) { 104 | this._s3Client = s3Client; 105 | } 106 | 107 | /** 108 | * The S3 config. 109 | */ 110 | get config(): S3ClientConfig { 111 | return this._config; 112 | } 113 | 114 | /** 115 | * The S3 config. 116 | */ 117 | set config(config: S3ClientConfig) { 118 | this._config = config; 119 | this._s3Client = new S3Client(config); 120 | } 121 | /** 122 | * The Drive base URL 123 | */ 124 | get baseUrl(): string { 125 | return this._baseUrl; 126 | } 127 | 128 | /** 129 | * The Drive base URL 130 | */ 131 | set baseUrl(url: string) { 132 | this._baseUrl = url; 133 | } 134 | 135 | /** 136 | * The Drive name getter 137 | */ 138 | get name(): string { 139 | return this._name; 140 | } 141 | 142 | /** 143 | * The Drive name setter 144 | */ 145 | set name(name: string) { 146 | this._name = name; 147 | 148 | // if name of drive is changed, the filebrowser needs to refresh its contents 149 | // as we are switching to another bucket 150 | this._fileChanged.emit({ 151 | type: 'new', 152 | oldValue: null, 153 | newValue: { path: '' } 154 | }); 155 | } 156 | 157 | /** 158 | * The Drive root getter 159 | */ 160 | get root(): string { 161 | return this._root; 162 | } 163 | 164 | /** 165 | * The Drive root setter 166 | */ 167 | set root(root: string) { 168 | this.formatRoot(root ?? '').then(root => (this._root = root)); 169 | } 170 | 171 | /** 172 | * Get the formatted root checker. 173 | */ 174 | get isRootFormatted(): boolean { 175 | return this._isRootFormatted; 176 | } 177 | 178 | /** 179 | * Set the formatted root checker. 180 | */ 181 | set isRootFormatted(isRootFormatted) { 182 | this._isRootFormatted = isRootFormatted; 183 | } 184 | 185 | /** 186 | * The Drive provider getter 187 | */ 188 | get provider(): string { 189 | return this._provider; 190 | } 191 | 192 | /** 193 | * The Drive provider setter */ 194 | set provider(name: string) { 195 | this._provider = name; 196 | } 197 | 198 | /** 199 | * The Drive region getter 200 | */ 201 | get region(): string { 202 | return this._region; 203 | } 204 | 205 | /** 206 | * The Drive region setter 207 | */ 208 | set region(region: string) { 209 | this._region = region; 210 | } 211 | 212 | /** 213 | * The Drive creationDate getter 214 | */ 215 | get creationDate(): string { 216 | return this._creationDate; 217 | } 218 | 219 | /** 220 | * The Drive creationDate setter 221 | */ 222 | set creationDate(date: string) { 223 | this._creationDate = date; 224 | } 225 | 226 | /** 227 | * The registered file types 228 | */ 229 | get registeredFileTypes(): IRegisteredFileTypes { 230 | return this._registeredFileTypes; 231 | } 232 | 233 | /** 234 | * The registered file types 235 | */ 236 | set registeredFileTypes(fileTypes: IRegisteredFileTypes) { 237 | this._registeredFileTypes = fileTypes; 238 | } 239 | 240 | /** 241 | * Settings for the notebook server. 242 | */ 243 | get serverSettings(): ServerConnection.ISettings { 244 | return this._serverSettings; 245 | } 246 | 247 | /** 248 | * A signal emitted when a file operation takes place. 249 | */ 250 | get fileChanged(): ISignal { 251 | return this._fileChanged; 252 | } 253 | 254 | /** 255 | * Test whether the manager has been disposed. 256 | */ 257 | get isDisposed(): boolean { 258 | return this._isDisposed; 259 | } 260 | 261 | /** 262 | * A signal emitted when the drive is disposed. 263 | */ 264 | get disposed(): ISignal { 265 | return this._disposed; 266 | } 267 | 268 | /** 269 | * Dispose of the resources held by the manager. 270 | */ 271 | dispose(): void { 272 | if (this.isDisposed) { 273 | return; 274 | } 275 | if (this._secretsManager) { 276 | this._secretsManager.detachAll(Private.getToken(), AUTH_FILEBROWSER_ID); 277 | } 278 | this._isDisposed = true; 279 | this._disposed.emit(); 280 | Signal.clearData(this); 281 | } 282 | 283 | /** 284 | * Get an encoded download url given a file path. 285 | * 286 | * @param path - An absolute POSIX file path on the server. 287 | * 288 | * #### Notes 289 | * It is expected that the path contains no relative paths, 290 | * use [[ContentsManager.getAbsolutePath]] to get an absolute 291 | * path if necessary. 292 | */ 293 | async getDownloadUrl(path: string): Promise { 294 | const url = await presignedS3Url(this._s3Client, this._name, path); 295 | return url; 296 | } 297 | 298 | /** 299 | * Get a file or directory. 300 | * 301 | * @param localPath: The path to the file. 302 | * 303 | * @param options: The options used to fetch the file. 304 | * 305 | * @returns A promise which resolves with the file content. 306 | * 307 | * Uses the [Jupyter Notebook API](https://petstore.swagger.io/?url=https://raw.githubusercontent.com/jupyter-server/jupyter_server/main/jupyter_server/services/api/api.yaml#!/contents) and validates the response model. 308 | */ 309 | async get( 310 | path: string, 311 | options?: Contents.IFetchOptions 312 | ): Promise { 313 | path = path.replace(this._name + '/', ''); 314 | 315 | // format root the first time contents are retrieved 316 | if (!this._isRootFormatted) { 317 | this.formatRoot(this._root ?? '').then((root: string) => { 318 | this._root = root; 319 | }); 320 | this._isRootFormatted = true; 321 | } 322 | 323 | // listing the contents of a directory or retriving the contents of a file 324 | data = await listS3Contents( 325 | this._s3Client, 326 | this._name, 327 | this.root, 328 | this.registeredFileTypes, 329 | path 330 | ); 331 | 332 | Contents.validateContentsModel(data); 333 | return data; 334 | } 335 | 336 | /** 337 | * Create a new untitled file or directory in the specified directory path. 338 | * 339 | * @param options: The options used to create the file. 340 | * 341 | * @returns A promise which resolves with the created file content when the 342 | * file is created. 343 | */ 344 | async newUntitled( 345 | options: Contents.ICreateOptions = {} 346 | ): Promise { 347 | // get current list of contents of drive 348 | const old_data = await listS3Contents( 349 | this._s3Client, 350 | this._name, 351 | this._root, 352 | this.registeredFileTypes, 353 | options.path 354 | ); 355 | 356 | if (options.type !== undefined) { 357 | // get incremented untitled name 358 | const name = this.incrementUntitledName(old_data, options); 359 | data = await createS3Object( 360 | this._s3Client, 361 | this._name, 362 | this._root, 363 | name, 364 | options.path ? PathExt.join(options.path, name) : name, 365 | '', // create new file with empty body, 366 | this.registeredFileTypes, 367 | options.type === 'directory' ? true : false 368 | ); 369 | } else { 370 | console.warn('Type of new element is undefined'); 371 | } 372 | 373 | Contents.validateContentsModel(data); 374 | this._fileChanged.emit({ 375 | type: 'new', 376 | oldValue: null, 377 | newValue: data 378 | }); 379 | return data; 380 | } 381 | 382 | protected incrementUntitledName( 383 | contents: Contents.IModel, 384 | options: Contents.ICreateOptions 385 | ): string { 386 | const content: Array = contents.content; 387 | let name: string = ''; 388 | let countText = 0; 389 | let countDir = 0; 390 | let countNotebook = 0; 391 | if (options.type === 'notebook') { 392 | options.ext = 'ipynb'; 393 | } 394 | 395 | content.forEach(item => { 396 | if (options.ext !== undefined) { 397 | if (item.name.includes('untitled') && item.name.includes('.txt')) { 398 | countText = countText + 1; 399 | } else if ( 400 | item.name.includes('Untitled') && 401 | item.name.includes('.ipynb') 402 | ) { 403 | countNotebook = countNotebook + 1; 404 | } 405 | } else if (item.name.includes('Untitled Folder')) { 406 | countDir = countDir + 1; 407 | } 408 | }); 409 | 410 | if (options.ext === 'txt') { 411 | if (countText === 0) { 412 | name = 'untitled' + '.' + options.ext; 413 | } else { 414 | name = 'untitled' + countText + '.' + options.ext; 415 | } 416 | } 417 | if (options.ext === 'ipynb') { 418 | if (countNotebook === 0) { 419 | name = 'Untitled' + '.' + options.ext; 420 | } else { 421 | name = 'Untitled' + countNotebook + '.' + options.ext; 422 | } 423 | } else if (options.type === 'directory') { 424 | if (countDir === 0) { 425 | name = 'Untitled Folder'; 426 | } else { 427 | name = 'Untitled Folder ' + countDir; 428 | } 429 | } 430 | return name; 431 | } 432 | 433 | /** 434 | * Delete a file. 435 | * 436 | * @param path - The path to the file. 437 | * 438 | * @returns A promise which resolves when the file is deleted. 439 | */ 440 | async delete(localPath: string): Promise { 441 | await deleteS3Objects(this._s3Client, this._name, this._root, localPath); 442 | 443 | this._fileChanged.emit({ 444 | type: 'delete', 445 | oldValue: { path: localPath }, 446 | newValue: { path: undefined } 447 | }); 448 | } 449 | 450 | /** 451 | * Rename a file or directory. 452 | * 453 | * @param oldLocalPath - The original file path. 454 | * 455 | * @param newLocalPath - The new file path. 456 | * 457 | * @returns A promise which resolves with the new file contents model when 458 | * the file is renamed. 459 | * 460 | * #### Notes 461 | * Uses the [Jupyter Notebook API](https://petstore.swagger.io/?url=https://raw.githubusercontent.com/jupyter-server/jupyter_server/main/jupyter_server/services/api/api.yaml#!/contents) and validates the response model. 462 | */ 463 | async rename( 464 | oldLocalPath: string, 465 | newLocalPath: string, 466 | options: Contents.ICreateOptions = {} 467 | ): Promise { 468 | let newFileName = PathExt.basename(newLocalPath); 469 | const isDir: boolean = await isDirectory( 470 | this._s3Client, 471 | this._name, 472 | oldLocalPath 473 | ); 474 | 475 | try { 476 | await checkS3Object( 477 | this._s3Client, 478 | this._name, 479 | this._root, 480 | newLocalPath, 481 | isDir 482 | ); 483 | newFileName = await this.incrementName(newLocalPath, this._name, isDir); 484 | } catch (error) { 485 | // HEAD request failed for this file name, continue, as name doesn't already exist. 486 | } finally { 487 | data = await renameS3Objects( 488 | this._s3Client, 489 | this._name, 490 | this._root, 491 | oldLocalPath, 492 | newLocalPath, 493 | newFileName, 494 | isDir, 495 | this._registeredFileTypes 496 | ); 497 | } 498 | 499 | Contents.validateContentsModel(data); 500 | this._fileChanged.emit({ 501 | type: 'rename', 502 | oldValue: { path: oldLocalPath }, 503 | newValue: data 504 | }); 505 | return data; 506 | } 507 | 508 | /** 509 | * Helping function to increment name of existing files or directorties. 510 | * 511 | * @param localPath - Path to file. 512 | * 513 | * @param bucketName - The name of the bucket where content is moved. 514 | * 515 | * @param isDir - Whether the object is a directory or a file. 516 | */ 517 | protected async incrementName( 518 | localPath: string, 519 | bucketName: string, 520 | isDir: boolean 521 | ) { 522 | let fileExtension: string = ''; 523 | let originalName: string = ''; 524 | 525 | // check if we are dealing with a directory 526 | if (isDir === true) { 527 | localPath = 528 | localPath[localPath.length - 1] === '/' 529 | ? localPath.substring(0, localPath.length - 1) 530 | : localPath; 531 | originalName = PathExt.basename(localPath); 532 | } 533 | // dealing with a file 534 | else { 535 | // extract name from path 536 | originalName = PathExt.basename(localPath); 537 | // eliminate file extension 538 | fileExtension = PathExt.extname(originalName); 539 | originalName = 540 | originalName.split('.')[originalName.split('.').length - 2]; 541 | } 542 | 543 | const counter = await countS3ObjectNameAppearances( 544 | this._s3Client, 545 | bucketName, 546 | this._root, 547 | localPath, 548 | originalName 549 | ); 550 | let newName = counter ? originalName + counter : originalName; 551 | newName = isDir ? newName + '/' : newName + fileExtension; 552 | 553 | return newName; 554 | } 555 | 556 | /** 557 | * Save a file. 558 | * 559 | * @param localPath - The desired file path. 560 | * 561 | * @param options - Optional overrides to the model. 562 | * 563 | * @returns A promise which resolves with the file content model when the 564 | * file is saved. 565 | * 566 | * #### Notes 567 | * Ensure that `model.content` is populated for the file. 568 | * 569 | * Uses the [Jupyter Notebook API](https://petstore.swagger.io/?url=https://raw.githubusercontent.com/jupyter-server/jupyter_server/main/jupyter_server/services/api/api.yaml#!/contents) and validates the response model. 570 | */ 571 | async save( 572 | localPath: string, 573 | options: Partial = {} 574 | ): Promise { 575 | const fileName = PathExt.basename(localPath); 576 | 577 | data = await createS3Object( 578 | this._s3Client, 579 | this._name, 580 | this._root, 581 | fileName, 582 | localPath, 583 | options.content, 584 | this._registeredFileTypes, 585 | true, 586 | options 587 | ); 588 | 589 | Contents.validateContentsModel(data); 590 | this._fileChanged.emit({ 591 | type: 'save', 592 | oldValue: null, 593 | newValue: data 594 | }); 595 | return data; 596 | } 597 | 598 | /** 599 | * Copy a file into a given directory. 600 | * 601 | * @param path - The original file path. 602 | * 603 | * @param bucketName - The name of the bucket where content is moved. 604 | * 605 | * @returns A promise which resolves with the new name when the 606 | * file is copied. 607 | */ 608 | protected async incrementCopyName( 609 | copiedItemPath: string, 610 | bucketName: string 611 | ) { 612 | const isDir: boolean = await isDirectory( 613 | this._s3Client, 614 | bucketName, 615 | copiedItemPath 616 | ); 617 | 618 | // extracting original file name 619 | const originalFileName = PathExt.basename(copiedItemPath); 620 | 621 | // constructing new file name and path with -Copy string 622 | const newFileName = isDir 623 | ? originalFileName + '-Copy' 624 | : originalFileName.split('.')[0] + 625 | '-Copy.' + 626 | originalFileName.split('.')[1]; 627 | 628 | const newFilePath = 629 | copiedItemPath.substring(0, copiedItemPath.lastIndexOf('/') + 1) + 630 | newFileName + 631 | (isDir ? '/' : ''); 632 | 633 | // getting incremented name of Copy in case of duplicates 634 | const incrementedName = await this.incrementName( 635 | newFilePath, 636 | bucketName, 637 | isDir 638 | ); 639 | 640 | return incrementedName; 641 | } 642 | 643 | /** 644 | * Copy a file into a given directory. 645 | * 646 | * @param path - The original file path. 647 | * 648 | * @param toDir - The destination directory path. 649 | * 650 | * @returns A promise which resolves with the new contents model when the 651 | * file is copied. 652 | */ 653 | async copy( 654 | path: string, 655 | toDir: string, 656 | options: Contents.ICreateOptions = {} 657 | ): Promise { 658 | // construct new file or directory name for the copy 659 | const newFileName = await this.incrementCopyName(path, this._name); 660 | 661 | data = await copyS3Objects( 662 | this._s3Client, 663 | this._name, 664 | this._root, 665 | newFileName, 666 | path, 667 | toDir, 668 | this._registeredFileTypes 669 | ); 670 | 671 | Contents.validateContentsModel(data); 672 | this._fileChanged.emit({ 673 | type: 'new', 674 | oldValue: null, 675 | newValue: data 676 | }); 677 | return data; 678 | } 679 | 680 | /** 681 | * Copy a file into another bucket. 682 | * 683 | * @param path - The original file path. 684 | * 685 | * @param toDir - The destination directory path. 686 | * 687 | * @param bucketName - The name of the bucket where content is moved. 688 | * 689 | * @returns A promise which resolves with the new contents model when the 690 | * file is copied. 691 | */ 692 | async copyToAnotherBucket( 693 | path: string, 694 | toDir: string, 695 | bucketName: string, 696 | options: Contents.ICreateOptions = {} 697 | ): Promise { 698 | // construct new file or directory name for the copy 699 | const newFileName = await this.incrementCopyName(path, bucketName); 700 | 701 | data = await copyS3Objects( 702 | this._s3Client, 703 | this._name, 704 | this._root, 705 | newFileName, 706 | path, 707 | toDir, 708 | this._registeredFileTypes, 709 | bucketName 710 | ); 711 | 712 | this._fileChanged.emit({ 713 | type: 'new', 714 | oldValue: null, 715 | newValue: data 716 | }); 717 | Contents.validateContentsModel(data); 718 | return data; 719 | } 720 | 721 | /** 722 | * Create a checkpoint for a file. 723 | * 724 | * @param path - The path of the file. 725 | * 726 | * @returns A promise which resolves with the new checkpoint model when the 727 | * checkpoint is created. 728 | */ 729 | async createCheckpoint(path: string): Promise { 730 | const emptyCheckpoint: Contents.ICheckpointModel = { 731 | id: '', 732 | last_modified: '' 733 | }; 734 | return Promise.resolve(emptyCheckpoint); 735 | } 736 | 737 | /** 738 | * List available checkpoints for a file. 739 | * 740 | * @param path - The path of the file. 741 | * 742 | * @returns A promise which resolves with a list of checkpoint models for 743 | * the file. 744 | */ 745 | listCheckpoints(path: string): Promise { 746 | return Promise.resolve([]); 747 | } 748 | 749 | /** 750 | * Restore a file to a known checkpoint state. 751 | * 752 | * @param path - The path of the file. 753 | * 754 | * @param checkpointID - The id of the checkpoint to restore. 755 | * 756 | * @returns A promise which resolves when the checkpoint is restored. 757 | */ 758 | restoreCheckpoint(path: string, checkpointID: string): Promise { 759 | return Promise.reject('Repository is read only'); 760 | } 761 | 762 | /** 763 | * Delete a checkpoint for a file. 764 | * 765 | * @param path - The path of the file. 766 | * 767 | * @param checkpointID - The id of the checkpoint to delete. 768 | * 769 | * @returns A promise which resolves when the checkpoint is deleted. 770 | */ 771 | deleteCheckpoint(path: string, checkpointID: string): Promise { 772 | return Promise.reject('Read only'); 773 | } 774 | 775 | /** 776 | * Helping function for extracting region of bucket. 777 | * @returns region of Bucket 778 | */ 779 | private async getRegion() { 780 | const response = await this._s3Client.send( 781 | new GetBucketLocationCommand({ 782 | Bucket: this._name 783 | }) 784 | ); 785 | return (response?.LocationConstraint as string) ?? ''; 786 | } 787 | 788 | /** 789 | * Get all registered file types and store them accordingly with their file 790 | * extension (e.g.: .txt, .pdf, .jpeg), file mimetype (e.g.: text/plain, application/pdf) 791 | * and file format (e.g.: base64, text). 792 | * 793 | * @param app 794 | */ 795 | getRegisteredFileTypes(app: JupyterFrontEnd) { 796 | // get called when instating the toolbar 797 | const registeredFileTypes = app.docRegistry.fileTypes(); 798 | 799 | for (const fileType of registeredFileTypes) { 800 | // check if we are dealing with a directory 801 | if (fileType.extensions.length === 0) { 802 | this._registeredFileTypes[''] = { 803 | fileType: 'directory', 804 | fileFormat: 'json', 805 | fileMimeTypes: ['text/directory'] 806 | }; 807 | } 808 | 809 | // store the mimetype and fileformat for each file extension 810 | fileType.extensions.forEach(extension => { 811 | if (!this._registeredFileTypes[extension]) { 812 | this._registeredFileTypes[extension] = { 813 | fileType: fileType.name, 814 | fileMimeTypes: [...fileType.mimeTypes], 815 | fileFormat: fileType.fileFormat ? fileType.fileFormat : '' 816 | }; 817 | } 818 | }); 819 | } 820 | } 821 | 822 | /** 823 | * Helping function which formats root by removing all leading or trailing 824 | * backslashes and checking if given path to directory exists. 825 | * 826 | * @param root 827 | * @returns formatted root 828 | */ 829 | private async formatRoot(root: string) { 830 | // if root is empty, no formatting needed 831 | if (root === '') { 832 | return root; 833 | } 834 | 835 | // reformat the path to arbitrary root so it has no leading or trailing / 836 | root = PathExt.removeSlash(PathExt.normalize(root)); 837 | // check if directory exists within bucket 838 | try { 839 | await checkS3Object(this._s3Client, this._name, root); 840 | // the directory exists, root is formatted correctly 841 | return root; 842 | } catch (error) { 843 | console.log("Given path to root directory doesn't exist within bucket."); 844 | return ''; 845 | } 846 | } 847 | 848 | private _serverSettings: ServerConnection.ISettings; 849 | private _s3Client: S3Client; 850 | private _name: string = ''; 851 | private _root: string = ''; 852 | private _config: S3ClientConfig = ''; 853 | private _isRootFormatted: boolean = false; 854 | private _provider: string = ''; 855 | private _baseUrl: string = ''; 856 | private _region: string = ''; 857 | private _creationDate: string = ''; 858 | private _fileChanged = new Signal(this); 859 | private _isDisposed: boolean = false; 860 | private _disposed = new Signal(this); 861 | private _registeredFileTypes: IRegisteredFileTypes = {}; 862 | private _secretsManager: ISecretsManager | undefined; 863 | } 864 | 865 | export namespace Drive { 866 | /** 867 | * The options used to initialize a `Drive`. 868 | */ 869 | export interface IOptions { 870 | /** 871 | * S3 client configuration if available 872 | */ 873 | config?: S3ClientConfig; 874 | 875 | /** 876 | * The name for the `Drive`, which is used in file 877 | * paths to disambiguate it from other drives. 878 | */ 879 | name: string; 880 | 881 | /** 882 | * Path to directory from drive, which acts as root. 883 | */ 884 | root: string; 885 | 886 | /** 887 | * Secrets manager used for config. 888 | */ 889 | secretsManager?: ISecretsManager; 890 | 891 | /** 892 | * Token used together with the secrets manager. 893 | */ 894 | token?: symbol; 895 | 896 | /** 897 | * The server settings for the server. 898 | */ 899 | serverSettings?: ServerConnection.ISettings; 900 | } 901 | } 902 | 903 | namespace Private { 904 | /** 905 | * The secrets manager token. 906 | */ 907 | let token: symbol; 908 | 909 | /** 910 | * Set secrets manager token. 911 | */ 912 | export function setToken(value: symbol): void { 913 | token = value; 914 | } 915 | 916 | /** 917 | * Get secrets manager token. 918 | */ 919 | export function getToken(): symbol { 920 | return token; 921 | } 922 | } 923 | -------------------------------------------------------------------------------- /src/s3.ts: -------------------------------------------------------------------------------- 1 | import { Contents } from '@jupyterlab/services'; 2 | import { PathExt } from '@jupyterlab/coreutils'; 3 | import { 4 | CopyObjectCommand, 5 | DeleteObjectCommand, 6 | ListObjectsV2Command, 7 | GetObjectCommand, 8 | PutObjectCommand, 9 | HeadObjectCommand, 10 | S3Client 11 | } from '@aws-sdk/client-s3'; 12 | 13 | import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; 14 | 15 | export interface IRegisteredFileTypes { 16 | [fileExtension: string]: { 17 | fileType: string; 18 | fileMimeTypes: string[]; 19 | fileFormat: string; 20 | }; 21 | } 22 | 23 | interface IContentsList { 24 | [fileName: string]: Contents.IModel; 25 | } 26 | 27 | let data: Contents.IModel = { 28 | name: '', 29 | path: '', 30 | last_modified: '', 31 | created: '', 32 | content: null, 33 | format: null, 34 | mimetype: '', 35 | size: 0, 36 | writable: true, 37 | type: '' 38 | }; 39 | 40 | /** 41 | * Get the presigned URL for an S3 object. 42 | * 43 | * @param s3Client: The S3Client used to send commands. 44 | * @param bucketName: The name of bucket. 45 | * @param path: The path to the object. 46 | * 47 | * @returns: A promise which resolves with presigned URL. 48 | */ 49 | export const presignedS3Url = async ( 50 | s3Client: S3Client, 51 | bucketName: string, 52 | path: string 53 | ): Promise => { 54 | // retrieve object from S3 bucket 55 | const getCommand = new GetObjectCommand({ 56 | Bucket: bucketName, 57 | Key: path, 58 | ResponseContentDisposition: 'attachment', 59 | ResponseContentType: 'application/octet-stream' 60 | }); 61 | await s3Client.send(getCommand); 62 | 63 | // get pre-signed URL of S3 file 64 | const presignedUrl = await getSignedUrl(s3Client, getCommand); 65 | return presignedUrl; 66 | }; 67 | 68 | /** 69 | * Get list of contents of root or directory. 70 | * 71 | * @param s3Client: The S3Client used to send commands. 72 | * @param bucketName: The bucket name. 73 | * @param root: The path to the directory acting as root. 74 | * @param registeredFileTypes: The list containing all registered file types. 75 | * @param path: The path to the directory (optional). 76 | * 77 | * @returns: A promise which resolves with the contents model. 78 | */ 79 | export const listS3Contents = async ( 80 | s3Client: S3Client, 81 | bucketName: string, 82 | root: string, 83 | registeredFileTypes: IRegisteredFileTypes, 84 | path?: string 85 | ): Promise => { 86 | let isFile: boolean = false; 87 | const fileList: IContentsList = {}; 88 | const prefix = path ? PathExt.join(root, path) : root; 89 | 90 | // listing contents of folder 91 | const command = new ListObjectsV2Command({ 92 | Bucket: bucketName, 93 | Prefix: prefix + (prefix ? '/' : '') 94 | }); 95 | 96 | let isTruncated: boolean | undefined = true; 97 | 98 | while (isTruncated) { 99 | const { Contents, IsTruncated, NextContinuationToken } = 100 | await s3Client.send(command); 101 | 102 | if (Contents) { 103 | Contents.forEach(c => { 104 | // check if we are dealing with the files inside a subfolder 105 | if ( 106 | c.Key !== root + '/' && 107 | c.Key !== path + '/' && 108 | c.Key !== root + '/' + path + '/' 109 | ) { 110 | let fileName = c.Key!.replace( 111 | (root ? root + '/' : '') + (path ? path + '/' : ''), 112 | '' 113 | ); 114 | const isDir: boolean = 115 | fileName === fileName.split('/')[0] ? false : true; 116 | fileName = fileName.split('/')[0]; 117 | const [fileType, fileMimeType, fileFormat] = Private.getFileType( 118 | PathExt.extname(PathExt.basename(fileName)), 119 | isDir, 120 | registeredFileTypes 121 | ); 122 | 123 | fileList[fileName] = fileList[fileName] ?? { 124 | name: fileName, 125 | path: path ? PathExt.join(path, fileName) : fileName, 126 | last_modified: c.LastModified!.toISOString(), 127 | created: '', 128 | content: !fileName.split('.')[1] ? [] : null, 129 | format: fileFormat as Contents.FileFormat, 130 | mimetype: fileMimeType, 131 | size: c.Size!, 132 | writable: true, 133 | type: fileType 134 | }; 135 | } 136 | }); 137 | } else { 138 | isFile = true; 139 | data = await getS3FileContents( 140 | s3Client, 141 | bucketName, 142 | root, 143 | path!, 144 | registeredFileTypes 145 | ); 146 | } 147 | 148 | if (isTruncated) { 149 | isTruncated = IsTruncated; 150 | } 151 | command.input.ContinuationToken = NextContinuationToken; 152 | } 153 | 154 | if (isFile === false) { 155 | data = { 156 | name: path ? PathExt.basename(path) : bucketName, 157 | path: path ? path + '/' : bucketName, 158 | last_modified: '', 159 | created: '', 160 | content: Object.values(fileList), 161 | format: 'json', 162 | mimetype: '', 163 | size: undefined, 164 | writable: true, 165 | type: 'directory' 166 | }; 167 | } 168 | 169 | return data; 170 | }; 171 | 172 | /** 173 | * Retrieve contents of a file. 174 | * 175 | * @param s3Client: The S3Client used to send commands. 176 | * @param bucketName: The bucket name. 177 | * @param root: The path to the directory acting as root. 178 | * @param path: The path to to file. 179 | * @param registeredFileTypes: The list containing all registered file types. 180 | * 181 | * @returns: A promise which resolves with the file contents model. 182 | */ 183 | export const getS3FileContents = async ( 184 | s3Client: S3Client, 185 | bucketName: string, 186 | root: string, 187 | path: string, 188 | registeredFileTypes: IRegisteredFileTypes 189 | ): Promise => { 190 | // retrieving contents and metadata of file 191 | const command = new GetObjectCommand({ 192 | Bucket: bucketName, 193 | Key: PathExt.join(root, path) 194 | }); 195 | 196 | const response = await s3Client.send(command); 197 | 198 | if (response) { 199 | const date: string = response.LastModified!.toISOString(); 200 | const [fileType, fileMimeType, fileFormat] = Private.getFileType( 201 | PathExt.extname(PathExt.basename(path)), 202 | false, 203 | registeredFileTypes 204 | ); 205 | 206 | let fileContents: string | Uint8Array; 207 | 208 | // for certain media type files, extract content as byte array and decode to base64 to view in JupyterLab 209 | if (fileFormat === 'base64' || fileType === 'PDF') { 210 | fileContents = await response.Body!.transformToByteArray(); 211 | fileContents = btoa( 212 | fileContents.reduce( 213 | (data, byte) => data + String.fromCharCode(byte), 214 | '' 215 | ) 216 | ); 217 | } else { 218 | fileContents = await response.Body!.transformToString(); 219 | } 220 | 221 | data = { 222 | name: PathExt.basename(path), 223 | path: PathExt.join(root, path), 224 | last_modified: date, 225 | created: '', 226 | content: fileContents, 227 | format: fileFormat as Contents.FileFormat, 228 | mimetype: fileMimeType, 229 | size: response.ContentLength!, 230 | writable: true, 231 | type: fileType 232 | }; 233 | } 234 | 235 | return data; 236 | }; 237 | 238 | /** 239 | * Create a new file or directory or save a file. 240 | * 241 | * When saving a file, the options parameter is needed. 242 | * 243 | * @param s3Client: The S3Client used to send commands. 244 | * @param bucketName: The bucket name. 245 | * @param root: The path to the directory acting as root. 246 | * @param name: The name of file or directory to be created or saved. 247 | * @param path: The path to to file or directory. 248 | * @param body: The new contents of the file. 249 | * @param registeredFileTypes: The list containing all registered file types. 250 | * @param options: The optional parameteres of saving a file or directory (optional). 251 | * 252 | * @returns A promise which resolves with the new file or directory contents model. 253 | */ 254 | export const createS3Object = async ( 255 | s3Client: S3Client, 256 | bucketName: string, 257 | root: string, 258 | name: string, 259 | path: string, 260 | body: string | Blob, 261 | registeredFileTypes: IRegisteredFileTypes, 262 | isDir: boolean, 263 | options?: Partial 264 | ): Promise => { 265 | path = PathExt.join(root, path); 266 | 267 | const [fileType, fileMimeType, fileFormat] = Private.getFileType( 268 | PathExt.extname(PathExt.basename(name)), 269 | isDir, 270 | registeredFileTypes 271 | ); 272 | 273 | // checking if we are creating a new file or saving an existing one (overwrriting) 274 | if (options) { 275 | body = Private.formatBody(options, fileFormat, fileType, fileMimeType); 276 | } 277 | let lastModified; 278 | 279 | await s3Client 280 | .send( 281 | new PutObjectCommand({ 282 | Bucket: bucketName, 283 | Key: path + (PathExt.extname(name) === '' ? '/' : ''), 284 | Body: body, 285 | CacheControl: options ? 'no-cache' : undefined 286 | }) 287 | ) 288 | .then(async () => { 289 | const newFileInfo = await s3Client.send( 290 | new HeadObjectCommand({ 291 | Bucket: bucketName, 292 | Key: path + (PathExt.extname(name) === '' ? '/' : '') 293 | }) 294 | ); 295 | lastModified = newFileInfo.LastModified?.toISOString(); 296 | }) 297 | .catch(error => { 298 | console.error('Failed saving or creating the S3 object: ', error); 299 | }); 300 | 301 | data = { 302 | name: name, 303 | path: PathExt.join(path, name), 304 | last_modified: lastModified ?? new Date().toISOString(), 305 | created: lastModified ?? new Date().toISOString(), 306 | content: path.split('.').length === 1 ? [] : body, 307 | format: fileFormat as Contents.FileFormat, 308 | mimetype: fileMimeType, 309 | size: typeof body === 'string' ? body.length : body.size, 310 | writable: true, 311 | type: fileType 312 | }; 313 | 314 | return data; 315 | }; 316 | 317 | /** 318 | * Deleting a file or directory. 319 | * 320 | * @param s3Client: The S3Client used to send commands. 321 | * @param bucketName: The bucket name. 322 | * @param root: The path to the directory acting as root. 323 | * @param path: The path to to file or directory. 324 | */ 325 | export const deleteS3Objects = async ( 326 | s3Client: S3Client, 327 | bucketName: string, 328 | root: string, 329 | path: string 330 | ): Promise => { 331 | path = PathExt.join(root, path); 332 | 333 | // get list of contents with given prefix (path) 334 | const command = new ListObjectsV2Command({ 335 | Bucket: bucketName, 336 | Prefix: PathExt.extname(path) === '' ? path + '/' : path 337 | }); 338 | 339 | let isTruncated: boolean | undefined = true; 340 | 341 | while (isTruncated) { 342 | const { Contents, IsTruncated, NextContinuationToken } = 343 | await s3Client.send(command); 344 | 345 | if (Contents) { 346 | await Promise.all( 347 | Contents.map(async c => { 348 | // delete each file with given path 349 | return Private.deleteFile(s3Client, bucketName, c.Key!); 350 | }) 351 | ); 352 | } 353 | if (isTruncated) { 354 | isTruncated = IsTruncated; 355 | } 356 | command.input.ContinuationToken = NextContinuationToken; 357 | } 358 | }; 359 | 360 | /** 361 | * Check whether an object (file or directory) exists within given S3 bucket. 362 | * 363 | * Used before renaming a file to avoid overwriting and when setting the root. 364 | * 365 | * @param s3Client: The S3Client used to send commands. 366 | * @param bucketName: The bucket name. 367 | * @param root: The path to the directory acting as root. 368 | * @param path: The path to to file or directory. 369 | * 370 | * @returns A promise which resolves or rejects depending on the existance of the object. 371 | */ 372 | export const checkS3Object = async ( 373 | s3Client: S3Client, 374 | bucketName: string, 375 | root: string, 376 | path?: string, 377 | isDir?: boolean 378 | ): Promise => { 379 | // checking the existance of an S3 object or if root folder exists 380 | const prefix: string = path 381 | ? PathExt.join(root, path) + (isDir === true ? '/' : '') 382 | : root + '/'; 383 | const { Contents } = await s3Client.send( 384 | new ListObjectsV2Command({ 385 | Bucket: bucketName, 386 | Prefix: prefix, 387 | MaxKeys: 1 388 | }) 389 | ); 390 | if (Contents) { 391 | return Promise.resolve(); 392 | } else { 393 | return Promise.reject(); 394 | } 395 | }; 396 | 397 | /** 398 | * Rename a file or directory. 399 | * 400 | * @param s3Client: The S3Client used to send commands. 401 | * @param bucketName: The bucket name. 402 | * @param root: The path to the directory acting as root. 403 | * @param oldLocalPath: The old path of the object. 404 | * @param newLocalPath: The new path of the object. 405 | * @param newFileName: The new object name. 406 | * @param isDir: Whether the object is a directory or a file. 407 | * @param registeredFileTypes: The list containing all registered file types. 408 | * 409 | * @returns A promise which resolves with the new object contents model. 410 | */ 411 | export const renameS3Objects = async ( 412 | s3Client: S3Client, 413 | bucketName: string, 414 | root: string, 415 | oldLocalPath: string, 416 | newLocalPath: string, 417 | newFileName: string, 418 | isDir: boolean, 419 | registeredFileTypes: IRegisteredFileTypes 420 | ): Promise => { 421 | newLocalPath = PathExt.join(root, newLocalPath); 422 | oldLocalPath = PathExt.join(root, oldLocalPath); 423 | 424 | if (isDir) { 425 | newLocalPath = newLocalPath.substring(0, newLocalPath.length - 1); 426 | } 427 | newLocalPath = 428 | newLocalPath.substring(0, newLocalPath.lastIndexOf('/') + 1) + newFileName; 429 | 430 | const [fileType, fileMimeType, fileFormat] = Private.getFileType( 431 | PathExt.extname(PathExt.basename(newFileName)), 432 | isDir, 433 | registeredFileTypes 434 | ); 435 | 436 | // list contents of path - contents of directory or one file 437 | const command = new ListObjectsV2Command({ 438 | Bucket: bucketName, 439 | Prefix: oldLocalPath 440 | }); 441 | 442 | let isTruncated: boolean | undefined = true; 443 | 444 | while (isTruncated) { 445 | const { Contents, IsTruncated, NextContinuationToken } = 446 | await s3Client.send(command); 447 | 448 | if (Contents) { 449 | // retrieve content of file or directory 450 | const oldFileContents = await s3Client.send( 451 | new GetObjectCommand({ 452 | Bucket: bucketName, 453 | Key: Contents[0].Key! 454 | }) 455 | ); 456 | const body = await oldFileContents.Body?.transformToString(); 457 | 458 | const promises = Contents.map(async c => { 459 | const remainingFilePath = c.Key!.substring(oldLocalPath.length); 460 | // wait for copy action to resolve, delete original file only if it succeeds 461 | await Private.copyFile( 462 | s3Client, 463 | bucketName, 464 | remainingFilePath, 465 | oldLocalPath, 466 | newLocalPath 467 | ); 468 | return Private.deleteFile( 469 | s3Client, 470 | bucketName, 471 | oldLocalPath + remainingFilePath 472 | ); 473 | }); 474 | await Promise.all(promises); 475 | 476 | let lastModifiedDate = new Date().toISOString(); 477 | if (!isDir) { 478 | // retrieve last modified time for new file, does not apply to remaming directory 479 | const newFileMetadata = await s3Client.send( 480 | new HeadObjectCommand({ 481 | Bucket: bucketName, 482 | Key: newLocalPath 483 | }) 484 | ); 485 | lastModifiedDate = newFileMetadata.LastModified!.toISOString(); 486 | } 487 | 488 | data = { 489 | name: newFileName, 490 | path: newLocalPath.replace(root, ''), 491 | last_modified: lastModifiedDate, 492 | created: '', 493 | content: body ? body : [], 494 | format: fileFormat as Contents.FileFormat, 495 | mimetype: fileMimeType, 496 | size: oldFileContents.ContentLength!, 497 | writable: true, 498 | type: fileType 499 | }; 500 | } 501 | if (isTruncated) { 502 | isTruncated = IsTruncated; 503 | } 504 | command.input.ContinuationToken = NextContinuationToken; 505 | } 506 | return data; 507 | }; 508 | 509 | /** 510 | * Copy a file or directory to a new location within the bucket or to another bucket. 511 | * 512 | * If no additional bucket name is provided, the content will be copied to the default bucket. 513 | * 514 | * @param s3Client: The S3Client used to send commands. 515 | * @param bucketName: The bucket name. 516 | * @param root: The path to the directory acting as root. 517 | * @param name: The new object name. 518 | * @param path: The original path to the object to be copied. 519 | * @param toDir: The new path where object should be copied. 520 | * @param registeredFileTypes: The list containing all registered file types. 521 | * @param newBucketName: The name of the bucket where to copy the object (optional). 522 | * 523 | * @returns A promise which resolves with the new object contents model. 524 | */ 525 | export const copyS3Objects = async ( 526 | s3Client: S3Client, 527 | bucketName: string, 528 | root: string, 529 | name: string, 530 | path: string, 531 | toDir: string, 532 | registeredFileTypes: IRegisteredFileTypes, 533 | newBucketName?: string 534 | ): Promise => { 535 | const isDir: boolean = await isDirectory(s3Client, bucketName, path); 536 | let suffix: string = ''; 537 | 538 | path = PathExt.join(root, path); 539 | toDir = PathExt.join(root, toDir); 540 | name = PathExt.join(toDir, name); 541 | path = path + (isDir ? '/' : ''); 542 | 543 | // list contents of path - contents of directory or one file 544 | const command = new ListObjectsV2Command({ 545 | Bucket: bucketName, 546 | Prefix: path 547 | }); 548 | 549 | let isTruncated: boolean | undefined = true; 550 | 551 | while (isTruncated) { 552 | const { Contents, IsTruncated, NextContinuationToken } = 553 | await s3Client.send(command); 554 | 555 | if (Contents) { 556 | const promises = Contents.map(c => { 557 | if (!suffix && c.Key!.search('/.emptyFolderPlaceholder') !== -1) { 558 | suffix = '.emptyFolderPlaceholder'; 559 | } 560 | const remainingFilePath = c.Key!.substring(path.length); 561 | // copy each file from old directory to new location 562 | return Private.copyFile( 563 | s3Client, 564 | bucketName, 565 | remainingFilePath, 566 | path, 567 | name, 568 | newBucketName 569 | ); 570 | }); 571 | await Promise.all(promises); 572 | } 573 | if (isTruncated) { 574 | isTruncated = IsTruncated; 575 | } 576 | command.input.ContinuationToken = NextContinuationToken; 577 | } 578 | 579 | const [fileType, fileMimeType, fileFormat] = Private.getFileType( 580 | PathExt.extname(PathExt.basename(name)), 581 | isDir, 582 | registeredFileTypes 583 | ); 584 | 585 | try { 586 | const newFileContents = await s3Client.send( 587 | new GetObjectCommand({ 588 | Bucket: newBucketName ?? bucketName, 589 | Key: name + (suffix ? suffix : '') 590 | }) 591 | ); 592 | 593 | data = { 594 | name: PathExt.basename(name), 595 | path: name, 596 | last_modified: newFileContents.LastModified!.toISOString(), 597 | created: new Date().toISOString(), 598 | content: await newFileContents.Body!.transformToString(), 599 | format: fileFormat as Contents.FileFormat, 600 | mimetype: fileMimeType, 601 | size: newFileContents.ContentLength!, 602 | writable: true, 603 | type: fileType 604 | }; 605 | } catch { 606 | // object directory itself doesn't exist 607 | data = { 608 | name: PathExt.basename(name), 609 | path: name, 610 | last_modified: new Date().toISOString(), 611 | created: new Date().toISOString(), 612 | content: [], 613 | format: fileFormat as Contents.FileFormat, 614 | mimetype: fileMimeType, 615 | size: 0, 616 | writable: true, 617 | type: fileType 618 | }; 619 | } 620 | 621 | return data; 622 | }; 623 | 624 | /** 625 | * Count number of appeareances of object name. 626 | * 627 | * @param s3Client: The S3Client used to send commands. 628 | * @param bucketName: The bucket name. 629 | * @param root: The path to the directory acting as root. 630 | * @param path: The path to the object. 631 | * @param originalName: The original name of the object (before it was incremented). 632 | * 633 | * @returns A promise which resolves with the number of appeareances of object. 634 | */ 635 | export const countS3ObjectNameAppearances = async ( 636 | s3Client: S3Client, 637 | bucketName: string, 638 | root: string, 639 | path: string, 640 | originalName: string 641 | ): Promise => { 642 | let counter: number = 0; 643 | path = PathExt.join(root, path); 644 | 645 | // count number of name appearances 646 | const command = new ListObjectsV2Command({ 647 | Bucket: bucketName, 648 | Prefix: path.substring(0, path.lastIndexOf('/')) 649 | }); 650 | 651 | let isTruncated: boolean | undefined = true; 652 | 653 | while (isTruncated) { 654 | const { Contents, IsTruncated, NextContinuationToken } = 655 | await s3Client.send(command); 656 | 657 | if (Contents) { 658 | Contents.forEach(c => { 659 | let fileName = 660 | c.Key![c.Key!.length - 1] === '/' 661 | ? c.Key!.substring(0, c.Key!.length - 1) 662 | : c.Key!; 663 | fileName = fileName 664 | .replace((root ? root + '/' : '') + (path ? path + '/' : ''), '') 665 | .split('/')[0]; 666 | if ( 667 | fileName.substring(0, originalName.length + 1).includes(originalName) 668 | ) { 669 | counter += 1; 670 | } 671 | }); 672 | } 673 | if (isTruncated) { 674 | isTruncated = IsTruncated; 675 | } 676 | command.input.ContinuationToken = NextContinuationToken; 677 | } 678 | 679 | return counter; 680 | }; 681 | 682 | /** 683 | * This is a helper function that resolves whether a given path 684 | * is a directory, because the S3 API does not provide this in listings. 685 | */ 686 | export async function isDirectory( 687 | s3Client: S3Client, 688 | bucketName: string, 689 | objectPath: string 690 | ): Promise { 691 | let isDir: boolean = false; 692 | 693 | // listing contents given a path, to check if it is a directory 694 | const command = new ListObjectsV2Command({ 695 | Bucket: bucketName, 696 | Prefix: 697 | objectPath[objectPath.length - 1] === '/' ? objectPath : objectPath + '/', 698 | MaxKeys: 1 699 | }); 700 | 701 | const { Contents } = await s3Client.send(command); 702 | if (Contents) { 703 | isDir = true; 704 | } 705 | 706 | return isDir; 707 | } 708 | 709 | namespace Private { 710 | /** 711 | * Helping function to define file type, mimetype and format based on file extension. 712 | * @param extension: File extension (e.g.: txt, ipynb, csv) 713 | * @param isDir: Boolean showing if the object is a directory or a file 714 | * @param registeredFileTypes: The list containing all registered file types. 715 | * @returns The object type, mimetype and format. 716 | */ 717 | export function getFileType( 718 | extension: string, 719 | isDir: boolean, 720 | registeredFileTypes: IRegisteredFileTypes 721 | ) { 722 | let fileType: string = isDir === false ? 'text' : 'directory'; 723 | let fileMimetype: string = 724 | isDir === false ? 'text/plain' : 'text/directory'; 725 | let fileFormat: string = isDir === false ? 'text' : 'json'; 726 | 727 | if (isDir === false && registeredFileTypes[extension]) { 728 | fileType = registeredFileTypes[extension].fileType; 729 | fileMimetype = registeredFileTypes[extension].fileMimeTypes[0]; 730 | fileFormat = registeredFileTypes[extension].fileFormat; 731 | } 732 | 733 | // the file format for notebooks appears as json, but should be text 734 | if (extension === '.ipynb') { 735 | fileFormat = 'text'; 736 | } 737 | 738 | return [fileType, fileMimetype, fileFormat]; 739 | } 740 | 741 | /** 742 | * Helping function for deleting files inside 743 | * a directory, in the case of deleting the directory. 744 | * 745 | * @param filePath complete path of file to delete 746 | */ 747 | export async function deleteFile( 748 | s3Client: S3Client, 749 | bucketName: string, 750 | filePath: string 751 | ) { 752 | await s3Client.send( 753 | new DeleteObjectCommand({ 754 | Bucket: bucketName, 755 | Key: filePath 756 | }) 757 | ); 758 | } 759 | 760 | /** 761 | * Helping function for copying the files inside a directory 762 | * to a new location, in the case of renaming or copying a directory. 763 | * 764 | * @param remainingFilePath remaining path of file to be copied 765 | * @param oldPath old path of file 766 | * @param newPath new path of file 767 | */ 768 | export async function copyFile( 769 | s3Client: S3Client, 770 | bucketName: string, 771 | remainingFilePath: string, 772 | oldPath: string, 773 | newPath: string, 774 | newBucketName?: string 775 | ) { 776 | await s3Client.send( 777 | new CopyObjectCommand({ 778 | Bucket: newBucketName ? newBucketName : bucketName, 779 | CopySource: PathExt.join(bucketName, oldPath, remainingFilePath), 780 | Key: PathExt.join(newPath, remainingFilePath) 781 | }) 782 | ); 783 | } 784 | 785 | /** 786 | * Helping function used for formatting the body of files. 787 | * 788 | * @param options: The parameteres for saving a file. 789 | * @param fileFormat: The registered file format. 790 | * @param fileType: The registered file type. 791 | * @param fileMimeType: The registered file mimetype. 792 | * 793 | * @returns The formatted content (body). 794 | */ 795 | export function formatBody( 796 | options: Partial, 797 | fileFormat: string, 798 | fileType: string, 799 | fileMimeType: string 800 | ) { 801 | let body: string | Blob; 802 | if (options.format === 'json') { 803 | body = JSON.stringify(options.content, null, 2); 804 | } else if ( 805 | options.format === 'base64' && 806 | (fileFormat === 'base64' || fileType === 'PDF') 807 | ) { 808 | // transform base64 encoding to a utf-8 array for saving and storing in S3 bucket 809 | const byteCharacters = atob(options.content); 810 | const byteArrays: Uint8Array[] = []; 811 | 812 | for (let offset = 0; offset < byteCharacters.length; offset += 512) { 813 | const slice = byteCharacters.slice(offset, offset + 512); 814 | const byteNumbers = new Array(slice.length); 815 | for (let i = 0; i < slice.length; i++) { 816 | byteNumbers[i] = slice.charCodeAt(i); 817 | } 818 | const byteArray = new Uint8Array(byteNumbers); 819 | byteArrays.push(byteArray); 820 | } 821 | 822 | body = new Blob(byteArrays, { type: fileMimeType }); 823 | } else { 824 | body = options.content; 825 | } 826 | return body; 827 | } 828 | } 829 | --------------------------------------------------------------------------------