├── .flake8 ├── .github ├── FUNDING.yml ├── ISSUE_TEMPLATE │ ├── bug_form.yml │ ├── config.yml │ ├── new_feature.yml │ └── use_case.yml ├── pull_request_template.md └── workflows │ ├── main.yml │ └── release-and-publish.yml ├── .gitignore ├── .pre-commit-config.yaml ├── CHANGELOG.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── SETUP.md ├── assets ├── fixtures.sql ├── macOS_accessibility.png ├── macOS_input_monitoring.png ├── macOS_permissions_alert.png ├── macOS_screen_recording.png └── visualize.png ├── chrome_extension ├── background.js ├── content.js ├── icons │ └── logo.png └── manifest.json ├── deploy ├── .env.example ├── README.md ├── deploy │ └── models │ │ └── omniparser │ │ ├── .dockerignore │ │ ├── Dockerfile │ │ ├── client.py │ │ └── deploy.py └── pyproject.toml ├── experiments ├── describe_actions.py ├── fastsamsom.py ├── gpt4o_seg.py ├── handle_similar_segments.py └── imagesimilarity.py ├── install ├── install_openadapt.ps1 └── install_openadapt.sh ├── openadapt ├── __init__.py ├── adapters │ ├── __init__.py │ ├── prompt.py │ ├── replicate.py │ ├── som.py │ └── ultralytics.py ├── alembic.ini ├── alembic │ ├── README │ ├── context_loader.py │ ├── env.py │ ├── script.py.mako │ └── versions │ │ ├── 104d4a614d95_add_performancestat.py │ │ ├── 186316d4f3ca_add_scrubbed_columns.py │ │ ├── 20f9c2afb42c_rename_input_event_to_action_event.py │ │ ├── 2a8a241785f2_add_original_recording_id.py │ │ ├── 30a5ba9d6453_add_active_segment_description_and_.py │ │ ├── 5139d7df38f6_add_recording_task_description.py │ │ ├── 530f0663324e_add_recording_video_start_time.py │ │ ├── 57d78d23087a_add_windowevent_state.py │ │ ├── 607d1380b5ae_add_memorystat.py │ │ ├── 8495f5471e23_add_recording_config.py │ │ ├── 8713b142f5de_add_png_diff_data_and_png_diff_mask_data.py │ │ ├── 87a78a84a8bf_remove_recording_timestamp_fks.py │ │ ├── 98505a067995_add_browserevent_table.py │ │ ├── 98c8851a5321_add_audio_info.py │ │ ├── a29b537fabe6_add_disabled_field_to_action_event.py │ │ ├── b206c80f7640_add_recording_inputevent_screenshot_.py │ │ ├── b2dc41850120_window_window_id.py │ │ ├── bb25e889ad71_generate_unique_user_id.py │ │ ├── c24abb5455d3_create_new_recording_fks.py │ │ ├── d63569e4fb90_actionevent_element_state.py │ │ ├── d714cc86fce8_add_scrubbed_recording_model.py │ │ ├── ec337f277666_datetime_timestamp.py │ │ └── f9586c10a561_migrate_data_to_new_fks.py ├── app │ ├── __init__.py │ ├── assets │ │ ├── logo.ico │ │ ├── logo.png │ │ └── logo_inverted.png │ ├── dashboard │ │ ├── .eslintrc.json │ │ ├── .gitignore │ │ ├── .nvmrc │ │ ├── .prettierrc.json │ │ ├── README.md │ │ ├── __init__.py │ │ ├── api.ts │ │ ├── api │ │ │ ├── action_events.py │ │ │ ├── index.py │ │ │ ├── recordings.py │ │ │ ├── scrubbing.py │ │ │ └── settings.py │ │ ├── app │ │ │ ├── favicon.ico │ │ │ ├── globals.css │ │ │ ├── layout.tsx │ │ │ ├── onboarding │ │ │ │ └── page.tsx │ │ │ ├── providers.tsx │ │ │ ├── recordings │ │ │ │ ├── RawRecordings.tsx │ │ │ │ ├── ScrubbedRecordings.tsx │ │ │ │ ├── detail │ │ │ │ │ └── page.tsx │ │ │ │ └── page.tsx │ │ │ ├── routes.ts │ │ │ ├── scrubbing │ │ │ │ ├── ScrubbingUpdates.tsx │ │ │ │ └── page.tsx │ │ │ ├── settings │ │ │ │ ├── (api_keys) │ │ │ │ │ ├── form.tsx │ │ │ │ │ └── page.tsx │ │ │ │ ├── SettingsHeader.tsx │ │ │ │ ├── layout.tsx │ │ │ │ ├── navbar.tsx │ │ │ │ ├── record_and_replay │ │ │ │ │ ├── form.tsx │ │ │ │ │ └── page.tsx │ │ │ │ ├── scrubbing │ │ │ │ │ ├── form.tsx │ │ │ │ │ └── page.tsx │ │ │ │ └── utils.ts │ │ │ └── utils.ts │ │ ├── assets │ │ │ └── logo.png │ │ ├── components │ │ │ ├── ActionEvent │ │ │ │ ├── ActionEvent.tsx │ │ │ │ ├── ActionEvents.tsx │ │ │ │ ├── RemoveActionEvent.tsx │ │ │ │ ├── Screenshots.tsx │ │ │ │ └── index.tsx │ │ │ ├── Navbar │ │ │ │ ├── Navbar.tsx │ │ │ │ └── index.tsx │ │ │ ├── Onboarding │ │ │ │ └── steps │ │ │ │ │ ├── BookACall.tsx │ │ │ │ │ ├── RegisterForUpdates.tsx │ │ │ │ │ └── Tutorial.tsx │ │ │ ├── RecordingDetails │ │ │ │ ├── RecordingDetails.tsx │ │ │ │ └── index.tsx │ │ │ ├── Shell │ │ │ │ ├── Shell.tsx │ │ │ │ └── index.tsx │ │ │ └── SimpleTable │ │ │ │ ├── SimpleTable.tsx │ │ │ │ └── index.tsx │ │ ├── entrypoint.ps1 │ │ ├── entrypoint.sh │ │ ├── index.js │ │ ├── next.config.js │ │ ├── package-lock.json │ │ ├── package.json │ │ ├── postcss.config.js │ │ ├── run.py │ │ ├── tailwind.config.js │ │ ├── tsconfig.json │ │ └── types │ │ │ ├── action-event.ts │ │ │ ├── recording.ts │ │ │ └── scrubbing.ts │ └── tray.py ├── browser.py ├── build.py ├── build_utils.py ├── cache.py ├── capture │ ├── __init__.py │ ├── __main__.py │ ├── _linux.py │ ├── _macos.py │ └── _windows.py ├── common.py ├── config.defaults.json ├── config.py ├── contrib │ ├── __init__.py │ └── som │ │ ├── __init__.py │ │ └── visualizer.py ├── custom_logger.py ├── db │ ├── __init__.py │ ├── crud.py │ ├── db.py │ ├── list.py │ └── remove.py ├── deprecated │ └── app │ │ └── cards.py ├── drivers │ ├── anthropic.py │ ├── google.py │ └── openai.py ├── entrypoint.py ├── error_reporting.py ├── events.py ├── extensions │ └── synchronized_queue.py ├── models.py ├── playback.py ├── plotting.py ├── privacy │ ├── __init__.py │ ├── base.py │ └── providers │ │ ├── __init__.py │ │ ├── aws_comprehend.py │ │ ├── presidio.py │ │ └── private_ai.py ├── productivity.py ├── prompts │ ├── apply_replay_instructions.j2 │ ├── describe_recording--segment.j2 │ ├── describe_recording.j2 │ ├── description.j2 │ ├── generate_action_event--segment.j2 │ ├── generate_action_event.j2 │ ├── is_action_complete.j2 │ └── system.j2 ├── record.py ├── replay.py ├── scripts │ ├── __init__.py │ ├── reset_db.py │ └── scrub.py ├── scrub.py ├── share.py ├── spacy_model_helpers │ ├── __init__.py │ ├── download_model.py │ └── spacy_model_init.py ├── start.py ├── strategies │ ├── __init__.py │ ├── base.py │ ├── demo.py │ ├── mixins │ │ ├── ascii.py │ │ ├── huggingface.py │ │ ├── ocr.py │ │ ├── openai.py │ │ ├── sam.py │ │ └── summary.py │ ├── naive.py │ ├── segment.py │ ├── stateful.py │ ├── vanilla.py │ ├── visual.py │ └── visual_browser.py ├── utils.py ├── video.py ├── vision.py ├── visualize.py └── window │ ├── __init__.py │ ├── _linux.py │ ├── _macos.py │ └── _windows.py ├── permissions_in_macOS.md ├── poetry.lock ├── pyproject.toml ├── scripts ├── downloads.py └── postinstall.py ├── setup.py └── tests ├── assets ├── calculator.png ├── excel.png ├── sample_llc_1.pdf ├── test_emr_image.png ├── test_image_redaction_presidio.png └── test_image_redaction_privateai.png ├── conftest.py └── openadapt ├── drivers ├── test_anthropic.py ├── test_google.py └── test_openai.py ├── privacy ├── providers │ ├── test_comprehend_scrub.py │ ├── test_presidio_scrub.py │ └── test_private_ai_scrub.py └── test_providers.py ├── test_browser.py ├── test_crop.py ├── test_crud.py ├── test_events.py ├── test_models.py ├── test_share.py ├── test_summary.py ├── test_utils.py ├── test_video.py └── test_vision.py /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | exclude = alembic,.venv,venv,contrib,.cache,.git 3 | docstring-convention = google 4 | max-line-length = 88 5 | extend-ignore = ANN101, E203 6 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: [OpenAdaptAI] 4 | patreon: # Replace with a single Patreon username 5 | open_collective: # Replace with a single Open Collective username 6 | ko_fi: # Replace with a single Ko-fi username 7 | tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel 8 | community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry 9 | liberapay: # Replace with a single Liberapay username 10 | issuehunt: # Replace with a single IssueHunt username 11 | lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry 12 | polar: # Replace with a single Polar username 13 | buy_me_a_coffee: # Replace with a single Buy Me a Coffee username 14 | custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] 15 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_form.yml: -------------------------------------------------------------------------------- 1 | name: Bug Report 2 | description: File a bug report 3 | title: "[Bug]: " 4 | labels: bug 5 | body: 6 | - type: markdown 7 | attributes: 8 | value: | 9 | Thanks for taking the time to fill out this bug report! 10 | - type: textarea 11 | id: describe-bug 12 | attributes: 13 | label: Describe the bug 14 | description: What behaviour was expected and what actually happened? Include screenshots and log output where useful. 15 | placeholder: I expected ... but what actually happened was ... 16 | validations: 17 | required: true 18 | - type: textarea 19 | id: steps 20 | attributes: 21 | label: To Reproduce 22 | description: What Operating System did you use and steps would we take to reproduce the behaviour? 23 | placeholder: "I use [macOS/Microsoft Windows]. 24 | Steps: 25 | 1. Go to '...' 26 | 2. Click on '....' 27 | 3. Scroll down to '....' " 28 | validations: 29 | required: true 30 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: true 2 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/new_feature.yml: -------------------------------------------------------------------------------- 1 | name: "Feature Request" 2 | description: Submit a proposal for a new OpenAdapt feature 3 | labels: enhancement 4 | body: 5 | - type: textarea 6 | id: feature-request 7 | validations: 8 | required: true 9 | attributes: 10 | label: Feature request 11 | description: A clear and concise description of the feature proposal. Please provide links to any relevant resources. 12 | 13 | - type: textarea 14 | id: motivation 15 | validations: 16 | required: false 17 | attributes: 18 | label: Motivation 19 | description: Please outline the purpose for the proposal (e.g., is it related to a problem?). Add any relevant links (e.g. GitHub issues). 20 | placeholder: I'm always frustrated when [...] so this feature would [...]. 21 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/use_case.yml: -------------------------------------------------------------------------------- 1 | name: Use Case 2 | description: Describe a new use case for OpenAdapt 3 | title: "Use Case: " 4 | labels: usecase 5 | body: 6 | - type: markdown 7 | attributes: 8 | value: | 9 | Thanks for taking the time to add a new use case! Default text extracted from https://github.com/MLDSAI/OpenAdapt/issues/72#issuecomment-1601991759. 10 | - type: input 11 | id: role 12 | attributes: 13 | label: Role 14 | description: Who is this use case about? 15 | placeholder: Travel Agent 16 | validations: 17 | required: true 18 | - type: textarea 19 | id: task 20 | attributes: 21 | label: Task 22 | description: What is this use case about? 23 | placeholder: Book a flight 24 | validations: 25 | required: true 26 | - type: textarea 27 | id: context 28 | attributes: 29 | label: Context 30 | description: What background information is there? 31 | placeholder: There are 4 people traveling to Barcelona, 2 of which require first class tickets 32 | validations: 33 | required: true 34 | - type: textarea 35 | id: workflow 36 | attributes: 37 | label: Workflow 38 | description: How is the task accomplished? Please place each step on a new line. 39 | placeholder: | 40 | 1. Open Chrome 41 | 2. Navigate to Google Flights 42 | 3. Set the departure city 43 | 4. ... 44 | validations: 45 | required: true 46 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | **What kind of change does this PR introduce?** 4 | 5 | 6 | 7 | **Summary** 8 | 9 | 10 | 11 | **Checklist** 12 | * [ ] My code follows the style guidelines of OpenAdapt 13 | * [ ] I have performed a self-review of my code 14 | * [ ] If applicable, I have added tests to prove my fix is functional/effective 15 | * [ ] I have linted my code locally prior to submission 16 | * [ ] I have commented my code, particularly in hard-to-understand areas 17 | * [ ] I have made corresponding changes to the documentation (e.g. README.md, requirements.txt) 18 | * [ ] New and existing unit tests pass locally with my changes 19 | 20 | **How can your code be run and tested?** 21 | 22 | 23 | 24 | 25 | **Other information** 26 | 27 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: Python CI 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - '**' 7 | 8 | jobs: 9 | run-ci: 10 | runs-on: ${{ matrix.os }} 11 | defaults: 12 | run: 13 | shell: bash 14 | 15 | strategy: 16 | matrix: 17 | # TODO: add windows matrix 18 | os: [macos-latest] 19 | 20 | env: 21 | REPO: ${{ github.event.pull_request.head.repo.full_name }} 22 | BRANCH: ${{ github.event.pull_request.head.ref }} 23 | SKIP_POETRY_SHELL: 1 24 | 25 | steps: 26 | - name: Checkout code 27 | uses: actions/checkout@v3 28 | with: 29 | ref: ${{ env.BRANCH }} 30 | repository: ${{ env.REPO }} 31 | 32 | - name: Set up Python 33 | uses: actions/setup-python@v5 34 | with: 35 | python-version: '3.10' 36 | 37 | - name: Run tests using the shell script (macOS compatible). 38 | if: matrix.os == 'macos-latest' 39 | run: sh install/install_openadapt.sh 40 | 41 | - name: Install poetry 42 | uses: snok/install-poetry@v1 43 | with: 44 | version: 1.5.1 45 | virtualenvs-create: true 46 | virtualenvs-in-project: true 47 | 48 | - name: Cache deps 49 | id: cache-deps 50 | uses: actions/cache@v2 51 | with: 52 | path: .venv 53 | key: pydeps-${{ hashFiles('**/poetry.lock') }} 54 | 55 | - run: poetry install --no-interaction --no-root 56 | if: steps.cache-deps.outputs.cache-hit != 'true' 57 | 58 | - name: Activate virtualenv 59 | run: source .venv/bin/activate 60 | if: steps.cache-deps.outputs.cache-hit == 'true' 61 | 62 | - name: Check formatting with Black 63 | run: poetry run black --preview --check . --exclude '/(alembic|\.cache|\.venv|venv|contrib|__pycache__)/' 64 | 65 | - name: Run Flake8 66 | run: poetry run flake8 --exclude=alembic,.venv,venv,contrib,.cache,.git 67 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Envirionment 2 | .env 3 | config.json 4 | data 5 | 6 | # Python 7 | __pycache__ 8 | .cache 9 | *.egg-info 10 | *~ 11 | .venv 12 | 13 | # Vim 14 | *.sw[m-p] 15 | 16 | # db 17 | *.db 18 | *.db-journal 19 | 20 | # VSCode 21 | .VSCode 22 | .vsCode 23 | 24 | # Idea 25 | .idea 26 | 27 | # Generated performance charts 28 | performance 29 | 30 | # Generated when adding editable dependencies in requirements.txt (-e) 31 | src 32 | 33 | # MacOS file 34 | .DS_Store 35 | 36 | *.pyc 37 | *.pt 38 | 39 | dist/ 40 | build/ 41 | 42 | OpenAdapt.spec 43 | build_scripts/OpenAdapt.iss 44 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v2.3.0 4 | hooks: 5 | - id: check-yaml 6 | - id: end-of-file-fixer 7 | - id: trailing-whitespace 8 | - repo: https://github.com/psf/black 9 | rev: 23.7.0 10 | hooks: 11 | - id: black 12 | args: [--preview] 13 | - repo: https://github.com/pycqa/isort 14 | rev: 5.12.0 15 | hooks: 16 | - id: isort 17 | name: isort (python) 18 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 MLDSAI Inc., Richard Abrich, and contributors. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /SETUP.md: -------------------------------------------------------------------------------- 1 | # Setup Instructions for the following: 2 | 3 | 1. Python 3.10 4 | 2. Git 5 | 3. Tesseract (for OCR) 6 | 7 | 8 | 9 | ## Python 3.10 10 | 11 | #### Windows: 12 | - Download the Python 3.10 Binary File: https://www.python.org/ftp/python/3.10.0/python-3.10.0-amd64.exe 13 | - Follow the steps for Installation here: https://www.digitalocean.com/community/tutorials/install-python-windows-10 14 | 15 | 16 | #### macOS: 17 | - Download the Python 3.10 Package Installer: https://www.python.org/ftp/python/3.10.11/python-3.10.11-macos11.pkg 18 | - Follow the steps for Installation here: https://www.codingforentrepreneurs.com/guides/install-python-on-macos/ 19 | 20 | 21 | #### Ubuntu 22 | - Open Terminal and type the following command: 23 | - `sudo apt install python3.10-full` 24 | 25 | 26 | 27 | 28 | ## Git 29 | 30 | #### Windows: 31 | - Download system appropriate binary (exe) file from here: https://git-scm.com/download/win 32 | - Install the setup file. 33 | 34 | 35 | #### macOS: 36 | - Follow the steps here: https://git-scm.com/download/mac 37 | 38 | 39 | #### Ubuntu (Linux and Unix): 40 | - Follow the steps here: https://git-scm.com/download/linux 41 | 42 | 43 | 44 | 45 | ## Tesseract OCR 46 | 47 | #### Windows: 48 | - Download the following binary (.exe) file: 49 | [Tesseract Setup](https://digi.bib.uni-mannheim.de/tesseract/tesseract-ocr-w64-setup-5.3.1.20230401.exe) 50 | - Run the downloaded file to install Tesseract OCR 51 | - Add Tesseract to User and System PATH variable. 52 | - See Step 5 and 6 in https://linuxhint.com/install-tesseract-windows/ 53 | 54 | 55 | #### macOS, Ubuntu and Other Operating System: 56 | - Follow the OS Specific installation here: https://tesseract-ocr.github.io/tessdoc/Installation.html 57 | 58 | 59 | ## nvm (Node Version Manager) 60 | 61 | #### Windows: 62 | - Follow this guide: https://www.xda-developers.com/how-install-nvm-windows/ 63 | 64 | #### macOS, Ubuntu and Other Operating System: 65 | - Follow the OS Specific installation here: https://collabnix.com/how-to-install-and-configure-nvm-on-mac-os/ 66 | -------------------------------------------------------------------------------- /assets/fixtures.sql: -------------------------------------------------------------------------------- 1 | -- assets/fixtures.sql 2 | 3 | -- Insert sample recordings 4 | INSERT INTO recording (timestamp, monitor_width, monitor_height, double_click_interval_seconds, double_click_distance_pixels, platform, task_description) 5 | VALUES 6 | (1689889605.9053426, 1920, 1080, 0.5, 4, 'win32', 'type l'); 7 | 8 | -- Insert sample action_events 9 | INSERT INTO action_event (name, timestamp, recording_timestamp, screenshot_timestamp, window_event_timestamp, mouse_x, mouse_y, mouse_dx, mouse_dy, mouse_button_name, mouse_pressed, key_name, key_char, key_vk, canonical_key_name, canonical_key_char, canonical_key_vk, parent_id, element_state) 10 | VALUES 11 | ('press', 1690049582.7713714, 1689889605.9053426, 1690049582.7686925, 1690049556.2166219, NULL, NULL, NULL, NULL, NULL, NULL, NULL, 'l', '76', NULL, 'l', NULL, NULL, 'null'), 12 | ('release', 1690049582.826782, 1689889605.9053426, 1690049582.7686925, 1690049556.2166219, NULL, NULL, NULL, NULL, NULL, NULL, NULL, 'l', '76', NULL, 'l', NULL, NULL, 'null'); 13 | 14 | -- Insert sample screenshots 15 | INSERT INTO screenshot (recording_timestamp, timestamp, png_data) 16 | VALUES 17 | (1689889605.9053426, 1690042711.774856, x'89504E470D0A1A0A0000000D49484452000000010000000108060000009077BF8A0000000A4944415408D7636000000005000000008D2B4233000000000049454E44AE426082'); 18 | -- PNG data represents a 1x1 pixel image with a white pixel 19 | 20 | -- Insert sample window_events 21 | INSERT INTO window_event (recording_timestamp, timestamp, state, title, left, top, width, height, window_id) 22 | VALUES 23 | (1689889605.9053426, 1690042703.7413292, '{"title": "recording.txt - openadapt - Visual Studio Code", "left": -9, "top": -9, "width": 1938, "height": 1048, "meta": {"class_name": "Chrome_WidgetWin_1", "control_id": 0, "rectangle": {"left": 0, "top": 0, "right": 1920, "bottom": 1030}, "is_visible": true, "is_enabled": true, "control_count": 0}}', 'recording.txt - openadapt - Visual Studio Code', -9, -9, 1938, 1048, '0'); 24 | 25 | -- Insert sample performance_stats 26 | INSERT INTO performance_stat (recording_timestamp, event_type, start_time, end_time, window_id) 27 | VALUES 28 | (1689889605.9053426, 'action', 1690042703, 1690042711, 1), 29 | (1689889605.9053426, 'action', 1690042712, 1690042720, 1); 30 | -- Add more rows as needed for additional performance_stats 31 | 32 | -- Insert sample memory_stats 33 | INSERT INTO memory_stat (recording_timestamp, memory_usage_bytes, timestamp) 34 | VALUES 35 | (1689889605.9053426, 524288, 1690042703), 36 | (1689889605.9053426, 1048576, 1690042711); 37 | -- Add more rows as needed for additional memory_stats 38 | -------------------------------------------------------------------------------- /assets/macOS_accessibility.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenAdaptAI/OpenAdapt/HEAD/assets/macOS_accessibility.png -------------------------------------------------------------------------------- /assets/macOS_input_monitoring.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenAdaptAI/OpenAdapt/HEAD/assets/macOS_input_monitoring.png -------------------------------------------------------------------------------- /assets/macOS_permissions_alert.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenAdaptAI/OpenAdapt/HEAD/assets/macOS_permissions_alert.png -------------------------------------------------------------------------------- /assets/macOS_screen_recording.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenAdaptAI/OpenAdapt/HEAD/assets/macOS_screen_recording.png -------------------------------------------------------------------------------- /assets/visualize.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenAdaptAI/OpenAdapt/HEAD/assets/visualize.png -------------------------------------------------------------------------------- /chrome_extension/icons/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenAdaptAI/OpenAdapt/HEAD/chrome_extension/icons/logo.png -------------------------------------------------------------------------------- /chrome_extension/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "openadapt", 3 | "description": "Uses sockets to expose DOM events to OpenAdapt", 4 | "version": "1.0", 5 | "manifest_version": 3, 6 | "icons": { 7 | "48": "icons/logo.png" 8 | }, 9 | "action": { 10 | "default_icon": "icons/logo.png" 11 | }, 12 | "background": { 13 | "service_worker": "background.js" 14 | }, 15 | "permissions": ["activeTab", "tabs", "scripting"], 16 | "host_permissions": [""], 17 | "content_scripts": [ 18 | { 19 | "matches": [""], 20 | "js": ["content.js"] 21 | } 22 | ] 23 | } 24 | -------------------------------------------------------------------------------- /deploy/.env.example: -------------------------------------------------------------------------------- 1 | AWS_ACCESS_KEY_ID= 2 | AWS_SECRET_ACCESS_KEY= 3 | AWS_REGION= 4 | -------------------------------------------------------------------------------- /deploy/README.md: -------------------------------------------------------------------------------- 1 | ``` 2 | # First time setup 3 | cd deploy 4 | uv venv 5 | source .venv/bin/activate 6 | uv pip install -e . 7 | 8 | # Subsequent usage 9 | python deploy/models/omniparser/deploy.py start 10 | ``` 11 | -------------------------------------------------------------------------------- /deploy/deploy/models/omniparser/.dockerignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | *.pyc 3 | *.pyo 4 | *.pyd 5 | .Python 6 | env 7 | pip-log.txt 8 | pip-delete-this-directory.txt 9 | .tox 10 | .coverage 11 | .coverage.* 12 | .cache 13 | nosetests.xml 14 | coverage.xml 15 | *.cover 16 | *.log 17 | .pytest_cache 18 | .env 19 | .venv 20 | .DS_Store 21 | -------------------------------------------------------------------------------- /deploy/deploy/models/omniparser/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM nvidia/cuda:12.3.1-devel-ubuntu22.04 2 | 3 | RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ 4 | git-lfs \ 5 | wget \ 6 | libgl1 \ 7 | libglib2.0-0 \ 8 | && apt-get clean \ 9 | && rm -rf /var/lib/apt/lists/* \ 10 | && git lfs install 11 | 12 | RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh && \ 13 | bash miniconda.sh -b -p /opt/conda && \ 14 | rm miniconda.sh 15 | ENV PATH="/opt/conda/bin:$PATH" 16 | 17 | RUN conda create -n omni python=3.12 && \ 18 | echo "source activate omni" > ~/.bashrc 19 | ENV CONDA_DEFAULT_ENV=omni 20 | ENV PATH="/opt/conda/envs/omni/bin:$PATH" 21 | 22 | WORKDIR /app 23 | 24 | RUN git clone https://github.com/microsoft/OmniParser.git && \ 25 | cd OmniParser && \ 26 | git lfs install && \ 27 | git lfs pull 28 | 29 | WORKDIR /app/OmniParser 30 | 31 | RUN . /opt/conda/etc/profile.d/conda.sh && conda activate omni && \ 32 | pip uninstall -y opencv-python opencv-python-headless && \ 33 | pip install --no-cache-dir opencv-python-headless==4.8.1.78 && \ 34 | pip install -r requirements.txt && \ 35 | pip install huggingface_hub fastapi uvicorn 36 | 37 | # Download V2 weights 38 | RUN . /opt/conda/etc/profile.d/conda.sh && conda activate omni && \ 39 | mkdir -p /app/OmniParser/weights && \ 40 | cd /app/OmniParser && \ 41 | rm -rf weights/icon_detect weights/icon_caption weights/icon_caption_florence && \ 42 | for folder in icon_caption icon_detect; do \ 43 | huggingface-cli download microsoft/OmniParser-v2.0 --local-dir weights --repo-type model --include "$folder/*"; \ 44 | done && \ 45 | mv weights/icon_caption weights/icon_caption_florence 46 | 47 | # Pre-download OCR models during build 48 | RUN . /opt/conda/etc/profile.d/conda.sh && conda activate omni && \ 49 | cd /app/OmniParser && \ 50 | python3 -c "import easyocr; reader = easyocr.Reader(['en']); print('Downloaded EasyOCR model')" && \ 51 | python3 -c "from paddleocr import PaddleOCR; ocr = PaddleOCR(lang='en', use_angle_cls=False, use_gpu=False, show_log=False); print('Downloaded PaddleOCR model')" 52 | 53 | CMD ["python3", "/app/OmniParser/omnitool/omniparserserver/omniparserserver.py", \ 54 | "--som_model_path", "/app/OmniParser/weights/icon_detect/model.pt", \ 55 | "--caption_model_path", "/app/OmniParser/weights/icon_caption_florence", \ 56 | "--device", "cuda", \ 57 | "--BOX_TRESHOLD", "0.05", \ 58 | "--host", "0.0.0.0", \ 59 | "--port", "8000"] 60 | -------------------------------------------------------------------------------- /deploy/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools>=61.0"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [project] 6 | name = "deploy" 7 | version = "0.1.0" 8 | authors = [ 9 | { name="Richard Abrich", email="richard@openadapt.ai" }, 10 | ] 11 | description = "Deployment tools for OpenAdapt models" 12 | requires-python = ">=3.10" 13 | dependencies = [ 14 | "boto3>=1.36.22", 15 | "fire>=0.7.0", 16 | "loguru>=0.7.0", 17 | "paramiko>=3.5.1", 18 | "pillow>=11.1.0", 19 | "pydantic>=2.10.6", 20 | "pydantic-settings>=2.7.1", 21 | "requests>=2.32.3", 22 | ] 23 | -------------------------------------------------------------------------------- /experiments/fastsamsom.py: -------------------------------------------------------------------------------- 1 | """SoM with Ultralytics FastSAM.""" 2 | 3 | from pprint import pformat 4 | 5 | from PIL import Image 6 | import numpy as np 7 | 8 | from openadapt import adapters, config, contrib, utils, vision 9 | from openadapt.custom_logger import logger 10 | 11 | CONTRAST_FACTOR = 10000 12 | DEBUG = False 13 | 14 | 15 | def main() -> None: 16 | """Main.""" 17 | image_file_path = config.ROOT_DIR_PATH / "../tests/assets/excel.png" 18 | image = Image.open(image_file_path) 19 | if DEBUG: 20 | image.show() 21 | 22 | image_contrasted = utils.increase_contrast(image, CONTRAST_FACTOR) 23 | if DEBUG: 24 | image_contrasted.show() 25 | 26 | segmentation_adapter = adapters.get_default_segmentation_adapter() 27 | segmented_image = segmentation_adapter.fetch_segmented_image( 28 | image, 29 | # threshold below which boxes will be filtered out 30 | conf=0, 31 | # discards all overlapping boxes with IoU > iou_threshold 32 | iou=0.05, 33 | ) 34 | if DEBUG: 35 | segmented_image.show() 36 | 37 | masks = vision.get_masks_from_segmented_image(segmented_image, sort_by_area=True) 38 | # refined_masks = vision.refine_masks(masks) 39 | 40 | image_arr = np.asarray(image) 41 | 42 | # https://github.com/microsoft/SoM/blob/main/task_adapter/sam/tasks/inference_sam_m2m_auto.py 43 | # metadata = MetadataCatalog.get('coco_2017_train_panoptic') 44 | metadata = None 45 | visual = contrib.som.visualizer.Visualizer(image_arr, metadata=metadata) 46 | mask_map = np.zeros(image_arr.shape, dtype=np.uint8) 47 | label_mode = "1" 48 | alpha = 0.1 49 | anno_mode = [ 50 | "Mask", 51 | # 'Mark', 52 | ] 53 | for i, mask in enumerate(masks): 54 | label = i + 1 55 | demo = visual.draw_binary_mask_with_number( 56 | mask, 57 | text=str(label), 58 | label_mode=label_mode, 59 | alpha=alpha, 60 | anno_mode=anno_mode, 61 | ) 62 | mask_map[mask == 1] = label 63 | 64 | im = demo.get_image() 65 | image_som = Image.fromarray(im) 66 | image_som.show() 67 | 68 | results = [] 69 | 70 | prompt_adapter = adapters.get_default_prompt_adapter() 71 | text = ( 72 | "What are the values of the dates in the leftmost column? What about the" 73 | " horizontal column headings?" 74 | ) 75 | output = prompt_adapter.prompt( 76 | text, 77 | images=[ 78 | # no marks seem to perform just as well as with marks on spreadsheets 79 | # image_som, 80 | image, 81 | ], 82 | ) 83 | logger.info(output) 84 | results.append((text, output)) 85 | 86 | text = "\n".join( 87 | [ 88 | ( 89 | "Consider the dates along the leftmost column and the horizontal" 90 | " column headings:" 91 | ), 92 | output, 93 | "What are the values in the corresponding cells?", 94 | ] 95 | ) 96 | output = prompt_adapter.prompt(text, images=[image_som]) 97 | logger.info(output) 98 | results.append((text, output)) 99 | 100 | text = "What are the contents of cells A2, B2, and C2?" 101 | output = prompt_adapter.prompt(text, images=[image_som]) 102 | logger.info(output) 103 | results.append((text, output)) 104 | 105 | logger.info(f"results=\n{pformat(results)}") 106 | 107 | 108 | if __name__ == "__main__": 109 | main() 110 | -------------------------------------------------------------------------------- /experiments/gpt4o_seg.py: -------------------------------------------------------------------------------- 1 | """Generate segmentations directly with LLM.""" 2 | 3 | from pprint import pformat 4 | import os 5 | import sys 6 | import time 7 | 8 | from PIL import Image 9 | 10 | from openadapt import cache, config, models, plotting, utils 11 | from openadapt.adapters import openai 12 | from openadapt.custom_logger import logger 13 | 14 | 15 | @cache.cache(force_refresh=False) 16 | def get_window_image(window_search_str: str) -> tuple: 17 | """Get window image.""" 18 | logger.info(f"Waiting for window with title containing {window_search_str=}...") 19 | while True: 20 | window_event = models.WindowEvent.get_active_window_event() 21 | window_title = window_event.title 22 | if window_search_str.lower() in window_title.lower(): 23 | logger.info(f"found {window_title=}") 24 | break 25 | time.sleep(0.1) 26 | 27 | screenshot = models.Screenshot.take_screenshot() 28 | image = screenshot.crop_active_window(window_event=window_event) 29 | return window_event, image 30 | 31 | 32 | def main(window_search_str: str | None) -> None: 33 | """Main.""" 34 | if window_search_str: 35 | window_event, image = get_window_image(window_search_str) 36 | window_dict = window_event.to_prompt_dict() 37 | window_dict = utils.normalize_positions( 38 | window_dict, -window_event.left, -window_event.top 39 | ) 40 | else: 41 | image_file_path = os.path.join( 42 | config.ROOT_DIR_PATH, "../tests/assets/calculator.png" 43 | ) 44 | image = Image.open(image_file_path) 45 | window_dict = None 46 | 47 | system_prompt = utils.render_template_from_file( 48 | "prompts/system.j2", 49 | ) 50 | 51 | if window_dict: 52 | window_prompt = ( 53 | f"Consider the corresponding window state:\n```{pformat(window_dict)}```" 54 | ) 55 | else: 56 | window_prompt = "" 57 | 58 | prompt = f"""You are a master GUI understander. 59 | Your task is to locate all interactable elements in the supplied screenshot. 60 | {window_prompt} 61 | Return JSON containing an array of segments with the following properties: 62 | - "name": a unique identifier 63 | - "description": enough context to be able to differentiate between similar segments 64 | - "top": top coordinate of bounding box 65 | - "left": left coordinate of bounding box 66 | - "width": width of bouding box 67 | - "height": height of bounding box 68 | Provide as much detail as possible. My career depends on this. Lives are at stake. 69 | Respond with JSON ONLY AND NOTHING ELSE. 70 | """ 71 | 72 | result = openai.prompt( 73 | prompt, 74 | system_prompt, 75 | [image], 76 | ) 77 | segment_dict = utils.parse_code_snippet(result) 78 | plotting.plot_segments(image, segment_dict) 79 | 80 | window_dict = window_event.to_prompt_dict() 81 | import ipdb 82 | 83 | ipdb.set_trace() 84 | 85 | 86 | if __name__ == "__main__": 87 | main(sys.argv[1]) 88 | -------------------------------------------------------------------------------- /openadapt/__init__.py: -------------------------------------------------------------------------------- 1 | """OpenAdapt package. 2 | 3 | This package contains modules for the OpenAdapt project. 4 | """ 5 | 6 | from pathlib import Path 7 | 8 | PROJECT_DIR = Path(__file__).parent 9 | -------------------------------------------------------------------------------- /openadapt/adapters/__init__.py: -------------------------------------------------------------------------------- 1 | """Adapters for completion and segmentation.""" 2 | 3 | from types import ModuleType 4 | 5 | from openadapt.config import config 6 | 7 | from . import prompt, replicate, som, ultralytics 8 | 9 | 10 | # TODO: remove 11 | def get_default_prompt_adapter() -> ModuleType: 12 | """Returns the default prompt adapter module. 13 | 14 | Returns: 15 | The module corresponding to the default prompt adapter. 16 | """ 17 | return prompt 18 | 19 | 20 | # TODO: refactor to follow adapters.prompt 21 | def get_default_segmentation_adapter() -> ModuleType: 22 | """Returns the default image segmentation adapter module. 23 | 24 | Returns: 25 | The module corresponding to the default segmentation adapter. 26 | """ 27 | return { 28 | "som": som, 29 | "replicate": replicate, 30 | "ultralytics": ultralytics, 31 | }[config.DEFAULT_SEGMENTATION_ADAPTER] 32 | 33 | 34 | __all__ = ["anthropic", "openai", "replicate", "som", "ultralytics", "google"] 35 | -------------------------------------------------------------------------------- /openadapt/adapters/prompt.py: -------------------------------------------------------------------------------- 1 | """Adapter for prompting foundation models.""" 2 | 3 | from typing import Type 4 | 5 | from PIL import Image 6 | 7 | from openadapt.custom_logger import logger 8 | from openadapt.drivers import anthropic, google, openai 9 | 10 | # Define a list of drivers in the order they should be tried 11 | DRIVER_ORDER: list[Type] = [openai, google, anthropic] 12 | 13 | 14 | def prompt( 15 | text: str, 16 | images: list[Image.Image] | None = None, 17 | system_prompt: str | None = None, 18 | ) -> str: 19 | """Attempt to fetch a prompt completion from various services in order of priority. 20 | 21 | Args: 22 | text: The main text prompt. 23 | images: list of images to include in the prompt. 24 | system_prompt: An optional system-level prompt. 25 | 26 | Returns: 27 | The result from the first successful driver. 28 | """ 29 | text = text.strip() 30 | for driver in DRIVER_ORDER: 31 | try: 32 | logger.info(f"Trying driver: {driver.__name__}") 33 | return driver.prompt(text, images=images, system_prompt=system_prompt) 34 | except Exception as e: 35 | logger.exception(e) 36 | logger.error(f"Driver {driver.__name__} failed with error: {e}") 37 | import ipdb 38 | 39 | ipdb.set_trace() 40 | continue 41 | raise Exception("All drivers failed to provide a response") 42 | 43 | 44 | if __name__ == "__main__": 45 | # This could be extended to use command-line arguments or other input methods 46 | print(prompt("Describe the solar system.")) 47 | -------------------------------------------------------------------------------- /openadapt/adapters/som.py: -------------------------------------------------------------------------------- 1 | """Adapter for segmenting images via Set-of-Mark server. 2 | 3 | See https://github.com/microsoft/SoM for server implementation. 4 | """ 5 | 6 | import os 7 | import tempfile 8 | 9 | from PIL import Image 10 | import fire 11 | import gradio_client 12 | 13 | from openadapt.config import config 14 | from openadapt.custom_logger import logger 15 | 16 | 17 | def save_image_to_temp_file(img: Image.Image) -> str: 18 | """Save PIL.Image to a temporary file and return the file path. 19 | 20 | Args: 21 | img: the PIL Image to save 22 | 23 | Returns: 24 | Name of temporary file containing saved image. 25 | """ 26 | temp_file = tempfile.NamedTemporaryFile(delete=False, suffix=".png") 27 | img.save(temp_file, format="PNG") 28 | temp_file.close() # Important to close the file so others can access it 29 | logger.info(f"Image saved to temporary file: {temp_file.name}") 30 | 31 | return temp_file.name 32 | 33 | 34 | def predict( 35 | file_path: str = None, 36 | server_url: str = config.SOM_SERVER_URL, 37 | granularity: float = 2.7, 38 | segmentation_mode: str = "Automatic", 39 | mask_alpha: float = 0.8, 40 | mark_mode: str = "Number", 41 | annotation_mode: list = ["Mask"], 42 | api_name: str = "/inference", 43 | ) -> str: 44 | """Make a prediction using the Gradio client with the provided IP address. 45 | 46 | Args: 47 | server_url (str): The URL of the SoM Gradio server. 48 | file_path (str): File path of temp image (.png) copy of a PIL image object. 49 | granularity (float): Granularity value for the operation. 50 | segmentation_mode (str): Mode of segmentation ('Automatic' or 'Interactive'). 51 | mask_alpha (float): Alpha value for the mask. 52 | mark_mode (str): Mark mode, either 'Number' or 'Alphabet'. 53 | annotation_mode (list): List of annotation modes. 54 | api_name (str): API endpoint name for inference. 55 | 56 | Returns: 57 | Path to segmented image. 58 | """ 59 | assert server_url, server_url 60 | assert server_url.startswith("http"), server_url 61 | client = gradio_client.Client(server_url) 62 | result = client.predict( 63 | { 64 | "background": gradio_client.file(file_path), 65 | }, 66 | granularity, 67 | segmentation_mode, 68 | mask_alpha, 69 | mark_mode, 70 | annotation_mode, 71 | api_name=api_name, 72 | ) 73 | logger.info(result) 74 | 75 | return result 76 | 77 | 78 | def fetch_segmented_image(image: Image.Image) -> Image.Image: 79 | """Process an image using PIL.Image.Image object and predict using Gradio client. 80 | 81 | Args: 82 | image: The PIL Image to segment. 83 | 84 | Returns: 85 | The segmented PIL Image. 86 | """ 87 | raise NotImplementedError( 88 | "SoM server currently compresses the segmented image, " 89 | "resulting in many more colors than masks." 90 | ) 91 | img_temp_path = save_image_to_temp_file(image) # Save the image to a temp file 92 | segmented_image_path = predict(file_path=img_temp_path) # Perform prediction 93 | os.remove(img_temp_path) # Delete the temp file after prediction 94 | image = Image.open(segmented_image_path) 95 | os.remove(segmented_image_path) 96 | return image 97 | 98 | 99 | if __name__ == "__main__": 100 | fire.Fire(predict) 101 | -------------------------------------------------------------------------------- /openadapt/alembic/README: -------------------------------------------------------------------------------- 1 | Generic single-database configuration. 2 | -------------------------------------------------------------------------------- /openadapt/alembic/context_loader.py: -------------------------------------------------------------------------------- 1 | """Context loader for alembic migrations.""" 2 | 3 | 4 | import pathlib 5 | 6 | from alembic.config import Config 7 | 8 | from alembic import command 9 | 10 | 11 | def load_alembic_context(): 12 | ALEMBIC_INI = pathlib.Path(__file__).parent.parent / "alembic.ini" 13 | 14 | config = Config(ALEMBIC_INI) 15 | config.set_main_option("script_location", str(pathlib.Path(__file__).parent)) 16 | command.upgrade(config, "head") 17 | 18 | 19 | if __name__ == "__main__": 20 | load_alembic_context() 21 | -------------------------------------------------------------------------------- /openadapt/alembic/env.py: -------------------------------------------------------------------------------- 1 | """Alembic Environment Configuration. 2 | 3 | This module provides the environment configuration for Alembic. 4 | """ 5 | 6 | from logging.config import fileConfig 7 | 8 | from sqlalchemy import engine_from_config, pool 9 | 10 | from alembic import context 11 | from openadapt.config import config 12 | from openadapt.db import db 13 | from openadapt.models import ForceFloat 14 | 15 | # This is the Alembic Config object, which provides 16 | # access to the values within the .ini file in use. 17 | alembic_config = context.config 18 | 19 | # Interpret the config file for Python logging. 20 | # This line sets up loggers basically. 21 | if alembic_config.config_file_name is not None: 22 | fileConfig(alembic_config.config_file_name) 23 | 24 | # Add your model's MetaData object here 25 | # for 'autogenerate' support 26 | # from myapp import mymodel 27 | # target_metadata = mymodel.Base.metadata 28 | target_metadata = db.Base.metadata 29 | 30 | 31 | def get_url() -> str: 32 | """Get the database URL. 33 | 34 | Returns: 35 | str: The database URL. 36 | """ 37 | print(f"DB_URL={config.DB_URL}") 38 | return config.DB_URL 39 | 40 | 41 | def process_revision_directives(context, revision, directives): 42 | script = directives[0] 43 | script.imports.add("import openadapt") 44 | 45 | 46 | def run_migrations_offline() -> None: 47 | """Run migrations in 'offline' mode. 48 | 49 | This configures the context with just a URL 50 | and not an Engine, though an Engine is acceptable 51 | here as well. By skipping the Engine creation, 52 | we don't even need a DBAPI to be available. 53 | 54 | Calls to context.execute() here emit the given string to the 55 | script output. 56 | """ 57 | url = get_url() 58 | context.configure( 59 | url=url, 60 | target_metadata=target_metadata, 61 | literal_binds=True, 62 | dialect_opts={"paramstyle": "named"}, 63 | render_as_batch=True, 64 | process_revision_directives=process_revision_directives, 65 | ) 66 | 67 | with context.begin_transaction(): 68 | context.run_migrations() 69 | 70 | 71 | def run_migrations_online() -> None: 72 | """Run migrations in 'online' mode. 73 | 74 | In this scenario, we need to create an Engine 75 | and associate a connection with the context. 76 | """ 77 | configuration = alembic_config.get_section(alembic_config.config_ini_section) 78 | configuration["sqlalchemy.url"] = get_url() 79 | connectable = engine_from_config( 80 | configuration=configuration, 81 | prefix="sqlalchemy.", 82 | poolclass=pool.NullPool, 83 | ) 84 | 85 | with connectable.connect() as connection: 86 | context.configure( 87 | connection=connection, 88 | target_metadata=target_metadata, 89 | render_as_batch=True, 90 | process_revision_directives=process_revision_directives, 91 | ) 92 | 93 | with context.begin_transaction(): 94 | context.run_migrations() 95 | 96 | 97 | if context.is_offline_mode(): 98 | run_migrations_offline() 99 | else: 100 | run_migrations_online() 101 | -------------------------------------------------------------------------------- /openadapt/alembic/script.py.mako: -------------------------------------------------------------------------------- 1 | """${message} 2 | 3 | Revision ID: ${up_revision} 4 | Revises: ${down_revision | comma,n} 5 | Create Date: ${create_date} 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | ${imports if imports else ""} 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = ${repr(up_revision)} 14 | down_revision = ${repr(down_revision)} 15 | branch_labels = ${repr(branch_labels)} 16 | depends_on = ${repr(depends_on)} 17 | 18 | 19 | def upgrade() -> None: 20 | ${upgrades if upgrades else "pass"} 21 | 22 | 23 | def downgrade() -> None: 24 | ${downgrades if downgrades else "pass"} 25 | -------------------------------------------------------------------------------- /openadapt/alembic/versions/104d4a614d95_add_performancestat.py: -------------------------------------------------------------------------------- 1 | """add PerformanceStat 2 | 3 | Revision ID: 104d4a614d95 4 | Revises: b2dc41850120 5 | Create Date: 2023-05-27 02:59:14.032373 6 | 7 | """ 8 | import sqlalchemy as sa 9 | 10 | from alembic import op 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "104d4a614d95" 14 | down_revision = "b2dc41850120" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade() -> None: 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.create_table( 22 | "performance_stat", 23 | sa.Column("id", sa.Integer(), nullable=False), 24 | sa.Column("recording_timestamp", sa.Integer(), nullable=True), 25 | sa.Column("event_type", sa.String(), nullable=True), 26 | sa.Column("start_time", sa.Integer(), nullable=True), 27 | sa.Column("end_time", sa.Integer(), nullable=True), 28 | sa.Column("window_id", sa.String(), nullable=True), 29 | sa.PrimaryKeyConstraint("id", name=op.f("pk_performance_stat")), 30 | ) 31 | # ### end Alembic commands ### 32 | 33 | 34 | def downgrade() -> None: 35 | # ### commands auto generated by Alembic - please adjust! ### 36 | op.drop_table("performance_stat") 37 | # ### end Alembic commands ### 38 | -------------------------------------------------------------------------------- /openadapt/alembic/versions/186316d4f3ca_add_scrubbed_columns.py: -------------------------------------------------------------------------------- 1 | """add_scrubbed_columns 2 | 3 | Revision ID: 186316d4f3ca 4 | Revises: 2a8a241785f2 5 | Create Date: 2024-04-26 02:23:19.422051 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | # revision identifiers, used by Alembic. 12 | revision = "186316d4f3ca" 13 | down_revision = "2a8a241785f2" 14 | branch_labels = None 15 | depends_on = None 16 | 17 | 18 | def upgrade() -> None: 19 | # ### commands auto generated by Alembic - please adjust! ### 20 | with op.batch_alter_table("action_event", schema=None) as batch_op: 21 | batch_op.add_column(sa.Column("scrubbed_text", sa.String(), nullable=True)) 22 | batch_op.add_column( 23 | sa.Column("scrubbed_canonical_text", sa.String(), nullable=True) 24 | ) 25 | 26 | # ### end Alembic commands ### 27 | 28 | 29 | def downgrade() -> None: 30 | # ### commands auto generated by Alembic - please adjust! ### 31 | with op.batch_alter_table("action_event", schema=None) as batch_op: 32 | batch_op.drop_column("scrubbed_canonical_text") 33 | batch_op.drop_column("scrubbed_text") 34 | 35 | # ### end Alembic commands ### 36 | -------------------------------------------------------------------------------- /openadapt/alembic/versions/20f9c2afb42c_rename_input_event_to_action_event.py: -------------------------------------------------------------------------------- 1 | """rename input_event to action_event 2 | 3 | Revision ID: 20f9c2afb42c 4 | Revises: 5139d7df38f6 5 | Create Date: 2023-05-10 11:22:37.266810 6 | 7 | """ 8 | import sqlalchemy as sa 9 | 10 | from alembic import op 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "20f9c2afb42c" 14 | down_revision = "5139d7df38f6" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade() -> None: 20 | op.rename_table("input_event", "action_event") 21 | 22 | 23 | def downgrade() -> None: 24 | op.rename_table("action_event", "input_event") 25 | -------------------------------------------------------------------------------- /openadapt/alembic/versions/2a8a241785f2_add_original_recording_id.py: -------------------------------------------------------------------------------- 1 | """add_original_recording_id 2 | 3 | Revision ID: 2a8a241785f2 4 | Revises: 87a78a84a8bf 5 | Create Date: 2024-04-25 10:25:54.685135 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | # revision identifiers, used by Alembic. 12 | revision = "2a8a241785f2" 13 | down_revision = "87a78a84a8bf" 14 | branch_labels = None 15 | depends_on = None 16 | 17 | 18 | def upgrade() -> None: 19 | # ### commands auto generated by Alembic - please adjust! ### 20 | with op.batch_alter_table("recording", schema=None) as batch_op: 21 | batch_op.add_column( 22 | sa.Column("original_recording_id", sa.Integer(), nullable=True) 23 | ) 24 | batch_op.create_foreign_key( 25 | batch_op.f("fk_recording_original_recording_id_recording"), 26 | "recording", 27 | ["original_recording_id"], 28 | ["id"], 29 | ) 30 | 31 | # ### end Alembic commands ### 32 | 33 | 34 | def downgrade() -> None: 35 | # ### commands auto generated by Alembic - please adjust! ### 36 | with op.batch_alter_table("recording", schema=None) as batch_op: 37 | batch_op.drop_constraint( 38 | batch_op.f("fk_recording_original_recording_id_recording"), 39 | type_="foreignkey", 40 | ) 41 | batch_op.drop_column("original_recording_id") 42 | 43 | # ### end Alembic commands ### 44 | -------------------------------------------------------------------------------- /openadapt/alembic/versions/30a5ba9d6453_add_active_segment_description_and_.py: -------------------------------------------------------------------------------- 1 | """add active_segment_description and available_segment_descriptions 2 | 3 | Revision ID: 30a5ba9d6453 4 | Revises: 530f0663324e 5 | Create Date: 2024-04-05 12:02:57.843244 6 | 7 | """ 8 | import sqlalchemy as sa 9 | 10 | from alembic import op 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "30a5ba9d6453" 14 | down_revision = "530f0663324e" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade() -> None: 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | with op.batch_alter_table("action_event", schema=None) as batch_op: 22 | batch_op.add_column( 23 | sa.Column("active_segment_description", sa.String(), nullable=True) 24 | ) 25 | batch_op.add_column( 26 | sa.Column("available_segment_descriptions", sa.String(), nullable=True) 27 | ) 28 | 29 | # ### end Alembic commands ### 30 | 31 | 32 | def downgrade() -> None: 33 | # ### commands auto generated by Alembic - please adjust! ### 34 | with op.batch_alter_table("action_event", schema=None) as batch_op: 35 | batch_op.drop_column("available_segment_descriptions") 36 | batch_op.drop_column("active_segment_description") 37 | 38 | # ### end Alembic commands ### 39 | -------------------------------------------------------------------------------- /openadapt/alembic/versions/5139d7df38f6_add_recording_task_description.py: -------------------------------------------------------------------------------- 1 | """add Recording.task_description 2 | 3 | Revision ID: 5139d7df38f6 4 | Revises: b206c80f7640 5 | Create Date: 2023-04-17 13:28:07.525123 6 | 7 | """ 8 | import sqlalchemy as sa 9 | 10 | from alembic import op 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "5139d7df38f6" 14 | down_revision = "b206c80f7640" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade() -> None: 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | with op.batch_alter_table("recording", schema=None) as batch_op: 22 | batch_op.add_column(sa.Column("task_description", sa.String(), nullable=True)) 23 | 24 | # ### end Alembic commands ### 25 | 26 | 27 | def downgrade() -> None: 28 | # ### commands auto generated by Alembic - please adjust! ### 29 | with op.batch_alter_table("recording", schema=None) as batch_op: 30 | batch_op.drop_column("task_description") 31 | 32 | # ### end Alembic commands ### 33 | -------------------------------------------------------------------------------- /openadapt/alembic/versions/530f0663324e_add_recording_video_start_time.py: -------------------------------------------------------------------------------- 1 | """add Recording.video_start_time 2 | 3 | Revision ID: 530f0663324e 4 | Revises: 8713b142f5de 5 | Create Date: 2024-02-16 10:31:05.020772 6 | 7 | """ 8 | import sqlalchemy as sa 9 | 10 | from alembic import op 11 | from openadapt.models import ForceFloat 12 | 13 | # revision identifiers, used by Alembic. 14 | revision = "530f0663324e" 15 | down_revision = "8713b142f5de" 16 | branch_labels = None 17 | depends_on = None 18 | 19 | 20 | def upgrade() -> None: 21 | # ### commands auto generated by Alembic - please adjust! ### 22 | with op.batch_alter_table("recording", schema=None) as batch_op: 23 | batch_op.add_column( 24 | sa.Column( 25 | "video_start_time", 26 | ForceFloat(precision=10, scale=2, asdecimal=False), 27 | nullable=True, 28 | ) 29 | ) 30 | 31 | # ### end Alembic commands ### 32 | 33 | 34 | def downgrade() -> None: 35 | # ### commands auto generated by Alembic - please adjust! ### 36 | with op.batch_alter_table("recording", schema=None) as batch_op: 37 | batch_op.drop_column("video_start_time") 38 | 39 | # ### end Alembic commands ### 40 | -------------------------------------------------------------------------------- /openadapt/alembic/versions/57d78d23087a_add_windowevent_state.py: -------------------------------------------------------------------------------- 1 | """add WindowEvent.state 2 | 3 | Revision ID: 57d78d23087a 4 | Revises: 20f9c2afb42c 5 | Create Date: 2023-05-14 18:32:57.473479 6 | 7 | """ 8 | import sqlalchemy as sa 9 | 10 | from alembic import op 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "57d78d23087a" 14 | down_revision = "20f9c2afb42c" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade() -> None: 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | with op.batch_alter_table("window_event", schema=None) as batch_op: 22 | batch_op.add_column(sa.Column("state", sa.JSON(), nullable=True)) 23 | 24 | # ### end Alembic commands ### 25 | 26 | 27 | def downgrade() -> None: 28 | # ### commands auto generated by Alembic - please adjust! ### 29 | with op.batch_alter_table("window_event", schema=None) as batch_op: 30 | batch_op.drop_column("state") 31 | 32 | # ### end Alembic commands ### 33 | -------------------------------------------------------------------------------- /openadapt/alembic/versions/607d1380b5ae_add_memorystat.py: -------------------------------------------------------------------------------- 1 | """add MemoryStat 2 | 3 | Revision ID: 607d1380b5ae 4 | Revises: 104d4a614d95 5 | Create Date: 2023-06-28 11:54:36.749072 6 | 7 | """ 8 | import sqlalchemy as sa 9 | 10 | from alembic import op 11 | from openadapt.models import ForceFloat 12 | 13 | # revision identifiers, used by Alembic. 14 | revision = "607d1380b5ae" 15 | down_revision = "104d4a614d95" 16 | branch_labels = None 17 | depends_on = None 18 | 19 | 20 | def upgrade() -> None: 21 | # ### commands auto generated by Alembic - please adjust! ### 22 | op.create_table( 23 | "memory_stat", 24 | sa.Column("id", sa.Integer(), nullable=False), 25 | sa.Column("recording_timestamp", sa.Integer(), nullable=True), 26 | sa.Column( 27 | "memory_usage_bytes", 28 | ForceFloat(precision=10, scale=2, asdecimal=False), 29 | nullable=True, 30 | ), 31 | sa.Column( 32 | "timestamp", 33 | ForceFloat(precision=10, scale=2, asdecimal=False), 34 | nullable=True, 35 | ), 36 | sa.PrimaryKeyConstraint("id", name=op.f("pk_memory_stat")), 37 | ) 38 | # ### end Alembic commands ### 39 | 40 | 41 | def downgrade() -> None: 42 | # ### commands auto generated by Alembic - please adjust! ### 43 | op.drop_table("memory_stat") 44 | # ### end Alembic commands ### 45 | -------------------------------------------------------------------------------- /openadapt/alembic/versions/8495f5471e23_add_recording_config.py: -------------------------------------------------------------------------------- 1 | """add Recording.config 2 | 3 | Revision ID: 8495f5471e23 4 | Revises: 30a5ba9d6453 5 | Create Date: 2024-05-02 15:08:30.109181 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "8495f5471e23" 14 | down_revision = "30a5ba9d6453" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade() -> None: 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | with op.batch_alter_table("recording", schema=None) as batch_op: 22 | batch_op.add_column(sa.Column("config", sa.JSON(), nullable=True)) 23 | 24 | # ### end Alembic commands ### 25 | 26 | 27 | def downgrade() -> None: 28 | # ### commands auto generated by Alembic - please adjust! ### 29 | with op.batch_alter_table("recording", schema=None) as batch_op: 30 | batch_op.drop_column("config") 31 | 32 | # ### end Alembic commands ### 33 | -------------------------------------------------------------------------------- /openadapt/alembic/versions/8713b142f5de_add_png_diff_data_and_png_diff_mask_data.py: -------------------------------------------------------------------------------- 1 | """Add png_diff_data and png_diff_mask_data 2 | 3 | Revision ID: 8713b142f5de 4 | Revises: 607d1380b5ae 5 | Create Date: 2023-07-09 15:31:28.462388 6 | 7 | """ 8 | import sqlalchemy as sa 9 | 10 | from alembic import op 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "8713b142f5de" 14 | down_revision = "607d1380b5ae" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade() -> None: 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | with op.batch_alter_table("screenshot", schema=None) as batch_op: 22 | batch_op.add_column(sa.Column("png_diff_data", sa.LargeBinary(), nullable=True)) 23 | batch_op.add_column( 24 | sa.Column("png_diff_mask_data", sa.LargeBinary(), nullable=True) 25 | ) 26 | 27 | # ### end Alembic commands ### 28 | 29 | 30 | def downgrade() -> None: 31 | # ### commands auto generated by Alembic - please adjust! ### 32 | with op.batch_alter_table("screenshot", schema=None) as batch_op: 33 | batch_op.drop_column("png_diff_mask_data") 34 | batch_op.drop_column("png_diff_data") 35 | 36 | # ### end Alembic commands ### 37 | -------------------------------------------------------------------------------- /openadapt/alembic/versions/87a78a84a8bf_remove_recording_timestamp_fks.py: -------------------------------------------------------------------------------- 1 | """remove_recording_timestamp_fks 2 | 3 | Revision ID: 87a78a84a8bf 4 | Revises: f9586c10a561 5 | Create Date: 2024-04-24 20:16:31.970666 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | # revision identifiers, used by Alembic. 12 | revision = "87a78a84a8bf" 13 | down_revision = "f9586c10a561" 14 | branch_labels = None 15 | depends_on = None 16 | 17 | 18 | def upgrade() -> None: 19 | # ### commands auto generated by Alembic - please adjust! ### 20 | with op.batch_alter_table("action_event", schema=None) as batch_op: 21 | batch_op.drop_constraint( 22 | "fk_input_event_recording_timestamp_recording", type_="foreignkey" 23 | ) 24 | batch_op.drop_constraint( 25 | "fk_input_event_window_event_timestamp_window_event", type_="foreignkey" 26 | ) 27 | batch_op.drop_constraint( 28 | "fk_input_event_screenshot_timestamp_screenshot", type_="foreignkey" 29 | ) 30 | 31 | with op.batch_alter_table("screenshot", schema=None) as batch_op: 32 | batch_op.drop_constraint( 33 | "fk_screenshot_recording_timestamp_recording", type_="foreignkey" 34 | ) 35 | 36 | with op.batch_alter_table("window_event", schema=None) as batch_op: 37 | batch_op.drop_constraint( 38 | "fk_window_event_recording_timestamp_recording", type_="foreignkey" 39 | ) 40 | 41 | # ### end Alembic commands ### 42 | 43 | 44 | def downgrade() -> None: 45 | # ### commands auto generated by Alembic - please adjust! ### 46 | with op.batch_alter_table("window_event", schema=None) as batch_op: 47 | batch_op.create_foreign_key( 48 | "fk_window_event_recording_timestamp_recording", 49 | "recording", 50 | ["recording_timestamp"], 51 | ["timestamp"], 52 | ) 53 | 54 | with op.batch_alter_table("screenshot", schema=None) as batch_op: 55 | batch_op.create_foreign_key( 56 | "fk_screenshot_recording_timestamp_recording", 57 | "recording", 58 | ["recording_timestamp"], 59 | ["timestamp"], 60 | ) 61 | 62 | with op.batch_alter_table("action_event", schema=None) as batch_op: 63 | batch_op.create_foreign_key( 64 | "fk_input_event_screenshot_timestamp_screenshot", 65 | "screenshot", 66 | ["screenshot_timestamp"], 67 | ["timestamp"], 68 | ) 69 | batch_op.create_foreign_key( 70 | "fk_input_event_window_event_timestamp_window_event", 71 | "window_event", 72 | ["window_event_timestamp"], 73 | ["timestamp"], 74 | ) 75 | batch_op.create_foreign_key( 76 | "fk_input_event_recording_timestamp_recording", 77 | "recording", 78 | ["recording_timestamp"], 79 | ["timestamp"], 80 | ) 81 | 82 | # ### end Alembic commands ### 83 | -------------------------------------------------------------------------------- /openadapt/alembic/versions/98505a067995_add_browserevent_table.py: -------------------------------------------------------------------------------- 1 | """add BrowserEvent table 2 | 3 | Revision ID: 98505a067995 4 | Revises: bb25e889ad71 5 | Create Date: 2024-08-28 16:51:10.592340 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | import openadapt 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = '98505a067995' 14 | down_revision = 'bb25e889ad71' 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade() -> None: 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | op.create_table('browser_event', 22 | sa.Column('id', sa.Integer(), nullable=False), 23 | sa.Column('recording_timestamp', openadapt.models.ForceFloat(precision=10, scale=2, asdecimal=False), nullable=True), 24 | sa.Column('recording_id', sa.Integer(), nullable=True), 25 | sa.Column('message', sa.JSON(), nullable=True), 26 | sa.Column('timestamp', openadapt.models.ForceFloat(precision=10, scale=2, asdecimal=False), nullable=True), 27 | sa.ForeignKeyConstraint(['recording_id'], ['recording.id'], name=op.f('fk_browser_event_recording_id_recording')), 28 | sa.PrimaryKeyConstraint('id', name=op.f('pk_browser_event')) 29 | ) 30 | with op.batch_alter_table('action_event', schema=None) as batch_op: 31 | batch_op.add_column(sa.Column('browser_event_timestamp', openadapt.models.ForceFloat(precision=10, scale=2, asdecimal=False), nullable=True)) 32 | batch_op.add_column(sa.Column('browser_event_id', sa.Integer(), nullable=True)) 33 | batch_op.create_foreign_key(batch_op.f('fk_action_event_browser_event_id_browser_event'), 'browser_event', ['browser_event_id'], ['id']) 34 | 35 | # ### end Alembic commands ### 36 | 37 | 38 | def downgrade() -> None: 39 | # ### commands auto generated by Alembic - please adjust! ### 40 | with op.batch_alter_table('action_event', schema=None) as batch_op: 41 | batch_op.drop_constraint(batch_op.f('fk_action_event_browser_event_id_browser_event'), type_='foreignkey') 42 | batch_op.drop_column('browser_event_id') 43 | batch_op.drop_column('browser_event_timestamp') 44 | 45 | op.drop_table('browser_event') 46 | # ### end Alembic commands ### 47 | -------------------------------------------------------------------------------- /openadapt/alembic/versions/98c8851a5321_add_audio_info.py: -------------------------------------------------------------------------------- 1 | """add_audio_info 2 | 3 | Revision ID: 98c8851a5321 4 | Revises: d714cc86fce8 5 | Create Date: 2024-05-29 16:56:25.832333 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | import openadapt 12 | 13 | # revision identifiers, used by Alembic. 14 | revision = "98c8851a5321" 15 | down_revision = "d714cc86fce8" 16 | branch_labels = None 17 | depends_on = None 18 | 19 | 20 | def upgrade() -> None: 21 | # ### commands auto generated by Alembic - please adjust! ### 22 | op.create_table( 23 | "audio_info", 24 | sa.Column("id", sa.Integer(), nullable=False), 25 | sa.Column( 26 | "timestamp", 27 | openadapt.models.ForceFloat(precision=10, scale=2, asdecimal=False), 28 | nullable=True, 29 | ), 30 | sa.Column("flac_data", sa.LargeBinary(), nullable=True), 31 | sa.Column("transcribed_text", sa.String(), nullable=True), 32 | sa.Column( 33 | "recording_timestamp", 34 | openadapt.models.ForceFloat(precision=10, scale=2, asdecimal=False), 35 | nullable=True, 36 | ), 37 | sa.Column("recording_id", sa.Integer(), nullable=True), 38 | sa.Column("sample_rate", sa.Integer(), nullable=True), 39 | sa.Column("words_with_timestamps", sa.Text(), nullable=True), 40 | sa.ForeignKeyConstraint( 41 | ["recording_id"], 42 | ["recording.id"], 43 | name=op.f("fk_audio_info_recording_id_recording"), 44 | ), 45 | sa.PrimaryKeyConstraint("id", name=op.f("pk_audio_info")), 46 | ) 47 | # ### end Alembic commands ### 48 | 49 | 50 | def downgrade() -> None: 51 | # ### commands auto generated by Alembic - please adjust! ### 52 | op.drop_table("audio_info") 53 | # ### end Alembic commands ### 54 | -------------------------------------------------------------------------------- /openadapt/alembic/versions/a29b537fabe6_add_disabled_field_to_action_event.py: -------------------------------------------------------------------------------- 1 | """add_disabled_field_to_action_event 2 | 3 | Revision ID: a29b537fabe6 4 | Revises: 98c8851a5321 5 | Create Date: 2024-05-28 11:28:50.353928 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | # revision identifiers, used by Alembic. 12 | revision = "a29b537fabe6" 13 | down_revision = "98c8851a5321" 14 | branch_labels = None 15 | depends_on = None 16 | 17 | 18 | def upgrade() -> None: 19 | # ### commands auto generated by Alembic - please adjust! ### 20 | with op.batch_alter_table("action_event", schema=None) as batch_op: 21 | batch_op.add_column(sa.Column("disabled", sa.Boolean(), nullable=True)) 22 | 23 | # ### end Alembic commands ### 24 | 25 | 26 | def downgrade() -> None: 27 | # ### commands auto generated by Alembic - please adjust! ### 28 | with op.batch_alter_table("action_event", schema=None) as batch_op: 29 | batch_op.drop_column("disabled") 30 | 31 | # ### end Alembic commands ### 32 | -------------------------------------------------------------------------------- /openadapt/alembic/versions/b2dc41850120_window_window_id.py: -------------------------------------------------------------------------------- 1 | """Window.window_id 2 | 3 | Revision ID: b2dc41850120 4 | Revises: d63569e4fb90 5 | Create Date: 2023-05-17 12:50:35.125610 6 | 7 | """ 8 | import sqlalchemy as sa 9 | 10 | from alembic import op 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "b2dc41850120" 14 | down_revision = "d63569e4fb90" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade() -> None: 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | with op.batch_alter_table("window_event", schema=None) as batch_op: 22 | batch_op.add_column(sa.Column("window_id", sa.String(), nullable=True)) 23 | 24 | # ### end Alembic commands ### 25 | 26 | 27 | def downgrade() -> None: 28 | # ### commands auto generated by Alembic - please adjust! ### 29 | with op.batch_alter_table("window_event", schema=None) as batch_op: 30 | batch_op.drop_column("window_id") 31 | 32 | # ### end Alembic commands ### 33 | -------------------------------------------------------------------------------- /openadapt/alembic/versions/bb25e889ad71_generate_unique_user_id.py: -------------------------------------------------------------------------------- 1 | """generate_unique_user_id 2 | 3 | Revision ID: bb25e889ad71 4 | Revises: a29b537fabe6 5 | Create Date: 2024-06-11 17:16:28.009900 6 | 7 | """ 8 | from uuid import uuid4 9 | import json 10 | 11 | from alembic import op 12 | import sqlalchemy as sa 13 | 14 | from openadapt.config import config 15 | 16 | # revision identifiers, used by Alembic. 17 | revision = "bb25e889ad71" 18 | down_revision = "a29b537fabe6" 19 | branch_labels = None 20 | depends_on = None 21 | 22 | 23 | def upgrade() -> None: 24 | # ### commands auto generated by Alembic - please adjust! ### 25 | config.UNIQUE_USER_ID = str(uuid4()) 26 | # ### end Alembic commands ### 27 | 28 | 29 | def downgrade() -> None: 30 | # ### commands auto generated by Alembic - please adjust! ### 31 | config.UNIQUE_USER_ID = "" 32 | # ### end Alembic commands ### 33 | -------------------------------------------------------------------------------- /openadapt/alembic/versions/d63569e4fb90_actionevent_element_state.py: -------------------------------------------------------------------------------- 1 | """ActionEvent.element_state 2 | 3 | Revision ID: d63569e4fb90 4 | Revises: ec337f277666 5 | Create Date: 2023-05-16 21:43:00.120143 6 | 7 | """ 8 | import sqlalchemy as sa 9 | 10 | from alembic import op 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "d63569e4fb90" 14 | down_revision = "ec337f277666" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade() -> None: 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | with op.batch_alter_table("action_event", schema=None) as batch_op: 22 | batch_op.add_column(sa.Column("element_state", sa.JSON(), nullable=True)) 23 | 24 | # ### end Alembic commands ### 25 | 26 | 27 | def downgrade() -> None: 28 | # ### commands auto generated by Alembic - please adjust! ### 29 | with op.batch_alter_table("action_event", schema=None) as batch_op: 30 | batch_op.drop_column("element_state") 31 | 32 | # ### end Alembic commands ### 33 | -------------------------------------------------------------------------------- /openadapt/alembic/versions/d714cc86fce8_add_scrubbed_recording_model.py: -------------------------------------------------------------------------------- 1 | """add_scrubbed_recording_model 2 | 3 | Revision ID: d714cc86fce8 4 | Revises: 186316d4f3ca 5 | Create Date: 2024-04-29 09:17:04.237108 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | 11 | from openadapt.models import ForceFloat 12 | 13 | # revision identifiers, used by Alembic. 14 | revision = "d714cc86fce8" 15 | down_revision = "186316d4f3ca" 16 | branch_labels = None 17 | depends_on = None 18 | 19 | 20 | def upgrade() -> None: 21 | # ### commands auto generated by Alembic - please adjust! ### 22 | op.create_table( 23 | "scrubbed_recording", 24 | sa.Column("id", sa.Integer(), nullable=False), 25 | sa.Column( 26 | "timestamp", 27 | ForceFloat(precision=10, scale=2, asdecimal=False), 28 | nullable=True, 29 | ), 30 | sa.Column("recording_id", sa.Integer(), nullable=True), 31 | sa.Column("provider", sa.String(), nullable=True), 32 | sa.Column("scrubbed", sa.Boolean(), nullable=True), 33 | sa.ForeignKeyConstraint( 34 | ["recording_id"], 35 | ["recording.id"], 36 | name=op.f("fk_scrubbed_recording_recording_id_recording"), 37 | ), 38 | sa.PrimaryKeyConstraint("id", name=op.f("pk_scrubbed_recording")), 39 | ) 40 | # ### end Alembic commands ### 41 | 42 | 43 | def downgrade() -> None: 44 | # ### commands auto generated by Alembic - please adjust! ### 45 | op.drop_table("scrubbed_recording") 46 | # ### end Alembic commands ### 47 | -------------------------------------------------------------------------------- /openadapt/alembic/versions/ec337f277666_datetime_timestamp.py: -------------------------------------------------------------------------------- 1 | """DateTime timestamp 2 | 3 | Revision ID: ec337f277666 4 | Revises: 57d78d23087a 5 | Create Date: 2023-05-16 17:51:00.061604 6 | 7 | """ 8 | import sqlalchemy as sa 9 | 10 | from alembic import op 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "ec337f277666" 14 | down_revision = "57d78d23087a" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade() -> None: 20 | # ### commands auto generated by Alembic - please adjust! ### 21 | with op.batch_alter_table("screenshot", schema=None) as batch_op: 22 | batch_op.create_foreign_key( 23 | batch_op.f("fk_screenshot_recording_timestamp_recording"), 24 | "recording", 25 | ["recording_timestamp"], 26 | ["timestamp"], 27 | ) 28 | 29 | with op.batch_alter_table("window_event", schema=None) as batch_op: 30 | batch_op.create_foreign_key( 31 | batch_op.f("fk_window_event_recording_timestamp_recording"), 32 | "recording", 33 | ["recording_timestamp"], 34 | ["timestamp"], 35 | ) 36 | 37 | # ### end Alembic commands ### 38 | 39 | 40 | def downgrade() -> None: 41 | # ### commands auto generated by Alembic - please adjust! ### 42 | with op.batch_alter_table("window_event", schema=None) as batch_op: 43 | batch_op.drop_constraint( 44 | batch_op.f("fk_window_event_recording_timestamp_recording"), 45 | type_="foreignkey", 46 | ) 47 | 48 | with op.batch_alter_table("screenshot", schema=None) as batch_op: 49 | batch_op.drop_constraint( 50 | batch_op.f("fk_screenshot_recording_timestamp_recording"), 51 | type_="foreignkey", 52 | ) 53 | 54 | # ### end Alembic commands ### 55 | -------------------------------------------------------------------------------- /openadapt/alembic/versions/f9586c10a561_migrate_data_to_new_fks.py: -------------------------------------------------------------------------------- 1 | """migrate_data_to_new_fks 2 | 3 | Revision ID: f9586c10a561 4 | Revises: c24abb5455d3 5 | Create Date: 2024-04-24 19:34:00.000152 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | from sqlalchemy import text 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = "f9586c10a561" 14 | down_revision = "c24abb5455d3" 15 | branch_labels = None 16 | depends_on = None 17 | 18 | 19 | def upgrade() -> None: 20 | bind = op.get_bind() 21 | session = sa.orm.Session(bind=bind) 22 | 23 | for table in [ 24 | "action_event", 25 | "window_event", 26 | "screenshot", 27 | "memory_stat", 28 | "performance_stat", 29 | ]: 30 | session.execute( 31 | text( 32 | f"UPDATE {table} SET recording_id = (SELECT id FROM recording WHERE" 33 | f" recording.timestamp = {table}.recording_timestamp)" 34 | ) 35 | ) 36 | 37 | session.execute( 38 | text( 39 | "UPDATE action_event SET window_event_id = (SELECT id FROM window_event WHERE" 40 | " window_event.timestamp = action_event.window_event_timestamp)" 41 | ) 42 | ) 43 | session.execute( 44 | text( 45 | "UPDATE action_event SET screenshot_id = (SELECT id FROM screenshot WHERE" 46 | " screenshot.timestamp = action_event.screenshot_timestamp)" 47 | ) 48 | ) 49 | 50 | session.commit() 51 | 52 | 53 | def downgrade() -> None: 54 | pass 55 | -------------------------------------------------------------------------------- /openadapt/app/__init__.py: -------------------------------------------------------------------------------- 1 | """openadapt.app module. 2 | 3 | This module provides core functionality for the OpenAdapt application. 4 | """ 5 | 6 | from datetime import datetime 7 | import multiprocessing 8 | import pathlib 9 | import time 10 | 11 | from openadapt.record import record 12 | from openadapt.utils import WrapStdout 13 | 14 | __all__ = [ 15 | "RecordProc", 16 | "record_proc", 17 | "stop_record", 18 | "is_recording", 19 | "quick_record", 20 | "FPATH", 21 | ] 22 | 23 | # Define FPATH 24 | FPATH = pathlib.Path(__file__).parent 25 | 26 | 27 | class RecordProc: 28 | """Class to manage the recording process.""" 29 | 30 | def __init__(self) -> None: 31 | """Initialize the RecordProc class.""" 32 | self.terminate_processing = multiprocessing.Event() 33 | self.terminate_recording = multiprocessing.Event() 34 | self.record_proc: multiprocessing.Process = None 35 | self.has_initiated_stop = False 36 | 37 | def set_terminate_processing(self) -> multiprocessing.Event: 38 | """Set the terminate event.""" 39 | return self.terminate_processing.set() 40 | 41 | def terminate(self) -> None: 42 | """Terminate the recording process.""" 43 | self.record_proc.terminate() 44 | 45 | def reset(self) -> None: 46 | """Reset the recording process.""" 47 | self.terminate_processing.clear() 48 | self.terminate_recording.clear() 49 | self.record_proc = None 50 | self.has_initiated_stop = False 51 | 52 | def wait(self) -> None: 53 | """Wait for the recording process to finish.""" 54 | while True: 55 | if self.terminate_recording.is_set(): 56 | self.record_proc.terminate() 57 | return 58 | time.sleep(0.1) 59 | 60 | def is_running(self) -> bool: 61 | """Check if the recording process is running.""" 62 | if self.record_proc is not None and not self.record_proc.is_alive(): 63 | self.reset() 64 | return self.record_proc is not None 65 | 66 | def start(self, func: callable, args: tuple, kwargs: dict) -> None: 67 | """Start the recording process.""" 68 | self.record_proc = multiprocessing.Process( 69 | target=WrapStdout(func), 70 | args=args, 71 | kwargs=kwargs, 72 | ) 73 | self.record_proc.start() 74 | 75 | 76 | record_proc = RecordProc() 77 | 78 | 79 | def stop_record() -> None: 80 | """Stop the current recording session.""" 81 | global record_proc 82 | if record_proc.is_running() and not record_proc.has_initiated_stop: 83 | record_proc.set_terminate_processing() 84 | 85 | # wait for process to terminate 86 | record_proc.wait() 87 | record_proc.reset() 88 | 89 | 90 | def is_recording() -> bool: 91 | """Check if a recording session is currently active.""" 92 | global record_proc 93 | return record_proc.is_running() 94 | 95 | 96 | def quick_record( 97 | task_description: str | None = None, 98 | status_pipe: multiprocessing.connection.Connection | None = None, 99 | ) -> None: 100 | """Run a recording session.""" 101 | global record_proc 102 | task_description = task_description or datetime.now().strftime("%d/%m/%Y %H:%M:%S") 103 | record_proc.start( 104 | record, 105 | ( 106 | task_description, 107 | record_proc.terminate_processing, 108 | record_proc.terminate_recording, 109 | status_pipe, 110 | ), 111 | { 112 | "log_memory": False, 113 | }, 114 | ) 115 | -------------------------------------------------------------------------------- /openadapt/app/assets/logo.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenAdaptAI/OpenAdapt/HEAD/openadapt/app/assets/logo.ico -------------------------------------------------------------------------------- /openadapt/app/assets/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenAdaptAI/OpenAdapt/HEAD/openadapt/app/assets/logo.png -------------------------------------------------------------------------------- /openadapt/app/assets/logo_inverted.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenAdaptAI/OpenAdapt/HEAD/openadapt/app/assets/logo_inverted.png -------------------------------------------------------------------------------- /openadapt/app/dashboard/.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "next/core-web-vitals" 3 | } 4 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | /.pnp 6 | .pnp.js 7 | 8 | # testing 9 | /coverage 10 | 11 | # next.js 12 | /.next/ 13 | /out/ 14 | 15 | # production 16 | /build 17 | 18 | # misc 19 | .DS_Store 20 | *.pem 21 | 22 | # debug 23 | npm-debug.log* 24 | yarn-debug.log* 25 | yarn-error.log* 26 | 27 | # local env files 28 | .env*.local 29 | 30 | # vercel 31 | .vercel 32 | 33 | # typescript 34 | *.tsbuildinfo 35 | next-env.d.ts 36 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/.nvmrc: -------------------------------------------------------------------------------- 1 | 21 2 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/.prettierrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "trailingComma": "es5", 3 | "tabWidth": 4, 4 | "semi": false, 5 | "singleQuote": true 6 | } 7 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/README.md: -------------------------------------------------------------------------------- 1 |

2 | 3 | 4 |

Next.js FastAPI Starter

5 | 6 |

7 | 8 |

Simple Next.js boilerplate that uses FastAPI as the API backend.

9 | 10 |
11 | 12 | ## Introduction 13 | 14 | This is a hybrid Next.js + Python app that uses Next.js as the frontend and FastAPI as the API backend. One great use case of this is to write Next.js apps that use Python AI libraries on the backend. 15 | 16 | ## How It Works 17 | 18 | The Python/FastAPI server is mapped into to Next.js app under `/api/`. 19 | 20 | This is implemented using [`next.config.js` rewrites](https://github.com/digitros/nextjs-fastapi/blob/main/next.config.js) to map any request to `/api/:path*` to the FastAPI API, which is hosted in the `/api` folder. 21 | 22 | On localhost, the rewrite will be made to the `127.0.0.1:8000` port, which is where the FastAPI server is running. 23 | 24 | In production, the FastAPI server is hosted as [Python serverless functions](https://vercel.com/docs/concepts/functions/serverless-functions/runtimes/python) on Vercel. 25 | 26 | ## Demo 27 | 28 | https://nextjs-fastapi-starter.vercel.app/ 29 | 30 | ## Deploy Your Own 31 | 32 | You can clone & deploy it to Vercel with one click: 33 | 34 | [![Deploy with Vercel](https://vercel.com/button)](https://vercel.com/new/clone?repository-url=https%3A%2F%2Fgithub.com%2Fdigitros%2Fnextjs-fastapi%2Ftree%2Fmain) 35 | 36 | ## Developing Locally 37 | 38 | You can clone & create this repo with the following command 39 | 40 | ```bash 41 | npx create-next-app nextjs-fastapi --example "https://github.com/digitros/nextjs-fastapi" 42 | ``` 43 | 44 | ## Getting Started 45 | 46 | First, install the dependencies: 47 | 48 | ```bash 49 | npm install 50 | # or 51 | yarn 52 | # or 53 | pnpm install 54 | ``` 55 | 56 | Then, run the development server: 57 | 58 | ```bash 59 | npm run dev 60 | # or 61 | yarn dev 62 | # or 63 | pnpm dev 64 | ``` 65 | 66 | Open [http://localhost:3000](http://localhost:3000) with your browser to see the result. 67 | 68 | The FastApi server will be running on [http://127.0.0.1:8000](http://127.0.0.1:8000) – feel free to change the port in `package.json` (you'll also need to update it in `next.config.js`). 69 | 70 | ## Learn More 71 | 72 | To learn more about Next.js, take a look at the following resources: 73 | 74 | - [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API. 75 | - [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial. 76 | - [FastAPI Documentation](https://fastapi.tiangolo.com/) - learn about FastAPI features and API. 77 | 78 | You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js/) - your feedback and contributions are welcome! 79 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/__init__.py: -------------------------------------------------------------------------------- 1 | """Script to install dependencies needed for the dashboard.""" 2 | 3 | import os 4 | import subprocess 5 | import sys 6 | 7 | 8 | def _run(bash_script: str) -> int: 9 | return subprocess.call(bash_script, shell=True) 10 | 11 | 12 | def entrypoint() -> None: 13 | """Entrypoint for the installation script.""" 14 | cwd = os.path.dirname(os.path.realpath(__file__)) 15 | os.chdir(cwd) 16 | 17 | if sys.platform == "win32": 18 | _run("powershell -File entrypoint.ps1") 19 | return 20 | _run("source ./entrypoint.sh") 21 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/api.ts: -------------------------------------------------------------------------------- 1 | export async function get(url: string, options: Partial = {}): Promise { 2 | return fetch(url, options).then((res) => res.json()); 3 | } 4 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/api/action_events.py: -------------------------------------------------------------------------------- 1 | """API endpoints for recordings.""" 2 | 3 | from fastapi import APIRouter 4 | 5 | from openadapt.custom_logger import logger 6 | from openadapt.db import crud 7 | 8 | 9 | class ActionEventsAPI: 10 | """API endpoints for action events.""" 11 | 12 | def __init__(self) -> None: 13 | """Initialize the ActionEventsAPI class.""" 14 | self.app = APIRouter() 15 | 16 | def attach_routes(self) -> APIRouter: 17 | """Attach routes to the FastAPI app.""" 18 | self.app.add_api_route("/{event_id}", self.disable_event, methods=["DELETE"]) 19 | return self.app 20 | 21 | @staticmethod 22 | def disable_event(event_id: int) -> dict[str, str]: 23 | """Disable an action event. 24 | 25 | Args: 26 | event_id (int): The ID of the event to disable. 27 | 28 | Returns: 29 | dict: The response message and status code. 30 | """ 31 | if not crud.acquire_db_lock(): 32 | return {"message": "Database is locked", "status": "error"} 33 | session = crud.get_new_session(read_and_write=True) 34 | try: 35 | crud.disable_action_event(session, event_id) 36 | except Exception as e: 37 | logger.error(f"Error deleting event: {e}") 38 | session.rollback() 39 | crud.release_db_lock() 40 | return {"message": "Error deleting event", "status": "error"} 41 | crud.release_db_lock() 42 | return {"message": "Event deleted", "status": "success"} 43 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/api/index.py: -------------------------------------------------------------------------------- 1 | """API endpoints for the dashboard.""" 2 | 3 | from pathlib import Path 4 | import os 5 | 6 | from fastapi import APIRouter, FastAPI 7 | from fastapi.responses import FileResponse 8 | from fastapi.staticfiles import StaticFiles 9 | import uvicorn 10 | 11 | from openadapt.app.dashboard.api.action_events import ActionEventsAPI 12 | from openadapt.app.dashboard.api.recordings import RecordingsAPI 13 | from openadapt.app.dashboard.api.scrubbing import ScrubbingAPI 14 | from openadapt.app.dashboard.api.settings import SettingsAPI 15 | from openadapt.build_utils import is_running_from_executable 16 | from openadapt.config import config 17 | from openadapt.custom_logger import logger 18 | 19 | app = FastAPI() 20 | 21 | api = APIRouter() 22 | 23 | action_events_app = ActionEventsAPI().attach_routes() 24 | recordings_app = RecordingsAPI().attach_routes() 25 | scrubbing_app = ScrubbingAPI().attach_routes() 26 | settings_app = SettingsAPI().attach_routes() 27 | 28 | api.include_router(action_events_app, prefix="/action-events") 29 | api.include_router(recordings_app, prefix="/recordings") 30 | api.include_router(scrubbing_app, prefix="/scrubbing") 31 | api.include_router(settings_app, prefix="/settings") 32 | 33 | app.include_router(api, prefix="/api") 34 | 35 | 36 | def run_app() -> None: 37 | """Run the dashboard.""" 38 | if is_running_from_executable(): 39 | build_directory = Path(__file__).parent.parent / "out" 40 | 41 | def add_route(path: str) -> None: 42 | """Add a route to the dashboard.""" 43 | 44 | def route() -> FileResponse: 45 | return FileResponse(build_directory / path) 46 | 47 | stripped_path = f'/{path.replace(".html", "")}' 48 | logger.info(f"Adding route: {stripped_path}") 49 | app.get(stripped_path)(route) 50 | 51 | for root, _, files in os.walk(build_directory): 52 | for file in files: 53 | if file.endswith(".html"): 54 | path = os.path.relpath(os.path.join(root, file), build_directory) 55 | add_route(path) 56 | 57 | app.mount("/", StaticFiles(directory=build_directory), name="static") 58 | 59 | uvicorn.run(app, port=config.DASHBOARD_SERVER_PORT) 60 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/api/settings.py: -------------------------------------------------------------------------------- 1 | """API endpoints for settings.""" 2 | 3 | from typing import Any, Literal 4 | 5 | from fastapi import APIRouter 6 | 7 | from openadapt.config import Config, config, persist_config 8 | 9 | 10 | class SettingsAPI: 11 | """API endpoints for settings.""" 12 | 13 | def __init__(self) -> None: 14 | """Initialize the SettingsAPI class.""" 15 | self.app = APIRouter() 16 | 17 | def attach_routes(self) -> APIRouter: 18 | """Attach routes to the FastAPI app.""" 19 | self.app.add_api_route("", self.get_settings, methods=["GET"]) 20 | self.app.add_api_route("", self.set_settings, methods=["POST"]) 21 | return self.app 22 | 23 | Category = Literal[ 24 | "api_keys", "scrubbing", "record_and_replay", "general", "onboarding" 25 | ] 26 | 27 | @staticmethod 28 | def get_settings(category: Category) -> dict[str, Any]: 29 | """Get all settings.""" 30 | compact_settings = dict() 31 | for key in Config.classifications[category]: 32 | compact_settings[key] = getattr(config, key) 33 | return compact_settings 34 | 35 | @staticmethod 36 | def set_settings(body: Config) -> dict[str, str]: 37 | """Set settings.""" 38 | persist_config(body) 39 | return {"status": "success"} 40 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/app/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenAdaptAI/OpenAdapt/HEAD/openadapt/app/dashboard/app/favicon.ico -------------------------------------------------------------------------------- /openadapt/app/dashboard/app/globals.css: -------------------------------------------------------------------------------- 1 | @layer mantine tailwind; 2 | 3 | @import '@mantine/core/styles.layer.css'; 4 | @import '@mantine/notifications/styles.layer.css'; 5 | @import '@mantine/carousel/styles.layer.css'; 6 | 7 | @layer tailwind { 8 | @tailwind utilities; 9 | } 10 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/app/layout.tsx: -------------------------------------------------------------------------------- 1 | import './globals.css' 2 | 3 | import { ColorSchemeScript, MantineProvider } from '@mantine/core' 4 | import { Notifications } from '@mantine/notifications'; 5 | import { Shell } from '@/components/Shell' 6 | import { CSPostHogProvider } from './providers'; 7 | 8 | export const metadata = { 9 | title: 'OpenAdapt.AI', 10 | } 11 | 12 | export default function RootLayout({ 13 | children, 14 | }: { 15 | children: React.ReactNode 16 | }) { 17 | return ( 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | {children} 28 | 29 | 30 | 31 | 32 | 33 | ) 34 | } 35 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/app/onboarding/page.tsx: -------------------------------------------------------------------------------- 1 | import { BookACall } from "@/components/Onboarding/steps/BookACall"; 2 | import { RegisterForUpdates } from "@/components/Onboarding/steps/RegisterForUpdates"; 3 | import { Tutorial } from "@/components/Onboarding/steps/Tutorial"; 4 | import { Box, Divider } from "@mantine/core"; 5 | 6 | export default function Onboarding() { 7 | return ( 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | ); 16 | } 17 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/app/providers.tsx: -------------------------------------------------------------------------------- 1 | 'use client' 2 | import { get } from '@/api' 3 | import posthog from 'posthog-js' 4 | import { PostHogProvider } from 'posthog-js/react' 5 | import { useEffect } from 'react' 6 | 7 | if (typeof window !== 'undefined') { 8 | if (process.env.NEXT_PUBLIC_MODE !== "development") { 9 | posthog.init(process.env.NEXT_PUBLIC_POSTHOG_PUBLIC_KEY as string, { 10 | api_host: process.env.NEXT_PUBLIC_POSTHOG_HOST, 11 | }) 12 | } 13 | } 14 | 15 | async function getSettings(): Promise> { 16 | return get('/api/settings?category=general', { 17 | cache: 'no-store', 18 | }) 19 | } 20 | 21 | 22 | export function CSPostHogProvider({ children }: { children: React.ReactNode }) { 23 | useEffect(() => { 24 | if (process.env.NEXT_PUBLIC_MODE !== "development") { 25 | getSettings().then((settings) => { 26 | posthog.identify(settings['UNIQUE_USER_ID']) 27 | }) 28 | } 29 | }, []) 30 | if (process.env.NEXT_PUBLIC_MODE === "development") { 31 | return <>{children}; 32 | } 33 | return {children} 34 | } 35 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/app/recordings/RawRecordings.tsx: -------------------------------------------------------------------------------- 1 | import { SimpleTable } from '@/components/SimpleTable'; 2 | import { Recording } from '@/types/recording'; 3 | import React, { useEffect, useState } from 'react' 4 | import { timeStampToDateString } from '../utils'; 5 | import { useRouter } from 'next/navigation'; 6 | 7 | export const RawRecordings = () => { 8 | const [recordings, setRecordings] = useState([]); 9 | const router = useRouter(); 10 | 11 | function fetchRecordings() { 12 | fetch('/api/recordings').then(res => { 13 | if (res.ok) { 14 | res.json().then((data) => { 15 | setRecordings(data.recordings); 16 | }); 17 | } 18 | }) 19 | } 20 | 21 | useEffect(() => { 22 | fetchRecordings(); 23 | }, []); 24 | 25 | function onClickRow(recording: Recording) { 26 | return () => router.push(`/recordings/detail/?id=${recording.id}`); 27 | } 28 | 29 | return ( 30 | recording.video_start_time ? timeStampToDateString(recording.video_start_time) : 'N/A'}, 35 | {name: 'Timestamp', accessor: (recording: Recording) => recording.timestamp ? timeStampToDateString(recording.timestamp) : 'N/A'}, 36 | {name: 'Monitor Width/Height', accessor: (recording: Recording) => `${recording.monitor_width}/${recording.monitor_height}`}, 37 | {name: 'Double Click Interval Seconds/Pixels', accessor: (recording: Recording) => `${recording.double_click_interval_seconds}/${recording.double_click_distance_pixels}`}, 38 | ]} 39 | data={recordings} 40 | refreshData={fetchRecordings} 41 | onClickRow={onClickRow} 42 | /> 43 | ) 44 | } 45 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/app/recordings/ScrubbedRecordings.tsx: -------------------------------------------------------------------------------- 1 | import { SimpleTable } from '@/components/SimpleTable'; 2 | import { Recording, ScrubbedRecording } from '@/types/recording'; 3 | import { useRouter } from 'next/navigation'; 4 | import React, { useEffect, useState } from 'react' 5 | 6 | export const ScrubbedRecordings = () => { 7 | const [recordings, setRecordings] = useState([]); 8 | const router = useRouter(); 9 | 10 | function fetchScrubbedRecordings() { 11 | fetch('/api/recordings/scrubbed').then(res => { 12 | if (res.ok) { 13 | res.json().then((data) => { 14 | setRecordings(data.recordings); 15 | }); 16 | } 17 | }) 18 | } 19 | 20 | function onClickRow(recording: ScrubbedRecording) { 21 | return () => router.push(`/recordings/detail/?id=${recording.recording_id}`); 22 | } 23 | 24 | useEffect(() => { 25 | fetchScrubbedRecordings(); 26 | }, []); 27 | 28 | return ( 29 | recording.recording_id}, 32 | {name: 'Description', accessor: (recording: ScrubbedRecording) => recording.recording.task_description}, 33 | {name: 'Provider', accessor: 'provider'}, 34 | {name: 'Original Recording', accessor: (recording: ScrubbedRecording) => recording.original_recording.task_description}, 35 | ]} 36 | data={recordings} 37 | refreshData={fetchScrubbedRecordings} 38 | onClickRow={onClickRow} 39 | /> 40 | ) 41 | } 42 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/app/recordings/page.tsx: -------------------------------------------------------------------------------- 1 | 'use client'; 2 | 3 | import { Box, Button, Tabs } from "@mantine/core"; 4 | import { useEffect, useState } from "react"; 5 | import { RecordingStatus } from "@/types/recording"; 6 | import { ScrubbedRecordings } from "./ScrubbedRecordings"; 7 | import { RawRecordings } from "./RawRecordings"; 8 | 9 | 10 | export default function Recordings() { 11 | const [recordingStatus, setRecordingStatus] = useState(RecordingStatus.UNKNOWN); 12 | function startRecording() { 13 | if (recordingStatus === RecordingStatus.RECORDING) { 14 | return; 15 | } 16 | fetch('/api/recordings/start').then(res => { 17 | if (res.ok) { 18 | setRecordingStatus(RecordingStatus.RECORDING); 19 | } 20 | }); 21 | } 22 | function stopRecording() { 23 | if (recordingStatus === RecordingStatus.STOPPED) { 24 | return; 25 | } 26 | setRecordingStatus(RecordingStatus.UNKNOWN); 27 | fetch('/api/recordings/stop').then(res => { 28 | if (res.ok) { 29 | setRecordingStatus(RecordingStatus.STOPPED); 30 | } 31 | }); 32 | } 33 | 34 | function fetchRecordingStatus() { 35 | fetch('/api/recordings/status').then(res => { 36 | if (res.ok) { 37 | res.json().then((data) => { 38 | if (data.recording) { 39 | setRecordingStatus(RecordingStatus.RECORDING); 40 | } else { 41 | setRecordingStatus(RecordingStatus.STOPPED); 42 | } 43 | }); 44 | } 45 | }); 46 | } 47 | 48 | useEffect(() => { 49 | fetchRecordingStatus(); 50 | }, []); 51 | return ( 52 | 53 | {recordingStatus === RecordingStatus.RECORDING && ( 54 | 57 | )} 58 | {recordingStatus === RecordingStatus.STOPPED && ( 59 | 62 | )} 63 | {recordingStatus === RecordingStatus.UNKNOWN && ( 64 | 67 | )} 68 | 69 | 70 | 71 | Recordings 72 | 73 | 74 | Scrubbed recordings 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | ) 87 | } 88 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/app/routes.ts: -------------------------------------------------------------------------------- 1 | type Route = { 2 | name: string 3 | path: string 4 | } 5 | 6 | export const routes: Route[] = [ 7 | { 8 | name: 'Recordings', 9 | path: '/recordings', 10 | }, 11 | { 12 | name: 'Settings', 13 | path: '/settings', 14 | }, 15 | { 16 | name: 'Scrubbing', 17 | path: '/scrubbing', 18 | }, 19 | { 20 | name: 'Onboarding', 21 | path: '/onboarding', 22 | } 23 | ] 24 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/app/scrubbing/ScrubbingUpdates.tsx: -------------------------------------------------------------------------------- 1 | import { ScrubbingUpdate } from '@/types/scrubbing'; 2 | import { Box, Button, Container, Progress, Stack, Text } from '@mantine/core'; 3 | import React from 'react' 4 | 5 | type Props = { 6 | data?: ScrubbingUpdate; 7 | resetScrubbingStatus: () => void; 8 | } 9 | 10 | export const ScrubbingUpdates = ({ data, resetScrubbingStatus }: Props) => { 11 | if (!data) { 12 | return null; 13 | } 14 | const isScrubbingComplete = ( 15 | data.total_action_events > 0 && 16 | data.total_window_events > 0 && 17 | data.total_screenshots > 0 && 18 | data.num_action_events_scrubbed === data.total_action_events && 19 | data.num_window_events_scrubbed === data.total_window_events && 20 | data.num_screenshots_scrubbed === data.total_screenshots 21 | ) || data.error; 22 | return ( 23 | 24 | 25 | Scrubbing updates for recording {data.recording.task_description} 26 | Provider: {data.provider} 27 | {data.copying_recording ? ( 28 | Copying recording (this may take a while if Spacy dependencies need to be downloaded on the first run)... 29 | ) : data.error ? ( 30 | {data.error} 31 | ) : ( 32 | 33 | 34 | 35 | {data.num_action_events_scrubbed} / {data.total_action_events} action events scrubbed 36 | 37 | 38 | 39 | {data.num_window_events_scrubbed} / {data.total_window_events} window events scrubbed 40 | 41 | 42 | 43 | {data.num_screenshots_scrubbed} / {data.total_screenshots} screenshots scrubbed 44 | 45 | 46 | )} 47 | {isScrubbingComplete && ( 48 | 49 | Scrubbing complete! 50 | 51 | 52 | )} 53 | 54 | 55 | ) 56 | } 57 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/app/settings/(api_keys)/form.tsx: -------------------------------------------------------------------------------- 1 | 'use client'; 2 | 3 | import { Button, Fieldset, Flex, Grid, Stack, TextInput } from '@mantine/core'; 4 | import { useForm } from '@mantine/form'; 5 | import React, { useEffect } from 'react' 6 | import { saveSettings } from '../utils'; 7 | 8 | type Props = { 9 | settings: Record, 10 | } 11 | 12 | export const Form = ({ 13 | settings, 14 | }: Props) => { 15 | const form = useForm({ 16 | initialValues: JSON.parse(JSON.stringify(settings)), 17 | }) 18 | 19 | useEffect(() => { 20 | form.setValues(JSON.parse(JSON.stringify(settings))); 21 | form.setInitialValues(JSON.parse(JSON.stringify(settings))); 22 | }, [settings]); 23 | 24 | function resetForm() { 25 | form.reset(); 26 | } 27 | return ( 28 |
29 | 30 | 31 |
32 | 33 | 34 | 35 |
36 | 37 | 38 |
39 | 40 | 41 | 42 |
43 | 44 | 45 |
46 | 47 | 48 | 49 | 50 | 51 |
52 | 53 | 54 | 55 | 58 | 61 | 62 |
63 | ) 64 | } 65 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/app/settings/(api_keys)/page.tsx: -------------------------------------------------------------------------------- 1 | 'use client'; 2 | 3 | import React, { useEffect, useState } from 'react' 4 | import { get } from '@/api'; 5 | import { Form } from './form'; 6 | 7 | 8 | async function getSettings(): Promise> { 9 | return get('/api/settings?category=api_keys', { 10 | cache: 'no-store', 11 | }) 12 | } 13 | 14 | export default function APIKeys () { 15 | const [settings, setSettings] = useState({}); 16 | useEffect(() => { 17 | getSettings().then(setSettings); 18 | }, []) 19 | 20 | return ( 21 |
22 | ) 23 | } 24 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/app/settings/SettingsHeader.tsx: -------------------------------------------------------------------------------- 1 | 'use client'; 2 | 3 | import { Text } from '@mantine/core'; 4 | import { usePathname } from 'next/navigation'; 5 | import React from 'react' 6 | 7 | 8 | type Props = { 9 | routes: { name: string; path: string }[]; 10 | } 11 | 12 | export const SettingsHeader = ({ routes }: Props) => { 13 | const currentRoute = usePathname() 14 | return ( 15 | Settings | {routes.find(route => route.path === currentRoute)?.name} 16 | ) 17 | } 18 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/app/settings/layout.tsx: -------------------------------------------------------------------------------- 1 | import { Box, Flex } from "@mantine/core" 2 | import { Navbar } from "./navbar" 3 | import { SettingsHeader } from "./SettingsHeader" 4 | 5 | const routes = [ 6 | { name: 'API Keys', path: '/settings' }, 7 | { name: 'Scrubbing', path: '/settings/scrubbing' }, 8 | { name: 'Record & Replay', path: '/settings/record_and_replay' }, 9 | ] 10 | 11 | export default function RootLayout({ 12 | children, 13 | }: { 14 | children: React.ReactNode 15 | }) { 16 | return ( 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | {children} 25 | 26 | 27 | 28 | ) 29 | } 30 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/app/settings/navbar.tsx: -------------------------------------------------------------------------------- 1 | 'use client'; 2 | 3 | import { Stack } from '@mantine/core'; 4 | import { IconChevronRight } from '@tabler/icons-react'; 5 | import Link from 'next/link'; 6 | import { usePathname } from 'next/navigation'; 7 | import React from 'react' 8 | 9 | type Props = { 10 | routes: { name: string; path: string }[] 11 | } 12 | 13 | export const Navbar = ({ 14 | routes 15 | }: Props) => { 16 | const currentRoute = usePathname() 17 | return ( 18 | 19 | {routes.map((route) => ( 20 | 28 | {route.name} 29 | 30 | ))} 31 | 32 | ) 33 | } 34 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/app/settings/record_and_replay/form.tsx: -------------------------------------------------------------------------------- 1 | 'use client'; 2 | 3 | import { Button, Checkbox, Flex, Grid, TextInput } from '@mantine/core'; 4 | import { useForm } from '@mantine/form'; 5 | import React, { useEffect } from 'react' 6 | import { saveSettings, validateRecordAndReplaySettings } from '../utils'; 7 | 8 | type Props = { 9 | settings: Record, 10 | } 11 | 12 | export const Form = ({ 13 | settings, 14 | }: Props) => { 15 | const form = useForm({ 16 | initialValues: JSON.parse(JSON.stringify(settings)), 17 | validate: (values) => { 18 | return validateRecordAndReplaySettings(values); 19 | }, 20 | }) 21 | 22 | useEffect(() => { 23 | form.setValues(JSON.parse(JSON.stringify(settings))); 24 | form.setInitialValues(JSON.parse(JSON.stringify(settings))); 25 | }, [settings]); 26 | 27 | function resetForm() { 28 | form.reset(); 29 | } 30 | 31 | return ( 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 57 | 60 | 61 | 62 | ) 63 | } 64 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/app/settings/record_and_replay/page.tsx: -------------------------------------------------------------------------------- 1 | 'use client'; 2 | 3 | import React, { useEffect, useState } from 'react' 4 | import { get } from '@/api'; 5 | import { Form } from './form'; 6 | 7 | 8 | async function getSettings(): Promise> { 9 | return get('/api/settings?category=record_and_replay', { 10 | cache: 'no-store', 11 | }) 12 | } 13 | 14 | export default function APIKeys () { 15 | const [settings, setSettings] = useState({}); 16 | useEffect(() => { 17 | getSettings().then(setSettings) 18 | }, []) 19 | return ( 20 |
21 | ) 22 | } 23 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/app/settings/scrubbing/form.tsx: -------------------------------------------------------------------------------- 1 | 'use client'; 2 | 3 | import { Button, Checkbox, ColorInput, Flex, Grid, TextInput } from '@mantine/core'; 4 | import { UseFormReturnType, useForm } from '@mantine/form'; 5 | import React, { useEffect } from 'react' 6 | import { saveSettings, validateScrubbingSettings } from '../utils'; 7 | 8 | type Props = { 9 | settings: Record, 10 | } 11 | 12 | export const Form = ({ 13 | settings, 14 | }: Props) => { 15 | const form = useForm({ 16 | initialValues: getSettingsCopy(settings), 17 | validate: (values) => { 18 | return validateScrubbingSettings(values); 19 | }, 20 | }) 21 | 22 | useEffect(() => { 23 | form.setValues(getSettingsCopy(settings)); 24 | form.setInitialValues(getSettingsCopy(settings)); 25 | }, [settings]); 26 | 27 | function resetForm() { 28 | form.reset(); 29 | } 30 | 31 | function _onSubmit(values: Record) { 32 | saveSettings(form)({ 33 | ...values, 34 | SCRUB_FILL_COLOR: convertHexToBGRInt(values.SCRUB_FILL_COLOR), 35 | }); 36 | } 37 | 38 | return ( 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 58 | 61 | 62 | 63 | ) 64 | } 65 | 66 | function toHex(value: number): string { 67 | return value.toString(16).padStart(2, '0'); 68 | } 69 | 70 | function toInt(value: string): number { 71 | return parseInt(value, 16); 72 | } 73 | 74 | function convertBGRIntToHex(color: number): string { 75 | const r = color & 255; 76 | const g = (color >> 8) & 255; 77 | const b = (color >> 16) & 255; 78 | return `#${toHex(r)}${toHex(g)}${toHex(b)}`; 79 | } 80 | 81 | function convertHexToBGRInt(color: string): number { 82 | const hex = color.replace('#', ''); 83 | const r = toInt(hex.slice(0, 2)); 84 | const g = toInt(hex.slice(2, 4)); 85 | const b = toInt(hex.slice(4, 6)); 86 | return r | (g << 8) | (b << 16); 87 | } 88 | 89 | function getSettingsCopy(settings: Record) { 90 | return JSON.parse(JSON.stringify({ 91 | ...settings, 92 | SCRUB_FILL_COLOR: convertBGRIntToHex(parseInt(settings.SCRUB_FILL_COLOR)), 93 | })); 94 | } 95 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/app/settings/scrubbing/page.tsx: -------------------------------------------------------------------------------- 1 | 'use client'; 2 | 3 | import React, { useEffect, useState } from 'react' 4 | import { get } from '@/api'; 5 | import { Form } from './form'; 6 | 7 | 8 | function getSettings(): Promise> { 9 | return get('/api/settings?category=scrubbing', { 10 | cache: 'no-store', 11 | }) 12 | } 13 | 14 | export default function APIKeys () { 15 | const [settings, setSettings] = useState({}); 16 | useEffect(() => { 17 | getSettings().then(setSettings) 18 | }, []) 19 | return ( 20 |
21 | ) 22 | } 23 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/app/settings/utils.ts: -------------------------------------------------------------------------------- 1 | import { UseFormReturnType } from '@mantine/form'; 2 | import { notifications } from '@mantine/notifications'; 3 | 4 | 5 | export function validateScrubbingSettings(settings: Record) { 6 | const errors: Record = {} 7 | if (settings.SCRUB_ENABLED) { 8 | return errors; 9 | } 10 | if (settings.SCRUB_CHAR.length === 0) { 11 | errors.SCRUB_CHAR = 'Scrubbing character is required' 12 | } 13 | if (settings.SCRUB_CHAR.length > 1) { 14 | errors.SCRUB_CHAR = 'Scrubbing character must be a single character' 15 | } 16 | if (settings.SCRUB_LANGUAGE.length === 0) { 17 | errors.SCRUB_LANGUAGE = 'Scrubbing language is required' 18 | } 19 | if (settings.SCRUB_LANGUAGE.length > 2) { 20 | errors.SCRUB_LANGUAGE = 'Scrubbing language must be a two character language code' 21 | } 22 | 23 | return errors 24 | } 25 | 26 | 27 | export function validateRecordAndReplaySettings(settings: Record) { 28 | const errors: Record = {} 29 | if (settings.VIDEO_PIXEL_FORMAT.length === 0) { 30 | errors.VIDEO_PIXEL_FORMAT = 'Video pixel format is required' 31 | } 32 | return errors 33 | } 34 | 35 | 36 | export function saveSettings( 37 | form: UseFormReturnType, 38 | ) { 39 | return function(values: Record) { 40 | fetch('/api/settings', { 41 | method: 'POST', 42 | headers: { 43 | 'Content-Type': 'application/json', 44 | }, 45 | body: JSON.stringify(values), 46 | }).then(resp => { 47 | if (resp.ok) { 48 | notifications.show({ 49 | title: 'Settings saved', 50 | message: 'Your settings have been saved', 51 | color: 'green', 52 | }); 53 | return resp.json(); 54 | } else { 55 | notifications.show({ 56 | title: 'Failed to save settings', 57 | message: 'Please try again', 58 | color: 'red', 59 | }) 60 | return null; 61 | } 62 | 63 | }).then((resp) => { 64 | if (!resp) { 65 | return; 66 | } 67 | form.setInitialValues(values); 68 | form.setDirty({}); 69 | }); 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/app/utils.ts: -------------------------------------------------------------------------------- 1 | import moment from 'moment'; 2 | 3 | 4 | export const timeStampToDateString = (timeStamp: number) => { 5 | if (!timeStamp) { 6 | return 'N/A'; 7 | } 8 | return moment.unix(timeStamp).format('DD/MM/YYYY HH:mm:ss'); 9 | } 10 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/assets/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenAdaptAI/OpenAdapt/HEAD/openadapt/app/dashboard/assets/logo.png -------------------------------------------------------------------------------- /openadapt/app/dashboard/components/ActionEvent/ActionEvents.tsx: -------------------------------------------------------------------------------- 1 | 'use client'; 2 | 3 | import { ActionEvent as ActionEventType } from "@/types/action-event"; 4 | import { Accordion } from "@mantine/core"; 5 | import { useState } from "react"; 6 | import { ActionEvent } from "./ActionEvent"; 7 | import { IconChevronDown } from "@tabler/icons-react"; 8 | import { Screenshots } from "./Screenshots"; 9 | 10 | type Props = { 11 | events: ActionEventType[] 12 | } 13 | 14 | export const ActionEvents = ({ 15 | events 16 | }: Props) => { 17 | const [values, setValues] = useState([]); 18 | const [isScreenshotModalOpen, setIsScreenshotModalOpen] = useState(false); 19 | const toggleScreenshotModal = () => setIsScreenshotModalOpen(prev => !prev); 20 | return ( 21 | <> 22 | }> 23 | {events.map((event) => ( 24 | 25 | ))} 26 | 27 | 28 | 29 | ) 30 | } 31 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/components/ActionEvent/RemoveActionEvent.tsx: -------------------------------------------------------------------------------- 1 | import { ActionEvent } from '@/types/action-event'; 2 | import { Button, Text } from '@mantine/core'; 3 | import { modals } from '@mantine/modals'; 4 | import { notifications } from '@mantine/notifications'; 5 | import React from 'react' 6 | 7 | type Props = { 8 | event: ActionEvent; 9 | } 10 | 11 | export const RemoveActionEvent = ({ 12 | event 13 | }: Props) => { 14 | const openModal = (e: React.MouseEvent) => { 15 | e.stopPropagation(); 16 | modals.openConfirmModal({ 17 | title: 'Please confirm your action', 18 | children: ( 19 | 20 | Are you sure you want to delete this action event? This action cannot be undone. 21 | 22 | ), 23 | labels: { confirm: 'Confirm', cancel: 'Cancel' }, 24 | onCancel: () => {}, 25 | onConfirm: deleteActionEvent, 26 | confirmProps: { color: 'red' }, 27 | }); 28 | } 29 | 30 | const deleteActionEvent = () => { 31 | fetch(`/api/action-events/${event.id}`, { 32 | method: 'DELETE', 33 | }).then(res => res.json()).then(data => { 34 | const { message, status } = data; 35 | if (status === 'success') { 36 | window.location.reload(); 37 | } else { 38 | notifications.show({ 39 | title: 'Error', 40 | message, 41 | color: 'red', 42 | }) 43 | } 44 | }); 45 | } 46 | if (event.isComputed || !event.isOriginal) return null; 47 | return ( 48 | 51 | ) 52 | } 53 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/components/ActionEvent/Screenshots.tsx: -------------------------------------------------------------------------------- 1 | import { ActionEvent } from '@/types/action-event' 2 | import { Carousel } from '@mantine/carousel'; 3 | import { Image, Modal } from '@mantine/core'; 4 | import React, { useState } from 'react' 5 | 6 | type Props = { 7 | events: ActionEvent[]; 8 | isOpen: boolean; 9 | onClose: () => void; 10 | } 11 | 12 | export const Screenshots = ({ 13 | events, 14 | isOpen, 15 | onClose, 16 | }: Props) => { 17 | const aspectRatio = 16 / 9; 18 | const height = window.innerHeight * 0.8; 19 | const width = height * aspectRatio; 20 | return ( 21 | 22 | 23 | {events.map((event) => ( 24 | event.screenshot && 25 | screenshot 33 | ))} 34 | 35 | 36 | ) 37 | } 38 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/components/ActionEvent/index.tsx: -------------------------------------------------------------------------------- 1 | export { ActionEvent } from './ActionEvent'; 2 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/components/Navbar/Navbar.tsx: -------------------------------------------------------------------------------- 1 | 'use client' 2 | 3 | import { usePathname } from 'next/navigation' 4 | import { routes } from '@/app/routes' 5 | import { Stack } from '@mantine/core' 6 | import Link from 'next/link' 7 | import React from 'react' 8 | import { IconChevronRight } from '@tabler/icons-react' 9 | 10 | export const Navbar = () => { 11 | const currentRoute = usePathname() 12 | return ( 13 | 14 | {routes.map((route) => ( 15 | 23 | {route.name} 24 | 25 | ))} 26 | 27 | ) 28 | } 29 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/components/Navbar/index.tsx: -------------------------------------------------------------------------------- 1 | export { Navbar } from './Navbar' 2 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/components/Onboarding/steps/BookACall.tsx: -------------------------------------------------------------------------------- 1 | import { Box, Text } from '@mantine/core' 2 | import Link from 'next/link' 3 | import React from 'react' 4 | 5 | export const BookACall = () => { 6 | return ( 7 | 8 | 9 | 10 | Book a call with us 11 | 12 | to discuss how OpenAdapt can help your team 13 | 14 | 15 | ) 16 | } 17 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/components/Onboarding/steps/RegisterForUpdates.tsx: -------------------------------------------------------------------------------- 1 | 'use client'; 2 | 3 | 4 | import { Box, Button, Stack, Text, TextInput } from '@mantine/core' 5 | import { isNotEmpty, useForm } from '@mantine/form' 6 | import { notifications } from '@mantine/notifications' 7 | import React, { useEffect } from 'react' 8 | 9 | export const RegisterForUpdates = () => { 10 | const onboardingForm = useForm({ 11 | initialValues: { 12 | email: '', 13 | }, 14 | validate: { 15 | email: isNotEmpty('Email is required'), 16 | } 17 | }) 18 | useEffect(() => { 19 | fetch('/api/settings', { 20 | method: 'POST', 21 | headers: { 22 | 'Content-Type': 'application/json', 23 | }, 24 | body: JSON.stringify({ 25 | REDIRECT_TO_ONBOARDING: false 26 | }), 27 | }) 28 | }, []) 29 | function onSubmit({ email }: { email: string }) { 30 | fetch('https://openadapt.ai/form.html', { 31 | method: 'POST', 32 | mode: 'no-cors', 33 | headers: { 34 | 'Content-Type': 'application/x-www-form-urlencoded', 35 | }, 36 | body: new URLSearchParams({ 37 | email, 38 | 'form-name': 'email', 39 | 'bot-field': '', 40 | }).toString(), 41 | }).then(() => { 42 | notifications.show({ 43 | title: 'Thank you!', 44 | message: 'You have been registered for updates', 45 | color: 'green', 46 | }) 47 | }) 48 | } 49 | return ( 50 | 51 | 52 | 53 | 60 | 63 | 64 | 65 | 66 | ) 67 | } 68 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/components/Onboarding/steps/Tutorial.tsx: -------------------------------------------------------------------------------- 1 | 'use client'; 2 | 3 | import { Box, Text } from '@mantine/core' 4 | import { algora, type AlgoraOutput } from '@algora/sdk'; 5 | import React, { useEffect } from 'react' 6 | import Link from 'next/link'; 7 | 8 | type Bounty = AlgoraOutput['bounty']['list']['items'][number]; 9 | 10 | function BountyCard(props: { bounty: Bounty }) { 11 | return ( 12 | 18 |
19 |
20 | {props.bounty.reward_formatted} 21 |
22 |
23 | {props.bounty.task.repo_name}#{props.bounty.task.number} 24 |
25 |
26 | {props.bounty.task.title} 27 |
28 |
29 | 30 | ); 31 | } 32 | 33 | const featuredBountyId = 'clxi7tk210002l20aqlz58ram'; 34 | 35 | async function getFeaturedBounty() { 36 | const bounty: Bounty = await algora.bounty.get.query({ id: featuredBountyId }); 37 | return bounty; 38 | } 39 | 40 | export const Tutorial = () => { 41 | const [featuredBounty, setFeaturedBounty] = React.useState(null); 42 | useEffect(() => { 43 | getFeaturedBounty().then(setFeaturedBounty); 44 | }, []); 45 | return ( 46 | 47 | 48 | Welcome to OpenAdapt! Thank you for joining us on our mission to build open source desktop AI. Your feedback is extremely valuable! 49 | 50 | 51 | To start, please watch the demonstration below. Then try it yourself! If you have any issues, please submit a Github Issue. 52 | 53 | 54 | 55 | If {"you'd"} like to contribute directly to our development, please consider the following open Bounties (no development experience required): 56 | {featuredBounty && } 57 | 58 | 59 | ) 60 | } 61 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/components/RecordingDetails/RecordingDetails.tsx: -------------------------------------------------------------------------------- 1 | 'use client'; 2 | 3 | import { timeStampToDateString } from '@/app/utils'; 4 | import { Recording } from '@/types/recording' 5 | import { Table } from '@mantine/core' 6 | import React from 'react' 7 | 8 | type Props = { 9 | recording: Recording; 10 | } 11 | 12 | const TableRowWithBorder = ({ children }: { children: React.ReactNode }) => ( 13 | 14 | {children} 15 | 16 | ) 17 | 18 | const TableCellWithBorder = ({ children }: { children: React.ReactNode }) => ( 19 | 20 | {children} 21 | 22 | ) 23 | 24 | export const RecordingDetails = ({ 25 | recording 26 | }: Props) => { 27 | return ( 28 | 29 | 30 | 31 | Recording ID 32 | {recording.id} 33 | 34 | 35 | timestamp 36 | {timeStampToDateString(recording.timestamp)} 37 | 38 | 39 | monitor width 40 | {recording.monitor_width} 41 | 42 | 43 | monitor height 44 | {recording.monitor_height} 45 | 46 | 47 | double click interval seconds 48 | {recording.double_click_interval_seconds} 49 | 50 | 51 | double click distance pixels 52 | {recording.double_click_distance_pixels} 53 | 54 | 55 | platform 56 | {recording.platform} 57 | 58 | 59 | task description 60 | {recording.task_description} 61 | 62 | 63 | video start time 64 | {recording.video_start_time} 65 | 66 | 67 |
68 | ) 69 | } 70 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/components/RecordingDetails/index.tsx: -------------------------------------------------------------------------------- 1 | export { RecordingDetails } from './RecordingDetails'; 2 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/components/Shell/Shell.tsx: -------------------------------------------------------------------------------- 1 | 'use client' 2 | 3 | import { AppShell, Box, Burger, Image, Text } from '@mantine/core' 4 | import React from 'react' 5 | import { Navbar } from '../Navbar' 6 | import { useDisclosure } from '@mantine/hooks' 7 | import logo from '../../assets/logo.png' 8 | 9 | type Props = { 10 | children: React.ReactNode 11 | } 12 | 13 | export const Shell = ({ children }: Props) => { 14 | const [opened, { toggle }] = useDisclosure() 15 | return ( 16 | 26 | 27 | 33 | 34 | OpenAdapt 35 | 36 | OpenAdapt.AI 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | {children} 46 | 47 | ) 48 | } 49 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/components/Shell/index.tsx: -------------------------------------------------------------------------------- 1 | export { Shell } from './Shell' 2 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/components/SimpleTable/SimpleTable.tsx: -------------------------------------------------------------------------------- 1 | import { Button, Flex, Table } from "@mantine/core" 2 | import { IconRefresh } from '@tabler/icons-react' 3 | import React from "react" 4 | 5 | type Props> = { 6 | columns: { 7 | name: string, 8 | accessor: string | ((row: T) => React.ReactNode), 9 | }[]; 10 | data: T[], 11 | refreshData: () => void, 12 | onClickRow: (row: T) => (event: React.MouseEvent) => void, 13 | } 14 | 15 | export function SimpleTable>({ 16 | columns, 17 | data, 18 | refreshData, 19 | onClickRow, 20 | }: Props) { 21 | return ( 22 | 23 | 26 | 27 | 28 | 29 | {columns.map(({name}) => ( 30 | {name} 31 | ))} 32 | 33 | 34 | 35 | {data.map((row, rowIndex) => ( 36 | 37 | {columns.map(({accessor}, accesorIndex) => ( 38 | 39 | {typeof accessor === 'string' ? row[accessor] : accessor(row)} 40 | 41 | ))} 42 | 43 | ))} 44 | 45 |
46 | 47 | ) 48 | } 49 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/components/SimpleTable/index.tsx: -------------------------------------------------------------------------------- 1 | export { SimpleTable } from './SimpleTable'; 2 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/entrypoint.ps1: -------------------------------------------------------------------------------- 1 | npm install 2 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export NVM_DIR="$HOME/.nvm" 4 | [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm 5 | [ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion" # This loads nvm bash_completion 6 | 7 | nvm install 21 8 | nvm use 21 9 | npm install 10 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/index.js: -------------------------------------------------------------------------------- 1 | // run `npm run dev` as a child process 2 | 3 | const { spawn } = require('child_process') 4 | 5 | const net = require('net') 6 | const checkPort = (port) => { 7 | return new Promise((resolve, reject) => { 8 | const server = net.createServer() 9 | server.unref() 10 | server.on('error', reject) 11 | server.listen(port, () => { 12 | server.close(() => { 13 | resolve(port) 14 | }) 15 | }) 16 | }) 17 | } 18 | 19 | // check if both ports are not being used 20 | const { DASHBOARD_CLIENT_PORT, DASHBOARD_SERVER_PORT, REDIRECT_TO_ONBOARDING } = process.env 21 | Promise.all([checkPort(DASHBOARD_CLIENT_PORT), checkPort(DASHBOARD_SERVER_PORT)]) 22 | .then(([clientPort, serverPort]) => { 23 | if (clientPort !== DASHBOARD_CLIENT_PORT) { 24 | console.error(`Port ${DASHBOARD_CLIENT_PORT} is already in use`) 25 | process.exit(1) 26 | } 27 | if (serverPort !== DASHBOARD_SERVER_PORT) { 28 | console.error(`Port ${DASHBOARD_SERVER_PORT} is already in use`) 29 | process.exit(1) 30 | } 31 | spawnChildProcess() 32 | }) 33 | 34 | function spawnChildProcess() { 35 | // Spawn child process to run `npm run dev` 36 | let childProcess; 37 | 38 | if (process.platform === 'win32') { 39 | childProcess = spawn('npm', ['run', 'dev:windows'], { stdio: 'inherit', shell: true }) 40 | } else { 41 | childProcess = spawn('npm', ['run', 'dev'], { stdio: 'inherit' }) 42 | } 43 | 44 | childProcess.on('spawn', () => { 45 | // wait for 3 seconds before opening the browser 46 | setTimeout(() => { 47 | import('open').then(({ default: open }) => { 48 | let url = `http://localhost:${DASHBOARD_CLIENT_PORT}` 49 | if (REDIRECT_TO_ONBOARDING === 'true') { 50 | url += '/onboarding' 51 | } 52 | open(url) 53 | }) 54 | }, 3000) 55 | }) 56 | 57 | // Handle SIG signals 58 | const signals = ['SIGINT', 'SIGTERM', 'SIGQUIT', 'SIGHUP']; 59 | 60 | signals.forEach(signal => { 61 | process.on(signal, () => { 62 | childProcess.kill(signal) 63 | }) 64 | }) 65 | 66 | // Listen for child process error event 67 | childProcess.on('error', (err) => { 68 | console.error('Error executing child process:', err) 69 | }) 70 | } 71 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/next.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('next').NextConfig} */ 2 | // get values from environment variables 3 | const { DASHBOARD_SERVER_PORT } = process.env 4 | 5 | const nextConfig = { 6 | rewrites: async () => { 7 | return [ 8 | { 9 | source: '/', 10 | destination: `/recordings`, 11 | }, 12 | { 13 | source: '/api/:path*', 14 | destination: `http://127.0.0.1:${DASHBOARD_SERVER_PORT}/api/:path*` 15 | }, 16 | { 17 | source: '/docs', 18 | destination: 19 | process.env.NODE_ENV === 'development' 20 | ? `http://127.0.0.1:${DASHBOARD_SERVER_PORT}/docs` 21 | : '/api/docs', 22 | }, 23 | { 24 | source: '/openapi.json', 25 | destination: 26 | process.env.NODE_ENV === 'development' 27 | ? `http://127.0.0.1:${DASHBOARD_SERVER_PORT}/openapi.json` 28 | : '/api/openapi.json', 29 | }, 30 | ] 31 | }, 32 | output: 'export', 33 | reactStrictMode: false, 34 | } 35 | 36 | module.exports = nextConfig 37 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "nextjs-fastapi", 3 | "version": "0.1.0", 4 | "private": true, 5 | "scripts": { 6 | "fastapi-dev": "python3 -m uvicorn api.index:app --port $DASHBOARD_SERVER_PORT --reload", 7 | "fastapi-dev:windows": "python -m uvicorn api.index:app --port %DASHBOARD_SERVER_PORT% --reload", 8 | "next-dev": "next dev -p $DASHBOARD_CLIENT_PORT", 9 | "next-dev:windows": "next dev -p %DASHBOARD_CLIENT_PORT%", 10 | "dev": "concurrently \"npm run next-dev\" \"npm run fastapi-dev\"", 11 | "dev:windows": "concurrently \"npm run next-dev:windows\" \"npm run fastapi-dev:windows\"", 12 | "build": "next build", 13 | "start": "next start", 14 | "lint": "next lint", 15 | "format": "prettier --write ." 16 | }, 17 | "dependencies": { 18 | "@algora/sdk": "^0.2.0", 19 | "@mantine/carousel": "7.7.1", 20 | "@mantine/core": "7.7.1", 21 | "@mantine/form": "7.7.1", 22 | "@mantine/hooks": "7.7.1", 23 | "@mantine/modals": "^7.7.1", 24 | "@mantine/notifications": "7.7.1", 25 | "@tabler/icons-react": "^3.1.0", 26 | "@types/node": "20.2.4", 27 | "@types/react": "18.2.7", 28 | "@types/react-dom": "18.2.4", 29 | "autoprefixer": "10.4.14", 30 | "concurrently": "^8.0.1", 31 | "eslint": "8.41.0", 32 | "eslint-config-next": "^14.1.4", 33 | "moment": "^2.30.1", 34 | "next": "^14.1.4", 35 | "postcss": "8.4.23", 36 | "posthog-js": "^1.128.3", 37 | "react": "^18.2.0", 38 | "react-dom": "^18.2.0", 39 | "tailwindcss": "3.3.2", 40 | "typescript": "5.0.4" 41 | }, 42 | "devDependencies": { 43 | "prettier": "^3.2.5" 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/postcss.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | plugins: { 3 | tailwindcss: {}, 4 | autoprefixer: {}, 5 | }, 6 | } 7 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/run.py: -------------------------------------------------------------------------------- 1 | """This module contains the functions to run the dashboard web application.""" 2 | 3 | from threading import Thread 4 | import os 5 | import pathlib 6 | import subprocess 7 | import webbrowser 8 | 9 | from openadapt.build_utils import is_running_from_executable 10 | from openadapt.config import POSTHOG_HOST, POSTHOG_PUBLIC_KEY, config 11 | from openadapt.custom_logger import logger 12 | 13 | from .api.index import run_app 14 | 15 | dashboard_process = None 16 | 17 | 18 | def run() -> Thread: 19 | """Run the dashboard web application.""" 20 | # change to the client directory 21 | cur_dir = pathlib.Path(__file__).parent 22 | 23 | def run_client() -> subprocess.Popen: 24 | """The entry point for the thread that runs the dashboard client.""" 25 | if is_running_from_executable(): 26 | if config.REDIRECT_TO_ONBOARDING: 27 | url = f"http://localhost:{config.DASHBOARD_SERVER_PORT}/onboarding" 28 | else: 29 | url = f"http://localhost:{config.DASHBOARD_SERVER_PORT}/recordings" 30 | webbrowser.open(url) 31 | run_app() 32 | return 33 | 34 | global dashboard_process 35 | dashboard_process = subprocess.Popen( 36 | ["node", "index.js"], 37 | cwd=cur_dir, 38 | env={ 39 | **os.environ, 40 | "DASHBOARD_CLIENT_PORT": str(config.DASHBOARD_CLIENT_PORT), 41 | "DASHBOARD_SERVER_PORT": str(config.DASHBOARD_SERVER_PORT), 42 | "NEXT_PUBLIC_POSTHOG_HOST": POSTHOG_HOST, 43 | "NEXT_PUBLIC_POSTHOG_PUBLIC_KEY": POSTHOG_PUBLIC_KEY, 44 | "REDIRECT_TO_ONBOARDING": ( 45 | "true" if config.REDIRECT_TO_ONBOARDING else "false" 46 | ), 47 | "NEXT_PUBLIC_MODE": ( 48 | "production" if is_running_from_executable() else "development" 49 | ), 50 | }, 51 | ) 52 | 53 | return Thread( 54 | target=run_client, 55 | daemon=True, 56 | args=(), 57 | ) 58 | 59 | 60 | def cleanup() -> None: 61 | """Cleanup the dashboard web application process.""" 62 | logger.debug("Terminating the dashboard client.") 63 | global dashboard_process 64 | if dashboard_process: 65 | dashboard_process.terminate() 66 | dashboard_process.wait() 67 | logger.debug("Dashboard client terminated.") 68 | 69 | 70 | if __name__ == "__main__": 71 | dashboard_thread = run() 72 | dashboard_thread.start() 73 | try: 74 | while True: 75 | pass 76 | except KeyboardInterrupt: 77 | cleanup() 78 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/tailwind.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('tailwindcss').Config} */ 2 | module.exports = { 3 | content: [ 4 | './components/**/*.{js,ts,jsx,tsx,mdx}', 5 | './app/**/*.{js,ts,jsx,tsx,mdx}', 6 | ], 7 | theme: { 8 | extend: { 9 | backgroundImage: { 10 | 'gradient-radial': 'radial-gradient(var(--tw-gradient-stops))', 11 | 'gradient-conic': 12 | 'conic-gradient(from 180deg at 50% 50%, var(--tw-gradient-stops))', 13 | }, 14 | }, 15 | }, 16 | plugins: [], 17 | } 18 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es5", 4 | "lib": ["dom", "dom.iterable", "esnext"], 5 | "allowJs": true, 6 | "skipLibCheck": true, 7 | "strict": true, 8 | "forceConsistentCasingInFileNames": true, 9 | "noEmit": true, 10 | "esModuleInterop": true, 11 | "module": "esnext", 12 | "moduleResolution": "node", 13 | "resolveJsonModule": true, 14 | "isolatedModules": true, 15 | "jsx": "preserve", 16 | "incremental": true, 17 | "plugins": [ 18 | { 19 | "name": "next" 20 | } 21 | ], 22 | "paths": { 23 | "@/*": ["./*"] 24 | } 25 | }, 26 | "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], 27 | "exclude": ["node_modules"] 28 | } 29 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/types/action-event.ts: -------------------------------------------------------------------------------- 1 | export type ActionEvent = { 2 | id: number | string; 3 | name?: string; 4 | timestamp: number; 5 | recording_timestamp: number; 6 | screenshot_timestamp?: number; 7 | window_event_timestamp: number; 8 | browser_event_timestamp: number; 9 | mouse_x: number | null; 10 | mouse_y: number | null; 11 | mouse_dx: number | null; 12 | mouse_dy: number | null; 13 | mouse_button_name: string | null; 14 | mouse_pressed: boolean | null; 15 | key_name: string | null; 16 | key_char: string | null; 17 | key_vk: number | null; 18 | canonical_key_name: string | null; 19 | canonical_key_char: string | null; 20 | canonical_key_vk: number | null; 21 | text?: string; 22 | canonical_text?: string; 23 | parent_id?: number; 24 | element_state: Record; 25 | screenshot: string | null; 26 | diff: string | null; 27 | mask: string | null; 28 | dimensions?: { width: number, height: number }; 29 | children?: ActionEvent[]; 30 | words?: string[]; 31 | isComputed?: boolean; 32 | isOriginal?: boolean; 33 | } 34 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/types/recording.ts: -------------------------------------------------------------------------------- 1 | export type Recording = { 2 | id: number; 3 | timestamp: number; 4 | monitor_width: number; 5 | monitor_height: number; 6 | double_click_interval_seconds: number; 7 | double_click_distance_pixels: number; 8 | platform: string; 9 | task_description: string; 10 | video_start_time: number | null; 11 | original_recording_id: number | null; 12 | } 13 | 14 | export enum RecordingStatus { 15 | STOPPED = 'STOPPED', 16 | RECORDING = 'RECORDING', 17 | UNKNOWN = 'UNKNOWN', 18 | } 19 | 20 | 21 | export type ScrubbedRecording = { 22 | id: number; 23 | recording_id: number; 24 | recording: Pick; 25 | provider: string; 26 | original_recording: Pick; 27 | scrubbed: boolean; 28 | } 29 | -------------------------------------------------------------------------------- /openadapt/app/dashboard/types/scrubbing.ts: -------------------------------------------------------------------------------- 1 | import { Recording } from "./recording"; 2 | 3 | export enum ScrubbingStatus { 4 | STOPPED = 'STOPPED', 5 | SCRUBBING = 'SCRUBBING', 6 | UNKNOWN = 'UNKNOWN', 7 | } 8 | 9 | export type ScrubbingUpdate = { 10 | num_action_events_scrubbed: number; 11 | num_screenshots_scrubbed: number; 12 | num_window_events_scrubbed: number; 13 | total_action_events: number; 14 | total_screenshots: number; 15 | total_window_events: number; 16 | recording: Pick; 17 | provider: string; 18 | copying_recording?: boolean; 19 | error?: string; 20 | } 21 | -------------------------------------------------------------------------------- /openadapt/build_utils.py: -------------------------------------------------------------------------------- 1 | """Utility functions for the build process.""" 2 | 3 | import importlib.metadata 4 | import pathlib 5 | import sys 6 | import time 7 | 8 | 9 | def get_root_dir_path() -> pathlib.Path: 10 | """Get the path to the project root directory.""" 11 | if not is_running_from_executable(): 12 | return pathlib.Path(__file__).parent 13 | if sys.platform == "darwin": 14 | # if macos, get the path to the /Users/username/Library/Preferences 15 | # and set the path for all user preferences 16 | path = pathlib.Path.home() / "Library" / "Preferences" / "openadapt" 17 | if not path.exists(): 18 | path.mkdir(parents=True, exist_ok=True) 19 | return path 20 | elif sys.platform == "win32": 21 | # if windows, get the path to the %APPDATA% directory and set the path 22 | # for all user preferences 23 | path = pathlib.Path.home() / "AppData" / "Roaming" / "openadapt" 24 | if not path.exists(): 25 | path.mkdir(parents=True, exist_ok=True) 26 | return path 27 | else: 28 | print(f"WARNING: openadapt.build_utils is not yet supported on {sys.platform=}") 29 | 30 | 31 | def is_running_from_executable() -> bool: 32 | """Check if the script is running from an executable.""" 33 | return getattr(sys, "frozen", False) 34 | 35 | 36 | class RedirectOutput: 37 | """Context manager to redirect stdout and stderr to /dev/null.""" 38 | 39 | def __enter__(self) -> "RedirectOutput": 40 | """Redirect stdout and stderr to /dev/null.""" 41 | if is_running_from_executable(): 42 | log_file_path = get_log_file_path() 43 | log_stream = open(log_file_path, "a") 44 | self.old_stdout = sys.stdout 45 | self.old_stderr = sys.stderr 46 | sys.stdout = sys.stderr = log_stream 47 | return self 48 | 49 | def __exit__(self, exc_type: type, exc_value: Exception, traceback: type) -> None: 50 | """Restore stdout and stderr.""" 51 | if is_running_from_executable(): 52 | sys.stdout.close() 53 | sys.stderr.close() 54 | sys.stdout = self.old_stdout 55 | sys.stderr = self.old_stderr 56 | 57 | 58 | def redirect_stdout_stderr() -> RedirectOutput: 59 | """Get the RedirectOutput instance for use as a context manager.""" 60 | return RedirectOutput() 61 | 62 | 63 | def get_log_file_path() -> str: 64 | """Get the path to the log file. 65 | 66 | Returns: 67 | str: The path to the log file. 68 | """ 69 | version = importlib.metadata.version("openadapt") 70 | date = time.strftime("%Y-%m-%d") 71 | path = get_root_dir_path() / "data" / "logs" / version / date / "openadapt.log" 72 | path.parent.mkdir(parents=True, exist_ok=True) 73 | return str(path) 74 | -------------------------------------------------------------------------------- /openadapt/capture/__init__.py: -------------------------------------------------------------------------------- 1 | """Capture the screen, audio, and camera as a video on macOS and Windows. 2 | 3 | Module: capture.py 4 | """ 5 | 6 | import sys 7 | 8 | if sys.platform == "darwin": 9 | from . import _macos as impl 10 | elif sys.platform == "win32": 11 | from . import _windows as impl 12 | elif sys.platform.startswith("linux"): 13 | from . import _linux as impl 14 | else: 15 | raise Exception(f"Unsupported platform: {sys.platform}") 16 | 17 | device = impl.Capture() 18 | 19 | 20 | def get_capture() -> impl.Capture: 21 | """Get the capture object. 22 | 23 | Returns: 24 | Capture: The capture object. 25 | """ 26 | return device 27 | 28 | 29 | def start(audio: bool = False, camera: bool = False) -> None: 30 | """Start the capture.""" 31 | device.start(audio=audio, camera=camera) 32 | 33 | 34 | def stop() -> None: 35 | """Stop the capture.""" 36 | device.stop() 37 | 38 | 39 | def test() -> None: 40 | """Test the capture.""" 41 | device.start() 42 | input("Press enter to stop") 43 | device.stop() 44 | 45 | 46 | if __name__ in ("__main__", "capture"): 47 | test() 48 | -------------------------------------------------------------------------------- /openadapt/capture/__main__.py: -------------------------------------------------------------------------------- 1 | """Entrypoint for video capture module. 2 | 3 | Usage: 4 | 5 | python -m openadapt.capture 6 | """ 7 | 8 | from . import test 9 | 10 | if __name__ == "__main__": 11 | test() 12 | -------------------------------------------------------------------------------- /openadapt/common.py: -------------------------------------------------------------------------------- 1 | """This module defines common constants used in OpenAdapt.""" 2 | 3 | RAW_PRECISE_MOUSE_EVENTS = ( 4 | "move", 5 | "click", 6 | ) 7 | # location of cursor doesn't matter as much when scrolling compared to moving/clicking 8 | RAW_IMPRECISE_MOUSE_EVENTS = ("scroll",) 9 | RAW_MOUSE_EVENTS = tuple( 10 | list(RAW_PRECISE_MOUSE_EVENTS) + list(RAW_IMPRECISE_MOUSE_EVENTS) 11 | ) 12 | FUSED_MOUSE_EVENTS = ( 13 | "singleclick", 14 | "doubleclick", 15 | ) 16 | MOUSE_EVENTS = tuple(list(RAW_MOUSE_EVENTS) + list(FUSED_MOUSE_EVENTS)) 17 | MOUSE_CLICK_EVENTS = (event for event in MOUSE_EVENTS if event.endswith("click")) 18 | PRECISE_MOUSE_EVENTS = tuple(list(RAW_PRECISE_MOUSE_EVENTS) + list(FUSED_MOUSE_EVENTS)) 19 | 20 | RAW_KEY_EVENTS = ( 21 | "press", 22 | "release", 23 | ) 24 | FUSED_KEY_EVENTS = ("type",) 25 | KEY_EVENTS = tuple(list(RAW_KEY_EVENTS) + list(FUSED_KEY_EVENTS)) 26 | 27 | ALL_EVENTS = tuple(list(MOUSE_EVENTS) + list(KEY_EVENTS)) 28 | -------------------------------------------------------------------------------- /openadapt/config.defaults.json: -------------------------------------------------------------------------------- 1 | { 2 | "PRIVATE_AI_API_KEY": "", 3 | "REPLICATE_API_TOKEN": "", 4 | "DEFAULT_ADAPTER": "openai", 5 | "DEFAULT_SEGMENTATION_ADAPTER": "ultralytics", 6 | "OPENAI_API_KEY": "", 7 | "ANTHROPIC_API_KEY": "", 8 | "GOOGLE_API_KEY": "", 9 | "SOM_SERVER_URL": "", 10 | "CACHE_DIR_PATH": ".cache", 11 | "CACHE_ENABLED": true, 12 | "CACHE_VERBOSITY": 0, 13 | "DB_ECHO": false, 14 | "ERROR_REPORTING_ENABLED": true, 15 | "OPENAI_MODEL_NAME": "gpt-3.5-turbo", 16 | "RECORD_WINDOW_DATA": false, 17 | "RECORD_READ_ACTIVE_ELEMENT_STATE": false, 18 | "REPLAY_STRIP_ELEMENT_STATE": true, 19 | "RECORD_VIDEO": true, 20 | "RECORD_AUDIO": false, 21 | "RECORD_BROWSER_EVENTS": false, 22 | "RECORD_FULL_VIDEO": false, 23 | "RECORD_IMAGES": false, 24 | "LOG_MEMORY": false, 25 | "STOP_SEQUENCES": [ 26 | [ 27 | "o", 28 | "a", 29 | ".", 30 | "s", 31 | "t", 32 | "o", 33 | "p" 34 | ], 35 | [ 36 | "ctrl", 37 | "ctrl", 38 | "ctrl" 39 | ] 40 | ], 41 | "IGNORE_WARNINGS": false, 42 | "MAX_NUM_WARNINGS_PER_SECOND": 5, 43 | "WARNING_SUPPRESSION_PERIOD": 1, 44 | "MESSAGES_TO_FILTER": [ 45 | "Cannot pickle Objective-C objects" 46 | ], 47 | "ACTION_TEXT_SEP": "-", 48 | "ACTION_TEXT_NAME_PREFIX": "<", 49 | "ACTION_TEXT_NAME_SUFFIX": ">", 50 | "PLOT_PERFORMANCE": true, 51 | "APP_DARK_MODE": false, 52 | "SCRUB_ENABLED": false, 53 | "SCRUB_CHAR": "*", 54 | "SCRUB_LANGUAGE": "en", 55 | "SCRUB_FILL_COLOR": "0x0000FF", 56 | "SCRUB_CONFIG_TRF": { 57 | "nlp_engine_name": "spacy", 58 | "models": [ 59 | { 60 | "lang_code": "en", 61 | "model_name": "en_core_web_trf" 62 | } 63 | ] 64 | }, 65 | "SCRUB_PRESIDIO_IGNORE_ENTITIES": [], 66 | "SCRUB_KEYS_HTML": [ 67 | "text", 68 | "canonical_text", 69 | "title", 70 | "state", 71 | "task_description", 72 | "key_char", 73 | "canonical_key_char", 74 | "key_vk", 75 | "children" 76 | ], 77 | "VISUALIZE_DARK_MODE": false, 78 | "VISUALIZE_RUN_NATIVELY": true, 79 | "VISUALIZE_DENSE_TREES": true, 80 | "VISUALIZE_ANIMATIONS": true, 81 | "VISUALIZE_EXPAND_ALL": false, 82 | "VISUALIZE_MAX_TABLE_CHILDREN": 10, 83 | "SAVE_SCREENSHOT_DIFF": false, 84 | "SPACY_MODEL_NAME": "en_core_web_trf", 85 | "DASHBOARD_CLIENT_PORT": 5173, 86 | "DASHBOARD_SERVER_PORT": 8080, 87 | "BROWSER_WEBSOCKET_PORT": 8765, 88 | "BROWSER_WEBSOCKET_SERVER_IP": "localhost", 89 | "UNIQUE_USER_ID": "", 90 | "REDIRECT_TO_ONBOARDING": true 91 | } 92 | -------------------------------------------------------------------------------- /openadapt/contrib/__init__.py: -------------------------------------------------------------------------------- 1 | """Third-party modules.""" 2 | 3 | from . import som 4 | -------------------------------------------------------------------------------- /openadapt/contrib/som/__init__.py: -------------------------------------------------------------------------------- 1 | """Set-of-Mark modules.""" 2 | 3 | from . import visualizer 4 | -------------------------------------------------------------------------------- /openadapt/custom_logger.py: -------------------------------------------------------------------------------- 1 | """Module for log message filtering, excluding strings & limiting warnings.""" 2 | 3 | from collections import defaultdict 4 | import time 5 | 6 | from loguru import logger as loguru_logger 7 | 8 | from openadapt.build_utils import get_log_file_path, is_running_from_executable 9 | from openadapt.config import config 10 | 11 | MESSAGE_TIMESTAMPS = defaultdict(list) 12 | 13 | # TODO: move utils.configure_logging to here 14 | 15 | 16 | def filter_log_messages(data: dict) -> bool: 17 | """Filter log messages based on the defined criteria. 18 | 19 | Args: 20 | data: The log message data from a loguru logger. 21 | 22 | Returns: 23 | bool: True if the log message should not be ignored, False otherwise. 24 | """ 25 | # TODO: ultimately, we want to fix the underlying issues, but for now, 26 | # we can ignore these messages 27 | for msg in config.MESSAGES_TO_FILTER: 28 | if msg in data["message"]: 29 | if config.MAX_NUM_WARNINGS_PER_SECOND > 0: 30 | current_timestamp = time.time() 31 | MESSAGE_TIMESTAMPS[msg].append(current_timestamp) 32 | timestamps = MESSAGE_TIMESTAMPS[msg] 33 | 34 | # Remove timestamps older than 1 second 35 | timestamps = [ 36 | ts 37 | for ts in timestamps 38 | if current_timestamp - ts <= config.WARNING_SUPPRESSION_PERIOD 39 | ] 40 | 41 | if len(timestamps) > config.MAX_NUM_WARNINGS_PER_SECOND: 42 | return False 43 | 44 | MESSAGE_TIMESTAMPS[msg] = timestamps 45 | 46 | return True 47 | 48 | 49 | logger = loguru_logger 50 | if is_running_from_executable(): 51 | logger.remove() 52 | logger.add( 53 | get_log_file_path(), 54 | ) 55 | -------------------------------------------------------------------------------- /openadapt/db/__init__.py: -------------------------------------------------------------------------------- 1 | """Package for interacting with the OpenAdapt database.""" 2 | 3 | from .db import export_recording # noqa: F401 4 | -------------------------------------------------------------------------------- /openadapt/db/list.py: -------------------------------------------------------------------------------- 1 | """Lists all recordings in the database. 2 | 3 | Usage: python -m openadapt.db.list 4 | """ 5 | 6 | from sys import stdout 7 | 8 | from openadapt.custom_logger import logger 9 | from openadapt.db import crud 10 | 11 | 12 | def main() -> None: 13 | """Prints all recordings in the database.""" 14 | logger.remove() 15 | logger.add( 16 | stdout, 17 | colorize=True, 18 | format="[DB] {message}", 19 | ) 20 | print() # newline 21 | 22 | session = crud.get_new_session(read_only=True) 23 | recordings = crud.get_all_recordings(session) 24 | 25 | if not recordings: 26 | logger.info("No recordings found.") 27 | 28 | for idx, recording in enumerate(recordings[::-1], start=1): 29 | logger.info( 30 | f"[{idx}]: {recording.task_description} | {recording.timestamp}" 31 | + (" [latest]" if idx == len(recordings) else "") 32 | ) 33 | 34 | 35 | if __name__ == "__main__": 36 | main() 37 | -------------------------------------------------------------------------------- /openadapt/db/remove.py: -------------------------------------------------------------------------------- 1 | """Removes recordings from the database. 2 | 3 | Usage: python -m openadapt.db.remove [--all | --latest | --id ] 4 | """ 5 | 6 | from sys import stdout 7 | 8 | import click 9 | 10 | from openadapt.custom_logger import logger 11 | from openadapt.db import crud 12 | 13 | print() # newline 14 | 15 | 16 | @click.command() 17 | @click.option("--all", is_flag=True, help="Remove all recordings.") 18 | @click.option("--latest", is_flag=True, help="Remove the latest recording.") 19 | @click.option("--id", "recording_id", type=int, help="Remove recording by ID.") 20 | def remove(all: str, latest: str, recording_id: int) -> int: 21 | """Removes a recording from the database.""" 22 | if not crud.acquire_db_lock(): 23 | logger.error("Failed to acquire database lock.") 24 | return 1 25 | 26 | def cleanup(return_code: int) -> int: 27 | """Releases the database lock and returns the given return code. 28 | 29 | Args: 30 | return_code (int): The return code to return. 31 | 32 | Returns: 33 | return_code (int): The given return code. 34 | """ 35 | crud.release_db_lock() 36 | return return_code 37 | 38 | read_only_session = crud.get_new_session(read_only=True) 39 | recordings = crud.get_all_recordings(read_only_session)[::-1] 40 | 41 | logger.remove() 42 | logger.add( 43 | stdout, 44 | colorize=True, 45 | format="[DB] {message}", 46 | ) 47 | 48 | if sum([all, latest, recording_id is not None]) > 1: 49 | logger.error("Invalid usage.") 50 | logger.error("Use --help for more information.") 51 | return cleanup(1) 52 | 53 | if not recordings: 54 | logger.error("No recordings found.") 55 | return cleanup(1) 56 | 57 | if all: 58 | if click.confirm("Are you sure you want to delete all recordings?"): 59 | with crud.get_new_session(read_and_write=True) as write_session: 60 | for r in recordings: 61 | logger.info(f"Deleting {r.task_description} | {r.timestamp}...") 62 | crud.delete_recording(write_session, r) 63 | logger.info("All recordings deleted.") 64 | else: 65 | logger.info("Aborting...") 66 | return cleanup(0) 67 | 68 | if latest: 69 | recording_id = len(recordings) 70 | 71 | elif recording_id is None or not (1 <= recording_id <= len(recordings)): 72 | logger.error("Invalid recording ID.") 73 | return cleanup(1) 74 | recording_to_delete = recordings[recording_id - 1] 75 | 76 | if click.confirm( 77 | "Are you sure you want to delete recording" 78 | f" {recording_to_delete.task_description} | {recording_to_delete.timestamp}?" 79 | ): 80 | with crud.get_new_session(read_and_write=True) as write_session: 81 | crud.delete_recording(write_session, recording_to_delete) 82 | crud.release_db_lock() 83 | logger.info("Recording deleted.") 84 | else: 85 | logger.info("Aborting...") 86 | return cleanup(0) 87 | 88 | 89 | if __name__ == "__main__": 90 | remove() 91 | -------------------------------------------------------------------------------- /openadapt/deprecated/app/cards.py: -------------------------------------------------------------------------------- 1 | """openadapt.deprecated.app.cards module. 2 | 3 | This module provides functions for managing UI cards in the OpenAdapt application. 4 | """ 5 | 6 | from datetime import datetime 7 | import multiprocessing 8 | import time 9 | 10 | from openadapt.record import record 11 | from openadapt.utils import WrapStdout 12 | 13 | 14 | class RecordProc: 15 | """Class to manage the recording process.""" 16 | 17 | def __init__(self) -> None: 18 | """Initialize the RecordProc class.""" 19 | self.terminate_processing = multiprocessing.Event() 20 | self.terminate_recording = multiprocessing.Event() 21 | self.record_proc: multiprocessing.Process = None 22 | self.has_initiated_stop = False 23 | 24 | def set_terminate_processing(self) -> multiprocessing.Event: 25 | """Set the terminate event.""" 26 | return self.terminate_processing.set() 27 | 28 | def terminate(self) -> None: 29 | """Terminate the recording process.""" 30 | self.record_proc.terminate() 31 | 32 | def reset(self) -> None: 33 | """Reset the recording process.""" 34 | self.terminate_processing.clear() 35 | self.terminate_recording.clear() 36 | self.record_proc = None 37 | record_proc.has_initiated_stop = False 38 | 39 | def wait(self) -> None: 40 | """Wait for the recording process to finish.""" 41 | while True: 42 | if self.terminate_recording.is_set(): 43 | self.record_proc.terminate() 44 | return 45 | time.sleep(0.1) 46 | 47 | def is_running(self) -> bool: 48 | """Check if the recording process is running.""" 49 | if self.record_proc is not None and not self.record_proc.is_alive(): 50 | self.reset() 51 | return self.record_proc is not None 52 | 53 | def start(self, func: callable, args: tuple, kwargs: dict) -> None: 54 | """Start the recording process.""" 55 | self.record_proc = multiprocessing.Process( 56 | target=WrapStdout(func), 57 | args=args, 58 | kwargs=kwargs, 59 | ) 60 | self.record_proc.start() 61 | 62 | 63 | record_proc = RecordProc() 64 | 65 | 66 | def stop_record() -> None: 67 | """Stop the current recording session.""" 68 | global record_proc 69 | if record_proc.is_running() and not record_proc.has_initiated_stop: 70 | record_proc.set_terminate_processing() 71 | 72 | # wait for process to terminate 73 | record_proc.wait() 74 | record_proc.reset() 75 | 76 | 77 | def is_recording() -> bool: 78 | """Check if a recording session is currently active.""" 79 | global record_proc 80 | return record_proc.is_running() 81 | 82 | 83 | def quick_record( 84 | task_description: str | None = None, 85 | status_pipe: multiprocessing.connection.Connection | None = None, 86 | ) -> None: 87 | """Run a recording session.""" 88 | global record_proc 89 | task_description = task_description or datetime.now().strftime("%d/%m/%Y %H:%M:%S") 90 | record_proc.start( 91 | record, 92 | ( 93 | task_description, 94 | record_proc.terminate_processing, 95 | record_proc.terminate_recording, 96 | status_pipe, 97 | ), 98 | { 99 | "log_memory": False, 100 | }, 101 | ) 102 | -------------------------------------------------------------------------------- /openadapt/drivers/google.py: -------------------------------------------------------------------------------- 1 | """Adapter for Google Gemini. 2 | 3 | See https://ai.google.dev/tutorials/python_quickstart for documentation. 4 | """ 5 | 6 | from pprint import pformat 7 | 8 | from PIL import Image 9 | import fire 10 | import google.generativeai as genai 11 | 12 | from openadapt import cache 13 | from openadapt.config import config 14 | from openadapt.custom_logger import logger 15 | 16 | MAX_TOKENS = 2**20 # 1048576 17 | MODEL_NAME = [ 18 | "gemini-pro-vision", 19 | "models/gemini-1.5-pro-latest", 20 | ][-1] 21 | # https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/send-multimodal-prompts 22 | MAX_IMAGES = { 23 | "gemini-pro-vision": 16, 24 | "models/gemini-1.5-pro-latest": 3000, 25 | }[MODEL_NAME] 26 | 27 | 28 | @cache.cache() 29 | def prompt( 30 | prompt: str, 31 | system_prompt: str | None = None, 32 | images: list[Image.Image] | None = None, 33 | # max_tokens: int | None = None, 34 | model_name: str = MODEL_NAME, 35 | timeout: int = 10, 36 | ) -> str: 37 | """Public method to get a response from the Google API with image support.""" 38 | full_prompt = "\n\n###\n\n".join([s for s in (system_prompt, prompt) if s]) 39 | # HACK 40 | full_prompt += "\nWhen responding in JSON, you MUST use double quotes around keys." 41 | 42 | genai.configure(api_key=config.GOOGLE_API_KEY) 43 | model = genai.GenerativeModel(model_name) 44 | response = model.generate_content( 45 | [full_prompt] + images, request_options={"timeout": timeout} 46 | ) 47 | response.resolve() 48 | logger.info(f"response=\n{pformat(response)}") 49 | return response.text 50 | 51 | 52 | def main(text: str, image_path: str | None = None) -> None: 53 | """Prompt Google Gemini with text and a path to an image.""" 54 | if image_path: 55 | image = Image.open(image_path) 56 | # Convert image to RGB if it's RGBA (to remove alpha channel) 57 | if image.mode in ("RGBA", "LA") or ( 58 | image.mode == "P" and "transparency" in image.info 59 | ): 60 | image = image.convert("RGB") 61 | else: 62 | image = None 63 | 64 | images = [image] if image else None 65 | output = prompt(text, images=images) 66 | logger.info(output) 67 | 68 | 69 | if __name__ == "__main__": 70 | fire.Fire(main) 71 | -------------------------------------------------------------------------------- /openadapt/entrypoint.py: -------------------------------------------------------------------------------- 1 | """Entrypoint for OpenAdapt.""" 2 | 3 | import multiprocessing 4 | 5 | if __name__ == "__main__": 6 | # This needs to be called before any code that uses multiprocessing 7 | multiprocessing.freeze_support() 8 | 9 | from openadapt.build_utils import redirect_stdout_stderr 10 | from openadapt.error_reporting import configure_error_reporting 11 | from openadapt.custom_logger import logger 12 | 13 | 14 | def run_openadapt() -> None: 15 | """Run OpenAdapt.""" 16 | with redirect_stdout_stderr(): 17 | try: 18 | from openadapt.alembic.context_loader import load_alembic_context 19 | from openadapt.app import tray 20 | from openadapt.config import print_config 21 | 22 | print_config() 23 | configure_error_reporting() 24 | load_alembic_context() 25 | tray._run() 26 | except Exception as exc: 27 | logger.exception(exc) 28 | 29 | 30 | if __name__ == "__main__": 31 | run_openadapt() 32 | -------------------------------------------------------------------------------- /openadapt/error_reporting.py: -------------------------------------------------------------------------------- 1 | """Module for error reporting logic.""" 2 | 3 | from typing import Any 4 | 5 | from loguru import logger 6 | from PySide6.QtGui import QIcon 7 | from PySide6.QtWidgets import QMessageBox, QPushButton 8 | import git 9 | import sentry_sdk 10 | import webbrowser 11 | 12 | from openadapt.build_utils import is_running_from_executable 13 | from openadapt.config import PARENT_DIR_PATH, config 14 | 15 | 16 | def configure_error_reporting() -> None: 17 | """Configure error reporting.""" 18 | logger.info(f"{config.ERROR_REPORTING_ENABLED=}") 19 | if not config.ERROR_REPORTING_ENABLED: 20 | return 21 | 22 | if is_running_from_executable(): 23 | is_reporting_branch = True 24 | else: 25 | active_branch_name = git.Repo(PARENT_DIR_PATH).active_branch.name 26 | logger.info(f"{active_branch_name=}") 27 | is_reporting_branch = active_branch_name == config.ERROR_REPORTING_BRANCH 28 | logger.info(f"{is_reporting_branch=}") 29 | 30 | if is_reporting_branch: 31 | sentry_sdk.init( 32 | dsn=config.ERROR_REPORTING_DSN, 33 | traces_sample_rate=1.0, 34 | before_send=before_send_event, 35 | ignore_errors=[KeyboardInterrupt], 36 | ) 37 | 38 | 39 | def show_alert() -> None: 40 | """Show an alert to the user.""" 41 | # TODO: move to config 42 | from openadapt.app.tray import ICON_PATH 43 | 44 | msg = QMessageBox() 45 | msg.setIcon(QMessageBox.Warning) 46 | msg.setWindowIcon(QIcon(ICON_PATH)) 47 | msg.setText( 48 | """ 49 | An error has occurred. The development team has been notified. 50 | Please join the discord server to get help or send an email to 51 | help@openadapt.ai 52 | """ 53 | ) 54 | discord_button = QPushButton("Join the discord server") 55 | discord_button.clicked.connect( 56 | lambda: webbrowser.open("https://discord.gg/yF527cQbDG") 57 | ) 58 | msg.addButton(discord_button, QMessageBox.ActionRole) 59 | msg.addButton(QMessageBox.Ok) 60 | msg.exec() 61 | 62 | 63 | def before_send_event(event: Any, hint: Any) -> Any: 64 | """Handle the event before sending it to Sentry.""" 65 | try: 66 | show_alert() 67 | except Exception: 68 | pass 69 | return event 70 | -------------------------------------------------------------------------------- /openadapt/privacy/__init__.py: -------------------------------------------------------------------------------- 1 | """Package for the Privacy API. 2 | 3 | Module: __init__.py 4 | """ 5 | -------------------------------------------------------------------------------- /openadapt/privacy/providers/__init__.py: -------------------------------------------------------------------------------- 1 | """Package for the Providers. 2 | 3 | Module: __init__.py 4 | """ 5 | 6 | 7 | class ScrubProvider: 8 | """A Class for Scrubbing Provider.""" 9 | 10 | PRESIDIO = "PRESIDIO" 11 | COMPREHEND = "COMPREHEND" 12 | PRIVATE_AI = "PRIVATE_AI" 13 | 14 | @classmethod 15 | def as_options(cls: "ScrubProvider") -> dict: 16 | """Return the available options.""" 17 | return { 18 | cls.PRESIDIO: "Presidio", 19 | # Comprehend does not support the scrub_image method 20 | # TODO: handle this 21 | # cls.COMPREHEND: "Comprehend", 22 | cls.PRIVATE_AI: "Private AI", 23 | } 24 | 25 | @classmethod 26 | def get_available_providers(cls: "ScrubProvider") -> list: 27 | """Return the available providers.""" 28 | return [cls.PRESIDIO, cls.PRIVATE_AI] 29 | 30 | @classmethod 31 | def get_scrubber(cls: "ScrubProvider", provider: str) -> "ScrubProvider": 32 | """Return the scrubber for the provider. 33 | 34 | Args: 35 | provider: The provider to get the scrubber for. 36 | 37 | Returns: 38 | The scrubber for the provider. 39 | """ 40 | if provider not in cls.get_available_providers(): 41 | raise ValueError(f"Provider {provider} is not supported.") 42 | if provider == cls.PRESIDIO: 43 | from openadapt.privacy.providers.presidio import PresidioScrubbingProvider 44 | 45 | return PresidioScrubbingProvider() 46 | elif provider == cls.PRIVATE_AI: 47 | from openadapt.privacy.providers.private_ai import ( 48 | PrivateAIScrubbingProvider, 49 | ) 50 | 51 | return PrivateAIScrubbingProvider() 52 | -------------------------------------------------------------------------------- /openadapt/prompts/apply_replay_instructions.j2: -------------------------------------------------------------------------------- 1 | Consider the actions in the recording: 2 | 3 | ```json 4 | {{ actions }} 5 | ``` 6 | 7 | Consider the user's replay instructions: 8 | ```text 9 | {{ replay_instructions }} 10 | ``` 11 | 12 | Provide an updated list of actions that are modified such that replaying them will 13 | accomplish the user's replay instructions. 14 | 15 | Do NOT provide available_segment_descriptions in your response. 16 | 17 | Respond with json and nothing else. 18 | 19 | {% if exceptions.length %} 20 | Your previous attempts at this produced the following exceptions: 21 | {% for exception in exceptions %} 22 | 23 | {{ exception }} 24 | 25 | {% endfor %} 26 | {% endif %} 27 | 28 | My career depends on this. Lives are at stake. 29 | -------------------------------------------------------------------------------- /openadapt/prompts/describe_recording--segment.j2: -------------------------------------------------------------------------------- 1 | Consider the actions in the recording and states of the active window immediately 2 | before each action was taken: 3 | 4 | ```json 5 | {{ action_windows }} 6 | ``` 7 | 8 | Consider the attached screenshots taken immediately before each action. The order of 9 | the screenshots matches the order of the actions above. 10 | 11 | Provide a detailed natural language description of everything that happened 12 | in this recording. This description will be embedded in the context for a future prompt 13 | to replay the recording (subject to proposed modifications in natural language) on a 14 | live system, so make sure to include everything you will need to know. 15 | 16 | My career depends on this. Lives are at stake. 17 | -------------------------------------------------------------------------------- /openadapt/prompts/describe_recording.j2: -------------------------------------------------------------------------------- 1 | Consider the actions in the recording and states of the active window immediately 2 | before each action was taken: 3 | 4 | ```json 5 | {{ action_windows }} 6 | ``` 7 | 8 | Consider the attached screenshots taken immediately before each action. The order of 9 | the screenshots matches the order of the actions above. 10 | 11 | Provide a detailed natural language description of everything that happened 12 | in this recording. This description will be embedded in the context for a future prompt 13 | to replay the recording (subject to proposed modifications in natural language) on a 14 | live system, so make sure to include everything you will need to know. 15 | 16 | My career depends on this. Lives are at stake. 17 | -------------------------------------------------------------------------------- /openadapt/prompts/description.j2: -------------------------------------------------------------------------------- 1 | You are a master GUI understander. I have attached a screenshot of a GUI, and a several 2 | segments of the screenshot. Please describe what GUI element is in each segment, 3 | in the order they are given. 4 | You MUST provide exactly one description per segment. There are {{ num_segments }}, 5 | therefore you must provide {{ num_segments }} descriptions. Include a monotonically 6 | increasing integer so you don't lose track. 7 | Respond in JSON, i.e.: 8 | ```json 9 | { 10 | "descriptions": [ 11 | [1, "..."], 12 | [2, "..."], 13 | ... 14 | ] 15 | } 16 | ``` 17 | {% if active_segment_description %} 18 | In particular, we are interested in a segment with the following description 19 | (i.e. the "active segment description"): 20 | ```text 21 | {{ active_segment_description }} 22 | ``` 23 | If any of the segments EXACTLY match this description, you MUST re-use this 24 | description *VERBATIM*, i.e. without any modifications. 25 | {% endif %} 26 | {% if exceptions %} 27 | Previously when you attempted this, the results generated these exceptions: 28 | {% for exception in exceptions %} 29 | {{ exception }} 30 | {% endfor %} 31 | YOU MUST RE-USE THE PROVIDED ACTIVE SEGMENT DESCRIPTION: 32 | ```text 33 | {{ active_segment_description }} 34 | ``` 35 | {% endif %} 36 | -------------------------------------------------------------------------------- /openadapt/prompts/generate_action_event--segment.j2: -------------------------------------------------------------------------------- 1 | {% if include_raw_recording %} 2 | Consider the previously recorded actions: 3 | 4 | ```json 5 | {{ recorded_actions }} 6 | ``` 7 | {% endif %} 8 | 9 | 10 | {% if include_raw_recording_description %} 11 | Consider the following description of the previously recorded actions: 12 | 13 | ``json 14 | {{ recording_description }} 15 | ``` 16 | {% endif %} 17 | 18 | 19 | {% if include_replay_instructions %} 20 | Consider the user's proposed modifications in natural language instructions: 21 | 22 | ```text 23 | {{ replay_instructions }} 24 | ``` 25 | {% endif %} 26 | 27 | 28 | {% if include_modified_recording %} 29 | Consider this updated list of actions that have been modified such that replaying them 30 | would have accomplished the user's instructions: 31 | 32 | ```json 33 | {{ modified_actions }} 34 | ``` 35 | {% endif %} 36 | 37 | 38 | {% if include_modified_recording_description %} 39 | Consider the following description of the updated list of actions that have been 40 | modified such that replaying them would have accomplished the user's instructions: 41 | 42 | ``json 43 | {{ modified_recording_description }} 44 | ``` 45 | {% endif %} 46 | 47 | 48 | Consider the actions you've produced (and we have played back) so far: 49 | 50 | ```json 51 | {{ replayed_actions }} 52 | ``` 53 | 54 | {% if include_active_window %} 55 | Consider the current active window: 56 | ```json 57 | {{ current_window }} 58 | ``` 59 | {% endif %} 60 | 61 | 62 | The attached image is a screenshot of the current state of the system. 63 | 64 | Provide the next action to be replayed in order to accomplish the user's replay 65 | instructions. 66 | 67 | Do NOT provide available_segment_descriptions in your response. 68 | 69 | Respond with JSON and nothing else. 70 | 71 | If you wish to terminate the recording, return an empty object. 72 | 73 | My career depends on this. Lives are at stake. 74 | -------------------------------------------------------------------------------- /openadapt/prompts/generate_action_event.j2: -------------------------------------------------------------------------------- 1 | Consider the previously recorded actions: 2 | 3 | ```json 4 | {{ recorded_actions }} 5 | ``` 6 | 7 | Consider the actions you've produced (and we have replayed) so far: 8 | 9 | ```json 10 | {{ replayed_actions }} 11 | ``` 12 | 13 | Consider the user's proposed modifications in natural language replay instructions: 14 | 15 | ```text 16 | {{ replay_instructions }} 17 | ``` 18 | 19 | Consider the current active window: 20 | ```json 21 | {{ current_window }} 22 | ``` 23 | 24 | The attached image is a screenshot of the current state of the system. 25 | 26 | Provide the next action to be replayed in order to accomplish the user's replay 27 | instructions. 28 | 29 | Respond with JSON and nothing else. 30 | 31 | If you wish to terminate the recording, return an empty object. 32 | 33 | My career depends on this. Lives are at stake. 34 | -------------------------------------------------------------------------------- /openadapt/prompts/is_action_complete.j2: -------------------------------------------------------------------------------- 1 | Consider the actions that you previously generated: 2 | 3 | ```json 4 | {{ actions }} 5 | ``` 6 | 7 | The attached image is a screenshot of the current state of the system, immediately 8 | after the last action in the sequence was played. 9 | 10 | Your task is to: 11 | 1. Describe what you would expect to see in the screenshot after the last action in the 12 | sequence is complete, and 13 | 2. Determine whether the the last action has completed by looking at the attached 14 | screenshot. For example, if you expect that the sequence of actions would result in 15 | opening a particular application, you should determine whether that application has 16 | finished opening. 17 | 18 | Respond with JSON and nothing else. The JSON should have the following keys: 19 | - "expected_state": Natural language description of what you would expect to see. 20 | - "is_complete": Boolean indicating whether the last action is complete or not. 21 | 22 | My career depends on this. Lives are at stake. 23 | -------------------------------------------------------------------------------- /openadapt/prompts/system.j2: -------------------------------------------------------------------------------- 1 | You are the cognitive engine powering OpenAdapt, the world's first and most advanced open-source AI-first process automation system. 2 | Your output directly manipulates a real keyboard/mouse controller on a live system. 3 | Your task is to accurately interpet previously recorded user actions (keyboard and mouse events) and associated screenshots and window states. 4 | Your responses will be parsed by a deterministic software system, and not read by a human (unless it is for debugging purposes). 5 | -------------------------------------------------------------------------------- /openadapt/scripts/__init__.py: -------------------------------------------------------------------------------- 1 | """Scripts package. 2 | 3 | This package contains scripts for various purposes. 4 | 5 | Module: __init__.py 6 | """ 7 | -------------------------------------------------------------------------------- /openadapt/scripts/reset_db.py: -------------------------------------------------------------------------------- 1 | """Reset Database Script. 2 | 3 | This script clears the database by removing the database file and 4 | running a database migration using Alembic. 5 | 6 | Module: reset_db.py 7 | """ 8 | 9 | from subprocess import PIPE, run 10 | import os 11 | 12 | from openadapt.config import config 13 | 14 | 15 | def reset_db() -> None: 16 | """Clears the database by removing the db file and running a db migration.""" 17 | if os.path.exists(config.DATABASE_FILE_PATH): 18 | os.remove(config.DATABASE_FILE_PATH) 19 | 20 | # Prevents duplicate logging of config values by piping stderr 21 | # and filtering the output. 22 | result = run(["alembic", "upgrade", "head"], stderr=PIPE, text=True) 23 | print(result.stderr[result.stderr.find("INFO [alembic") :]) # noqa: E203 24 | if result.returncode != 0: 25 | raise RuntimeError("Database migration failed.") 26 | else: 27 | print("Database cleared successfully.") 28 | 29 | 30 | if __name__ == "__main__": 31 | reset_db() 32 | -------------------------------------------------------------------------------- /openadapt/spacy_model_helpers/__init__.py: -------------------------------------------------------------------------------- 1 | """Init file for spacy_model_helpers package.""" 2 | 3 | from .download_model import download_spacy_model 4 | 5 | __all__ = ["download_spacy_model"] 6 | -------------------------------------------------------------------------------- /openadapt/spacy_model_helpers/spacy_model_init.py: -------------------------------------------------------------------------------- 1 | """Spacy model init file.""" 2 | 3 | from pathlib import Path 4 | 5 | from spacy.language import Language 6 | from spacy.util import get_model_meta, load_model_from_init_py 7 | 8 | __version__ = get_model_meta(Path(__file__).parent)["version"] 9 | 10 | 11 | def load(**overrides: dict) -> Language: 12 | """Load the model. 13 | 14 | Args: 15 | **overrides: Optional overrides for model settings. 16 | 17 | Returns: 18 | The loaded model. 19 | """ 20 | return load_model_from_init_py(__file__, **overrides) 21 | -------------------------------------------------------------------------------- /openadapt/start.py: -------------------------------------------------------------------------------- 1 | """Implements the code necessary to update the OpenAdapt app if required. 2 | 3 | Usage: 4 | python3 -m openadapt.start 5 | """ 6 | 7 | import subprocess 8 | 9 | from openadapt.custom_logger import logger 10 | 11 | 12 | def main() -> None: 13 | """The main function which runs the OpenAdapt app when it is updated.""" 14 | result = subprocess.run(["git", "status"], capture_output=True, text=True) 15 | 16 | if "unmerged" in result.stdout: 17 | logger.info("Please fix merge conflicts and try again") 18 | return 19 | 20 | subprocess.run(["git", "stash"]) 21 | 22 | if "git pull" in result.stdout: 23 | subprocess.run(["git", "pull", "-q"]) 24 | logger.info("Updated the OpenAdapt App") 25 | 26 | 27 | if __name__ == "__main__": 28 | main() 29 | -------------------------------------------------------------------------------- /openadapt/strategies/__init__.py: -------------------------------------------------------------------------------- 1 | """Package containing different replay strategies. 2 | 3 | Module: __init__.py 4 | """ 5 | 6 | # flake8: noqa 7 | 8 | from openadapt.strategies.base import BaseReplayStrategy 9 | from openadapt.strategies.visual_browser import VisualBrowserReplayStrategy 10 | 11 | # disabled because importing is expensive 12 | # from openadapt.strategies.demo import DemoReplayStrategy 13 | from openadapt.strategies.naive import NaiveReplayStrategy 14 | from openadapt.strategies.segment import SegmentReplayStrategy 15 | from openadapt.strategies.stateful import StatefulReplayStrategy 16 | from openadapt.strategies.vanilla import VanillaReplayStrategy 17 | from openadapt.strategies.visual import VisualReplayStrategy 18 | 19 | # add more strategies here 20 | -------------------------------------------------------------------------------- /openadapt/strategies/demo.py: -------------------------------------------------------------------------------- 1 | """Demonstration of HuggingFace, OCR, and ASCII ReplayStrategyMixins. 2 | 3 | Usage: 4 | 5 | $ python -m openadapt.replay DemoReplayStrategy 6 | """ 7 | 8 | from openadapt.custom_logger import logger 9 | from openadapt.db import crud 10 | from openadapt.models import Recording, Screenshot, WindowEvent 11 | from openadapt.strategies.base import BaseReplayStrategy 12 | from openadapt.strategies.mixins.ascii import ASCIIReplayStrategyMixin 13 | from openadapt.strategies.mixins.huggingface import ( 14 | MAX_INPUT_SIZE, 15 | HuggingFaceReplayStrategyMixin, 16 | ) 17 | from openadapt.strategies.mixins.ocr import OCRReplayStrategyMixin 18 | from openadapt.strategies.mixins.sam import SAMReplayStrategyMixin 19 | from openadapt.strategies.mixins.summary import SummaryReplayStrategyMixin 20 | 21 | 22 | class DemoReplayStrategy( 23 | HuggingFaceReplayStrategyMixin, 24 | OCRReplayStrategyMixin, 25 | ASCIIReplayStrategyMixin, 26 | SAMReplayStrategyMixin, 27 | SummaryReplayStrategyMixin, 28 | BaseReplayStrategy, 29 | ): 30 | """Demo replay strategy that combines HuggingFace, OCR, and ASCII mixins.""" 31 | 32 | def __init__( 33 | self, 34 | recording: Recording, 35 | ) -> None: 36 | """Initialize the DemoReplayStrategy. 37 | 38 | Args: 39 | recording (Recording): The recording to replay. 40 | """ 41 | super().__init__(recording) 42 | self.result_history = [] 43 | session = crud.get_new_session(read_only=True) 44 | self.screenshots = crud.get_screenshots(session, recording) 45 | self.screenshot_idx = 0 46 | 47 | def get_next_action_event( 48 | self, 49 | screenshot: Screenshot, 50 | window_event: WindowEvent, 51 | ) -> None: 52 | """Get the next action event based on the current screenshot and window event. 53 | 54 | Args: 55 | screenshot (Screenshot): The current screenshot. 56 | window_event (WindowEvent): The current window event. 57 | 58 | Returns: 59 | None: No action event is returned in this demo strategy. 60 | """ 61 | # ascii_text = self.get_ascii_text(screenshot) 62 | # logger.info(f"ascii_text=\n{ascii_text}") 63 | 64 | # ocr_text = self.get_ocr_text(screenshot) 65 | # logger.info(f"ocr_text=\n{ocr_text}") 66 | 67 | screenshot_bbox = self.get_screenshot_bbox(screenshot) 68 | logger.info(f"screenshot_bbox=\n{screenshot_bbox}") 69 | 70 | screenshot_click_event_bbox = self.get_click_event_bbox( 71 | self.screenshots[self.screenshot_idx] 72 | ) 73 | logger.info( 74 | "self.screenshots[self.screenshot_idx].action_event=\n" 75 | f"{screenshot_click_event_bbox}" 76 | ) 77 | event_strs = [f"<{event}>" for event in self.recording.action_events] 78 | history_strs = [f"<{completion}>" for completion in self.result_history] 79 | prompt = " ".join(event_strs + history_strs) 80 | N = max(0, len(prompt) - MAX_INPUT_SIZE) 81 | prompt = prompt[N:] 82 | # logger.info(f"{prompt=}") 83 | max_tokens = 10 84 | completion = self.get_completion(prompt, max_tokens) 85 | # logger.info(f"{completion=}") 86 | 87 | # only take the first <...> 88 | result = completion.split(">")[0].strip(" <>") 89 | # logger.info(f"{result=}") 90 | self.result_history.append(result) 91 | 92 | # TODO: parse result into ActionEvent(s) 93 | self.screenshot_idx += 1 94 | return None 95 | -------------------------------------------------------------------------------- /openadapt/strategies/mixins/ascii.py: -------------------------------------------------------------------------------- 1 | """Implements a ReplayStrategy mixin for converting images to ASCII. 2 | 3 | Usage: 4 | 5 | class MyReplayStrategy(ASCIIReplayStrategyMixin): 6 | ... 7 | """ 8 | 9 | from ascii_magic import AsciiArt 10 | 11 | from openadapt.custom_logger import logger 12 | from openadapt.models import Recording, Screenshot 13 | from openadapt.strategies.base import BaseReplayStrategy 14 | 15 | COLUMNS = 120 16 | WIDTH_RATIO = 2.2 17 | MONOCHROME = True 18 | 19 | 20 | class ASCIIReplayStrategyMixin(BaseReplayStrategy): 21 | """ReplayStrategy mixin for converting images to ASCII.""" 22 | 23 | def __init__( 24 | self, 25 | recording: Recording, 26 | ) -> None: 27 | """Initialize the ASCIIReplayStrategyMixin. 28 | 29 | Args: 30 | recording (Recording): The recording to replay. 31 | """ 32 | super().__init__(recording) 33 | 34 | def get_ascii_text( 35 | self, 36 | screenshot: Screenshot, 37 | monochrome: bool = MONOCHROME, 38 | columns: int = COLUMNS, 39 | width_ratio: float = WIDTH_RATIO, 40 | ) -> str: 41 | """Convert the screenshot image to ASCII text. 42 | 43 | Args: 44 | screenshot (Screenshot): The screenshot to convert. 45 | monochrome (bool): Flag to indicate monochrome conversion (default: True). 46 | columns (int): Number of columns for the ASCII text (default: 120). 47 | width_ratio (float): Width ratio for the ASCII text (default: 2.2). 48 | 49 | Returns: 50 | str: The ASCII representation of the screenshot image. 51 | """ 52 | ascii_art = AsciiArt.from_pillow_image(screenshot.image) 53 | ascii_text = ascii_art.to_ascii( 54 | monochrome=monochrome, 55 | columns=columns, 56 | width_ratio=width_ratio, 57 | ) 58 | logger.debug(f"ascii_text=\n{ascii_text}") 59 | return ascii_text 60 | -------------------------------------------------------------------------------- /openadapt/strategies/mixins/huggingface.py: -------------------------------------------------------------------------------- 1 | """Implements a ReplayStrategy mixin for generating completions with HuggingFace. 2 | 3 | Usage: 4 | 5 | class MyReplayStrategy(HuggingFaceReplayStrategyMixin): 6 | ... 7 | """ 8 | 9 | import transformers as tf # RIP TensorFlow 10 | 11 | from openadapt.custom_logger import logger 12 | from openadapt.models import Recording 13 | from openadapt.strategies.base import BaseReplayStrategy 14 | 15 | MODEL_NAME = "gpt2" # gpt2-xl is bigger and slower 16 | MAX_INPUT_SIZE = 1024 17 | 18 | 19 | class HuggingFaceReplayStrategyMixin(BaseReplayStrategy): 20 | """ReplayStrategy mixin for generating completions with HuggingFace.""" 21 | 22 | def __init__( 23 | self, 24 | recording: Recording, 25 | model_name: str = MODEL_NAME, 26 | max_input_size: int = MAX_INPUT_SIZE, 27 | ) -> None: 28 | """Initialize the HuggingFaceReplayStrategyMixin. 29 | 30 | Args: 31 | recording (Recording): The recording to replay. 32 | model_name (str): The name of the HuggingFace model to use 33 | (default: "gpt2"). 34 | max_input_size (int): The maximum input size for the model 35 | (default: 1024). 36 | """ 37 | super().__init__(recording) 38 | 39 | logger.info(f"{model_name=}") 40 | self.tokenizer = tf.AutoTokenizer.from_pretrained(model_name) 41 | self.model = tf.AutoModelForCausalLM.from_pretrained(model_name) 42 | self.max_input_size = max_input_size 43 | 44 | def get_completion( 45 | self, 46 | prompt: str, 47 | max_tokens: int, 48 | ) -> str: 49 | """Generate completion for a given prompt using the HuggingFace model. 50 | 51 | Args: 52 | prompt (str): The prompt for generating completion. 53 | max_tokens (int): The maximum number of tokens to generate. 54 | 55 | Returns: 56 | str: The generated completion. 57 | """ 58 | max_input_size = self.max_input_size 59 | if max_input_size and len(prompt) > max_input_size: 60 | logger.warning(f"Truncating from {len(prompt) =} to {max_input_size=}") 61 | prompt = prompt[-max_input_size:] 62 | logger.warning(f"Truncated {len(prompt)=}") 63 | 64 | logger.debug(f"{prompt=} {max_tokens=}") 65 | input_tokens = self.tokenizer(prompt, return_tensors="pt") 66 | pad_token_id = self.tokenizer.eos_token_id 67 | attention_mask = input_tokens["attention_mask"] 68 | output_tokens = self.model.generate( 69 | input_ids=input_tokens["input_ids"], 70 | attention_mask=attention_mask, 71 | max_length=input_tokens["input_ids"].shape[-1] + max_tokens, 72 | pad_token_id=pad_token_id, 73 | num_return_sequences=1, 74 | ) 75 | N = input_tokens["input_ids"].shape[-1] 76 | completion = self.tokenizer.decode( 77 | output_tokens[:, N:][0], 78 | clean_up_tokenization_spaces=True, 79 | ) 80 | logger.debug(f"{completion=}") 81 | return completion 82 | -------------------------------------------------------------------------------- /openadapt/strategies/mixins/summary.py: -------------------------------------------------------------------------------- 1 | """Implements a ReplayStrategy mixin which summarizes the content of texts. 2 | 3 | Usage: 4 | 5 | class MyReplayStrategy(SummaryReplayStrategyMixin): 6 | ... 7 | """ 8 | 9 | from sumy.nlp.stemmers import Stemmer 10 | from sumy.nlp.tokenizers import Tokenizer 11 | from sumy.parsers.plaintext import PlaintextParser 12 | from sumy.summarizers.lsa import LsaSummarizer 13 | from sumy.utils import get_stop_words 14 | import nltk 15 | 16 | from openadapt.custom_logger import logger 17 | from openadapt.models import Recording 18 | from openadapt.strategies.base import BaseReplayStrategy 19 | 20 | 21 | class SummaryReplayStrategyMixin(BaseReplayStrategy): 22 | """ReplayStrategy mixin for summarizing text content.""" 23 | 24 | def __init__( 25 | self, 26 | recording: Recording, 27 | ) -> None: 28 | """Initialize the SummaryReplayStrategyMixin. 29 | 30 | Args: 31 | recording (Recording): The recording object. 32 | 33 | Additional Attributes: 34 | - stemmer: The stemmer for text processing. 35 | - summarizer: The summarizer for generating text summaries. 36 | """ 37 | super().__init__(recording) 38 | self.stemmer = Stemmer("english") 39 | summarizer = LsaSummarizer(self.stemmer) 40 | summarizer.stop_words = get_stop_words("english") 41 | self.summarizer = summarizer 42 | 43 | def get_summary( 44 | self, 45 | text: str, 46 | num_sentences: int, 47 | ) -> str: 48 | """Generate a summary of the given text. 49 | 50 | Args: 51 | text (str): The text to summarize. 52 | num_sentences (int): The number of sentences to include in the summary. 53 | 54 | Returns: 55 | str: The summarized text. 56 | """ 57 | while True: 58 | try: 59 | Tokenizer("english") 60 | break 61 | except Exception as e: 62 | logger.info(e) 63 | logger.info("Downloading punkt now") 64 | nltk.download("punkt") 65 | 66 | parser = PlaintextParser.from_string(text, Tokenizer("english")) 67 | summarized = self.summarizer(parser.document, num_sentences) 68 | return summarized 69 | -------------------------------------------------------------------------------- /openadapt/window/__init__.py: -------------------------------------------------------------------------------- 1 | """Package for interacting with active window and elements across platforms. 2 | 3 | Module: __init__.py 4 | """ 5 | 6 | from typing import Any 7 | import sys 8 | 9 | from openadapt.config import config 10 | from openadapt.custom_logger import logger 11 | 12 | if sys.platform == "darwin": 13 | from . import _macos as impl 14 | elif sys.platform == "win32": 15 | from . import _windows as impl 16 | elif sys.platform.startswith("linux"): 17 | from . import _linux as impl 18 | else: 19 | raise Exception(f"Unsupported platform: {sys.platform}") 20 | 21 | 22 | def get_active_window_data( 23 | include_window_data: bool = config.RECORD_WINDOW_DATA, 24 | ) -> dict[str, Any] | None: 25 | """Get data of the active window. 26 | 27 | Args: 28 | include_window_data (bool): whether to include a11y data. 29 | 30 | Returns: 31 | dict or None: A dictionary containing information about the active window, 32 | or None if the state is not available. 33 | """ 34 | state = get_active_window_state(include_window_data) 35 | if not state: 36 | return {} 37 | title = state["title"] 38 | left = state["left"] 39 | top = state["top"] 40 | width = state["width"] 41 | height = state["height"] 42 | window_id = state["window_id"] 43 | window_data = { 44 | "title": title, 45 | "left": left, 46 | "top": top, 47 | "width": width, 48 | "height": height, 49 | "window_id": window_id, 50 | "state": state, 51 | } 52 | return window_data 53 | 54 | 55 | def get_active_window_state(read_window_data: bool) -> dict | None: 56 | """Get the state of the active window. 57 | 58 | Returns: 59 | dict or None: A dictionary containing the state of the active window, 60 | or None if the state is not available. 61 | """ 62 | # TODO: save window identifier (a window's title can change, or 63 | # multiple windows can have the same title) 64 | try: 65 | return impl.get_active_window_state(read_window_data) 66 | except Exception as exc: 67 | logger.warning(f"{exc=}") 68 | return None 69 | 70 | 71 | def get_active_element_state(x: int, y: int) -> dict | None: 72 | """Get the state of the active element at the specified coordinates. 73 | 74 | Args: 75 | x (int): The x-coordinate of the element. 76 | y (int): The y-coordinate of the element. 77 | 78 | Returns: 79 | dict or None: A dictionary containing the state of the active element, 80 | or None if the state is not available. 81 | """ 82 | try: 83 | return impl.get_active_element_state(x, y) 84 | except Exception as exc: 85 | logger.warning(f"{exc=}") 86 | return None 87 | -------------------------------------------------------------------------------- /permissions_in_macOS.md: -------------------------------------------------------------------------------- 1 | # Setting permissions in macOS 2 | 3 | If using macOS, you'll need to set up some permissions to ensure that OpenAdapt works as intended. The following instructions apply to macOS Ventura. Earlier version may present different menus. 4 | 5 | Note that while you will be prompted on first run for input monitoring and screen recording permissions, you won't be prompted for accessibility permissions needed for replay of actions. If permission is not granted in that case, replay of actions will silently fail. 6 | 7 | ## Enabling input monitoring 8 | 9 | Input monitoring must be enabled in Settings in order to allow capture of mouse and keyboard input. You can do so by enabling the Terminal application under Settings → Privacy and security → Input monitoring → Allow Terminal 10 | 11 | ![Enabling input monitoring](./assets/macOS_input_monitoring.png) 12 | 13 | ## Enabling screen recording 14 | 15 | Screen recoding must be enabled in Settings in order to allow capture of screenshots. You can do so by enabling the Terminal application under Settings → Privacy and security → Screen recording → Allow Terminal 16 | 17 | ![Enabling screen recording](./assets/macOS_screen_recording.png) 18 | 19 | ## Enabling replay of actions 20 | 21 | Mouse and keyboard control must be enabled in Settings in order to allow replay of actions by the framework. You can do so by enabling the Terminal application under Settings → Privay and security → Accessibility → Allow Terminal 22 | 23 | ![Enabling replay of actions](./assets/macOS_accessibility.png) 24 | -------------------------------------------------------------------------------- /scripts/postinstall.py: -------------------------------------------------------------------------------- 1 | """Consolidated post-install script for OpenAdapt.""" 2 | 3 | import os 4 | import subprocess 5 | import sys 6 | 7 | 8 | def install_detectron2() -> None: 9 | """Install Detectron2 from its GitHub repository.""" 10 | try: 11 | subprocess.check_call( 12 | [ 13 | sys.executable, 14 | "-m", 15 | "pip", 16 | "install", 17 | "git+https://github.com/facebookresearch/detectron2.git", 18 | "--no-build-isolation", 19 | ] 20 | ) 21 | except subprocess.CalledProcessError as e: 22 | print(f"Error installing Detectron2: {e}") 23 | sys.exit(1) 24 | 25 | 26 | def install_dashboard() -> None: 27 | """Install dashboard dependencies based on the operating system.""" 28 | original_directory = os.getcwd() 29 | print(f"Original directory: {original_directory}") 30 | 31 | dashboard_dir = os.path.join(original_directory, "openadapt", "app", "dashboard") 32 | print(f"Dashboard directory: {dashboard_dir}") 33 | 34 | if not os.path.exists(os.path.join(dashboard_dir, "package.json")): 35 | print("package.json not found in the dashboard directory.") 36 | sys.exit(1) 37 | 38 | try: 39 | os.chdir(dashboard_dir) 40 | print("Changed directory to:", os.getcwd()) 41 | 42 | if sys.platform.startswith("win"): 43 | try: 44 | subprocess.check_call(["nvm", "install", "21"]) 45 | subprocess.check_call(["nvm", "use", "21"]) 46 | except FileNotFoundError: 47 | if os.getenv("CI") == "true": 48 | print("nvm not found. Skipping installation.") 49 | else: 50 | print("nvm not found. Please install nvm.") 51 | sys.exit(1) 52 | subprocess.check_call(["powershell", "./entrypoint.ps1"]) 53 | else: 54 | subprocess.check_call(["bash", "./entrypoint.sh"]) 55 | except subprocess.CalledProcessError as e: 56 | print(f"Error running entrypoint script: {e}") 57 | sys.exit(1) 58 | finally: 59 | os.chdir(original_directory) 60 | print("Reverted to original directory:", os.getcwd()) 61 | 62 | 63 | def main() -> None: 64 | """Main function to install dependencies.""" 65 | try: 66 | install_detectron2() 67 | install_dashboard() 68 | except Exception as e: 69 | print(f"Unhandled error: {e}") 70 | sys.exit(1) 71 | 72 | 73 | if __name__ == "__main__": 74 | main() 75 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Install openadapt as a package that can be imported. 3 | 4 | Typical usage example: 5 | 6 | $ cd path/to/setup.py 7 | $ pip install -e . 8 | """ 9 | 10 | from setuptools import find_packages, setup 11 | 12 | MODULE_NAME = "openadapt" 13 | MODULE_VERSION = "0.1.0" 14 | MODULE_DESCRIPTION = "GUI Process Automation with Transformers" 15 | MODULE_AUTHOR_NAME = "Richard Abrich" 16 | MODULE_AUTHOR_EMAIL = "richard.abrich@mldsai.com" 17 | MODULE_REPO_URL = "https://github.com/MLDSAI/openadapt" 18 | MODULE_README_FNAME = "README.md" 19 | MODULE_LICENSE_FNAME = "LICENSE" 20 | 21 | 22 | readme = open(MODULE_README_FNAME).read() 23 | license = open(MODULE_LICENSE_FNAME).read() 24 | 25 | 26 | setup( 27 | name=MODULE_NAME, 28 | version=MODULE_VERSION, 29 | description=MODULE_DESCRIPTION, 30 | long_description=readme, 31 | author=MODULE_AUTHOR_NAME, 32 | author_email=MODULE_AUTHOR_EMAIL, 33 | url=MODULE_REPO_URL, 34 | license=license, 35 | packages=find_packages(exclude=("tests", "docs")), 36 | ) 37 | -------------------------------------------------------------------------------- /tests/assets/calculator.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenAdaptAI/OpenAdapt/HEAD/tests/assets/calculator.png -------------------------------------------------------------------------------- /tests/assets/excel.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenAdaptAI/OpenAdapt/HEAD/tests/assets/excel.png -------------------------------------------------------------------------------- /tests/assets/sample_llc_1.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenAdaptAI/OpenAdapt/HEAD/tests/assets/sample_llc_1.pdf -------------------------------------------------------------------------------- /tests/assets/test_emr_image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenAdaptAI/OpenAdapt/HEAD/tests/assets/test_emr_image.png -------------------------------------------------------------------------------- /tests/assets/test_image_redaction_presidio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenAdaptAI/OpenAdapt/HEAD/tests/assets/test_image_redaction_presidio.png -------------------------------------------------------------------------------- /tests/assets/test_image_redaction_privateai.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/OpenAdaptAI/OpenAdapt/HEAD/tests/assets/test_image_redaction_privateai.png -------------------------------------------------------------------------------- /tests/openadapt/drivers/test_anthropic.py: -------------------------------------------------------------------------------- 1 | """Tests for drivers.anthropic.""" 2 | 3 | from PIL import Image 4 | import pytest 5 | 6 | import anthropic 7 | 8 | from openadapt import drivers 9 | 10 | 11 | def test_prompt(calculator_image: Image) -> None: 12 | """Test image prompt.""" 13 | prompt = "What is this a screenshot of?" 14 | try: 15 | result = drivers.anthropic.prompt(prompt, images=[calculator_image]) 16 | assert "calculator" in result.lower(), result 17 | except anthropic.AuthenticationError as e: 18 | pytest.xfail(f"Anthropic AuthenticationError occurred: {e}") 19 | 20 | 21 | if __name__ == "__main__": 22 | pytest.main() 23 | -------------------------------------------------------------------------------- /tests/openadapt/drivers/test_google.py: -------------------------------------------------------------------------------- 1 | """Tests for drivers.google.""" 2 | 3 | from google.api_core.exceptions import DeadlineExceeded, InvalidArgument 4 | from PIL import Image 5 | import pytest 6 | 7 | from openadapt.drivers import google 8 | 9 | 10 | def test_prompt(calculator_image: Image) -> None: 11 | """Test image prompt.""" 12 | try: 13 | prompt = "What is this a screenshot of?" 14 | result = google.prompt(prompt, images=[calculator_image]) 15 | assert "calculator" in result.lower(), result 16 | except InvalidArgument: 17 | pytest.xfail("Invalid API key, expected failure.") 18 | except DeadlineExceeded: 19 | pytest.xfail("Request timeout, expected failure.") 20 | 21 | 22 | if __name__ == "__main__": 23 | pytest.main() 24 | -------------------------------------------------------------------------------- /tests/openadapt/drivers/test_openai.py: -------------------------------------------------------------------------------- 1 | """Tests for drivers.openai.""" 2 | 3 | import pytest 4 | 5 | from PIL import Image 6 | import requests 7 | 8 | from openadapt.drivers import openai 9 | 10 | 11 | def test_prompt(calculator_image: Image) -> None: 12 | """Test image prompt.""" 13 | prompt = "What is this a screenshot of?" 14 | try: 15 | result = openai.prompt(prompt, images=[calculator_image]) 16 | assert "calculator" in result.lower(), result 17 | except Exception as e: 18 | if "Incorrect API key" in str(e): 19 | pytest.xfail(f"ValueError due to incorrect API key: {e}") 20 | else: 21 | raise 22 | except requests.exceptions.HTTPError as e: 23 | if "Unauthorized" in str(e): 24 | pytest.xfail(f"HTTPError: {e}") 25 | 26 | 27 | if __name__ == "__main__": 28 | pytest.main() 29 | -------------------------------------------------------------------------------- /tests/openadapt/privacy/test_providers.py: -------------------------------------------------------------------------------- 1 | """A test module for the providers module.""" 2 | 3 | import pytest 4 | import spacy 5 | 6 | from openadapt.config import config 7 | 8 | if not spacy.util.is_package(config.SPACY_MODEL_NAME): # pylint: disable=no-member 9 | pytestmark = pytest.mark.skip(reason="SpaCy model not installed!") 10 | else: 11 | from openadapt.privacy.base import Modality, ScrubbingProviderFactory 12 | from openadapt.privacy.providers.aws_comprehend import ( # noqa: F401 13 | ComprehendScrubbingProvider, 14 | ) 15 | from openadapt.privacy.providers.presidio import ( # noqa: F401 16 | PresidioScrubbingProvider, 17 | ) 18 | from openadapt.privacy.providers.private_ai import ( # noqa: F401 19 | PrivateAIScrubbingProvider, 20 | ) 21 | 22 | 23 | def test_scrubbing_provider_factory() -> None: 24 | """Test the ScrubbingProviderFactory for Modality.TEXT.""" 25 | providers_list = ScrubbingProviderFactory.get_for_modality(Modality.TEXT) 26 | 27 | # Ensure that we get at least one provider 28 | assert providers_list 29 | 30 | for provider in providers_list: 31 | # Ensure that the provider supports Modality.TEXT 32 | assert Modality.TEXT in provider.capabilities 33 | -------------------------------------------------------------------------------- /tests/openadapt/test_crop.py: -------------------------------------------------------------------------------- 1 | """Module to test cropping functionality.""" 2 | 3 | from unittest import mock 4 | from unittest.mock import Mock 5 | 6 | from PIL import Image 7 | 8 | from openadapt.models import Screenshot 9 | 10 | 11 | def test_crop_active_window() -> None: 12 | """Test the crop_active_window function. 13 | 14 | This function creates a mock action event with a mock window event, 15 | sets up the necessary environment, performs the cropping operation, 16 | and verifies that the image size has been reduced. 17 | 18 | Returns: 19 | None 20 | """ 21 | # Create a mock action event with a mock window event 22 | action_event_mock = Mock() 23 | window_event_mock = Mock() 24 | 25 | # Define window_event's attributes 26 | window_event_mock.left = 100 27 | window_event_mock.top = 100 28 | window_event_mock.width = 300 29 | window_event_mock.height = 300 30 | action_event_mock.window_event = window_event_mock 31 | 32 | # Mock the utils.get_scale_ratios to return some fixed ratios 33 | with mock.patch("openadapt.utils.get_scale_ratios", return_value=(1, 1)): 34 | # Create a dummy image and put it in a Screenshot object 35 | image = Image.new("RGB", (500, 500), color="white") 36 | screenshot = Screenshot() 37 | screenshot._image = image 38 | 39 | # Store original image size for comparison 40 | original_size = screenshot._image.size 41 | 42 | # Perform the cropping operation 43 | cropped_image = screenshot.crop_active_window(action_event=action_event_mock) 44 | 45 | # Verify that the image size has been reduced 46 | assert (cropped_image.size[0] < original_size[0]) or ( 47 | cropped_image.size[1] < original_size[1] 48 | ) 49 | -------------------------------------------------------------------------------- /tests/openadapt/test_crud.py: -------------------------------------------------------------------------------- 1 | """Tests for the CRUD operations in the openadapt.db.crud module.""" 2 | 3 | from unittest.mock import patch 4 | 5 | import pytest 6 | import sqlalchemy as sa 7 | 8 | from openadapt.db import crud, db 9 | from openadapt.models import Recording 10 | 11 | 12 | def test_get_new_session_read_only(db_engine: sa.engine.Engine) -> None: 13 | """Test that get_new_session returns a read-only session when read_only=True. 14 | 15 | Args: 16 | db_engine (sa.engine.Engine): The test database engine. 17 | """ 18 | # patch the ReadOnlySession class to return a mock object 19 | with patch( 20 | "openadapt.db.db.get_read_only_session_maker", 21 | return_value=db.get_read_only_session_maker(db_engine), 22 | ): 23 | session = crud.get_new_session(read_only=True) 24 | recording = Recording( 25 | timestamp=0, 26 | monitor_width=1920, 27 | monitor_height=1080, 28 | double_click_interval_seconds=0, 29 | double_click_distance_pixels=0, 30 | platform="Windows", 31 | task_description="Task description", 32 | ) 33 | # with pytest.raises(PermissionError): 34 | # session.add(recording) 35 | with pytest.raises(PermissionError): 36 | session.commit() 37 | with pytest.raises(PermissionError): 38 | session.flush() 39 | with pytest.raises(PermissionError): 40 | session.delete(recording) 41 | -------------------------------------------------------------------------------- /tests/openadapt/test_models.py: -------------------------------------------------------------------------------- 1 | """Tests for openadapt.models.""" 2 | 3 | from openadapt import models 4 | 5 | 6 | def test_action_from_dict() -> None: 7 | """Test ActionEvent.from_dict().""" 8 | input_variations_by_expected_output = { 9 | # all named keys 10 | "--": [ 11 | # standard 12 | "--", 13 | # mal-formed 14 | "", 15 | # mixed 16 | "-", 17 | "-", 18 | ], 19 | # TODO: support malformed configurations below 20 | # all char keys 21 | "a-b-c": [ 22 | # standard 23 | "a-b-c", 24 | # malformed 25 | # "abc", 26 | # mixed 27 | # "a-bc", 28 | # "ab-c", 29 | ], 30 | # mixed named and char 31 | "-t": [ 32 | # standard 33 | "-t", 34 | # malformed 35 | # "t", 36 | ], 37 | } 38 | 39 | for ( 40 | expected_output, 41 | input_variations, 42 | ) in input_variations_by_expected_output.items(): 43 | for input_variation in input_variations: 44 | action_dict = { 45 | "name": "type", 46 | "text": input_variation, 47 | "canonical_text": input_variation, 48 | } 49 | print(f"{input_variation=}") 50 | action_event = models.ActionEvent.from_dict(action_dict) 51 | assert action_event.text == expected_output, action_event 52 | -------------------------------------------------------------------------------- /tests/openadapt/test_summary.py: -------------------------------------------------------------------------------- 1 | """Tests the summarization function in summary.py.""" 2 | 3 | from fuzzywuzzy import fuzz 4 | 5 | from openadapt.models import Recording 6 | from openadapt.strategies.mixins.summary import SummaryReplayStrategyMixin 7 | 8 | RECORDING = Recording() 9 | 10 | 11 | class SummaryReplayStrategy(SummaryReplayStrategyMixin): 12 | """Custom Replay Strategy to solely test the Summary Mixin.""" 13 | 14 | def __init__(self, recording: Recording) -> None: 15 | """Initialize the SummaryReplayStrategy object. 16 | 17 | Args: 18 | recording (Recording): The recording object. 19 | """ 20 | super().__init__(recording) 21 | 22 | def get_next_action_event(self) -> None: 23 | """Get the next action event.""" 24 | pass 25 | 26 | 27 | REPLAY = SummaryReplayStrategy(RECORDING) 28 | 29 | 30 | def test_summary_empty() -> None: 31 | """Test that an empty text returns an empty summary.""" 32 | empty_text = "" 33 | actual = REPLAY.get_summary(empty_text, 1) 34 | assert len(actual) == 0 35 | 36 | 37 | def test_summary_sentence() -> None: 38 | """Test the summarization of a sentence.""" 39 | story = ( 40 | "However, this bottle was not marked 'poison,' so Alice ventured to taste it," 41 | " and finding it very nice," 42 | " (it had, in fact, a sort of mixed flavour of cherry-tart," 43 | " custard, pine-apple, roast turkey, toffee, and hot buttered toast,)" 44 | " she very soon finished it off." 45 | ) 46 | actual = REPLAY.get_summary(story, 1) 47 | assert fuzz.WRatio(actual, story) > 50 48 | -------------------------------------------------------------------------------- /tests/openadapt/test_utils.py: -------------------------------------------------------------------------------- 1 | """Test openadapt.utils.""" 2 | 3 | from unittest.mock import patch 4 | 5 | from openadapt import utils 6 | from openadapt.config import config 7 | 8 | 9 | def test_get_scale_ratios() -> None: 10 | """Tests utils.get_scale_ratios.""" 11 | # TODO: pass in action event 12 | width, height = utils.get_scale_ratios() 13 | 14 | assert isinstance( 15 | width, (int, float) 16 | ), f"Expected width to be int or float, got {type(width).__name__}" 17 | assert isinstance( 18 | height, (int, float) 19 | ), f"Expected height to be int or float, got {type(height).__name__}" 20 | 21 | 22 | def test_posthog_capture() -> None: 23 | """Tests utils.get_posthog_instance.""" 24 | with patch("posthog.Posthog.capture") as mock_capture: 25 | with patch("importlib.metadata.version") as mock_version: 26 | mock_version.return_value = "1.0.0" 27 | posthog = utils.get_posthog_instance() 28 | posthog.capture(event="test_event", properties={"test_prop": "test_val"}) 29 | mock_capture.assert_called_once_with( 30 | event="test_event", 31 | properties={ 32 | "test_prop": "test_val", 33 | "version": "1.0.0", 34 | "is_development": True, 35 | }, 36 | distinct_id=config.UNIQUE_USER_ID, 37 | ) 38 | -------------------------------------------------------------------------------- /tests/openadapt/test_video.py: -------------------------------------------------------------------------------- 1 | """Module to test openadapt.video.""" 2 | 3 | # from openadapt import models, video 4 | 5 | 6 | # TODO: compare diff shown in deprecated.visualize(diff_video=True) 7 | --------------------------------------------------------------------------------