├── .env.example ├── .eslintignore ├── .eslintrc ├── .github ├── CODEOWNERS ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md ├── auto_assign.yml ├── dependabot.yml ├── stale.yml └── workflows │ ├── auto-assign.yml │ ├── build-zip.yml │ ├── greetings.yml │ └── test.yml ├── .gitignore ├── .husky ├── commit-msg └── pre-commit ├── .npmrc ├── .nvmrc ├── .prettierignore ├── .prettierrc ├── CONTRIBUTING.md ├── CONTRIBUTING_KNOWLEDGE.md ├── LICENSE ├── PULL_REQUEST_TEMPLATE.md ├── README.md ├── TROUBLESHOOTING.md ├── commitlint.config.js ├── jest.config.js ├── manifest.js ├── media └── web-annotation.png ├── package.json ├── package.lib.json ├── pnpm-lock.yaml ├── postcss.config.js ├── public └── icon-128.png ├── rollup.lib.config.js ├── src ├── assets │ ├── img │ │ └── logo.svg │ └── style │ │ └── theme.scss ├── common │ ├── App.tsx │ ├── AutosizeTextarea.tsx │ ├── CopyButton.tsx │ ├── CustomKnowledgeBase │ │ ├── DefaultKnowledge.tsx │ │ ├── DuplicateKnowledgeAlert.tsx │ │ ├── HostKnowledge.tsx │ │ ├── NewKnowledgeForm.tsx │ │ ├── NewKnowledgeJson.tsx │ │ ├── Notes.tsx │ │ └── index.tsx │ ├── RecommendedTasks.tsx │ ├── RunTaskButton.tsx │ ├── Settings.tsx │ ├── TaskHistory.tsx │ ├── TaskStatus.tsx │ ├── TaskUI.tsx │ ├── TokenCount.tsx │ ├── VoiceButton.tsx │ └── settings │ │ ├── AgentModeDropdown.tsx │ │ ├── ModelDropdown.tsx │ │ └── SetAPIKey.tsx ├── constants.ts ├── environment.d.ts ├── global.d.ts ├── helpers │ ├── aiSdkUtils.ts │ ├── browserUtils.ts │ ├── buildAnnotatedScreenshots.ts │ ├── chromeDebugger.ts │ ├── countTokens.ts │ ├── disableExtensions.ts │ ├── dom-agent │ │ ├── availableActions.ts │ │ ├── determineNextAction.ts │ │ └── parseResponse.ts │ ├── errorChecker.ts │ ├── index.ts │ ├── knowledge │ │ ├── db.json │ │ ├── index.ts │ │ └── redirects.json │ ├── rpc │ │ ├── domActions.ts │ │ ├── pageRPC.ts │ │ ├── performAction.ts │ │ ├── runtimeFunctionStrings.ts │ │ └── utils.ts │ ├── shrinkHTML │ │ ├── tagsSelfClose.ts │ │ ├── templatize.test.ts │ │ └── templatize.ts │ ├── simplifyDom.ts │ ├── utils.ts │ ├── vision-agent │ │ ├── determineNavigateAction.ts │ │ ├── determineNextAction.ts │ │ ├── parseResponse.ts │ │ └── tools.ts │ └── voiceControl.ts ├── pages │ ├── background │ │ └── index.ts │ ├── content │ │ ├── attachFile.ts │ │ ├── copyToClipboard.ts │ │ ├── domOperations.ts │ │ ├── drawLabels.ts │ │ ├── getAnnotatedDOM.ts │ │ ├── getViewportPercentage.ts │ │ ├── index.ts │ │ ├── injected.ts │ │ ├── mainWorld │ │ │ ├── index.ts │ │ │ └── mainWorld.ts │ │ ├── permission.ts │ │ ├── reverseMarkdown.ts │ │ ├── ripple.ts │ │ ├── style.global.scss │ │ └── style.scss │ ├── devtools │ │ ├── index.html │ │ └── index.ts │ ├── newtab │ │ ├── Newtab.css │ │ ├── Newtab.scss │ │ ├── Newtab.tsx │ │ ├── index.css │ │ ├── index.html │ │ └── index.tsx │ ├── options │ │ ├── Options.css │ │ ├── Options.tsx │ │ ├── index.css │ │ ├── index.html │ │ └── index.tsx │ ├── panel │ │ ├── Panel.css │ │ ├── Panel.tsx │ │ ├── index.css │ │ ├── index.html │ │ └── index.tsx │ ├── permission │ │ ├── index.html │ │ └── requestPermission.ts │ ├── popup │ │ ├── Popup.css │ │ ├── Popup.tsx │ │ ├── index.css │ │ ├── index.html │ │ └── index.tsx │ └── sidepanel │ │ ├── index.css │ │ ├── index.html │ │ └── index.tsx ├── shared │ ├── hoc │ │ ├── withErrorBoundary.tsx │ │ └── withSuspense.tsx │ ├── hooks │ │ └── useStorage.tsx │ ├── images │ │ └── mergeScreenshots.ts │ └── storages │ │ ├── base.ts │ │ └── exampleThemeStorage.ts ├── state │ ├── currentTask.ts │ ├── settings.ts │ ├── store.ts │ └── ui.ts └── vite-env.d.ts ├── tailwind.config.js ├── test-utils └── jest.setup.js ├── tsconfig.json ├── utils ├── log.ts ├── manifest-parser │ └── index.ts ├── plugins │ ├── add-hmr.ts │ ├── custom-dynamic-import.ts │ ├── inline-vite-preload-script.ts │ ├── make-manifest.ts │ └── watch-rebuild.ts └── reload │ ├── constant.ts │ ├── initReloadClient.ts │ ├── initReloadServer.ts │ ├── injections │ ├── script.ts │ └── view.ts │ ├── interpreter │ ├── index.ts │ └── types.ts │ ├── rollup.config.mjs │ └── utils.ts └── vite.config.ts /.env.example: -------------------------------------------------------------------------------- 1 | VITE_DEBUG_MODE=true 2 | -------------------------------------------------------------------------------- /.eslintignore: -------------------------------------------------------------------------------- 1 | dist 2 | node_modules 3 | -------------------------------------------------------------------------------- /.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "browser": true, 4 | "es6": true, 5 | "node": true 6 | }, 7 | "extends": [ 8 | "eslint:recommended", 9 | "plugin:react/recommended", 10 | "plugin:@typescript-eslint/recommended", 11 | "plugin:react-hooks/recommended", 12 | "plugin:import/recommended", 13 | "plugin:jsx-a11y/recommended", 14 | "prettier" 15 | ], 16 | "parser": "@typescript-eslint/parser", 17 | "parserOptions": { 18 | "ecmaFeatures": { 19 | "jsx": true 20 | }, 21 | "ecmaVersion": "latest", 22 | "sourceType": "module" 23 | }, 24 | "plugins": [ 25 | "react", 26 | "@typescript-eslint", 27 | "react-hooks", 28 | "import", 29 | "jsx-a11y", 30 | "prettier" 31 | ], 32 | "settings": { 33 | "react": { 34 | "version": "detect" 35 | } 36 | }, 37 | "rules": { 38 | "prefer-promise-reject-errors": "error", 39 | "no-throw-literal": "error", 40 | "react/react-in-jsx-scope": "off", 41 | "import/no-unresolved": "off" 42 | }, 43 | "globals": { 44 | "chrome": "readonly" 45 | }, 46 | "ignorePatterns": ["watch.js", "dist/**"] 47 | } 48 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @mondaychen -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: bug 6 | assignees: Jonghakseo 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Desktop (please complete the following information):** 27 | - OS: [e.g. Mac, Window, Linux] 28 | - Browser [e.g. chrome, firefox] 29 | - Node Version [e.g. 18.12.0] 30 | 31 | **Additional context** 32 | Add any other context about the problem here. 33 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: enhancement 6 | assignees: Jonghakseo 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.github/auto_assign.yml: -------------------------------------------------------------------------------- 1 | # Set to true to add reviewers to pull requests 2 | addReviewers: true 3 | 4 | # Set to true to add assignees to pull requests 5 | addAssignees: author 6 | 7 | # A list of reviewers to be added to pull requests (GitHub user name) 8 | reviewers: 9 | - mondaychen 10 | 11 | # A number of reviewers added to the pull request 12 | # Set 0 to add all the reviewers (default: 0) 13 | numberOfReviewers: 0 14 | 15 | # A list of assignees, overrides reviewers if set 16 | # assignees: 17 | # - assigneeA 18 | 19 | # A number of assignees to add to the pull request 20 | # Set to 0 to add all of the assignees. 21 | # Uses numberOfReviewers if unset. 22 | # numberOfAssignees: 2 23 | 24 | # A list of keywords to be skipped the process that add reviewers if pull requests include it 25 | # skipKeywords: 26 | # - wip 27 | 28 | filterLabels: 29 | exclude: 30 | - dependencies 31 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "npm" # See documentation for possible values 9 | directory: "/" # Location of package manifests 10 | schedule: 11 | interval: "weekly" 12 | -------------------------------------------------------------------------------- /.github/stale.yml: -------------------------------------------------------------------------------- 1 | # Number of days of inactivity before an Issue or Pull Request becomes stale 2 | daysUntilStale: 90 3 | # Number of days of inactivity before a stale Issue or Pull Request is closed 4 | daysUntilClose: 30 5 | # Issues or Pull Requests with these labels will never be considered stale. Set to `[]` to disable 6 | exemptLabels: 7 | - pinned 8 | - security 9 | # Label to use when marking as stale 10 | staleLabel: stale 11 | # Comment to post when marking as stale. Set to `false` to disable 12 | markComment: > 13 | This issue has been automatically marked as stale because it has not had 14 | recent activity. It will be closed if no further activity occurs. Thank you 15 | for your contributions. 16 | # Comment to post when removing the stale label. Set to `false` to disable 17 | unmarkComment: false 18 | # Comment to post when closing a stale Issue or Pull Request. Set to `false` to disable 19 | closeComment: ture 20 | # Limit to only `issues` or `pulls` 21 | only: issues 22 | -------------------------------------------------------------------------------- /.github/workflows/auto-assign.yml: -------------------------------------------------------------------------------- 1 | name: 'Auto Assign' 2 | on: 3 | pull_request: 4 | types: [opened, ready_for_review] 5 | 6 | jobs: 7 | add-reviews: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: kentaro-m/auto-assign-action@v1.2.5 11 | with: 12 | configuration-path: '.github/auto_assign.yml' 13 | -------------------------------------------------------------------------------- /.github/workflows/build-zip.yml: -------------------------------------------------------------------------------- 1 | name: Build And Upload Extension Zip Via Artifact 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | 8 | jobs: 9 | build: 10 | 11 | runs-on: ubuntu-latest 12 | 13 | steps: 14 | - uses: actions/checkout@v3 15 | 16 | - name: Setup Node.js 17 | uses: actions/setup-node@v3 18 | with: 19 | node-version-file: ".nvmrc" 20 | 21 | - uses: actions/cache@v3 22 | with: 23 | path: node_modules 24 | key: ${{ runner.OS }}-build-${{ hashFiles('**/package-lock.json') }} 25 | 26 | - uses: pnpm/action-setup@v2 27 | 28 | - run: pnpm install --frozen-lockfile 29 | 30 | - run: pnpm build 31 | 32 | - uses: actions/upload-artifact@v3 33 | with: 34 | name: fuji-extension 35 | path: dist/* 36 | -------------------------------------------------------------------------------- /.github/workflows/greetings.yml: -------------------------------------------------------------------------------- 1 | name: Greetings 2 | 3 | on: [pull_request_target, issues] 4 | 5 | jobs: 6 | greeting: 7 | runs-on: ubuntu-latest 8 | permissions: 9 | issues: write 10 | pull-requests: write 11 | steps: 12 | - uses: actions/first-interaction@v1 13 | with: 14 | repo-token: ${{ secrets.GITHUB_TOKEN }} 15 | issue-message: 'Thank you for your contribution. We will check and reply to you as soon as possible.' 16 | pr-message: 'Thank you for your contribution. We will check and reply to you as soon as possible.' 17 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | 8 | jobs: 9 | test: 10 | runs-on: ubuntu-latest 11 | 12 | steps: 13 | - uses: actions/checkout@v3 14 | 15 | - name: Setup Node.js 16 | uses: actions/setup-node@v3 17 | with: 18 | node-version-file: ".nvmrc" 19 | 20 | - uses: actions/cache@v3 21 | with: 22 | path: node_modules 23 | key: ${{ runner.OS }}-build-${{ hashFiles('**/package-lock.json') }} 24 | 25 | - uses: pnpm/action-setup@v2 26 | 27 | - run: pnpm install --frozen-lockfile 28 | 29 | - run: pnpm test 30 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # dependencies 2 | /node_modules 3 | 4 | # testing 5 | /coverage 6 | 7 | # build 8 | /dist 9 | /dist-lib 10 | 11 | # etc 12 | .DS_Store 13 | .env.local 14 | .env 15 | .idea 16 | 17 | # compiled 18 | utils/reload/*.js 19 | utils/reload/injections/*.js 20 | public/manifest.json 21 | -------------------------------------------------------------------------------- /.husky/commit-msg: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | . "$(dirname -- "$0")/_/husky.sh" 3 | 4 | npm run commitlint ${1} 5 | -------------------------------------------------------------------------------- /.husky/pre-commit: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | . "$(dirname -- "$0")/_/husky.sh" 3 | 4 | npx lint-staged 5 | -------------------------------------------------------------------------------- /.npmrc: -------------------------------------------------------------------------------- 1 | public-hoist-pattern[]=@testing-library/dom 2 | -------------------------------------------------------------------------------- /.nvmrc: -------------------------------------------------------------------------------- 1 | 22.11.0 2 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | dist 2 | node_modules 3 | .gitignore 4 | .github 5 | .eslintignore 6 | .husky 7 | .nvmrc 8 | .prettierignore 9 | LICENSE 10 | *.md 11 | pnpm-lock.yaml -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "semi": true, 3 | "printWidth": 80 4 | } 5 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to Fuji-Web 2 | 3 | Thank you for your interest in contributing to Fuji-Web! Fuji-Web is a tool that simplifies web interactions through the innovative use of multi-modal Large Language Models, offering users a more intuitive and efficient online experience. We welcome contributions from the community to help make Fuji-Web even better. 4 | 5 | ## How to Contribute 6 | 7 | There are many ways to contribute to Fuji-Web, from writing code to improving documentation, reporting bugs, and suggesting enhancements. Here's how you can get started: 8 | 9 | ### Spread the Word 10 | If you love Fuji-Web, you can make a big difference by telling others about it. Write a blog post, talk about it on social media, or share your experience with friends and colleagues. Every bit helps in growing our community and bringing new contributors on board. 11 | 12 | ### Reporting Bugs 13 | 14 | Before reporting a new bug, please ensure that the issue has not already been reported. You can do this by searching through the existing issues in our GitHub repository. 15 | 16 | If you encounter a bug while using Fuji-Web and it has not been reported yet, please report it by creating a new issue. Be sure to include: 17 | 18 | - A clear and descriptive title 19 | - A detailed description of the bug, including steps to reproduce it 20 | - Any relevant screenshots or error messages 21 | - Your Fuji-Web version and browser details 22 | 23 | ### Suggesting Enhancements 24 | 25 | We're always looking for ways to improve Fuji-Web. If you have an idea for a new feature or an enhancement to an existing one, please submit it as an issue, using a clear and concise title and description. Explain why this enhancement would be useful, and if possible, include examples of how it could be implemented. 26 | 27 | ### Contributing Code 28 | 29 | Confirm alignment on the proposed work. For small fixes or minor enhancements, make sure there is an open and accepted issue. For larger contributions, a design or plan should have been reviewed and agreed upon by the maintainers. 30 | 31 | Before submitting your first code contribution, please make sure to: 32 | 33 | 1. Clone the repository. 34 | 2. Follow the setup instructions in the README.md to get your development environment running. 35 | 4. Make your changes in a new git branch and test your changes locally. 36 | 5. Commit your changes using a clear and descriptive commit message. 37 | 6. Push your branch to GitHub and open a pull request against the `main` branch. In your pull request, include any relevant issue numbers and a description of the changes you've made. 38 | 39 | ### Pull Request Guidelines 40 | 41 | - Ensure that your code follows the project's coding conventions and is properly documented. 42 | - Include screenshots or animated GIFs in your pull request whenever possible, especially for UI-related changes. 43 | - Follow the [Pull Request Template](https://github.com/normal-computing/fuji-web/PULL_REQUEST_TEMPLATE.md) provided in the repository for the description of your pull request. 44 | 45 | ### Code Review Process 46 | 47 | After you submit a pull request, the project maintainers will review your proposed changes. This process helps to ensure the quality and consistency of the Fuji-Web codebase. The review may require some back-and-forth communication, so please be patient. We appreciate your contributions and will do our best to provide feedback and guidance as quickly as possible. 48 | 49 | ## Community and Conduct 50 | 51 | We are committed to providing a welcoming and inspiring community for all. We encourage all contributors to foster an open and welcoming environment, and to be respectful of differing viewpoints and experiences. 52 | 53 | ## Acknowledgements 54 | 55 | Your contributions help make Fuji-Web a better tool for everyone. We look forward to your ideas, feedback, and contributions. Thank you for being part of the Fuji-Web community! Happy contributing! 56 | -------------------------------------------------------------------------------- /CONTRIBUTING_KNOWLEDGE.md: -------------------------------------------------------------------------------- 1 | # Contributing to Prior Knowledge Augmentation 2 | 3 | Fuji-Web's Prior Knowledge Augmentation system is designed to enhance the tool's web navigation and task execution capabilities by leveraging a shared knowledge base. Contributions to this system help make Fuji-Web smarter and more capable. 4 | 5 | ## What Kind of Knowledge Are We Looking For? 6 | 7 | We seek knowledge that: 8 | - Enhances the understanding of specific web pages or actions, making task execution more reliable. 9 | - Includes insights into website layouts, common patterns, and user interfaces that are not immediately obvious. 10 | - Provides rules or annotations that help the AI better interpret the purpose of elements on a page. 11 | 12 | For example, if a website has two buttons with the same name but different functionalities, it's crucial to describe in notes how to distinguish between them. 13 | 14 | ## How to Add and Test New Knowledge 15 | 16 | We offer two convenient ways to add and test new knowledge in real-time: 17 | - Via Form: Within the Fuji-Web UI settings, navigate to the "Custom Knowledge Base" and select "Add Host Knowledge with Form" to input new knowledge using a user-friendly form. 18 | - Via JSON: If you prefer to work directly with JSON, choose "Add Host Knowledge with JSON" to enter your custom knowledge. 19 | 20 | You can test the new knowledge by running several tasks on the relevant web pages to ensure Fuji-Web behaves as expected. 21 | 22 | Once you've tested various knowledge inputs and are satisfied with the new knowledge's performance, you can then copy that knowledge into the db.json file. 23 | 24 | 1. Locate the `db.json` file in the `src/helpers/knowledge` directory of the Fuji-Web repository. 25 | 2. Add your knowledge in the JSON format, following the existing structure. `annotationRules` is optional. 26 | ```json 27 | { 28 | "example.com": { 29 | "rules": [ 30 | { 31 | "regexes": ["regular expression to match pathname (not host name)"], 32 | "knowledge": { 33 | "notes": ["Your insights or notes about this page or action"], 34 | "annotationRules": [ 35 | { 36 | "selector": "CSS selector", 37 | "allowInvisible": true, 38 | "allowCovered": true, 39 | "allowAriaHidden": true 40 | } 41 | ] 42 | } 43 | } 44 | ] 45 | } 46 | } 47 | ``` 48 | 3. Please ensure your contributions are clear and concise, with `regexes` and `selector` accurately defined. 49 | 50 | ## Submitting Your Contribution 51 | 52 | Please check out the [Contribution Guide](CONTRIBUTING.md). Share your testing process and results in your pull request to help reviewers understand the impact of your contribution. Specifically, describe how the new knowledge help Fuji-Web achieve something it previously cannot perform correctly. 53 | -------------------------------------------------------------------------------- /PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## Description 2 | 3 | Please include a summary of the changes you have made. Describe any new features, bug fixes, or improvements you have introduced. If your changes address an open issue, please include a reference to it (e.g., `Fixes #123`). 4 | 5 | ## Type of Change 6 | 7 | Please delete options that are not relevant. 8 | 9 | - [ ] New feature (non-breaking change which adds functionality) 10 | - [ ] Bug fix (non-breaking change which fixes an issue) 11 | - [ ] Documentation update 12 | - [ ] Code style update (formatting, renaming) 13 | - [ ] Refactoring (no functional changes, no API changes) 14 | - [ ] Build-related changes 15 | - [ ] Other (please describe): 16 | 17 | ## How Has This Been Tested? 18 | 19 | Please describe the tests that you ran to verify your changes. Provide instructions so we can reproduce the testing. Please also list any relevant details for your test configuration. 20 | 21 | - [ ] Test A 22 | - [ ] Test B 23 | 24 | ## Checklist: 25 | 26 | Before submitting your pull request, please review the following checklist: 27 | 28 | - [ ] I have proved my fix is effective or that my feature works. 29 | - [ ] I have performed a self-review of my own code. 30 | - [ ] I have commented on my code, particularly in hard-to-understand areas. 31 | - [ ] My changes generate no new warnings. 32 | 33 | ## Screenshots (if applicable) 34 | 35 | If your changes are visual and it helps to illustrate them, please include screenshots or GIFs here. 36 | 37 | ## Additional Context 38 | 39 | Provide any additional information about your pull request here. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Fuji-web Logo 2 | 3 | # Fuji-Web: AI based Full Browser Automation 🗻 4 | 5 | Fuji-Web is an intelligent AI partner that understands the user’s intent, navigates websites autonomously, and executes tasks on the user’s behalf while explaining each action step. 6 | 7 | ### Demo 8 | https://github.com/normal-computing/fuji-web/assets/1001890/88a2fa12-31d9-4856-be67-27dcf9f1e634 9 | 10 | ## How does it work? 11 | 12 | **Please read [our blog post](https://blog.normalcomputing.ai/posts/2024-05-22-introducing-fuji-web/fuji-web.html) for a demo video, benchmarks and deep-dive technical overview!** 13 | 14 | ## Installing and Running 15 | 16 | ### Download and Install the extension in your browser 17 | 18 | 1. Go to the [releases page](https://github.com/normal-computing/fuji-web/releases), find the latest version of the extension and download "fuji-extension.zip". 19 | 2. Unzip the file. 20 | 3. Load your extension on Chrome by doing the following: 21 | 1. Navigate to `chrome://extensions/` 22 | 2. Toggle `Developer mode` 23 | 3. Click on `Load unpacked extension` 24 | 4. Select the unzipped folder 25 | 26 | ### Use the extension 27 | 28 | *Please note that you may need to refresh the page for the extension to work.* 29 | 30 | 1. Find the Fuji icon in the top right corner of your browser and click on it to open the sidepanel. 31 | 2. Create or access an existing [OpenAI API Key](https://platform.openai.com/account/api-keys) or [Anthropic API key](https://console.anthropic.com/settings/keys) and paste it in the provided box. This key will be stored in your browser, and will not be uploaded to a third party. 32 | 3. Finally, navigate to a webpage you want Fuji-Web and type in the task you want it to perform. 33 | 34 | _Note: all prompts (text and image) are sent directly to the API of your selection. Fuji-Web does not attempt to collect any information from you._ 35 | 36 | ### Build the extension 37 | 38 | If you want to build the extension from source, follow these instructions: 39 | 40 | 1. Ensure you have [Node.js](https://nodejs.org/). The development was done on Node v20 but it should work with some lower versions. 41 | 2. Clone this repository 42 | 3. Install `pnpm` globally: `npm install -g pnpm` 43 | 4. Run `pnpm install` 44 | 5. Run `pnpm dev` to start the development server, or `pnpm build` to build the extension. 45 | 46 | When loading the extension, you will need to load the `dist` folder created by the build process. 47 | 48 | ## Roadmap 49 | 50 | - Expose API for easy integration with browser automation frameworks (e.g. Puppeteer, Playwright, Selenium) 51 | - Add support for more complex & cross-tab workflows 52 | - Add support for more browsing behaviors (select from dropdown, extract content from entire page etc.) 53 | - Add support for saving workflows 54 | - Add support for sharing workflows & instructions with others 55 | - Create wikipedia-like knowledge base where users can work together to create knowledge that can improve the Fuji-Web's performance 56 | 57 | ## Troubleshooting 58 | 59 | Check out our [Troubleshooting Guide](TROUBLESHOOTING.md) for help with common problems. 60 | 61 | ## Contributing 62 | 63 | Interested in contributing to Fuji-Web? We'd love your help! Check out our [Contribution Guide](CONTRIBUTING.md) for guidelines on how to contribute, report bugs, suggest enhancements, and more. 64 | 65 | We also have set up a dedicated channel for Fuji-Web feedback on Discord at https://discord.gg/yfMjZ8udb5. 66 | 67 | ## Credits 68 | 69 | - Fuji-Web's image annotation method was inspired by Microsoft's [UFO paper](https://arxiv.org/abs/2402.07939). 70 | - Fuji as a tool that lives in the browser sidepanel was inspired by [TaxyAI's browser extension](https://github.com/TaxyAI/browser-extension). We also used some of its UI code. 71 | - The Chrome extension set-up leveraged an awesome boilerplate project [Jonghakseo/chrome-extension-boilerplate-react-vite](https://github.com/Jonghakseo/chrome-extension-boilerplate-react-vite). 72 | - The Fuji logo is from [Toss Face](https://emojipedia.org/toss-face) Emoji design set. 73 | -------------------------------------------------------------------------------- /TROUBLESHOOTING.md: -------------------------------------------------------------------------------- 1 | # Troubleshooting Guide for Fuji-Web 2 | 3 | This guide aims to help you diagnose and resolve common problems you might encounter. If you're still facing difficulties after following these steps, please reach out to us through our [GitHub Issues](https://github.com/normal-computing/fuji-web/issues). 4 | 5 | 6 | ## Common Issues and Solutions 7 | 8 | ### Extension Not Loading 9 | 10 | **Symptom**: The Fuji-Web extension doesn't appear in your browser or won't load. 11 | 12 | **Solutions**: 13 | 1. Ensure your browser is compatible with Fuji-Web. Currently, Fuji-Web supports Chrome. 14 | 2. Verify that `Developer mode` is enabled in `chrome://extensions/` 15 | 3. (Only when you are building it from source) Make sure you've loaded the extension from the `dist` folder. 16 | 4. Restart your browser as this can resolve many loading issues. 17 | 18 | ### API Key Problems 19 | 20 | **Symptom**: Issues related to the OpenAI API key, such as authentication errors or features not working due to key issues. e.g., 404 The model `gpt-4-vision-preview` does not exist or you do not have access to it. 21 | 22 | **Solutions**: 23 | 1. Make sure you entered a valid OpenAI API key. Note that keys can expire, so verify if yours is still active. 24 | 2. Ensure that your OpenAI API key has the necessary permissions. Visit https://platform.openai.com/playground/chat to check your permissions. Lack of credits can also restrict access to certain models. 25 | 26 | ### Dom Actions Problems 27 | 28 | **Symptom**: Fuji-Web did not perform dom actions properly. 29 | 30 | **Solutions**: 31 | 1. Currently, Fuji-Web does not support running in the background. If you open a new tab or navigate away from the website that Fuji-Web is working on, some actions may fail. Please stay on the website where you execute Fuji-Web. 32 | 33 | ### Custom Knowledge Base Problems 34 | 35 | **Symptom**: After adding custom knowledge, Fuji-Web crashed or did not perform according to the new knowledge about the active tab. 36 | 37 | **Solutions**: 38 | 1. Currently, Fuji-Web only supports basic entry validation for custom knowledge. Make sure you entered the correct host name and correct regular expressions if using a custom URL Matching Pattern. 39 | 40 | ### Voice Mode Problems 41 | 42 | **Symptom**: Fuji-Web did not capture speech. 43 | 44 | **Solutions**: 45 | 1. Check if Fuji-Web has microphone access in the browser. When you turn on the voice mode in settings, the microphone access dialog should pop up in the browser; please select "allow". 46 | 2. If the dialog didn't pop up, right-click the Fuji-Web icon in the extensions group and select "View Web Permissions". Then select "Allow" for Microphone. 47 | 48 | 49 | ## Reporting New Issues 50 | 51 | If you encounter a problem not covered in this guide, please help us by reporting it. Provide as much detail as possible, including steps to reproduce the issue, browser version, and any error messages you receive. Your reports are invaluable in helping us improve Fuji-Web. -------------------------------------------------------------------------------- /commitlint.config.js: -------------------------------------------------------------------------------- 1 | export default { extends: ["@commitlint/config-conventional"] }; 2 | -------------------------------------------------------------------------------- /manifest.js: -------------------------------------------------------------------------------- 1 | import packageJson from "./package.json" with { type: "json" }; 2 | 3 | /** 4 | * After changing, please reload the extension at `chrome://extensions` 5 | * @type {chrome.runtime.ManifestV3} 6 | */ 7 | const manifest = { 8 | manifest_version: 3, 9 | name: "Fuji", 10 | version: packageJson.version, 11 | description: packageJson.description, 12 | permissions: [ 13 | "storage", 14 | "sidePanel", 15 | "tabs", 16 | "activeTab", 17 | "scripting", 18 | "clipboardWrite", 19 | "debugger", 20 | "management", 21 | ], 22 | host_permissions: [""], 23 | side_panel: { 24 | default_path: "src/pages/sidepanel/index.html", 25 | }, 26 | options_page: "src/pages/options/index.html", 27 | background: { 28 | service_worker: "src/pages/background/index.js", 29 | type: "module", 30 | }, 31 | action: { 32 | // default_popup: 'src/pages/popup/index.html', 33 | default_title: "Click to open side panel", 34 | default_icon: "icon-128.png", 35 | }, 36 | icons: { 37 | 128: "icon-128.png", 38 | }, 39 | content_scripts: [ 40 | { 41 | matches: ["http://*/*", "https://*/*", ""], 42 | js: ["src/pages/content/index.js"], 43 | css: ["assets/css/contentStyleGlobal.css"], 44 | run_at: "document_start", // load the js as soon as possible since it does not rely on the DOM 45 | }, 46 | ], 47 | // devtools_page: "src/pages/devtools/index.html", 48 | web_accessible_resources: [ 49 | { 50 | resources: [ 51 | "assets/js/*.js", 52 | "assets/css/*.css", 53 | "assets/fonts/*", 54 | "icon-128.png", 55 | "src/pages/permission/index.html", 56 | "src/pages/permission/requestPermissions.ts", 57 | ], 58 | matches: ["*://*/*"], 59 | }, 60 | ], 61 | }; 62 | 63 | export default manifest; 64 | -------------------------------------------------------------------------------- /media/web-annotation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/normal-computing/fuji-web/1aec509e4c437ca7764a5b4a56deaeba18691729/media/web-annotation.png -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "fuji-web", 3 | "version": "2.2.0", 4 | "description": "A tool that redefines web interaction, making complex online tasks as simple as uttering a single command.", 5 | "repository": { 6 | "type": "git", 7 | "url": "https://github.com/normal-computing/fuji-web" 8 | }, 9 | "scripts": { 10 | "build": "tsc --noEmit && vite build", 11 | "build:lib": "rollup --config rollup.lib.config.js && cp package.lib.json dist-lib/package.json", 12 | "build:firefox": "tsc --noEmit && cross-env __FIREFOX__=true vite build", 13 | "build:watch": "cross-env __DEV__=true vite build -w --mode development", 14 | "build:firefox:watch": "cross-env __DEV__=true __FIREFOX__=true vite build -w --mode development", 15 | "build:hmr": "rollup --config utils/reload/rollup.config.mjs", 16 | "wss": "node utils/reload/initReloadServer.js", 17 | "dev": "pnpm build:hmr && (run-p wss build:watch)", 18 | "dev:firefox": "pnpm build:hmr && (run-p wss build:firefox:watch)", 19 | "test": "exit 0", 20 | "commitlint": "commitlint --edit", 21 | "lint": "eslint src --ext .ts", 22 | "lint:fix": "pnpm lint --fix", 23 | "prettier": "prettier . --write", 24 | "prepare": "husky install" 25 | }, 26 | "type": "module", 27 | "dependencies": { 28 | "@anthropic-ai/sdk": "^0.19.1", 29 | "@chakra-ui/icons": "^2.1.1", 30 | "@chakra-ui/react": "^2.8.2", 31 | "@emotion/react": "^11.11.4", 32 | "@emotion/styled": "^11.11.5", 33 | "@google/generative-ai": "^0.19.0", 34 | "accname": "^1.1.0", 35 | "construct-style-sheets-polyfill": "3.1.0", 36 | "formik": "^2.4.5", 37 | "immer": "^10.0.3", 38 | "lodash": "^4.17.21", 39 | "openai": "^4.60.0", 40 | "react": "18.2.0", 41 | "react-dom": "18.2.0", 42 | "react-icons": "^5.3.0", 43 | "react-syntax-highlighter": "^15.5.0", 44 | "react-textarea-autosize": "^8.4.1", 45 | "react-use": "^17.4.0", 46 | "tailwindcss": "^3.4.4", 47 | "webextension-polyfill": "0.10.0", 48 | "zod": "^3.23.8", 49 | "zod-validation-error": "^3.3.1", 50 | "zustand": "^4.5.2" 51 | }, 52 | "devDependencies": { 53 | "@commitlint/cli": "19.5.0", 54 | "@commitlint/config-conventional": "18.1.0", 55 | "@jest/globals": "^29.7.0", 56 | "@rollup/plugin-typescript": "11.1.6", 57 | "@testing-library/react": "14.0.0", 58 | "@types/chrome": "0.0.251", 59 | "@types/dom-speech-recognition": "^0.0.4", 60 | "@types/jest": "29.5.7", 61 | "@types/lodash": "^4.17.7", 62 | "@types/node": "20.11.24", 63 | "@types/react": "18.2.37", 64 | "@types/react-dom": "18.2.22", 65 | "@types/ws": "8.5.10", 66 | "@typescript-eslint/eslint-plugin": "6.10.0", 67 | "@typescript-eslint/parser": "6.9.1", 68 | "@vitejs/plugin-react": "4.2.1", 69 | "autoprefixer": "^10.4.16", 70 | "chokidar": "3.6.0", 71 | "cross-env": "7.0.3", 72 | "eslint": "8.57.0", 73 | "eslint-config-airbnb-typescript": "17.1.0", 74 | "eslint-config-prettier": "9.1.0", 75 | "eslint-plugin-import": "2.29.1", 76 | "eslint-plugin-jsx-a11y": "6.8.0", 77 | "eslint-plugin-prettier": "5.1.3", 78 | "eslint-plugin-react": "7.35.0", 79 | "eslint-plugin-react-hooks": "4.6.0", 80 | "fs-extra": "11.1.1", 81 | "husky": "9.0.11", 82 | "jest": "29.7.0", 83 | "jest-environment-jsdom": "29.7.0", 84 | "lint-staged": "15.2.7", 85 | "npm-run-all": "4.1.5", 86 | "postcss": "^8.4.38", 87 | "prettier": "3.2.5", 88 | "rollup": "4.17.2", 89 | "rollup-plugin-dts": "^6.1.1", 90 | "rollup-plugin-esbuild": "^6.1.1", 91 | "sass": "1.72.0", 92 | "ts-jest": "29.2.5", 93 | "tslib": "2.6.2", 94 | "typescript": "5.5.3", 95 | "vite": "5.2.14", 96 | "ws": "8.18.0" 97 | }, 98 | "lint-staged": { 99 | "*.{js,jsx,ts,tsx}": [ 100 | "prettier --write", 101 | "eslint --fix" 102 | ] 103 | }, 104 | "packageManager": "pnpm@9.13.2" 105 | } 106 | -------------------------------------------------------------------------------- /package.lib.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "web-wand-lib", 3 | "version": "2.0.3", 4 | "description": "Helper library for Web Wand", 5 | "repository": { 6 | "type": "git", 7 | "url": "https://github.com/normal-computing/web-wand" 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /postcss.config.js: -------------------------------------------------------------------------------- 1 | export default { 2 | plugins: { 3 | tailwindcss: {}, 4 | autoprefixer: {}, 5 | }, 6 | }; 7 | -------------------------------------------------------------------------------- /public/icon-128.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/normal-computing/fuji-web/1aec509e4c437ca7764a5b4a56deaeba18691729/public/icon-128.png -------------------------------------------------------------------------------- /rollup.lib.config.js: -------------------------------------------------------------------------------- 1 | import dts from "rollup-plugin-dts"; 2 | import esbuild from "rollup-plugin-esbuild"; 3 | import { resolve, dirname } from "path"; 4 | import { fileURLToPath } from "url"; 5 | 6 | const __dirname = dirname(fileURLToPath(import.meta.url)); 7 | 8 | const OUT_DIR = "dist-lib"; 9 | const rootDir = resolve(__dirname); 10 | const srcDir = resolve(rootDir, "src"); 11 | const pagesDir = resolve(srcDir, "pages"); 12 | 13 | function createConfigPair(name, path) { 14 | return [ 15 | { 16 | input: path, 17 | plugins: [esbuild()], 18 | output: [ 19 | { 20 | file: `${OUT_DIR}/${name}.js`, 21 | format: "cjs", 22 | sourcemap: true, 23 | exports: "auto", 24 | }, 25 | ], 26 | }, 27 | { 28 | input: path, 29 | plugins: [dts()], 30 | output: { 31 | file: `${OUT_DIR}/${name}.d.ts`, 32 | format: "es", 33 | }, 34 | }, 35 | ]; 36 | } 37 | 38 | export default [ 39 | ...createConfigPair( 40 | "domOperations", 41 | resolve(pagesDir, "content", "domOperations.ts"), 42 | ), 43 | ...createConfigPair("helpers", resolve(srcDir, "helpers", "index.ts")), 44 | ]; 45 | -------------------------------------------------------------------------------- /src/assets/img/logo.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /src/assets/style/theme.scss: -------------------------------------------------------------------------------- 1 | .crx-class { 2 | color: pink; 3 | } 4 | -------------------------------------------------------------------------------- /src/common/App.tsx: -------------------------------------------------------------------------------- 1 | import { 2 | Link, 3 | Box, 4 | ChakraProvider, 5 | Heading, 6 | HStack, 7 | IconButton, 8 | Icon, 9 | } from "@chakra-ui/react"; 10 | import { SettingsIcon } from "@chakra-ui/icons"; 11 | import { FaDiscord, FaGithub } from "react-icons/fa6"; 12 | import { useState } from "react"; 13 | import { useAppState } from "../state/store"; 14 | import SetAPIKey from "./settings/SetAPIKey"; 15 | import TaskUI from "./TaskUI"; 16 | import Settings from "./Settings"; 17 | 18 | const App = () => { 19 | const hasAPIKey = useAppState( 20 | (state) => state.settings.anthropicKey || state.settings.openAIKey, 21 | ); 22 | const [inSettingsView, setInSettingsView] = useState(false); 23 | 24 | return ( 25 | 26 | 27 | 28 | 29 | Fuji 🗻 30 | 31 | {hasAPIKey && ( 32 | } 34 | onClick={() => setInSettingsView(true)} 35 | aria-label="open settings" 36 | /> 37 | )} 38 | 39 | {hasAPIKey ? ( 40 | inSettingsView ? ( 41 | 42 | ) : ( 43 | 44 | ) 45 | ) : ( 46 | 47 | )} 48 | 49 | 60 | 70 | 74 | About this project 75 | 76 | 77 | Leave Feedback 78 | 79 | 80 | GitHub 81 | 82 | 83 | Join Our Discord 84 | 85 | 86 | 87 | 88 | ); 89 | }; 90 | 91 | export default App; 92 | -------------------------------------------------------------------------------- /src/common/AutosizeTextarea.tsx: -------------------------------------------------------------------------------- 1 | import { Textarea, TextareaProps } from "@chakra-ui/react"; 2 | import ResizeTextarea from "react-textarea-autosize"; 3 | import React from "react"; 4 | 5 | const AutosizeTextarea = React.forwardRef( 6 | (props, ref) => { 7 | return ( 8 |