├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md └── workflows │ └── python-package.yml ├── .gitignore ├── Dockerfile ├── LICENSE.md ├── MANIFEST.in ├── README.md ├── chainforge ├── __init__.py ├── app.py ├── examples │ ├── animal-images.cforge │ ├── audit-bias.cforge │ ├── basic-comparison.cforge │ ├── basic-function-calls.cforge │ ├── basic-math.cforge │ ├── book-beginnings.cforge │ ├── chaining-prompts.cforge │ ├── chat-sycophancy.cforge │ ├── compare-prompts.cforge │ ├── comparing-formats.cforge │ ├── comparing-system-msg.cforge │ ├── custom_provider_cohere.py │ ├── mosquito-knowledge.cforge │ ├── prompt-injection-test.cforge │ ├── python-coding-eval.cforge │ ├── red-team-stereotypes.cforge │ ├── structured-outputs.cforge │ └── tweet-multi-eval.cforge ├── flask_app.py ├── oaievals │ ├── 2d_movement.cforge │ ├── README.md │ ├── adultery_state_laws.cforge │ ├── afrikaans-lexicon.cforge │ ├── aime_evaluation.cforge │ ├── algebra-word-problems.cforge │ ├── ambiguous-sentences.cforge │ ├── arithmetical_puzzles.cforge │ ├── asl-classifiers.cforge │ ├── atpl_exams.cforge │ ├── balance-chemical-equation.cforge │ ├── banking77.cforge │ ├── beam-analysis.cforge │ ├── belarusian-grammar.cforge │ ├── belarusian-lexicon.cforge │ ├── belarusian-proverbs.cforge │ ├── belarusian-russian-translation.cforge │ ├── belarusian-syllable-count.cforge │ ├── belarusian-synonyms.cforge │ ├── bitwise.cforge │ ├── body-movement.cforge │ ├── born-first.cforge │ ├── brazilian_laws.cforge │ ├── building_floorplan.cforge │ ├── bulgarian-lexicon.cforge │ ├── canto_wu_pronunciation.cforge │ ├── chess-piece-count.cforge │ ├── chess.cforge │ ├── chinese_tang_poetries.cforge │ ├── chinese_zodiac.cforge │ ├── color_theory_complementary.cforge │ ├── compare-countries-area.cforge │ ├── comprehensive-graph-reasoning.cforge │ ├── contents.json │ ├── count_intersections_polynomial.cforge │ ├── count_token_freq_dna.cforge │ ├── counterfactual-reasoning.cforge │ ├── countries.cforge │ ├── crepe.cforge │ ├── cricket_situations.cforge │ ├── crontab.cforge │ ├── cube-pack.cforge │ ├── date-booking.cforge │ ├── date-calculator.cforge │ ├── day-of-week-from-date.cforge │ ├── determinant.cforge │ ├── diagrammatic_logic.cforge │ ├── directions.cforge │ ├── dna-melting-calculation.cforge │ ├── dutch-lexicon.cforge │ ├── emoji-riddle.cforge │ ├── escher-sentences.cforge │ ├── european-date-format-challenge.cforge │ ├── fcc_amateur_extra.cforge │ ├── finance.cforge │ ├── financial-derivatives.cforge │ ├── find-letter.cforge │ ├── find-thirukkural.cforge │ ├── find_country_from_svg.cforge │ ├── finnish-rhyme.cforge │ ├── first-letters.cforge │ ├── food.cforge │ ├── formal-grammar-to-regex.cforge │ ├── french-lexicon.cforge │ ├── french-part-of-speech.cforge │ ├── geometry_puzzle.cforge │ ├── german-part-of-speech.cforge │ ├── gol.cforge │ ├── greek-vocabulary.cforge │ ├── guess-the-singer.cforge │ ├── heart-disease.cforge │ ├── hebrew-bible.cforge │ ├── hebrew-rhyme.cforge │ ├── hebrew-same-noun-gender.cforge │ ├── hindi_shuddha.cforge │ ├── hindi_words.cforge │ ├── historical-kana-orthography-reading.cforge │ ├── imperial_date_to_string.cforge │ ├── indonesian_numbers.cforge │ ├── infiniteloop-match.cforge │ ├── internal_representations.cforge │ ├── invert_word_wise.cforge │ ├── invoice_due_date_leap_day_adjustment.cforge │ ├── irony.cforge │ ├── islands.cforge │ ├── isosceles-right-triangle.cforge │ ├── italian-new-words.cforge │ ├── italian-rhyme.cforge │ ├── japanese-itpassport-exam01.cforge │ ├── japanese-national-medical-exam01.cforge │ ├── japanese-national-medical-exam02.cforge │ ├── japanese_driving_license.cforge │ ├── japanese_number_reading.cforge │ ├── japanese_populer_video_game_title_and_the_publisher.cforge │ ├── jee-math.cforge │ ├── job_listing_title_for_a_caregiver_in_japan.cforge │ ├── json_patch_object.cforge │ ├── kanji-idioms.cforge │ ├── korean-consonant-vowel-combination.cforge │ ├── korean_spelling.cforge │ ├── korean_yaminjeongeum.cforge │ ├── largest_country.cforge │ ├── last-word-nth.cforge │ ├── lat_long_identify.cforge │ ├── linear-equations.cforge │ ├── list_comparison_missing_name.cforge │ ├── logic-statements.cforge │ ├── logiqa.cforge │ ├── mandaliof-table.cforge │ ├── map-electronic-component-part-to-fact.cforge │ ├── mate-in-one.cforge │ ├── matrix-mult-rows.cforge │ ├── medmcqa.cforge │ ├── missing-operators.cforge │ ├── moral_exceptQA.cforge │ ├── multi-step-equations.cforge │ ├── multistep-word-problems.cforge │ ├── music-theory-chord-notes.cforge │ ├── music_theory_scale_modes.cforge │ ├── nepali-song-singer.cforge │ ├── newsology.cforge │ ├── next-val-series.cforge │ ├── norwegian-lexicon.cforge │ ├── number-pattern.cforge │ ├── number-reading.cforge │ ├── numbers_game.cforge │ ├── numeral-type-comparisons.cforge │ ├── ordered-history-events.cforge │ ├── partially_solved_crossword_clues.cforge │ ├── passing-balls.cforge │ ├── pattern_identification.cforge │ ├── ph_calculation.cforge │ ├── physics-interaction.cforge │ ├── points-on-line.cforge │ ├── poker_analysis.cforge │ ├── polish-lexicon.cforge │ ├── polish-syllable-count.cforge │ ├── portuguese-sarcasm.cforge │ ├── portuguese-syllable-count.cforge │ ├── probability_questions.cforge │ ├── pure_korean.cforge │ ├── python_list_comprehension.cforge │ ├── rare-and-loanwords-dutch-lexicon.cforge │ ├── regex-match.cforge │ ├── resistor-ohm-calculator.cforge │ ├── reverse-polish-notation.cforge │ ├── reverse-string.cforge │ ├── rhetorical-devices.cforge │ ├── rock-climbing.cforge │ ├── rot13.cforge │ ├── russian-english-homonym-context-resolution.cforge │ ├── russian-lexicon.cforge │ ├── russian-rhyme.cforge │ ├── russian_medical.cforge │ ├── seating_arrangements.cforge │ ├── shape-in-shape.cforge │ ├── shared-borders.cforge │ ├── shopping_discount_comparison.cforge │ ├── simple-knowledge-mongolian.cforge │ ├── simple_physics_engine.cforge │ ├── solve-for-variable.cforge │ ├── sort-numeric.cforge │ ├── south-african-bands.cforge │ ├── spanish_feminine_noun_masculine_article.cforge │ ├── split_chinese_characters.cforge │ ├── squares-gpt.cforge │ ├── stats-tests.cforge │ ├── svg_understanding.cforge │ ├── swap-words.cforge │ ├── swedish-spelling.cforge │ ├── swedish_sat.cforge │ ├── syllables_long_words.cforge │ ├── syntax-check.cforge │ ├── taxes.cforge │ ├── tempo_to_measure_count.cforge │ ├── test-comp-sci.cforge │ ├── test_japanese_radical.cforge │ ├── test_japanese_units.cforge │ ├── tetris.cforge │ ├── three-pt-mapping.cforge │ ├── track_objects.cforge │ ├── tracking-shuffled-objects.cforge │ ├── turkish_characters.cforge │ ├── unified-patch.cforge │ ├── unsolvable_questions.cforge │ ├── utility_price_parsing.cforge │ ├── vigenere.cforge │ ├── vintage_phone_keyboard_decode.cforge │ ├── which-is-heavier.cforge │ ├── wkt_understanding.cforge │ └── word_vector_over_reliance.cforge ├── providers │ ├── __init__.py │ └── protocol.py ├── react-server │ ├── .eslintrc.js │ ├── .gitignore │ ├── .nvmrc │ ├── .prettierignore │ ├── .prettierrc.json │ ├── README.md │ ├── craco.config.js │ ├── package-lock.json │ ├── package.json │ ├── public │ │ ├── favicon.ico │ │ ├── index.html │ │ ├── logo192.png │ │ ├── logo512.png │ │ ├── manifest.json │ │ └── robots.txt │ ├── src │ │ ├── AiPopover.tsx │ │ ├── AlertModal.tsx │ │ ├── App.css │ │ ├── App.tsx │ │ ├── AreYouSureModal.tsx │ │ ├── BaseNode.tsx │ │ ├── ChatHistoryView.tsx │ │ ├── CodeEvaluatorNode.tsx │ │ ├── ColorThemeProvider.tsx │ │ ├── CommentNode.tsx │ │ ├── EditableTable.tsx │ │ ├── EvalGen │ │ │ ├── EvalGenWizard.tsx │ │ │ ├── FeedbackStep.tsx │ │ │ ├── GradeResponsesStep.tsx │ │ │ ├── GradingView.tsx │ │ │ ├── PickCriteriaStep.tsx │ │ │ ├── ReportCardStep.tsx │ │ │ └── WelcomeStep.tsx │ │ ├── ExampleFlowsModal.tsx │ │ ├── FlowSidebar.tsx │ │ ├── GlobalSettingsModal.tsx │ │ ├── ImagePreviewModal.tsx │ │ ├── InspectFooter.tsx │ │ ├── InspectorNode.tsx │ │ ├── ItemsNode.tsx │ │ ├── JoinNode.tsx │ │ ├── LLMEvalNode.tsx │ │ ├── LLMItemButtonGroup.tsx │ │ ├── LLMListComponent.tsx │ │ ├── LLMListItem.tsx │ │ ├── LLMResponseInspector.tsx │ │ ├── LLMResponseInspectorDrawer.tsx │ │ ├── LLMResponseInspectorModal.tsx │ │ ├── MediaNode.tsx │ │ ├── ModelSettingSchemas.tsx │ │ ├── ModelSettingsModal.tsx │ │ ├── MultiEvalNode.tsx │ │ ├── NestedMenu.tsx │ │ ├── NodeLabelComponent.tsx │ │ ├── PlotLegend.tsx │ │ ├── PromptNode.tsx │ │ ├── RemoveEdge.tsx │ │ ├── RenameValueModal.tsx │ │ ├── RequestClarificationModal.tsx │ │ ├── ResponseBoxes.tsx │ │ ├── ResponseRatingToolbar.tsx │ │ ├── ScriptNode.tsx │ │ ├── SimpleEvalNode.tsx │ │ ├── SplitNode.tsx │ │ ├── StatusIndicatorComponent.tsx │ │ ├── StrictModeDroppable.js │ │ ├── TabularDataNode.tsx │ │ ├── TemplateHooksComponent.tsx │ │ ├── TextFieldsNode.tsx │ │ ├── UploadFileModal.tsx │ │ ├── VisNode.tsx │ │ ├── backend │ │ │ ├── __test__ │ │ │ │ ├── ai.test.ts │ │ │ │ ├── aiSuggestionsManager.test.ts │ │ │ │ ├── backend.test.ts │ │ │ │ ├── cache.test.ts │ │ │ │ ├── query.test.ts │ │ │ │ ├── setUtils.test.ts │ │ │ │ ├── template.test.ts │ │ │ │ └── utils.test.ts │ │ │ ├── ai.ts │ │ │ ├── aiSuggestionsManager.ts │ │ │ ├── backend.ts │ │ │ ├── cache.ts │ │ │ ├── canceler.ts │ │ │ ├── errors.ts │ │ │ ├── evalgen │ │ │ │ ├── README.md │ │ │ │ ├── executor.ts │ │ │ │ ├── oai_utils.ts │ │ │ │ ├── test.ts │ │ │ │ ├── typing.ts │ │ │ │ └── utils.ts │ │ │ ├── models.ts │ │ │ ├── pyodide │ │ │ │ ├── exec-py.js │ │ │ │ └── exec-py.worker.js │ │ │ ├── query.ts │ │ │ ├── setUtils.ts │ │ │ ├── tableUtils.ts │ │ │ ├── template.ts │ │ │ ├── typing.ts │ │ │ └── utils.ts │ │ ├── example_flows.tsx │ │ ├── index.css │ │ ├── index.js │ │ ├── logo.svg │ │ ├── reportWebVitals.js │ │ ├── setupTests.js │ │ ├── store.tsx │ │ └── styles.css │ └── tsconfig.json ├── requirements.txt └── security │ ├── __init__.py │ ├── password_utils.py │ └── secure_save.py └── setup.py /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Desktop (please complete the following information):** 27 | - OS: [e.g. iOS] 28 | - Browser [e.g. chrome, safari] 29 | - Version [e.g. 22] 30 | 31 | **Smartphone (please complete the following information):** 32 | - Device: [e.g. iPhone6] 33 | - OS: [e.g. iOS8.1] 34 | - Browser [e.g. stock browser, safari] 35 | - Version [e.g. 22] 36 | 37 | **Additional context** 38 | Add any other context about the problem here. 39 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.github/workflows/python-package.yml: -------------------------------------------------------------------------------- 1 | name: Validate Install 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | jobs: 10 | test: 11 | runs-on: ${{ matrix.os }} 12 | 13 | strategy: 14 | matrix: 15 | os: [ubuntu-latest, windows-latest, macos-latest, macos-13, macos-14] 16 | python-version: [3.11, 3.12, 3.13] 17 | fail-fast: false 18 | 19 | steps: 20 | - name: Check out the repository 21 | uses: actions/checkout@v3 22 | 23 | - name: Set up Python ${{ matrix.python-version }} 24 | uses: actions/setup-python@v4 25 | with: 26 | python-version: ${{ matrix.python-version }} 27 | 28 | - name: Cache pip 29 | uses: actions/cache@v4 30 | with: 31 | path: ~/.cache/pip 32 | key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('chainforge/requirements.txt') }} 33 | restore-keys: | 34 | ${{ runner.os }}-pip-${{ matrix.python-version }}- 35 | 36 | - name: Upgrade pip 37 | run: python -m pip install --upgrade pip 38 | 39 | - name: Install dependencies 40 | run: | 41 | pip install -r chainforge/requirements.txt 42 | pip install . 43 | pip install pytest 44 | 45 | # TODO: Check that the server runs 46 | 47 | # TODO: Add pytest 48 | # - name: Run tests 49 | # run: pytest 50 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.DS_Store 2 | chainforge/cache 3 | chainforge/examples/oaievals/ 4 | chainforge/react-server/node_modules 5 | chainforge/react-server/build 6 | 7 | # == Below was generated by https://www.toptal.com/developers/gitignore/api/python == 8 | # Edit at https://www.toptal.com/developers/gitignore?templates=python 9 | 10 | ### Python ### 11 | # Byte-compiled / optimized / DLL files 12 | __pycache__/ 13 | *.py[cod] 14 | *$py.class 15 | 16 | # Docker 17 | packages/ 18 | jobs/ 19 | 20 | # C extensions 21 | *.so 22 | 23 | # Distribution / packaging 24 | .Python 25 | develop-eggs/ 26 | dist/ 27 | downloads/ 28 | eggs/ 29 | .eggs/ 30 | lib/ 31 | lib64/ 32 | parts/ 33 | sdist/ 34 | var/ 35 | wheels/ 36 | share/python-wheels/ 37 | *.egg-info/ 38 | .installed.cfg 39 | *.egg 40 | MANIFEST 41 | 42 | # PyInstaller 43 | # Usually these files are written by a python script from a template 44 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 45 | *.manifest 46 | *.spec 47 | 48 | # Installer logs 49 | pip-log.txt 50 | pip-delete-this-directory.txt 51 | 52 | # Unit test / coverage reports 53 | htmlcov/ 54 | .tox/ 55 | .nox/ 56 | .coverage 57 | .coverage.* 58 | .cache 59 | nosetests.xml 60 | coverage.xml 61 | *.cover 62 | *.py,cover 63 | .hypothesis/ 64 | .pytest_cache/ 65 | cover/ 66 | 67 | # Translations 68 | *.mo 69 | *.pot 70 | 71 | # Django stuff: 72 | *.log 73 | local_settings.py 74 | db.sqlite3 75 | db.sqlite3-journal 76 | 77 | # Flask stuff: 78 | instance/ 79 | .webassets-cache 80 | 81 | # Scrapy stuff: 82 | .scrapy 83 | 84 | # Sphinx documentation 85 | docs/_build/ 86 | 87 | # PyBuilder 88 | .pybuilder/ 89 | target/ 90 | 91 | # Jupyter Notebook 92 | .ipynb_checkpoints 93 | 94 | # IPython 95 | profile_default/ 96 | ipython_config.py 97 | 98 | # pyenv 99 | # For a library or package, you might want to ignore these files since the code is 100 | # intended to run in multiple environments; otherwise, check them in: 101 | # .python-version 102 | 103 | # pipenv 104 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 105 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 106 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 107 | # install all needed dependencies. 108 | #Pipfile.lock 109 | 110 | # poetry 111 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 112 | # This is especially recommended for binary packages to ensure reproducibility, and is more 113 | # commonly ignored for libraries. 114 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 115 | #poetry.lock 116 | 117 | # pdm 118 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 119 | #pdm.lock 120 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 121 | # in version control. 122 | # https://pdm.fming.dev/#use-with-ide 123 | .pdm.toml 124 | 125 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 126 | __pypackages__/ 127 | 128 | # Celery stuff 129 | celerybeat-schedule 130 | celerybeat.pid 131 | 132 | # SageMath parsed files 133 | *.sage.py 134 | 135 | # Environments 136 | .env 137 | .venv 138 | env/ 139 | venv/ 140 | ENV/ 141 | env.bak/ 142 | venv.bak/ 143 | 144 | # Spyder project settings 145 | .spyderproject 146 | .spyproject 147 | 148 | # Rope project settings 149 | .ropeproject 150 | 151 | # mkdocs documentation 152 | /site 153 | 154 | # mypy 155 | .mypy_cache/ 156 | .dmypy.json 157 | dmypy.json 158 | 159 | # Pyre type checker 160 | .pyre/ 161 | 162 | # pytype static type analyzer 163 | .pytype/ 164 | 165 | # Cython debug symbols 166 | cython_debug/ 167 | 168 | # PyCharm 169 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 170 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 171 | # and can be added to the global gitignore or merged into this file. For a more nuclear 172 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 173 | #.idea/ 174 | 175 | ### Python Patch ### 176 | # Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration 177 | poetry.toml 178 | 179 | # ruff 180 | .ruff_cache/ 181 | 182 | # LSP config files 183 | pyrightconfig.json 184 | 185 | # End of https://www.toptal.com/developers/gitignore/api/python 186 | chainforge_assets/ 187 | .vscode/ 188 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.12-slim AS builder 2 | 3 | RUN pip install --upgrade pip 4 | RUN pip install chainforge --no-cache-dir 5 | 6 | WORKDIR /chainforge 7 | 8 | EXPOSE 8000 9 | ENTRYPOINT [ "chainforge", "serve", "--host", "0.0.0.0" ] 10 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Ian Arawjo 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | graft chainforge/react-server/build 2 | graft chainforge/examples 3 | include README.md -------------------------------------------------------------------------------- /chainforge/__init__.py: -------------------------------------------------------------------------------- 1 | from .app import main 2 | -------------------------------------------------------------------------------- /chainforge/app.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from chainforge.flask_app import run_server 3 | import textwrap 4 | 5 | # Main Chainforge start 6 | def main(): 7 | parser = argparse.ArgumentParser(description='Chainforge command line tool') 8 | 9 | # Serve command 10 | subparsers = parser.add_subparsers(dest='serve') 11 | serve_parser = subparsers.add_parser('serve', help='Start Chainforge server', formatter_class=argparse.RawTextHelpFormatter) 12 | 13 | # TODO: Add this back 14 | # Turn on to disable all outbound LLM API calls and replace them with dummy calls 15 | # that return random strings of ASCII characters. Useful for testing interface without wasting $$. 16 | # serve_parser.add_argument('--dummy-responses', 17 | # help="""Disables queries to LLMs, replacing them with spoofed responses composed of random ASCII characters. 18 | # Produces each dummy response at random intervals between 0.1 and 3 seconds.""", 19 | # dest='dummy_responses', 20 | # action='store_true') 21 | 22 | # TODO: Reimplement this where the React server is given the backend's port before loading. 23 | serve_parser.add_argument('--port', 24 | help='The port to run the server on. Defaults to 8000.', 25 | type=int, default=8000, nargs='?') 26 | serve_parser.add_argument('--host', 27 | help="The host to run the server on. Defaults to 'localhost'.", 28 | type=str, default="localhost", nargs='?') 29 | serve_parser.add_argument('--dir', 30 | help=textwrap.dedent("""\ 31 | Set a custom directory to use for saving flows and autosaving. 32 | By default, ChainForge uses the user data location suggested by the `platformdirs` module. 33 | Should be an absolute path."""), 34 | type=str, 35 | default=None) 36 | serve_parser.add_argument('--secure', 37 | choices=["off", "settings", "all"], 38 | default="off", 39 | help=textwrap.dedent("""\ 40 | Encrypts locally stored files with a password. 41 | Encryption modes are: 42 | - off = no encryption (default) 43 | - settings = only encrypt the settings file (that may contain API keys entered via the UI) 44 | - all = encrypt all files (flows, settings, favorites, etc) 45 | You must provide a password at every startup. 46 | Ensure that you save your password somewhere, as it is not stored anywhere. 47 | If you lose it, you will not be able to access your files. 48 | NOTE: Clicking the 'Export' button in the UI will still export a non-encrypted flow, in case you need to share the file in the normal manner. 49 | This setting is only for local storage.""") 50 | ) 51 | 52 | args = parser.parse_args() 53 | 54 | # Currently only support the 'serve' command... 55 | if not args.serve: 56 | parser.print_help() 57 | exit(0) 58 | 59 | port = args.port if args.port else 8000 60 | host = args.host if args.host else "localhost" 61 | 62 | if args.dir: 63 | print(f"Using directory for storing flows: {args.dir}") 64 | 65 | print(f"Serving Flask server on {host} on port {port}...") 66 | run_server(host=host, port=port, flows_dir=args.dir, secure=args.secure) 67 | 68 | if __name__ == "__main__": 69 | main() -------------------------------------------------------------------------------- /chainforge/examples/custom_provider_cohere.py: -------------------------------------------------------------------------------- 1 | """ 2 | A simple custom model provider to add to the ChainForge interface, 3 | to support Cohere AI text completions through their Python API. 4 | 5 | NOTE: You must have the `cohere` package installed and an API key. 6 | """ 7 | from chainforge.providers import provider 8 | import cohere 9 | 10 | # Init the Cohere client (replace with your Cohere API Key) 11 | co = cohere.Client('') 12 | 13 | # JSON schemas to pass react-jsonschema-form, one for this endpoints' settings and one to describe the settings UI. 14 | COHERE_SETTINGS_SCHEMA = { 15 | "settings": { 16 | "temperature": { 17 | "type": "number", 18 | "title": "temperature", 19 | "description": "Controls the 'creativity' or randomness of the response.", 20 | "default": 0.75, 21 | "minimum": 0, 22 | "maximum": 5.0, 23 | "multipleOf": 0.01, 24 | }, 25 | "max_tokens": { 26 | "type": "integer", 27 | "title": "max_tokens", 28 | "description": "Maximum number of tokens to generate in the response.", 29 | "default": 100, 30 | "minimum": 1, 31 | "maximum": 1024, 32 | }, 33 | }, 34 | "ui": { 35 | "temperature": { 36 | "ui:help": "Defaults to 1.0.", 37 | "ui:widget": "range" 38 | }, 39 | "max_tokens": { 40 | "ui:help": "Defaults to 100.", 41 | "ui:widget": "range" 42 | }, 43 | } 44 | } 45 | 46 | # Our custom model provider for Cohere's text generation API. 47 | @provider(name="Cohere", 48 | emoji="🖇", 49 | models=['command', 'command-nightly', 'command-light', 'command-light-nightly'], 50 | rate_limit="sequential", # enter "sequential" for blocking; an integer N > 0 means N is the max mumber of requests per minute. 51 | settings_schema=COHERE_SETTINGS_SCHEMA) 52 | def CohereCompletion(prompt: str, model: str, temperature: float = 0.75, **kwargs) -> str: 53 | print(f"Calling Cohere model {model} with prompt '{prompt}'...") 54 | response = co.generate(model=model, prompt=prompt, temperature=temperature, **kwargs) 55 | return response.generations[0].text 56 | -------------------------------------------------------------------------------- /chainforge/oaievals/README.md: -------------------------------------------------------------------------------- 1 | # Preconverted OpenAI evals 2 | 3 | The ChainForge flows in this directory were derived from a subset of OpenAI's evals registry: https://github.com/openai/evals 4 | These files are _not_ included in the PyPI chainforge package, but rather fetched from GitHub on an as-needed basis. 5 | This is to avoid requiring users to install OpenAI evals (which requires Git LFS, Python 3.9+, and a large number of dependencies). 6 | 7 | OpenAI evals is under the MIT License: 8 | 9 | MIT License 10 | 11 | Copyright (c) 2023 OpenAI 12 | 13 | Permission is hereby granted, free of charge, to any person obtaining a copy 14 | of this software and associated documentation files (the "Software"), to deal 15 | in the Software without restriction, including without limitation the rights 16 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 17 | copies of the Software, and to permit persons to whom the Software is 18 | furnished to do so, subject to the following conditions: 19 | 20 | The above copyright notice and this permission notice shall be included in all 21 | copies or substantial portions of the Software. 22 | 23 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 24 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 25 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 26 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 27 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 28 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 29 | SOFTWARE. -------------------------------------------------------------------------------- /chainforge/oaievals/adultery_state_laws.cforge: -------------------------------------------------------------------------------- 1 | {"flow": {"nodes": [{"width": 312, "height": 311, "id": "prompt-adultery_state_laws", "type": "prompt", "data": {"prompt": "{prompt}", "n": 1, "llms": [{"key": "aa3c0f03-22bd-416e-af4d-4bf5c4278c99", "settings": {"system_msg": "You are an assistant with knowledge of U.S. state laws. Answer the questions accurately.", "temperature": 1, "functions": [], "function_call": "", "top_p": 1, "stop": [], "presence_penalty": 0, "frequency_penalty": 0}, "name": "GPT3.5", "emoji": "\ud83d\ude42", "model": "gpt-3.5-turbo", "base_model": "gpt-3.5-turbo", "temp": 1, "formData": {"shortname": "GPT3.5", "model": "gpt-3.5-turbo", "system_msg": "You are an assistant with knowledge of U.S. state laws. Answer the questions accurately.", "temperature": 1, "functions": "", "function_call": "", "top_p": 1, "stop": "", "presence_penalty": 0, "frequency_penalty": 0}}]}, "position": {"x": 448, "y": 224}, "selected": false, "positionAbsolute": {"x": 448, "y": 224}, "dragging": false}, {"width": 333, "height": 182, "id": "eval-adultery_state_laws", "type": "evaluator", "data": {"code": "function evaluate(response) {\n\tlet ideal = response.meta['Ideal'];\n\treturn response.text.startsWith(ideal);\n}", "language": "javascript"}, "position": {"x": 820, "y": 150}, "positionAbsolute": {"x": 820, "y": 150}}, {"width": 228, "height": 196, "id": "vis-adultery_state_laws", "type": "vis", "data": {"input": "eval-adultery_state_laws"}, "position": {"x": 1200, "y": 250}, "positionAbsolute": {"x": 1200, "y": 250}}, {"width": 302, "height": 260, "id": "inspect-adultery_state_laws", "type": "inspect", "data": {"input": "prompt-adultery_state_laws"}, "position": {"x": 820, "y": 400}, "positionAbsolute": {"x": 820, "y": 400}}, {"width": 423, "height": 417, "id": "table-adultery_state_laws", "type": "table", "data": {"rows": [{"prompt": "List the states where adultery is technically illegal. Only provide a list of states with no explanation.", "ideal": "Alabama, Arizona, Florida, Idaho, Illinois, Kansas, Michigan, Minnesota, Mississippi, New York, North Carolina, Oklahoma, Rhode Island, South Carolina, Virginia, Wisconsin, Georgia"}, {"prompt": "Is adultery illegal in Alabama?", "ideal": "Yes"}, {"prompt": "Is adultery illegal in Arizona?", "ideal": "Yes"}, {"prompt": "Is adultery illegal in Florida?", "ideal": "Yes"}, {"prompt": "Is adultery illegal in Georgia?", "ideal": "Yes"}, {"prompt": "Is adultery illegal in Idaho?", "ideal": "Yes"}, {"prompt": "Is adultery illegal in Illinois?", "ideal": "Yes"}, {"prompt": "Is adultery illegal in Kansas?", "ideal": "Yes"}, {"prompt": "Is adultery illegal in Michigan?", "ideal": "Yes"}, {"prompt": "Is adultery illegal in Minnesota?", "ideal": "Yes"}, {"prompt": "Is adultery illegal in Mississippi?", "ideal": "Yes"}, {"prompt": "Is adultery illegal in New York?", "ideal": "Yes"}, {"prompt": "Is adultery illegal in North Carolina?", "ideal": "Yes"}, {"prompt": "Is adultery illegal in Oklahoma?", "ideal": "Yes"}, {"prompt": "Is adultery illegal in Rhode Island?", "ideal": "Yes"}, {"prompt": "Is adultery illegal in South Carolina?", "ideal": "Yes"}, {"prompt": "Is adultery illegal in Virginia?", "ideal": "Yes"}, {"prompt": "Is adultery illegal in Wisconsin?", "ideal": "Yes"}, {"prompt": "Is adultery illegal in Alaska?", "ideal": "No"}, {"prompt": "List the states where adultery is illegal again. How was your list different now?", "ideal": "Alabama, Arizona, Florida, Idaho, Illinois, Kansas, Michigan, Minnesota, Mississippi, New York, North Carolina, Oklahoma, Rhode Island, South Carolina, Virginia, Wisconsin, Georgia. The list remains the same."}], "columns": [{"key": "prompt", "header": "Prompt"}, {"key": "ideal", "header": "Ideal"}]}, "position": {"x": -16, "y": 160}, "selected": false, "positionAbsolute": {"x": -16, "y": 160}, "dragging": false}], "edges": [{"source": "prompt-adultery_state_laws", "sourceHandle": "prompt", "target": "eval-adultery_state_laws", "targetHandle": "responseBatch", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-prompt-1686756357355prompt-eval-1686756357355responseBatch"}, {"source": "prompt-adultery_state_laws", "sourceHandle": "prompt", "target": "inspect-adultery_state_laws", "targetHandle": "input", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-prompt-1686756357355prompt-inspect-1686756357355input"}, {"source": "eval-adultery_state_laws", "sourceHandle": "output", "target": "vis-adultery_state_laws", "targetHandle": "input", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-eval-1686756357355output-vis-1686756357355input"}, {"source": "table-adultery_state_laws", "sourceHandle": "Prompt", "target": "prompt-adultery_state_laws", "targetHandle": "prompt", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-table-1686756385002Prompt-prompt-1686756357355prompt"}], "viewport": {"x": 144, "y": 37, "zoom": 1}}, "cache": {"eval-1686756357355.json": {}, "inspect-1686756357355.json": {}, "prompt-1686756357355.json": {}, "table-1686756385002.json": {}, "vis-1686756357355.json": {}}} -------------------------------------------------------------------------------- /chainforge/oaievals/algebra-word-problems.cforge: -------------------------------------------------------------------------------- 1 | {"flow": {"nodes": [{"width": 312, "height": 311, "id": "prompt-algebra-word-problems", "type": "prompt", "data": {"prompt": "{prompt}", "n": 1, "llms": [{"key": "aa3c0f03-22bd-416e-af4d-4bf5c4278c99", "settings": {"system_msg": "Answer the following question with a single number and no additional text. You are a helpful assistant.", "temperature": 1, "functions": [], "function_call": "", "top_p": 1, "stop": [], "presence_penalty": 0, "frequency_penalty": 0}, "name": "GPT3.5", "emoji": "\ud83d\ude42", "model": "gpt-3.5-turbo", "base_model": "gpt-3.5-turbo", "temp": 1, "formData": {"shortname": "GPT3.5", "model": "gpt-3.5-turbo", "system_msg": "Answer the following question with a single number and no additional text. You are a helpful assistant.", "temperature": 1, "functions": "", "function_call": "", "top_p": 1, "stop": "", "presence_penalty": 0, "frequency_penalty": 0}}]}, "position": {"x": 448, "y": 224}, "selected": false, "positionAbsolute": {"x": 448, "y": 224}, "dragging": false}, {"width": 333, "height": 182, "id": "eval-algebra-word-problems", "type": "evaluator", "data": {"code": "function evaluate(response) {\n\tlet ideal = response.meta['Ideal'];\n\treturn response.text.startsWith(ideal);\n}", "language": "javascript"}, "position": {"x": 820, "y": 150}, "positionAbsolute": {"x": 820, "y": 150}}, {"width": 228, "height": 196, "id": "vis-algebra-word-problems", "type": "vis", "data": {"input": "eval-algebra-word-problems"}, "position": {"x": 1200, "y": 250}, "positionAbsolute": {"x": 1200, "y": 250}}, {"width": 302, "height": 260, "id": "inspect-algebra-word-problems", "type": "inspect", "data": {"input": "prompt-algebra-word-problems"}, "position": {"x": 820, "y": 400}, "positionAbsolute": {"x": 820, "y": 400}}, {"width": 423, "height": 417, "id": "table-algebra-word-problems", "type": "table", "data": {"rows": [{"prompt": "If it takes 5 machines 5 minutes to make 5 devices, how long would it take 100 machines to make 100 devices?", "ideal": "5"}, {"prompt": "What is the sum of 60000, 5000, 400, and 3, with the third value multiplied by 5 before performing the operation?", "ideal": "67003"}, {"prompt": "If the sum of the smallest and largest of three consecutive even numbers is 28, what is the value of the second largest number in the series?", "ideal": "14"}, {"prompt": "John is trying to fill a 16 oz. bottle with water. If John fills the bottle at 1 oz per second and the bottle leaks .2 oz per second, how long would it take for John to fill the bottle?", "ideal": "20"}, {"prompt": "Annie is training for a marathon. She has a weekly training routine, training for five hours a day on some days and 3 hours a day on the other days. She trains a total of 27 hours in a seven day week. On how many days does she train for five hours?", "ideal": "3"}, {"prompt": "At the start of the year the ratio of boys to girls in a class is 2 : 1. But now, half a year later, four boys have left the class and there are two new girls. The ratio of boys to girls is now 4 : 3. How many students are there altogether now?", "ideal": "28"}], "columns": [{"key": "prompt", "header": "Prompt"}, {"key": "ideal", "header": "Ideal"}]}, "position": {"x": -16, "y": 160}, "selected": false, "positionAbsolute": {"x": -16, "y": 160}, "dragging": false}], "edges": [{"source": "prompt-algebra-word-problems", "sourceHandle": "prompt", "target": "eval-algebra-word-problems", "targetHandle": "responseBatch", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-prompt-1686756357355prompt-eval-1686756357355responseBatch"}, {"source": "prompt-algebra-word-problems", "sourceHandle": "prompt", "target": "inspect-algebra-word-problems", "targetHandle": "input", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-prompt-1686756357355prompt-inspect-1686756357355input"}, {"source": "eval-algebra-word-problems", "sourceHandle": "output", "target": "vis-algebra-word-problems", "targetHandle": "input", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-eval-1686756357355output-vis-1686756357355input"}, {"source": "table-algebra-word-problems", "sourceHandle": "Prompt", "target": "prompt-algebra-word-problems", "targetHandle": "prompt", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-table-1686756385002Prompt-prompt-1686756357355prompt"}], "viewport": {"x": 144, "y": 37, "zoom": 1}}, "cache": {"eval-1686756357355.json": {}, "inspect-1686756357355.json": {}, "prompt-1686756357355.json": {}, "table-1686756385002.json": {}, "vis-1686756357355.json": {}}} -------------------------------------------------------------------------------- /chainforge/oaievals/counterfactual-reasoning.cforge: -------------------------------------------------------------------------------- 1 | {"flow": {"nodes": [{"width": 312, "height": 311, "id": "prompt-counterfactual-reasoning", "type": "prompt", "data": {"prompt": "{prompt}", "n": 1, "llms": [{"key": "aa3c0f03-22bd-416e-af4d-4bf5c4278c99", "settings": {"system_msg": "You are a helpful assistant.", "temperature": 1, "functions": [], "function_call": "", "top_p": 1, "stop": [], "presence_penalty": 0, "frequency_penalty": 0}, "name": "GPT3.5", "emoji": "\ud83d\ude42", "model": "gpt-3.5-turbo", "base_model": "gpt-3.5-turbo", "temp": 1, "formData": {"shortname": "GPT3.5", "model": "gpt-3.5-turbo", "system_msg": "You are a helpful assistant.", "temperature": 1, "functions": "", "function_call": "", "top_p": 1, "stop": "", "presence_penalty": 0, "frequency_penalty": 0}}]}, "position": {"x": 448, "y": 224}, "selected": false, "positionAbsolute": {"x": 448, "y": 224}, "dragging": false}, {"width": 333, "height": 182, "id": "eval-counterfactual-reasoning", "type": "evaluator", "data": {"code": "function evaluate(response) {\n\tlet txt = response.text;\n\tlet ideal = response.meta['Ideal'];\n\treturn ideal.includes(txt) || txt.includes(ideal);\n}", "language": "javascript"}, "position": {"x": 820, "y": 150}, "positionAbsolute": {"x": 820, "y": 150}}, {"width": 228, "height": 196, "id": "vis-counterfactual-reasoning", "type": "vis", "data": {"input": "eval-counterfactual-reasoning"}, "position": {"x": 1200, "y": 250}, "positionAbsolute": {"x": 1200, "y": 250}}, {"width": 302, "height": 260, "id": "inspect-counterfactual-reasoning", "type": "inspect", "data": {"input": "prompt-counterfactual-reasoning"}, "position": {"x": 820, "y": 400}, "positionAbsolute": {"x": 820, "y": 400}}, {"width": 423, "height": 417, "id": "table-counterfactual-reasoning", "type": "table", "data": {"rows": [{"prompt": "If the sky flies in a bird, then what does the ground run on?", "ideal": "human"}, {"prompt": "If a song sings a bird, then what does a book read?", "ideal": "human"}, {"prompt": "If the river swims in a fish, then what does a bone chew?", "ideal": "dog"}, {"prompt": "If the earth flows on the river, then what does the sky hangs in?", "ideal": "the sun"}, {"prompt": "If the windmill blows the wind, then what does the grass moisten?", "ideal": "the rain"}, {"prompt": "If the lock unlocks the key, then what does the sheath go in?", "ideal": "the knife or the sword"}, {"prompt": "If the moon means the day, then what does the sun means?", "ideal": "the night"}, {"prompt": "If the black color means bad things, then what does the white color means?", "ideal": "good things"}, {"prompt": "If the black color means a low position, then what does the white color means?", "ideal": "a high position"}, {"prompt": "If the ice feels hot, then what does the fire feel?", "ideal": "cold"}, {"prompt": "If the moon is bigger than the earth, then who is bigger between the earth and the sun?", "ideal": "the earth"}, {"prompt": "If the moon is a cubic object, then what is the shape of the sun?", "ideal": "cube"}, {"prompt": "If chinese food matches Beijing, then what does american food match?", "ideal": "washington"}, {"prompt": "If 1 is less than 2, then is 3 bigger than 4?", "ideal": "yes"}, {"prompt": "If one matches eno, then what does two match?", "ideal": "owt"}], "columns": [{"key": "prompt", "header": "Prompt"}, {"key": "ideal", "header": "Ideal"}]}, "position": {"x": -16, "y": 160}, "selected": false, "positionAbsolute": {"x": -16, "y": 160}, "dragging": false}], "edges": [{"source": "prompt-counterfactual-reasoning", "sourceHandle": "prompt", "target": "eval-counterfactual-reasoning", "targetHandle": "responseBatch", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-prompt-1686756357355prompt-eval-1686756357355responseBatch"}, {"source": "prompt-counterfactual-reasoning", "sourceHandle": "prompt", "target": "inspect-counterfactual-reasoning", "targetHandle": "input", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-prompt-1686756357355prompt-inspect-1686756357355input"}, {"source": "eval-counterfactual-reasoning", "sourceHandle": "output", "target": "vis-counterfactual-reasoning", "targetHandle": "input", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-eval-1686756357355output-vis-1686756357355input"}, {"source": "table-counterfactual-reasoning", "sourceHandle": "Prompt", "target": "prompt-counterfactual-reasoning", "targetHandle": "prompt", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-table-1686756385002Prompt-prompt-1686756357355prompt"}], "viewport": {"x": 144, "y": 37, "zoom": 1}}, "cache": {"eval-1686756357355.json": {}, "inspect-1686756357355.json": {}, "prompt-1686756357355.json": {}, "table-1686756385002.json": {}, "vis-1686756357355.json": {}}} -------------------------------------------------------------------------------- /chainforge/oaievals/crontab.cforge: -------------------------------------------------------------------------------- 1 | {"flow": {"nodes": [{"width": 312, "height": 311, "id": "prompt-crontab", "type": "prompt", "data": {"prompt": "{prompt}", "n": 1, "llms": [{"key": "aa3c0f03-22bd-416e-af4d-4bf5c4278c99", "settings": {"system_msg": "Generate a cron expression with 5 fields from the given description. Output the cron expression ONLY and make your answer as short as possible.", "temperature": 1, "functions": [], "function_call": "", "top_p": 1, "stop": [], "presence_penalty": 0, "frequency_penalty": 0}, "name": "GPT3.5", "emoji": "\ud83d\ude42", "model": "gpt-3.5-turbo", "base_model": "gpt-3.5-turbo", "temp": 1, "formData": {"shortname": "GPT3.5", "model": "gpt-3.5-turbo", "system_msg": "Generate a cron expression with 5 fields from the given description. Output the cron expression ONLY and make your answer as short as possible.", "temperature": 1, "functions": "", "function_call": "", "top_p": 1, "stop": "", "presence_penalty": 0, "frequency_penalty": 0}}]}, "position": {"x": 448, "y": 224}, "selected": false, "positionAbsolute": {"x": 448, "y": 224}, "dragging": false}, {"width": 333, "height": 182, "id": "eval-crontab", "type": "evaluator", "data": {"code": "function evaluate(response) {\n\tlet ideal = response.meta['Ideal'];\n\treturn response.text.startsWith(ideal);\n}", "language": "javascript"}, "position": {"x": 820, "y": 150}, "positionAbsolute": {"x": 820, "y": 150}}, {"width": 228, "height": 196, "id": "vis-crontab", "type": "vis", "data": {"input": "eval-crontab"}, "position": {"x": 1200, "y": 250}, "positionAbsolute": {"x": 1200, "y": 250}}, {"width": 302, "height": 260, "id": "inspect-crontab", "type": "inspect", "data": {"input": "prompt-crontab"}, "position": {"x": 820, "y": 400}, "positionAbsolute": {"x": 820, "y": 400}}, {"width": 423, "height": 417, "id": "table-crontab", "type": "table", "data": {"rows": [{"prompt": "At 04:05.", "ideal": "5 4 * * *"}, {"prompt": "At 00:05 in August.", "ideal": "5 0 * 8 *"}, {"prompt": "At 14:15 on day-of-month 1.", "ideal": "15 14 1 * *"}, {"prompt": "At 22:00 on every day-of-week from Monday through Friday.", "ideal": "0 22 * * 1-5"}, {"prompt": "At minute 23 past every 2nd hour from 0 through 20.", "ideal": "23 0-20/2 * * *"}, {"prompt": "At minute 0 past hour 0 and 12 on day-of-month 1 in every 2nd month.", "ideal": "0 0,12 1 */2 *"}, {"prompt": "At 04:00 on every day-of-month from 8 through 14.", "ideal": "0 4 8-14 * *"}, {"prompt": "At 00:00 on day-of-month 1 and 15 and on Wednesday.", "ideal": "0 0 1,15 * 3"}, {"prompt": "At 04:05 on Sunday.", "ideal": "5 4 * * 0"}, {"prompt": "At every minute.", "ideal": "* * * * *"}, {"prompt": "At every 2nd minute.", "ideal": "*/2 * * * *"}, {"prompt": "At every 2nd minute from 1 through 59.", "ideal": "1-59/2 * * * *"}, {"prompt": "At every 3rd minute.", "ideal": "*/3 * * * *"}, {"prompt": "At every 4th minute.", "ideal": "*/4 * * * *"}, {"prompt": "At every 5th minute.", "ideal": "*/5 * * * *"}, {"prompt": "At minute 30.", "ideal": "30 * * * *"}, {"prompt": "At every 30th minute.", "ideal": "*/30 * * * *"}, {"prompt": "At 00:00 on day-of-month 1 in January.", "ideal": "0 0 1 1 *"}, {"prompt": "At 00:00 on day-of-month 1 in every 6th month.", "ideal": "0 0 1 */6 *"}, {"prompt": "At 00:00 on Saturday and Sunday.", "ideal": "0 0 * * 6,0"}, {"prompt": "At 00:00 on day-of-month 1 in every 3rd month.", "ideal": "0 0 1 */3 *"}], "columns": [{"key": "prompt", "header": "Prompt"}, {"key": "ideal", "header": "Ideal"}]}, "position": {"x": -16, "y": 160}, "selected": false, "positionAbsolute": {"x": -16, "y": 160}, "dragging": false}], "edges": [{"source": "prompt-crontab", "sourceHandle": "prompt", "target": "eval-crontab", "targetHandle": "responseBatch", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-prompt-1686756357355prompt-eval-1686756357355responseBatch"}, {"source": "prompt-crontab", "sourceHandle": "prompt", "target": "inspect-crontab", "targetHandle": "input", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-prompt-1686756357355prompt-inspect-1686756357355input"}, {"source": "eval-crontab", "sourceHandle": "output", "target": "vis-crontab", "targetHandle": "input", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-eval-1686756357355output-vis-1686756357355input"}, {"source": "table-crontab", "sourceHandle": "Prompt", "target": "prompt-crontab", "targetHandle": "prompt", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-table-1686756385002Prompt-prompt-1686756357355prompt"}], "viewport": {"x": 144, "y": 37, "zoom": 1}}, "cache": {"eval-1686756357355.json": {}, "inspect-1686756357355.json": {}, "prompt-1686756357355.json": {}, "table-1686756385002.json": {}, "vis-1686756357355.json": {}}} -------------------------------------------------------------------------------- /chainforge/oaievals/finnish-rhyme.cforge: -------------------------------------------------------------------------------- 1 | {"flow": {"nodes": [{"width": 312, "height": 311, "id": "prompt-finnish-rhyme", "type": "prompt", "data": {"prompt": "{prompt}", "n": 1, "llms": [{"key": "aa3c0f03-22bd-416e-af4d-4bf5c4278c99", "settings": {"system_msg": "For each pair of words, determine whether their Finnish translations rhyme. If they do, output the pair of rhyming words in Finnish. If not, output NONE.", "temperature": 1, "functions": [], "function_call": "", "top_p": 1, "stop": [], "presence_penalty": 0, "frequency_penalty": 0}, "name": "GPT3.5", "emoji": "\ud83d\ude42", "model": "gpt-3.5-turbo", "base_model": "gpt-3.5-turbo", "temp": 1, "formData": {"shortname": "GPT3.5", "model": "gpt-3.5-turbo", "system_msg": "For each pair of words, determine whether their Finnish translations rhyme. If they do, output the pair of rhyming words in Finnish. If not, output NONE.", "temperature": 1, "functions": "", "function_call": "", "top_p": 1, "stop": "", "presence_penalty": 0, "frequency_penalty": 0}}]}, "position": {"x": 448, "y": 224}, "selected": false, "positionAbsolute": {"x": 448, "y": 224}, "dragging": false}, {"width": 333, "height": 182, "id": "eval-finnish-rhyme", "type": "evaluator", "data": {"code": "function evaluate(response) {\n\tlet txt = response.text;\n\tlet ideal = response.meta['Ideal'];\n\treturn ideal.includes(txt) || txt.includes(ideal);\n}", "language": "javascript"}, "position": {"x": 820, "y": 150}, "positionAbsolute": {"x": 820, "y": 150}}, {"width": 228, "height": 196, "id": "vis-finnish-rhyme", "type": "vis", "data": {"input": "eval-finnish-rhyme"}, "position": {"x": 1200, "y": 250}, "positionAbsolute": {"x": 1200, "y": 250}}, {"width": 302, "height": 260, "id": "inspect-finnish-rhyme", "type": "inspect", "data": {"input": "prompt-finnish-rhyme"}, "position": {"x": 820, "y": 400}, "positionAbsolute": {"x": 820, "y": 400}}, {"width": 423, "height": 417, "id": "table-finnish-rhyme", "type": "table", "data": {"rows": [{"prompt": "boat, Go!", "ideal": "vene, Mene!"}, {"prompt": "snow, rubber", "ideal": "lumi, kumi"}, {"prompt": "car, tasteless", "ideal": "NONE"}, {"prompt": "flower, sock", "ideal": "kukka, sukka"}, {"prompt": "carpet, motto", "ideal": "NONE"}, {"prompt": "moth, butter", "ideal": "koi, voi"}, {"prompt": "ass (animal), glass", "ideal": "NONE"}, {"prompt": "meringue, spirit", "ideal": "marenki, henki"}, {"prompt": "marble, basket", "ideal": "marmori, kori"}, {"prompt": "sea, blood", "ideal": "meri, veri"}, {"prompt": "sea, pal", "ideal": "meri, kaveri"}, {"prompt": "wall, hay", "ideal": "sein\u00e4, hein\u00e4"}, {"prompt": "bottle, ball", "ideal": "NONE"}, {"prompt": "human being, delicious", "ideal": "ihminen, herkullinen"}, {"prompt": "pill, blood", "ideal": "pilleri, veri"}, {"prompt": "book, Norway", "ideal": "NONE"}, {"prompt": "slave, Norway", "ideal": "orja, Norja"}, {"prompt": "climate, mast", "ideal": "ilmasto, masto"}, {"prompt": "room, machine", "ideal": "NONE"}, {"prompt": "sabbath, watt", "ideal": "sapatti, watti"}], "columns": [{"key": "prompt", "header": "Prompt"}, {"key": "ideal", "header": "Ideal"}]}, "position": {"x": -16, "y": 160}, "selected": false, "positionAbsolute": {"x": -16, "y": 160}, "dragging": false}], "edges": [{"source": "prompt-finnish-rhyme", "sourceHandle": "prompt", "target": "eval-finnish-rhyme", "targetHandle": "responseBatch", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-prompt-1686756357355prompt-eval-1686756357355responseBatch"}, {"source": "prompt-finnish-rhyme", "sourceHandle": "prompt", "target": "inspect-finnish-rhyme", "targetHandle": "input", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-prompt-1686756357355prompt-inspect-1686756357355input"}, {"source": "eval-finnish-rhyme", "sourceHandle": "output", "target": "vis-finnish-rhyme", "targetHandle": "input", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-eval-1686756357355output-vis-1686756357355input"}, {"source": "table-finnish-rhyme", "sourceHandle": "Prompt", "target": "prompt-finnish-rhyme", "targetHandle": "prompt", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-table-1686756385002Prompt-prompt-1686756357355prompt"}], "viewport": {"x": 144, "y": 37, "zoom": 1}}, "cache": {"eval-1686756357355.json": {}, "inspect-1686756357355.json": {}, "prompt-1686756357355.json": {}, "table-1686756385002.json": {}, "vis-1686756357355.json": {}}} -------------------------------------------------------------------------------- /chainforge/oaievals/historical-kana-orthography-reading.cforge: -------------------------------------------------------------------------------- 1 | {"flow": {"nodes": [{"width": 312, "height": 311, "id": "prompt-historical-kana-orthography-reading", "type": "prompt", "data": {"prompt": "{prompt}", "n": 1, "llms": [{"key": "aa3c0f03-22bd-416e-af4d-4bf5c4278c99", "settings": {"system_msg": "Answer the reading of the word in historical kana orthography with modern kana orthography. Only use hiragana. Do not output any other output at all. If the input is the answer itself, please answer as is.", "temperature": 1, "functions": [], "function_call": "", "top_p": 1, "stop": [], "presence_penalty": 0, "frequency_penalty": 0}, "name": "GPT3.5", "emoji": "\ud83d\ude42", "model": "gpt-3.5-turbo", "base_model": "gpt-3.5-turbo", "temp": 1, "formData": {"shortname": "GPT3.5", "model": "gpt-3.5-turbo", "system_msg": "Answer the reading of the word in historical kana orthography with modern kana orthography. Only use hiragana. Do not output any other output at all. If the input is the answer itself, please answer as is.", "temperature": 1, "functions": "", "function_call": "", "top_p": 1, "stop": "", "presence_penalty": 0, "frequency_penalty": 0}}]}, "position": {"x": 448, "y": 224}, "selected": false, "positionAbsolute": {"x": 448, "y": 224}, "dragging": false}, {"width": 333, "height": 182, "id": "eval-historical-kana-orthography-reading", "type": "evaluator", "data": {"code": "function evaluate(response) {\n\tlet ideal = response.meta['Ideal'];\n\treturn response.text.startsWith(ideal);\n}", "language": "javascript"}, "position": {"x": 820, "y": 150}, "positionAbsolute": {"x": 820, "y": 150}}, {"width": 228, "height": 196, "id": "vis-historical-kana-orthography-reading", "type": "vis", "data": {"input": "eval-historical-kana-orthography-reading"}, "position": {"x": 1200, "y": 250}, "positionAbsolute": {"x": 1200, "y": 250}}, {"width": 302, "height": 260, "id": "inspect-historical-kana-orthography-reading", "type": "inspect", "data": {"input": "prompt-historical-kana-orthography-reading"}, "position": {"x": 820, "y": 400}, "positionAbsolute": {"x": 820, "y": 400}}, {"width": 423, "height": 417, "id": "table-historical-kana-orthography-reading", "type": "table", "data": {"rows": [{"prompt": "\u304a\u307b\u3057", "ideal": "\u304a\u304a\u3057"}, {"prompt": "\u3042\u3046\u3080", "ideal": "\u304a\u3046\u3080"}, {"prompt": "\u3044\u3046\u306a\u308a", "ideal": "\u3086\u3046\u306a\u308a"}, {"prompt": "\u3048\u3046\u3058", "ideal": "\u3088\u3046\u3058"}, {"prompt": "\u3046\u3064\u304f\u3057\u3046", "ideal": "\u3046\u3064\u304f\u3057\u3085\u3046"}, {"prompt": "\u3044\u3066\u3075", "ideal": "\u3044\u3061\u3087\u3046"}, {"prompt": "\u304f\u308f\u304b\u304f", "ideal": "\u304b\u304b\u304f"}, {"prompt": "\u305b\u3046\u3058\u3093", "ideal": "\u3057\u3087\u3046\u3058\u3093"}, {"prompt": "\u3072\u3080\u304c\u3057", "ideal": "\u3072\u3093\u304c\u3057"}, {"prompt": "\u304f\u308f\u3057", "ideal": "\u304b\u3057"}, {"prompt": "\u3042\u306f\u308c", "ideal": "\u3042\u308f\u308c"}, {"prompt": "\u3091\u3073\u3059", "ideal": "\u3048\u3073\u3059"}, {"prompt": "\u3067\u305b\u3046", "ideal": "\u3067\u3057\u3087\u3046"}, {"prompt": "\u304b\u306f", "ideal": "\u304b\u308f"}, {"prompt": "\u3090\u306a\u304b", "ideal": "\u3044\u306a\u304b"}, {"prompt": "\u3092\u3068\u3053", "ideal": "\u304a\u3068\u3053"}, {"prompt": "\u3066\u3046\u3065", "ideal": "\u3061\u3087\u3046\u305a"}, {"prompt": "\u3082\u3093\u3050\u308f\u3044", "ideal": "\u3082\u3093\u304c\u3044"}, {"prompt": "\u3042\u3075\u304e", "ideal": "\u304a\u3046\u304e"}, {"prompt": "\u3042\u3072\u3065\u3061", "ideal": "\u3042\u3044\u3065\u3061"}, {"prompt": "\u306b\u3064\u304d", "ideal": "\u306b\u3063\u304d"}], "columns": [{"key": "prompt", "header": "Prompt"}, {"key": "ideal", "header": "Ideal"}]}, "position": {"x": -16, "y": 160}, "selected": false, "positionAbsolute": {"x": -16, "y": 160}, "dragging": false}], "edges": [{"source": "prompt-historical-kana-orthography-reading", "sourceHandle": "prompt", "target": "eval-historical-kana-orthography-reading", "targetHandle": "responseBatch", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-prompt-1686756357355prompt-eval-1686756357355responseBatch"}, {"source": "prompt-historical-kana-orthography-reading", "sourceHandle": "prompt", "target": "inspect-historical-kana-orthography-reading", "targetHandle": "input", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-prompt-1686756357355prompt-inspect-1686756357355input"}, {"source": "eval-historical-kana-orthography-reading", "sourceHandle": "output", "target": "vis-historical-kana-orthography-reading", "targetHandle": "input", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-eval-1686756357355output-vis-1686756357355input"}, {"source": "table-historical-kana-orthography-reading", "sourceHandle": "Prompt", "target": "prompt-historical-kana-orthography-reading", "targetHandle": "prompt", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-table-1686756385002Prompt-prompt-1686756357355prompt"}], "viewport": {"x": 144, "y": 37, "zoom": 1}}, "cache": {"eval-1686756357355.json": {}, "inspect-1686756357355.json": {}, "prompt-1686756357355.json": {}, "table-1686756385002.json": {}, "vis-1686756357355.json": {}}} -------------------------------------------------------------------------------- /chainforge/oaievals/multi-step-equations.cforge: -------------------------------------------------------------------------------- 1 | {"flow": {"nodes": [{"width": 312, "height": 311, "id": "prompt-multi-step-equations", "type": "prompt", "data": {"prompt": "{prompt}", "n": 1, "llms": [{"key": "aa3c0f03-22bd-416e-af4d-4bf5c4278c99", "settings": {"system_msg": "Solve the following question for x. Answer with a single number and no additional text. You are a helpful assistant.", "temperature": 1, "functions": [], "function_call": "", "top_p": 1, "stop": [], "presence_penalty": 0, "frequency_penalty": 0}, "name": "GPT3.5", "emoji": "\ud83d\ude42", "model": "gpt-3.5-turbo", "base_model": "gpt-3.5-turbo", "temp": 1, "formData": {"shortname": "GPT3.5", "model": "gpt-3.5-turbo", "system_msg": "Solve the following question for x. Answer with a single number and no additional text. You are a helpful assistant.", "temperature": 1, "functions": "", "function_call": "", "top_p": 1, "stop": "", "presence_penalty": 0, "frequency_penalty": 0}}]}, "position": {"x": 448, "y": 224}, "selected": false, "positionAbsolute": {"x": 448, "y": 224}, "dragging": false}, {"width": 333, "height": 182, "id": "eval-multi-step-equations", "type": "evaluator", "data": {"code": "function evaluate(response) {\n\tlet ideal = response.meta['Ideal'];\n\treturn response.text.startsWith(ideal);\n}", "language": "javascript"}, "position": {"x": 820, "y": 150}, "positionAbsolute": {"x": 820, "y": 150}}, {"width": 228, "height": 196, "id": "vis-multi-step-equations", "type": "vis", "data": {"input": "eval-multi-step-equations"}, "position": {"x": 1200, "y": 250}, "positionAbsolute": {"x": 1200, "y": 250}}, {"width": 302, "height": 260, "id": "inspect-multi-step-equations", "type": "inspect", "data": {"input": "prompt-multi-step-equations"}, "position": {"x": 820, "y": 400}, "positionAbsolute": {"x": 820, "y": 400}}, {"width": 423, "height": 417, "id": "table-multi-step-equations", "type": "table", "data": {"rows": [{"prompt": "8x + 4x = 12", "ideal": "1"}, {"prompt": "-5x - 3x = 16", "ideal": "-2"}, {"prompt": "2x + 4 + 5 = 13", "ideal": "2"}, {"prompt": "-7x - 2x = 0", "ideal": "0"}, {"prompt": "8x + 8 = 8", "ideal": "0"}, {"prompt": "40 = 10x + 10x", "ideal": "2"}, {"prompt": "-15 = \u22122x + 5x", "ideal": "-5"}, {"prompt": "\u221210 = 4(1 + 4x) \u2212 2x", "ideal": "-1"}, {"prompt": "5(\u2212x + 5) = 15", "ideal": "2"}, {"prompt": "\u22122x \u2212 4(1 + 8x) = \u2212174", "ideal": "5"}, {"prompt": "\u22122(4x \u2212 2) = \u22122(2 + 3x)", "ideal": "4"}, {"prompt": "2x \u2212 4(1 + 2x) = \u221224 + 4x", "ideal": "2"}, {"prompt": "\u22122(\u22122x \u2212 4) = \u22124x + 184", "ideal": "22"}, {"prompt": "\u221210 \u2212 5x = 2(5x - 20)", "ideal": "2"}, {"prompt": "\u22122(6x \u2212 2) = \u22122(3 + 5x)", "ideal": "5"}], "columns": [{"key": "prompt", "header": "Prompt"}, {"key": "ideal", "header": "Ideal"}]}, "position": {"x": -16, "y": 160}, "selected": false, "positionAbsolute": {"x": -16, "y": 160}, "dragging": false}], "edges": [{"source": "prompt-multi-step-equations", "sourceHandle": "prompt", "target": "eval-multi-step-equations", "targetHandle": "responseBatch", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-prompt-1686756357355prompt-eval-1686756357355responseBatch"}, {"source": "prompt-multi-step-equations", "sourceHandle": "prompt", "target": "inspect-multi-step-equations", "targetHandle": "input", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-prompt-1686756357355prompt-inspect-1686756357355input"}, {"source": "eval-multi-step-equations", "sourceHandle": "output", "target": "vis-multi-step-equations", "targetHandle": "input", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-eval-1686756357355output-vis-1686756357355input"}, {"source": "table-multi-step-equations", "sourceHandle": "Prompt", "target": "prompt-multi-step-equations", "targetHandle": "prompt", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-table-1686756385002Prompt-prompt-1686756357355prompt"}], "viewport": {"x": 144, "y": 37, "zoom": 1}}, "cache": {"eval-1686756357355.json": {}, "inspect-1686756357355.json": {}, "prompt-1686756357355.json": {}, "table-1686756385002.json": {}, "vis-1686756357355.json": {}}} -------------------------------------------------------------------------------- /chainforge/oaievals/nepali-song-singer.cforge: -------------------------------------------------------------------------------- 1 | {"flow": {"nodes": [{"width": 312, "height": 311, "id": "prompt-nepali-song-singer", "type": "prompt", "data": {"prompt": "{prompt}", "n": 1, "llms": [{"key": "aa3c0f03-22bd-416e-af4d-4bf5c4278c99", "settings": {"system_msg": "A user will give you a English transliteration phrase of Nepali song. Give the name of the singer or band.", "temperature": 1, "functions": [], "function_call": "", "top_p": 1, "stop": [], "presence_penalty": 0, "frequency_penalty": 0}, "name": "GPT3.5", "emoji": "\ud83d\ude42", "model": "gpt-3.5-turbo", "base_model": "gpt-3.5-turbo", "temp": 1, "formData": {"shortname": "GPT3.5", "model": "gpt-3.5-turbo", "system_msg": "A user will give you a English transliteration phrase of Nepali song. Give the name of the singer or band.", "temperature": 1, "functions": "", "function_call": "", "top_p": 1, "stop": "", "presence_penalty": 0, "frequency_penalty": 0}}]}, "position": {"x": 448, "y": 224}, "selected": false, "positionAbsolute": {"x": 448, "y": 224}, "dragging": false}, {"width": 333, "height": 182, "id": "eval-nepali-song-singer", "type": "evaluator", "data": {"code": "function evaluate(response) {\n\tlet ideal = response.meta['Ideal'];\n\treturn response.text.includes(ideal);\n}", "language": "javascript"}, "position": {"x": 820, "y": 150}, "positionAbsolute": {"x": 820, "y": 150}}, {"width": 228, "height": 196, "id": "vis-nepali-song-singer", "type": "vis", "data": {"input": "eval-nepali-song-singer"}, "position": {"x": 1200, "y": 250}, "positionAbsolute": {"x": 1200, "y": 250}}, {"width": 302, "height": 260, "id": "inspect-nepali-song-singer", "type": "inspect", "data": {"input": "prompt-nepali-song-singer"}, "position": {"x": 820, "y": 400}, "positionAbsolute": {"x": 820, "y": 400}}, {"width": 423, "height": 417, "id": "table-nepali-song-singer", "type": "table", "data": {"rows": [{"prompt": "Yo maan ta mero nepali ho", "ideal": "1974 AD"}, {"prompt": "Timro maya le badhera rakha", "ideal": "Ashmita Adhikari"}, {"prompt": "Euta manchhe ko maya le", "ideal": "Narayan Gopal"}, {"prompt": "Kehi mitho baata gara", "ideal": "Narayan Gopal"}, {"prompt": "Mohani Lagla Hai", "ideal": "Narayan Gopal"}, {"prompt": "Biratako Chino", "ideal": "Narayan Gopal"}, {"prompt": "Baneko chha Paharale", "ideal": "Gopal Yonjan"}, {"prompt": "Siran ma photo cha", "ideal": "Nepathya"}, {"prompt": "Bheda ko oon jasto", "ideal": "Nepathya"}, {"prompt": "Thula thula mahal", "ideal": "Karna Das"}, {"prompt": "Anautho betha", "ideal": "Karna Das"}, {"prompt": "Ma Kahile Haseko Thiye", "ideal": "Yash Kumar"}, {"prompt": "Sanjha Pakha", "ideal": "Nabin K Bhattarai"}, {"prompt": "Yehi ho ta maya", "ideal": "Nabin K Bhattarai"}, {"prompt": "Ekanta chha thau", "ideal": "COD"}, {"prompt": "Yo Prasanga", "ideal": "Yama Buddha"}, {"prompt": "Orali lageko", "ideal": "Ram Krishna Dhakal"}, {"prompt": "Malai Yo Zindagi Le", "ideal": "Ram Krishna Dhakal"}, {"prompt": "Pheri Tyo Din", "ideal": "Sugam Pokhrel"}, {"prompt": "Dashain Tihar", "ideal": "Sugam Pokhrel"}, {"prompt": "Samhalincha Kahile Maan", "ideal": "Sugam Pokhrel"}, {"prompt": "Ma ta dur dekhi", "ideal": "Deep Shrestha"}, {"prompt": "Badal haru", "ideal": "Deep Shrestha"}, {"prompt": "Halla Chalechha", "ideal": "Raju Lama"}, {"prompt": "Sayad Timro Bato Ma", "ideal": "Raju Lama"}, {"prompt": "Timi Lai Dekhera", "ideal": "Raju Lama"}, {"prompt": "Aaja maan udhera bhagchha", "ideal": "Udit Narayan"}, {"prompt": "Kaha Hola Ghar Bara", "ideal": "Karma"}, {"prompt": "Ko hau timi", "ideal": "Karma"}, {"prompt": "Khaseka Tara", "ideal": "Albatross"}, {"prompt": "Timi Bhane", "ideal": "Albatross"}], "columns": [{"key": "prompt", "header": "Prompt"}, {"key": "ideal", "header": "Ideal"}]}, "position": {"x": -16, "y": 160}, "selected": false, "positionAbsolute": {"x": -16, "y": 160}, "dragging": false}], "edges": [{"source": "prompt-nepali-song-singer", "sourceHandle": "prompt", "target": "eval-nepali-song-singer", "targetHandle": "responseBatch", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-prompt-1686756357355prompt-eval-1686756357355responseBatch"}, {"source": "prompt-nepali-song-singer", "sourceHandle": "prompt", "target": "inspect-nepali-song-singer", "targetHandle": "input", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-prompt-1686756357355prompt-inspect-1686756357355input"}, {"source": "eval-nepali-song-singer", "sourceHandle": "output", "target": "vis-nepali-song-singer", "targetHandle": "input", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-eval-1686756357355output-vis-1686756357355input"}, {"source": "table-nepali-song-singer", "sourceHandle": "Prompt", "target": "prompt-nepali-song-singer", "targetHandle": "prompt", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-table-1686756385002Prompt-prompt-1686756357355prompt"}], "viewport": {"x": 144, "y": 37, "zoom": 1}}, "cache": {"eval-1686756357355.json": {}, "inspect-1686756357355.json": {}, "prompt-1686756357355.json": {}, "table-1686756385002.json": {}, "vis-1686756357355.json": {}}} -------------------------------------------------------------------------------- /chainforge/oaievals/passing-balls.cforge: -------------------------------------------------------------------------------- 1 | {"flow": {"nodes": [{"width": 312, "height": 311, "id": "prompt-passing-balls", "type": "prompt", "data": {"prompt": "{prompt}", "n": 1, "llms": [{"key": "aa3c0f03-22bd-416e-af4d-4bf5c4278c99", "settings": {"system_msg": "Imagine there are three players: player 1, 2, and 3. The players are passing a ball around. For a given starting player and passing sequence, determine the last player holding the ball. Output only the number of the player. Do not explain your reasoning.", "temperature": 1, "functions": [], "function_call": "", "top_p": 1, "stop": [], "presence_penalty": 0, "frequency_penalty": 0}, "name": "GPT3.5", "emoji": "\ud83d\ude42", "model": "gpt-3.5-turbo", "base_model": "gpt-3.5-turbo", "temp": 1, "formData": {"shortname": "GPT3.5", "model": "gpt-3.5-turbo", "system_msg": "Imagine there are three players: player 1, 2, and 3. The players are passing a ball around. For a given starting player and passing sequence, determine the last player holding the ball. Output only the number of the player. Do not explain your reasoning.", "temperature": 1, "functions": "", "function_call": "", "top_p": 1, "stop": "", "presence_penalty": 0, "frequency_penalty": 0}}]}, "position": {"x": 448, "y": 224}, "selected": false, "positionAbsolute": {"x": 448, "y": 224}, "dragging": false}, {"width": 333, "height": 182, "id": "eval-passing-balls", "type": "evaluator", "data": {"code": "function evaluate(response) {\n\tlet txt = response.text;\n\tlet ideal = response.meta['Ideal'];\n\treturn ideal.includes(txt) || txt.includes(ideal);\n}", "language": "javascript"}, "position": {"x": 820, "y": 150}, "positionAbsolute": {"x": 820, "y": 150}}, {"width": 228, "height": 196, "id": "vis-passing-balls", "type": "vis", "data": {"input": "eval-passing-balls"}, "position": {"x": 1200, "y": 250}, "positionAbsolute": {"x": 1200, "y": 250}}, {"width": 302, "height": 260, "id": "inspect-passing-balls", "type": "inspect", "data": {"input": "prompt-passing-balls"}, "position": {"x": 820, "y": 400}, "positionAbsolute": {"x": 820, "y": 400}}, {"width": 423, "height": 417, "id": "table-passing-balls", "type": "table", "data": {"rows": [{"prompt": "Starting player: 1. Passing sequence: 2", "ideal": "2"}, {"prompt": "Starting player: 1. Passing sequence: 23", "ideal": "3"}, {"prompt": "Starting player: 1. Passing sequence: 232", "ideal": "2"}, {"prompt": "Starting player: 1. Passing sequence: 2321", "ideal": "1"}, {"prompt": "Starting player: 1. Passing sequence: 23212", "ideal": "2"}, {"prompt": "Starting player: 1. Passing sequence: 232123", "ideal": "3"}, {"prompt": "Starting player: 1. Passing sequence: 231232", "ideal": "2"}, {"prompt": "Starting player: 1. Passing sequence: 2312321", "ideal": "1"}, {"prompt": "Starting player: 1. Passing sequence: 23123212", "ideal": "2"}, {"prompt": "Starting player: 1. Passing sequence: 231232123", "ideal": "3"}, {"prompt": "Starting player: 1. Passing sequence: 23123", "ideal": "3"}, {"prompt": "Starting player: 1. Passing sequence: 23123123", "ideal": "3"}, {"prompt": "Starting player: 1. Passing sequence: 23123123123", "ideal": "3"}, {"prompt": "Starting player: 1. Passing sequence: 23123123123123", "ideal": "3"}, {"prompt": "Starting player: 1. Passing sequence: 21231323121321", "ideal": "1"}, {"prompt": "Starting player: 1. Passing sequence: 23231323121312", "ideal": "2"}, {"prompt": "Starting player: 1. Passing sequence: 23123232132313", "ideal": "3"}], "columns": [{"key": "prompt", "header": "Prompt"}, {"key": "ideal", "header": "Ideal"}]}, "position": {"x": -16, "y": 160}, "selected": false, "positionAbsolute": {"x": -16, "y": 160}, "dragging": false}], "edges": [{"source": "prompt-passing-balls", "sourceHandle": "prompt", "target": "eval-passing-balls", "targetHandle": "responseBatch", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-prompt-1686756357355prompt-eval-1686756357355responseBatch"}, {"source": "prompt-passing-balls", "sourceHandle": "prompt", "target": "inspect-passing-balls", "targetHandle": "input", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-prompt-1686756357355prompt-inspect-1686756357355input"}, {"source": "eval-passing-balls", "sourceHandle": "output", "target": "vis-passing-balls", "targetHandle": "input", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-eval-1686756357355output-vis-1686756357355input"}, {"source": "table-passing-balls", "sourceHandle": "Prompt", "target": "prompt-passing-balls", "targetHandle": "prompt", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-table-1686756385002Prompt-prompt-1686756357355prompt"}], "viewport": {"x": 144, "y": 37, "zoom": 1}}, "cache": {"eval-1686756357355.json": {}, "inspect-1686756357355.json": {}, "prompt-1686756357355.json": {}, "table-1686756385002.json": {}, "vis-1686756357355.json": {}}} -------------------------------------------------------------------------------- /chainforge/oaievals/polish-syllable-count.cforge: -------------------------------------------------------------------------------- 1 | {"flow": {"nodes": [{"width": 312, "height": 311, "id": "prompt-polish-syllable-count", "type": "prompt", "data": {"prompt": "{prompt}", "n": 1, "llms": [{"key": "aa3c0f03-22bd-416e-af4d-4bf5c4278c99", "settings": {"system_msg": "Trzeba policzy\u0107 sylaby w polskim wyrazie i odpowiedzie\u0107 tylko jedn\u0105 cyfr\u0105. Wpisz tylko numer i nic wi\u0119cej.", "temperature": 1, "functions": [], "function_call": "", "top_p": 1, "stop": [], "presence_penalty": 0, "frequency_penalty": 0}, "name": "GPT3.5", "emoji": "\ud83d\ude42", "model": "gpt-3.5-turbo", "base_model": "gpt-3.5-turbo", "temp": 1, "formData": {"shortname": "GPT3.5", "model": "gpt-3.5-turbo", "system_msg": "Trzeba policzy\u0107 sylaby w polskim wyrazie i odpowiedzie\u0107 tylko jedn\u0105 cyfr\u0105. Wpisz tylko numer i nic wi\u0119cej.", "temperature": 1, "functions": "", "function_call": "", "top_p": 1, "stop": "", "presence_penalty": 0, "frequency_penalty": 0}}]}, "position": {"x": 448, "y": 224}, "selected": false, "positionAbsolute": {"x": 448, "y": 224}, "dragging": false}, {"width": 333, "height": 182, "id": "eval-polish-syllable-count", "type": "evaluator", "data": {"code": "function evaluate(response) {\n\tlet ideal = response.meta['Ideal'];\n\treturn response.text.startsWith(ideal);\n}", "language": "javascript"}, "position": {"x": 820, "y": 150}, "positionAbsolute": {"x": 820, "y": 150}}, {"width": 228, "height": 196, "id": "vis-polish-syllable-count", "type": "vis", "data": {"input": "eval-polish-syllable-count"}, "position": {"x": 1200, "y": 250}, "positionAbsolute": {"x": 1200, "y": 250}}, {"width": 302, "height": 260, "id": "inspect-polish-syllable-count", "type": "inspect", "data": {"input": "prompt-polish-syllable-count"}, "position": {"x": 820, "y": 400}, "positionAbsolute": {"x": 820, "y": 400}}, {"width": 423, "height": 417, "id": "table-polish-syllable-count", "type": "table", "data": {"rows": [{"prompt": "Wyraz: miasto", "ideal": "2"}, {"prompt": "Wyraz: cieszy\u0107", "ideal": "2"}, {"prompt": "Wyraz: kwiatek", "ideal": "2"}, {"prompt": "Wyraz: piosenkarka", "ideal": "4"}, {"prompt": "Wyraz: dziewanna", "ideal": "3"}, {"prompt": "Wyraz: niedowiarek", "ideal": "4"}, {"prompt": "Wyraz: nienormalny", "ideal": "4"}, {"prompt": "Wyraz: zmia\u017cd\u017cy\u0107", "ideal": "2"}, {"prompt": "Wyraz: cierpienie", "ideal": "3"}, {"prompt": "Wyraz: Paulina", "ideal": "3"}, {"prompt": "Wyraz: przyja\u017a\u0144", "ideal": "2"}, {"prompt": "Wyraz: chrz\u0105szcz", "ideal": "1"}, {"prompt": "Wyraz: wszystkich", "ideal": "2"}, {"prompt": "Wyraz: wi\u0119kszo\u015b\u0107", "ideal": "2"}, {"prompt": "Wyraz: ciecierzycy", "ideal": "4"}, {"prompt": "Wyraz: fiord", "ideal": "1"}], "columns": [{"key": "prompt", "header": "Prompt"}, {"key": "ideal", "header": "Ideal"}]}, "position": {"x": -16, "y": 160}, "selected": false, "positionAbsolute": {"x": -16, "y": 160}, "dragging": false}], "edges": [{"source": "prompt-polish-syllable-count", "sourceHandle": "prompt", "target": "eval-polish-syllable-count", "targetHandle": "responseBatch", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-prompt-1686756357355prompt-eval-1686756357355responseBatch"}, {"source": "prompt-polish-syllable-count", "sourceHandle": "prompt", "target": "inspect-polish-syllable-count", "targetHandle": "input", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-prompt-1686756357355prompt-inspect-1686756357355input"}, {"source": "eval-polish-syllable-count", "sourceHandle": "output", "target": "vis-polish-syllable-count", "targetHandle": "input", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-eval-1686756357355output-vis-1686756357355input"}, {"source": "table-polish-syllable-count", "sourceHandle": "Prompt", "target": "prompt-polish-syllable-count", "targetHandle": "prompt", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-table-1686756385002Prompt-prompt-1686756357355prompt"}], "viewport": {"x": 144, "y": 37, "zoom": 1}}, "cache": {"eval-1686756357355.json": {}, "inspect-1686756357355.json": {}, "prompt-1686756357355.json": {}, "table-1686756385002.json": {}, "vis-1686756357355.json": {}}} -------------------------------------------------------------------------------- /chainforge/oaievals/reverse-polish-notation.cforge: -------------------------------------------------------------------------------- 1 | {"flow": {"nodes": [{"width": 312, "height": 311, "id": "prompt-reverse-polish-notation", "type": "prompt", "data": {"prompt": "{prompt}", "n": 1, "llms": [{"key": "aa3c0f03-22bd-416e-af4d-4bf5c4278c99", "settings": {"system_msg": "Convert mathematical expression to reverse polish notation. Do not provide any explanation or commentary, output the result only.", "temperature": 1, "functions": [], "function_call": "", "top_p": 1, "stop": [], "presence_penalty": 0, "frequency_penalty": 0}, "name": "GPT3.5", "emoji": "\ud83d\ude42", "model": "gpt-3.5-turbo", "base_model": "gpt-3.5-turbo", "temp": 1, "formData": {"shortname": "GPT3.5", "model": "gpt-3.5-turbo", "system_msg": "Convert mathematical expression to reverse polish notation. Do not provide any explanation or commentary, output the result only.", "temperature": 1, "functions": "", "function_call": "", "top_p": 1, "stop": "", "presence_penalty": 0, "frequency_penalty": 0}}]}, "position": {"x": 448, "y": 224}, "selected": false, "positionAbsolute": {"x": 448, "y": 224}, "dragging": false}, {"width": 333, "height": 182, "id": "eval-reverse-polish-notation", "type": "evaluator", "data": {"code": "function evaluate(response) {\n\tlet ideal = response.meta['Ideal'];\n\treturn response.text.startsWith(ideal);\n}", "language": "javascript"}, "position": {"x": 820, "y": 150}, "positionAbsolute": {"x": 820, "y": 150}}, {"width": 228, "height": 196, "id": "vis-reverse-polish-notation", "type": "vis", "data": {"input": "eval-reverse-polish-notation"}, "position": {"x": 1200, "y": 250}, "positionAbsolute": {"x": 1200, "y": 250}}, {"width": 302, "height": 260, "id": "inspect-reverse-polish-notation", "type": "inspect", "data": {"input": "prompt-reverse-polish-notation"}, "position": {"x": 820, "y": 400}, "positionAbsolute": {"x": 820, "y": 400}}, {"width": 423, "height": 417, "id": "table-reverse-polish-notation", "type": "table", "data": {"rows": [{"prompt": "Mathematical expression: 1 + 1", "ideal": "1 1 +"}, {"prompt": "Mathematical expression: 2 * 4 + 8", "ideal": "2 4 * 8 +"}, {"prompt": "Mathematical expression: 2 * (4 + 8)", "ideal": "2 4 8 + *"}, {"prompt": "Mathematical expression: (1 + 2) * (3 * 4)", "ideal": "1 2 + 3 4 * *"}, {"prompt": "Mathematical expression: 1 + (2 - (3 + 4))", "ideal": "1 2 - 3 4 + +"}, {"prompt": "Mathematical expression: (3 + 5) * (7 - 2)", "ideal": "3 5 + 7 2 - *"}, {"prompt": "Mathematical expression: (1 + 2) * 4", "ideal": "1 2 + 4 *"}, {"prompt": "Mathematical expression: 0", "ideal": "0"}, {"prompt": "Mathematical expression: (1 + 2) * 5 + 7", "ideal": "1 2 + 5 * 7 +"}, {"prompt": "Mathematical expression: (10 + 6) * 3", "ideal": "10 6 + 3 *"}, {"prompt": "Mathematical expression: (2 / 8) * (5 - 6)", "ideal": "2 8 / 5 6 - *"}, {"prompt": "Mathematical expression: (x + y) * z", "ideal": "x y + z *"}, {"prompt": "Mathematical expression: 5 / (5 + y * x)", "ideal": "5 5 y x * + /"}, {"prompt": "Mathematical expression: 4 + (13 / 5)", "ideal": "4 13 5 / +"}, {"prompt": "Mathematical expression: 2^3", "ideal": "2 3 ^"}, {"prompt": "Mathematical expression: (3 + x * 8) / (1 + 3 * x)", "ideal": "3 x 8 * + 1 3 x * + /"}, {"prompt": "Mathematical expression: (y * 5 + 1) / (3 * x + 1)", "ideal": "y 5 * 1 + 3 x * 1 + /"}], "columns": [{"key": "prompt", "header": "Prompt"}, {"key": "ideal", "header": "Ideal"}]}, "position": {"x": -16, "y": 160}, "selected": false, "positionAbsolute": {"x": -16, "y": 160}, "dragging": false}], "edges": [{"source": "prompt-reverse-polish-notation", "sourceHandle": "prompt", "target": "eval-reverse-polish-notation", "targetHandle": "responseBatch", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-prompt-1686756357355prompt-eval-1686756357355responseBatch"}, {"source": "prompt-reverse-polish-notation", "sourceHandle": "prompt", "target": "inspect-reverse-polish-notation", "targetHandle": "input", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-prompt-1686756357355prompt-inspect-1686756357355input"}, {"source": "eval-reverse-polish-notation", "sourceHandle": "output", "target": "vis-reverse-polish-notation", "targetHandle": "input", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-eval-1686756357355output-vis-1686756357355input"}, {"source": "table-reverse-polish-notation", "sourceHandle": "Prompt", "target": "prompt-reverse-polish-notation", "targetHandle": "prompt", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-table-1686756385002Prompt-prompt-1686756357355prompt"}], "viewport": {"x": 144, "y": 37, "zoom": 1}}, "cache": {"eval-1686756357355.json": {}, "inspect-1686756357355.json": {}, "prompt-1686756357355.json": {}, "table-1686756385002.json": {}, "vis-1686756357355.json": {}}} -------------------------------------------------------------------------------- /chainforge/oaievals/rock-climbing.cforge: -------------------------------------------------------------------------------- 1 | {"flow": {"nodes": [{"width": 312, "height": 311, "id": "prompt-rock-climbing", "type": "prompt", "data": {"prompt": "{prompt}", "n": 1, "llms": [{"key": "aa3c0f03-22bd-416e-af4d-4bf5c4278c99", "settings": {"system_msg": "You are to answer each given question with a single non plural form word, all lowercase, and without punctuation.", "temperature": 1, "functions": [], "function_call": "", "top_p": 1, "stop": [], "presence_penalty": 0, "frequency_penalty": 0}, "name": "GPT3.5", "emoji": "\ud83d\ude42", "model": "gpt-3.5-turbo", "base_model": "gpt-3.5-turbo", "temp": 1, "formData": {"shortname": "GPT3.5", "model": "gpt-3.5-turbo", "system_msg": "You are to answer each given question with a single non plural form word, all lowercase, and without punctuation.", "temperature": 1, "functions": "", "function_call": "", "top_p": 1, "stop": "", "presence_penalty": 0, "frequency_penalty": 0}}]}, "position": {"x": 448, "y": 224}, "selected": false, "positionAbsolute": {"x": 448, "y": 224}, "dragging": false}, {"width": 333, "height": 182, "id": "eval-rock-climbing", "type": "evaluator", "data": {"code": "function evaluate(response) {\n\tlet ideal = response.meta['Ideal'];\n\treturn response.text.startsWith(ideal);\n}", "language": "javascript"}, "position": {"x": 820, "y": 150}, "positionAbsolute": {"x": 820, "y": 150}}, {"width": 228, "height": 196, "id": "vis-rock-climbing", "type": "vis", "data": {"input": "eval-rock-climbing"}, "position": {"x": 1200, "y": 250}, "positionAbsolute": {"x": 1200, "y": 250}}, {"width": 302, "height": 260, "id": "inspect-rock-climbing", "type": "inspect", "data": {"input": "prompt-rock-climbing"}, "position": {"x": 820, "y": 400}, "positionAbsolute": {"x": 820, "y": 400}}, {"width": 423, "height": 417, "id": "table-rock-climbing", "type": "table", "data": {"rows": [{"prompt": "The name actually comes from the way your fingers hold it so this climbing hold describes a small but positive edge you can only fit the top pad of your fingers on.", "ideal": "crimp"}, {"prompt": "Normally describes a large, deep climbing hold that you can wrap your whole hand around.", "ideal": "jug"}, {"prompt": "Often large, round climbing holds that do not have a positive gradient. This means you can\u2019t close your fingers around them at all and your hand remains quite open.", "ideal": "sloper"}, {"prompt": "You must pull sideways on this climbing hold rather than downwards. The place you put your fingers is on the side of the climbing hold which is facing away from you.", "ideal": "sidepull"}, {"prompt": "You must pull sideways on this climbing hold rather than downwards. The place you put your fingers is on the side of the climbing hold which is facing towards you.", "ideal": "gaston"}, {"prompt": "You must pull upwards on this climbing hold rather than downwards. The place you put your fingers is on the side of the climbing hold which is facing downwards.", "ideal": "undercling"}, {"prompt": "A climbing hold that requires your thumb to get in on the action as well. Often the squeezing action is what is required to grip and use this climbing hold.", "ideal": "pinch"}, {"prompt": "Rather than gripping a variation in angle on the wall, you are dealing with a climbing hold which is an actual hole that has been eroded in. This climbing hold comes in all shapes and sizes.", "ideal": "pocket"}], "columns": [{"key": "prompt", "header": "Prompt"}, {"key": "ideal", "header": "Ideal"}]}, "position": {"x": -16, "y": 160}, "selected": false, "positionAbsolute": {"x": -16, "y": 160}, "dragging": false}], "edges": [{"source": "prompt-rock-climbing", "sourceHandle": "prompt", "target": "eval-rock-climbing", "targetHandle": "responseBatch", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-prompt-1686756357355prompt-eval-1686756357355responseBatch"}, {"source": "prompt-rock-climbing", "sourceHandle": "prompt", "target": "inspect-rock-climbing", "targetHandle": "input", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-prompt-1686756357355prompt-inspect-1686756357355input"}, {"source": "eval-rock-climbing", "sourceHandle": "output", "target": "vis-rock-climbing", "targetHandle": "input", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-eval-1686756357355output-vis-1686756357355input"}, {"source": "table-rock-climbing", "sourceHandle": "Prompt", "target": "prompt-rock-climbing", "targetHandle": "prompt", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-table-1686756385002Prompt-prompt-1686756357355prompt"}], "viewport": {"x": 144, "y": 37, "zoom": 1}}, "cache": {"eval-1686756357355.json": {}, "inspect-1686756357355.json": {}, "prompt-1686756357355.json": {}, "table-1686756385002.json": {}, "vis-1686756357355.json": {}}} -------------------------------------------------------------------------------- /chainforge/oaievals/russian-rhyme.cforge: -------------------------------------------------------------------------------- 1 | {"flow": {"nodes": [{"width": 312, "height": 311, "id": "prompt-russian-rhyme", "type": "prompt", "data": {"prompt": "{prompt}", "n": 1, "llms": [{"key": "aa3c0f03-22bd-416e-af4d-4bf5c4278c99", "settings": {"system_msg": "For each pair of words, determine whether their Russian translations rhyme. If they do, output the pair of rhyming words in Russian. If not, output NONE.", "temperature": 1, "functions": [], "function_call": "", "top_p": 1, "stop": [], "presence_penalty": 0, "frequency_penalty": 0}, "name": "GPT3.5", "emoji": "\ud83d\ude42", "model": "gpt-3.5-turbo", "base_model": "gpt-3.5-turbo", "temp": 1, "formData": {"shortname": "GPT3.5", "model": "gpt-3.5-turbo", "system_msg": "For each pair of words, determine whether their Russian translations rhyme. If they do, output the pair of rhyming words in Russian. If not, output NONE.", "temperature": 1, "functions": "", "function_call": "", "top_p": 1, "stop": "", "presence_penalty": 0, "frequency_penalty": 0}}]}, "position": {"x": 448, "y": 224}, "selected": false, "positionAbsolute": {"x": 448, "y": 224}, "dragging": false}, {"width": 333, "height": 182, "id": "eval-russian-rhyme", "type": "evaluator", "data": {"code": "function evaluate(response) {\n\tlet txt = response.text;\n\tlet ideal = response.meta['Ideal'];\n\treturn ideal.includes(txt) || txt.includes(ideal);\n}", "language": "javascript"}, "position": {"x": 820, "y": 150}, "positionAbsolute": {"x": 820, "y": 150}}, {"width": 228, "height": 196, "id": "vis-russian-rhyme", "type": "vis", "data": {"input": "eval-russian-rhyme"}, "position": {"x": 1200, "y": 250}, "positionAbsolute": {"x": 1200, "y": 250}}, {"width": 302, "height": 260, "id": "inspect-russian-rhyme", "type": "inspect", "data": {"input": "prompt-russian-rhyme"}, "position": {"x": 820, "y": 400}, "positionAbsolute": {"x": 820, "y": 400}}, {"width": 423, "height": 417, "id": "table-russian-rhyme", "type": "table", "data": {"rows": [{"prompt": "cannon, toy", "ideal": "\u043f\u0443\u0448\u043a\u0430, \u0438\u0433\u0440\u0443\u0448\u043a\u0430"}, {"prompt": "life, organism", "ideal": "\u0436\u0438\u0437\u043d\u044c, \u043e\u0440\u0433\u0430\u043d\u0438\u0437\u043c"}, {"prompt": "human, escape", "ideal": "\u0447\u0435\u043b\u043e\u0432\u0435\u043a, \u043f\u043e\u0431\u0435\u0433"}, {"prompt": "soul, ramson", "ideal": "\u0434\u0443\u0448\u0430, \u0447\u0435\u0440\u0435\u043c\u0448\u0430"}, {"prompt": "peace, marshmallow", "ideal": "\u043c\u0438\u0440, \u0437\u0435\u0444\u0438\u0440"}, {"prompt": "sky, turnip", "ideal": "\u043d\u0435\u0431\u043e, \u0440\u0435\u043f\u0430"}, {"prompt": "summer, coin", "ideal": "\u043b\u0435\u0442\u043e, \u043c\u043e\u043d\u0435\u0442\u0430"}, {"prompt": "smile, violin", "ideal": "\u0443\u043b\u044b\u0431\u043a\u0430, \u0441\u043a\u0440\u0438\u043f\u043a\u0430"}, {"prompt": "sun, shaking", "ideal": "NONE"}, {"prompt": "rain, snow", "ideal": "NONE"}, {"prompt": "house, street", "ideal": "NONE"}, {"prompt": "flower, tree", "ideal": "NONE"}, {"prompt": "car, train", "ideal": "NONE"}, {"prompt": "mountain, valley", "ideal": "NONE"}, {"prompt": "day, night", "ideal": "NONE"}, {"prompt": "book, pen", "ideal": "NONE"}, {"prompt": "apple, orange", "ideal": "NONE"}], "columns": [{"key": "prompt", "header": "Prompt"}, {"key": "ideal", "header": "Ideal"}]}, "position": {"x": -16, "y": 160}, "selected": false, "positionAbsolute": {"x": -16, "y": 160}, "dragging": false}], "edges": [{"source": "prompt-russian-rhyme", "sourceHandle": "prompt", "target": "eval-russian-rhyme", "targetHandle": "responseBatch", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-prompt-1686756357355prompt-eval-1686756357355responseBatch"}, {"source": "prompt-russian-rhyme", "sourceHandle": "prompt", "target": "inspect-russian-rhyme", "targetHandle": "input", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-prompt-1686756357355prompt-inspect-1686756357355input"}, {"source": "eval-russian-rhyme", "sourceHandle": "output", "target": "vis-russian-rhyme", "targetHandle": "input", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-eval-1686756357355output-vis-1686756357355input"}, {"source": "table-russian-rhyme", "sourceHandle": "Prompt", "target": "prompt-russian-rhyme", "targetHandle": "prompt", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-table-1686756385002Prompt-prompt-1686756357355prompt"}], "viewport": {"x": 144, "y": 37, "zoom": 1}}, "cache": {"eval-1686756357355.json": {}, "inspect-1686756357355.json": {}, "prompt-1686756357355.json": {}, "table-1686756385002.json": {}, "vis-1686756357355.json": {}}} -------------------------------------------------------------------------------- /chainforge/oaievals/south-african-bands.cforge: -------------------------------------------------------------------------------- 1 | {"flow": {"nodes": [{"width": 312, "height": 311, "id": "prompt-south-african-bands", "type": "prompt", "data": {"prompt": "{prompt}", "n": 1, "llms": [{"key": "aa3c0f03-22bd-416e-af4d-4bf5c4278c99", "settings": {"system_msg": "A user will give you a name of a South African band. Name the lead vocalist, if they use a stage name answer with that instead of real name. Restrict answers to name only, with no further explanations being necessary.", "temperature": 1, "functions": [], "function_call": "", "top_p": 1, "stop": [], "presence_penalty": 0, "frequency_penalty": 0}, "name": "GPT3.5", "emoji": "\ud83d\ude42", "model": "gpt-3.5-turbo", "base_model": "gpt-3.5-turbo", "temp": 1, "formData": {"shortname": "GPT3.5", "model": "gpt-3.5-turbo", "system_msg": "A user will give you a name of a South African band. Name the lead vocalist, if they use a stage name answer with that instead of real name. Restrict answers to name only, with no further explanations being necessary.", "temperature": 1, "functions": "", "function_call": "", "top_p": 1, "stop": "", "presence_penalty": 0, "frequency_penalty": 0}}]}, "position": {"x": 448, "y": 224}, "selected": false, "positionAbsolute": {"x": 448, "y": 224}, "dragging": false}, {"width": 333, "height": 182, "id": "eval-south-african-bands", "type": "evaluator", "data": {"code": "function evaluate(response) {\n\tlet ideal = response.meta['Ideal'];\n\treturn response.text.startsWith(ideal);\n}", "language": "javascript"}, "position": {"x": 820, "y": 150}, "positionAbsolute": {"x": 820, "y": 150}}, {"width": 228, "height": 196, "id": "vis-south-african-bands", "type": "vis", "data": {"input": "eval-south-african-bands"}, "position": {"x": 1200, "y": 250}, "positionAbsolute": {"x": 1200, "y": 250}}, {"width": 302, "height": 260, "id": "inspect-south-african-bands", "type": "inspect", "data": {"input": "prompt-south-african-bands"}, "position": {"x": 820, "y": 400}, "positionAbsolute": {"x": 820, "y": 400}}, {"width": 423, "height": 417, "id": "table-south-african-bands", "type": "table", "data": {"rows": [{"prompt": "Watershed", "ideal": "Craig Hinds"}, {"prompt": "Ballyhoo", "ideal": "Derrick Dryan"}, {"prompt": "Hi-5", "ideal": "Niel Schoombee"}, {"prompt": "Jack Hammer", "ideal": "Piet Botha"}, {"prompt": "The Skylarks", "ideal": "Miriam Makeba"}, {"prompt": "Freshlyground", "ideal": "Zolani Mahola"}, {"prompt": "Savuka", "ideal": "Johnny Clegg"}, {"prompt": "Die Antwoord", "ideal": "Ninja"}, {"prompt": "Caf\u00e9 Society", "ideal": "Julia Jade Aston"}, {"prompt": "Beatenberg", "ideal": "Matthew Field"}, {"prompt": "The Kiffness", "ideal": "David Scott"}, {"prompt": "Mango Groove", "ideal": "Claire Johnston"}, {"prompt": "\u00c9Void", "ideal": "Erik Windrich"}, {"prompt": "The Bang", "ideal": "Joshua Gregg"}, {"prompt": "The Parlotones", "ideal": "Kahn Morbee"}, {"prompt": "Savuka", "ideal": "Johnny Clegg"}, {"prompt": "Henry Ate", "ideal": "Karma-Ann Swanepoel"}, {"prompt": "Lark", "ideal": "Inge Beckmann"}, {"prompt": "Boo!", "ideal": "Chris Chameleon"}, {"prompt": "Little Sister", "ideal": "Debbi Lonmon"}, {"prompt": "Prime Circle", "ideal": "Ross Learmonth"}, {"prompt": "Four Jacks and a Jill", "ideal": "Glenys Lynne"}, {"prompt": "Rabbitt", "ideal": "Trevor Rabin"}, {"prompt": "The Narrow", "ideal": "Hanu de Jong"}, {"prompt": "Just Jinjer", "ideal": "Ard Matthews"}, {"prompt": "Beast", "ideal": "Inge Beckmann"}, {"prompt": "Taxi Violence", "ideal": "George van der Spuy"}, {"prompt": "HONEYMOAN", "ideal": "Alison Rachel"}, {"prompt": "Fire Through the Window", "ideal": "Sinead Dennis"}, {"prompt": "Shortstraw", "ideal": "Alastair Thomas"}], "columns": [{"key": "prompt", "header": "Prompt"}, {"key": "ideal", "header": "Ideal"}]}, "position": {"x": -16, "y": 160}, "selected": false, "positionAbsolute": {"x": -16, "y": 160}, "dragging": false}], "edges": [{"source": "prompt-south-african-bands", "sourceHandle": "prompt", "target": "eval-south-african-bands", "targetHandle": "responseBatch", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-prompt-1686756357355prompt-eval-1686756357355responseBatch"}, {"source": "prompt-south-african-bands", "sourceHandle": "prompt", "target": "inspect-south-african-bands", "targetHandle": "input", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-prompt-1686756357355prompt-inspect-1686756357355input"}, {"source": "eval-south-african-bands", "sourceHandle": "output", "target": "vis-south-african-bands", "targetHandle": "input", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-eval-1686756357355output-vis-1686756357355input"}, {"source": "table-south-african-bands", "sourceHandle": "Prompt", "target": "prompt-south-african-bands", "targetHandle": "prompt", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-table-1686756385002Prompt-prompt-1686756357355prompt"}], "viewport": {"x": 144, "y": 37, "zoom": 1}}, "cache": {"eval-1686756357355.json": {}, "inspect-1686756357355.json": {}, "prompt-1686756357355.json": {}, "table-1686756385002.json": {}, "vis-1686756357355.json": {}}} -------------------------------------------------------------------------------- /chainforge/oaievals/spanish_feminine_noun_masculine_article.cforge: -------------------------------------------------------------------------------- 1 | {"flow": {"nodes": [{"width": 312, "height": 311, "id": "prompt-spanish_feminine_noun_masculine_article", "type": "prompt", "data": {"prompt": "{prompt}", "n": 1, "llms": [{"key": "aa3c0f03-22bd-416e-af4d-4bf5c4278c99", "settings": {"system_msg": "You will be given a singular feminine Spanish word. What article should come before this word \"La\" or \"El\"? Answer with either \"La\" or \"El\" only", "temperature": 1, "functions": [], "function_call": "", "top_p": 1, "stop": [], "presence_penalty": 0, "frequency_penalty": 0}, "name": "GPT3.5", "emoji": "\ud83d\ude42", "model": "gpt-3.5-turbo", "base_model": "gpt-3.5-turbo", "temp": 1, "formData": {"shortname": "GPT3.5", "model": "gpt-3.5-turbo", "system_msg": "You will be given a singular feminine Spanish word. What article should come before this word \"La\" or \"El\"? Answer with either \"La\" or \"El\" only", "temperature": 1, "functions": "", "function_call": "", "top_p": 1, "stop": "", "presence_penalty": 0, "frequency_penalty": 0}}]}, "position": {"x": 448, "y": 224}, "selected": false, "positionAbsolute": {"x": 448, "y": 224}, "dragging": false}, {"width": 333, "height": 182, "id": "eval-spanish_feminine_noun_masculine_article", "type": "evaluator", "data": {"code": "function evaluate(response) {\n\tlet ideals = JSON.parse(response.meta['Ideal']);\n\treturn ideals.some(i => response.text.startsWith(i));\n}", "language": "javascript"}, "position": {"x": 820, "y": 150}, "positionAbsolute": {"x": 820, "y": 150}}, {"width": 228, "height": 196, "id": "vis-spanish_feminine_noun_masculine_article", "type": "vis", "data": {"input": "eval-spanish_feminine_noun_masculine_article"}, "position": {"x": 1200, "y": 250}, "positionAbsolute": {"x": 1200, "y": 250}}, {"width": 302, "height": 260, "id": "inspect-spanish_feminine_noun_masculine_article", "type": "inspect", "data": {"input": "prompt-spanish_feminine_noun_masculine_article"}, "position": {"x": 820, "y": 400}, "positionAbsolute": {"x": 820, "y": 400}}, {"width": 423, "height": 417, "id": "table-spanish_feminine_noun_masculine_article", "type": "table", "data": {"rows": [{"prompt": "agua", "ideal": "[\"El\"]"}, {"prompt": "\u00e1guila", "ideal": "[\"El\"]"}, {"prompt": "ala", "ideal": "[\"El\"]"}, {"prompt": "alba", "ideal": "[\"El\"]"}, {"prompt": "alma", "ideal": "[\"El\"]"}, {"prompt": "ama de casa", "ideal": "[\"El\"]"}, {"prompt": "ancla", "ideal": "[\"El\"]"}, {"prompt": "arma", "ideal": "[\"El\"]"}, {"prompt": "arpa", "ideal": "[\"El\"]"}, {"prompt": "asma", "ideal": "[\"El\"]"}, {"prompt": "ave", "ideal": "[\"El\"]"}, {"prompt": "habla", "ideal": "[\"El\"]"}, {"prompt": "hacha", "ideal": "[\"El\"]"}, {"prompt": "hambre", "ideal": "[\"El\"]"}, {"prompt": "hampa", "ideal": "[\"El\"]"}, {"prompt": "hache", "ideal": "[\"La\"]"}, {"prompt": "a", "ideal": "[\"La\"]"}, {"prompt": "amapola", "ideal": "[\"La\"]"}], "columns": [{"key": "prompt", "header": "Prompt"}, {"key": "ideal", "header": "Ideal"}]}, "position": {"x": -16, "y": 160}, "selected": false, "positionAbsolute": {"x": -16, "y": 160}, "dragging": false}], "edges": [{"source": "prompt-spanish_feminine_noun_masculine_article", "sourceHandle": "prompt", "target": "eval-spanish_feminine_noun_masculine_article", "targetHandle": "responseBatch", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-prompt-1686756357355prompt-eval-1686756357355responseBatch"}, {"source": "prompt-spanish_feminine_noun_masculine_article", "sourceHandle": "prompt", "target": "inspect-spanish_feminine_noun_masculine_article", "targetHandle": "input", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-prompt-1686756357355prompt-inspect-1686756357355input"}, {"source": "eval-spanish_feminine_noun_masculine_article", "sourceHandle": "output", "target": "vis-spanish_feminine_noun_masculine_article", "targetHandle": "input", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-eval-1686756357355output-vis-1686756357355input"}, {"source": "table-spanish_feminine_noun_masculine_article", "sourceHandle": "Prompt", "target": "prompt-spanish_feminine_noun_masculine_article", "targetHandle": "prompt", "interactionWidth": 100, "markerEnd": {"type": "arrow", "width": "22px", "height": "22px"}, "id": "reactflow__edge-table-1686756385002Prompt-prompt-1686756357355prompt"}], "viewport": {"x": 144, "y": 37, "zoom": 1}}, "cache": {"eval-1686756357355.json": {}, "inspect-1686756357355.json": {}, "prompt-1686756357355.json": {}, "table-1686756385002.json": {}, "vis-1686756357355.json": {}}} -------------------------------------------------------------------------------- /chainforge/oaievals/syntax-check.cforge: -------------------------------------------------------------------------------- 1 | {"flow": {"nodes": [{"width": 312, "height": 311, "id": "prompt-syntax-check", "type": "prompt", "data": {"prompt": "{prompt}", "n": 1, "llms": [{"key": "aa3c0f03-22bd-416e-af4d-4bf5c4278c99", "settings": {"system_msg": "You are an IDE plugin assistant. Determine the named programming language or framework based on the partial code snippet. Respond only with the name.", "temperature": 1, "functions": [], "function_call": "", "top_p": 1, "stop": [], "presence_penalty": 0, "frequency_penalty": 0}, "name": "GPT3.5", "emoji": "\ud83d\ude42", "model": "gpt-3.5-turbo", "base_model": "gpt-3.5-turbo", "temp": 1, "formData": {"shortname": "GPT3.5", "model": "gpt-3.5-turbo", "system_msg": "You are an IDE plugin assistant. Determine the named programming language or framework based on the partial code snippet. Respond only with the name.", "temperature": 1, "functions": "", "function_call": "", "top_p": 1, "stop": "", "presence_penalty": 0, "frequency_penalty": 0}}]}, "position": {"x": 448, "y": 224}, "selected": false, "positionAbsolute": {"x": 448, "y": 224}, "dragging": false}, {"width": 333, "height": 182, "id": "eval-syntax-check", "type": "evaluator", "data": {"code": "function evaluate(response) {\n\tlet ideal = response.meta['Ideal'];\n\treturn response.text.startsWith(ideal);\n}", "language": "javascript"}, "position": {"x": 820, "y": 150}, "positionAbsolute": {"x": 820, "y": 150}}, {"width": 228, "height": 196, "id": "vis-syntax-check", "type": "vis", "data": {"input": "eval-syntax-check"}, "position": {"x": 1200, "y": 250}, "positionAbsolute": {"x": 1200, "y": 250}}, {"width": 302, "height": 260, "id": "inspect-syntax-check", "type": "inspect", "data": {"input": "prompt-syntax-check"}, "position": {"x": 820, "y": 400}, "positionAbsolute": {"x": 820, "y": 400}}, {"width": 423, "height": 417, "id": "table-syntax-check", "type": "table", "data": {"rows": [{"prompt": "const c=20;", "ideal": "JavaScript"}, {"prompt": "() => {}", "ideal": "JavaScript"}, {"prompt": "", "ideal": "HTML"}, {"prompt": "puts [9,1]", "ideal": "Ruby"}, {"prompt": "ActionController::Base.render json: {data: @data}", "ideal": "Ruby on Rails"}, {"prompt": "(1..10).sum", "ideal": "Ruby"}, {"prompt": "const [items, setItems] = useState([])", "ideal": "React"}, {"prompt": "i in range(10)", "ideal": "Python"}, {"prompt": "sum [1..10]", "ideal": "Haskell"}, {"prompt": "map { $x += $_ } @a;", "ideal": "Perl"}, {"prompt": "import os", "ideal": "Python"}, {"prompt": "

Welcome, {{ username|capitalize }}!

", "ideal": "Jinja"}, {"prompt": "