├── .coveragerc ├── .flake8 ├── .github └── workflows │ ├── black.yml │ ├── lint.yml │ ├── mypy.yml │ ├── publish.yml │ ├── test-js.yml │ └── unit-test.yml ├── .gitignore ├── .readthedocs.yaml ├── LICENSE ├── MANIFEST.in ├── README.md ├── arthur_bench ├── __init__.py ├── client │ ├── __init__.py │ ├── auth │ │ ├── __init__.py │ │ ├── helpers.py │ │ └── refresh.py │ ├── bench_client.py │ ├── http │ │ ├── __init__.py │ │ ├── helper.py │ │ ├── requests.py │ │ └── validation.py │ ├── local │ │ ├── __init__.py │ │ └── client.py │ ├── rest │ │ ├── __init__.py │ │ ├── admin │ │ │ ├── __init__.py │ │ │ ├── client.py │ │ │ └── models.py │ │ ├── bench │ │ │ ├── __init__.py │ │ │ └── client.py │ │ └── client.py │ └── utils.py ├── exceptions │ ├── __init__.py │ └── exceptions.py ├── logger │ ├── __init__.py │ └── logger.py ├── models │ ├── __init__.py │ ├── client.py │ ├── models.py │ └── scoring.py ├── run │ ├── __init__.py │ ├── testrun.py │ ├── testsuite.py │ └── utils.py ├── scoring │ ├── __init__.py │ ├── bertscore.py │ ├── exact_match.py │ ├── hallucination.py │ ├── hedging_language.py │ ├── prompts │ │ ├── qa_correctness.py │ │ └── summary_quality.py │ ├── python_unit_testing.py │ ├── qa_quality.py │ ├── readability.py │ ├── scorer.py │ ├── specificity.py │ ├── summary_quality.py │ ├── utils.py │ └── word_count_match.py ├── server │ ├── __init__.py │ ├── js │ │ ├── index.tsx │ │ ├── package.json │ │ ├── packages │ │ │ ├── api │ │ │ │ ├── comparators.types.ts │ │ │ │ ├── package.json │ │ │ │ └── useOnClickOutside.ts │ │ │ ├── arthur-axios │ │ │ │ ├── index.ts │ │ │ │ └── package.json │ │ │ ├── arthur-redux │ │ │ │ ├── config │ │ │ │ │ ├── configureStore.prod.ts │ │ │ │ │ ├── configureStore.ts │ │ │ │ │ ├── rootReducer.ts │ │ │ │ │ └── state.type.ts │ │ │ │ ├── index.ts │ │ │ │ ├── package.json │ │ │ │ └── slices │ │ │ │ │ ├── skin │ │ │ │ │ ├── skin.constants.ts │ │ │ │ │ ├── skin.reducers.ts │ │ │ │ │ └── skin.type.ts │ │ │ │ │ └── testSuites │ │ │ │ │ ├── actions.ts │ │ │ │ │ ├── constants.ts │ │ │ │ │ ├── reducers.ts │ │ │ │ │ └── types.ts │ │ │ ├── components │ │ │ │ ├── compound │ │ │ │ │ ├── Chip │ │ │ │ │ │ ├── Chip.tsx │ │ │ │ │ │ ├── index.ts │ │ │ │ │ │ ├── styles.ts │ │ │ │ │ │ └── types.ts │ │ │ │ │ ├── FilterButton │ │ │ │ │ │ ├── FilterButton.tsx │ │ │ │ │ │ ├── index.ts │ │ │ │ │ │ └── styles.ts │ │ │ │ │ ├── FilterComponent │ │ │ │ │ │ ├── FilterComponent.styles.ts │ │ │ │ │ │ ├── FilterComponent.tsx │ │ │ │ │ │ ├── FilterComponentProps.types.ts │ │ │ │ │ │ └── index.tsx │ │ │ │ │ ├── FloatingHelp │ │ │ │ │ │ ├── FloatingHelp.tsx │ │ │ │ │ │ ├── index.tsx │ │ │ │ │ │ └── styles.ts │ │ │ │ │ ├── InputsOutputsTable │ │ │ │ │ │ └── InputsOutputsTable.tsx │ │ │ │ │ ├── InsightCard │ │ │ │ │ │ ├── InsightCard.tsx │ │ │ │ │ │ ├── styles.ts │ │ │ │ │ │ ├── types.ts │ │ │ │ │ │ └── utils.ts │ │ │ │ │ ├── InsightHeadline │ │ │ │ │ │ ├── InsightHeadline.tsx │ │ │ │ │ │ ├── index.ts │ │ │ │ │ │ └── styles.ts │ │ │ │ │ ├── Layout │ │ │ │ │ │ ├── Layout.tsx │ │ │ │ │ │ └── styles.ts │ │ │ │ │ ├── Loading │ │ │ │ │ │ └── index.tsx │ │ │ │ │ ├── MetricsDropdownSelection │ │ │ │ │ │ ├── MetricDropdownSelection.tsx │ │ │ │ │ │ ├── index.ts │ │ │ │ │ │ ├── styles.ts │ │ │ │ │ │ └── typings.d.ts │ │ │ │ │ ├── ModeSelector │ │ │ │ │ │ ├── ModeSelector.tsx │ │ │ │ │ │ ├── index.ts │ │ │ │ │ │ └── styles.ts │ │ │ │ │ ├── MultipleSelect │ │ │ │ │ │ ├── MultipleSelect.tsx │ │ │ │ │ │ └── styles.ts │ │ │ │ │ ├── SortDropdown │ │ │ │ │ │ ├── SortDropdown.tsx │ │ │ │ │ │ └── styles.ts │ │ │ │ │ ├── SummaryVisualizations │ │ │ │ │ │ ├── AverageScores.tsx │ │ │ │ │ │ ├── CategoricalDistribution.tsx │ │ │ │ │ │ ├── RunDistributions.tsx │ │ │ │ │ │ ├── SummaryVisualizations.tsx │ │ │ │ │ │ └── styles.ts │ │ │ │ │ ├── TestRunDeepDive │ │ │ │ │ │ ├── CompareModal.tsx │ │ │ │ │ │ ├── CompareTable.tsx │ │ │ │ │ │ ├── Header.tsx │ │ │ │ │ │ ├── TestRunDeepDive.tsx │ │ │ │ │ │ └── styles.ts │ │ │ │ │ ├── TestRunTable │ │ │ │ │ │ ├── TestRunHeader.tsx │ │ │ │ │ │ ├── TestRunRow.tsx │ │ │ │ │ │ ├── TestRunTable.tsx │ │ │ │ │ │ └── styles.ts │ │ │ │ │ ├── TestSuiteCard │ │ │ │ │ │ ├── MethodTag.tsx │ │ │ │ │ │ ├── TestSuiteCard.tsx │ │ │ │ │ │ └── styles.ts │ │ │ │ │ ├── TestSuiteDetailsModal │ │ │ │ │ │ ├── GeneralInformationTab.tsx │ │ │ │ │ │ ├── ScorerInformationTab.tsx │ │ │ │ │ │ ├── TestSuiteDetailsModal.tsx │ │ │ │ │ │ └── styles.ts │ │ │ │ │ ├── TestSuitesHeader │ │ │ │ │ │ ├── TestSuitesHeader.tsx │ │ │ │ │ │ └── styles.ts │ │ │ │ │ └── WelcomeModal │ │ │ │ │ │ ├── WelcomeModal.tsx │ │ │ │ │ │ └── styles.ts │ │ │ │ ├── core │ │ │ │ │ ├── Breadcrumbs │ │ │ │ │ │ ├── Breadcrumbs.tsx │ │ │ │ │ │ ├── index.ts │ │ │ │ │ │ └── styles.ts │ │ │ │ │ ├── Button │ │ │ │ │ │ ├── Button.tsx │ │ │ │ │ │ ├── constants.ts │ │ │ │ │ │ ├── index.ts │ │ │ │ │ │ ├── styles.ts │ │ │ │ │ │ ├── typings.d.ts │ │ │ │ │ │ ├── typings.ts │ │ │ │ │ │ └── utils.ts │ │ │ │ │ ├── Charts │ │ │ │ │ │ ├── BarChart │ │ │ │ │ │ │ ├── BarChart.stories.tsx │ │ │ │ │ │ │ ├── BarChart.tsx │ │ │ │ │ │ │ └── index.ts │ │ │ │ │ │ ├── Legend │ │ │ │ │ │ │ ├── Legend.tsx │ │ │ │ │ │ │ └── styles.ts │ │ │ │ │ │ ├── LineChart │ │ │ │ │ │ │ ├── LineChart.stories.tsx │ │ │ │ │ │ │ ├── LineChart.tsx │ │ │ │ │ │ │ ├── components │ │ │ │ │ │ │ │ ├── MultiLineTooltip.tsx │ │ │ │ │ │ │ │ └── styles.ts │ │ │ │ │ │ │ ├── index.ts │ │ │ │ │ │ │ └── typings.d.ts │ │ │ │ │ │ └── constants.ts │ │ │ │ │ ├── Checkbox │ │ │ │ │ │ ├── Checkbox.styles.ts │ │ │ │ │ │ ├── Checkbox.tsx │ │ │ │ │ │ ├── index.ts │ │ │ │ │ │ └── typings.d.ts │ │ │ │ │ ├── Collapsible │ │ │ │ │ │ ├── Collapsible.tsx │ │ │ │ │ │ ├── index.ts │ │ │ │ │ │ ├── styles.ts │ │ │ │ │ │ └── typings.d.ts │ │ │ │ │ ├── Dropdown │ │ │ │ │ │ ├── Dropdown.tsx │ │ │ │ │ │ ├── index.ts │ │ │ │ │ │ └── styles.ts │ │ │ │ │ ├── Field │ │ │ │ │ │ ├── Field.styles.ts │ │ │ │ │ │ ├── Field.tsx │ │ │ │ │ │ ├── index.ts │ │ │ │ │ │ └── types.ts │ │ │ │ │ ├── HelpTile │ │ │ │ │ │ ├── HelpTile.tsx │ │ │ │ │ │ ├── index.ts │ │ │ │ │ │ ├── styles.ts │ │ │ │ │ │ └── typings.d.ts │ │ │ │ │ ├── Icon │ │ │ │ │ │ ├── Icon.tsx │ │ │ │ │ │ ├── constants.ts │ │ │ │ │ │ ├── index.ts │ │ │ │ │ │ ├── readme.md │ │ │ │ │ │ └── types.ts │ │ │ │ │ ├── Modal │ │ │ │ │ │ ├── Modal.tsx │ │ │ │ │ │ └── styles.ts │ │ │ │ │ ├── Paginator │ │ │ │ │ │ ├── Paginator.tsx │ │ │ │ │ │ ├── index.ts │ │ │ │ │ │ ├── styles.ts │ │ │ │ │ │ └── typings.d.ts │ │ │ │ │ ├── PopUp │ │ │ │ │ │ ├── PopUp.tsx │ │ │ │ │ │ ├── PopUpHeader │ │ │ │ │ │ │ ├── PopUpHeader.tsx │ │ │ │ │ │ │ └── styles.tsx │ │ │ │ │ │ └── styles.ts │ │ │ │ │ ├── Search │ │ │ │ │ │ ├── Search.styles.ts │ │ │ │ │ │ ├── Search.tsx │ │ │ │ │ │ ├── index.ts │ │ │ │ │ │ └── types.ts │ │ │ │ │ ├── StyledSelect │ │ │ │ │ │ ├── StyledSelect.tsx │ │ │ │ │ │ ├── index.ts │ │ │ │ │ │ └── styles.ts │ │ │ │ │ ├── Table │ │ │ │ │ │ ├── Table.tsx │ │ │ │ │ │ ├── components │ │ │ │ │ │ │ ├── ExpandableTableCell.tsx │ │ │ │ │ │ │ ├── TableBody.tsx │ │ │ │ │ │ │ ├── TableCell.tsx │ │ │ │ │ │ │ ├── TableFooter.tsx │ │ │ │ │ │ │ ├── TableHeader.tsx │ │ │ │ │ │ │ └── TableRow.tsx │ │ │ │ │ │ ├── index.ts │ │ │ │ │ │ ├── styles.ts │ │ │ │ │ │ └── typings.d.ts │ │ │ │ │ ├── Tabs │ │ │ │ │ │ ├── Tab.tsx │ │ │ │ │ │ ├── Tabs.tsx │ │ │ │ │ │ ├── index.ts │ │ │ │ │ │ └── styles.ts │ │ │ │ │ ├── Toggle │ │ │ │ │ │ ├── Toggle.styles.ts │ │ │ │ │ │ ├── Toggle.tsx │ │ │ │ │ │ ├── Toggle.types.ts │ │ │ │ │ │ └── index.ts │ │ │ │ │ └── Tooltip │ │ │ │ │ │ ├── Tooltip.styles.ts │ │ │ │ │ │ └── Tooltip.tsx │ │ │ │ ├── helpers │ │ │ │ │ └── compose.ts │ │ │ │ ├── package.json │ │ │ │ ├── tsconfig.json │ │ │ │ └── typings.d.ts │ │ │ ├── resources │ │ │ │ ├── colors │ │ │ │ │ └── Arthur │ │ │ │ │ │ ├── graphs.ts │ │ │ │ │ │ ├── primary.ts │ │ │ │ │ │ └── secondary.ts │ │ │ │ ├── fonts │ │ │ │ │ ├── Graphik │ │ │ │ │ │ ├── Graphik-Black.otf │ │ │ │ │ │ ├── Graphik-BlackItalic.otf │ │ │ │ │ │ ├── Graphik-BlackItalic.woff │ │ │ │ │ │ ├── Graphik-BlackItalic.woff2 │ │ │ │ │ │ ├── Graphik-Bold.otf │ │ │ │ │ │ ├── Graphik-Bold.woff │ │ │ │ │ │ ├── Graphik-Bold.woff2 │ │ │ │ │ │ ├── Graphik-BoldItalic.otf │ │ │ │ │ │ ├── Graphik-BoldItalic.woff │ │ │ │ │ │ ├── Graphik-BoldItalic.woff2 │ │ │ │ │ │ ├── Graphik-Extralight.otf │ │ │ │ │ │ ├── Graphik-Extralight.woff │ │ │ │ │ │ ├── Graphik-Extralight.woff2 │ │ │ │ │ │ ├── Graphik-ExtralightItalic.otf │ │ │ │ │ │ ├── Graphik-ExtralightItalic.woff │ │ │ │ │ │ ├── Graphik-ExtralightItalic.woff2 │ │ │ │ │ │ ├── Graphik-Light.otf │ │ │ │ │ │ ├── Graphik-Light.woff │ │ │ │ │ │ ├── Graphik-Light.woff2 │ │ │ │ │ │ ├── Graphik-LightItalic.otf │ │ │ │ │ │ ├── Graphik-LightItalic.woff │ │ │ │ │ │ ├── Graphik-LightItalic.woff2 │ │ │ │ │ │ ├── Graphik-Medium.otf │ │ │ │ │ │ ├── Graphik-Medium.woff │ │ │ │ │ │ ├── Graphik-Medium.woff2 │ │ │ │ │ │ ├── Graphik-MediumItalic.otf │ │ │ │ │ │ ├── Graphik-MediumItalic.woff │ │ │ │ │ │ ├── Graphik-MediumItalic.woff2 │ │ │ │ │ │ ├── Graphik-Regular.otf │ │ │ │ │ │ ├── Graphik-Regular.woff │ │ │ │ │ │ ├── Graphik-Regular.woff2 │ │ │ │ │ │ ├── Graphik-RegularItalic.otf │ │ │ │ │ │ ├── Graphik-Semibold.otf │ │ │ │ │ │ ├── Graphik-Semibold.woff │ │ │ │ │ │ ├── Graphik-Semibold.woff2 │ │ │ │ │ │ ├── Graphik-SemiboldItalic.otf │ │ │ │ │ │ ├── Graphik-SemiboldItalic.woff │ │ │ │ │ │ ├── Graphik-SemiboldItalic.woff2 │ │ │ │ │ │ ├── Graphik-Super.otf │ │ │ │ │ │ ├── Graphik-Super.woff │ │ │ │ │ │ ├── Graphik-Super.woff2 │ │ │ │ │ │ ├── Graphik-SuperItalic.otf │ │ │ │ │ │ ├── Graphik-SuperItalic.woff │ │ │ │ │ │ ├── Graphik-SuperItalic.woff2 │ │ │ │ │ │ ├── Graphik-Thin.otf │ │ │ │ │ │ ├── Graphik-Thin.woff │ │ │ │ │ │ ├── Graphik-Thin.woff2 │ │ │ │ │ │ ├── Graphik-ThinItalic.otf │ │ │ │ │ │ ├── Graphik-ThinItalic.woff │ │ │ │ │ │ └── Graphik-ThinItalic.woff2 │ │ │ │ │ ├── IBM_Plex_Mono │ │ │ │ │ │ ├── IBMPlexMono-Bold.ttf │ │ │ │ │ │ ├── IBMPlexMono-BoldItalic.ttf │ │ │ │ │ │ ├── IBMPlexMono-ExtraLight.ttf │ │ │ │ │ │ ├── IBMPlexMono-ExtraLightItalic.ttf │ │ │ │ │ │ ├── IBMPlexMono-Italic.ttf │ │ │ │ │ │ ├── IBMPlexMono-Light.ttf │ │ │ │ │ │ ├── IBMPlexMono-LightItalic.ttf │ │ │ │ │ │ ├── IBMPlexMono-Medium.ttf │ │ │ │ │ │ ├── IBMPlexMono-MediumItalic.ttf │ │ │ │ │ │ ├── IBMPlexMono-Regular.ttf │ │ │ │ │ │ ├── IBMPlexMono-SemiBold.ttf │ │ │ │ │ │ ├── IBMPlexMono-SemiBoldItalic.ttf │ │ │ │ │ │ ├── IBMPlexMono-Thin.ttf │ │ │ │ │ │ ├── IBMPlexMono-ThinItalic.ttf │ │ │ │ │ │ └── OFL.txt │ │ │ │ │ ├── Lato │ │ │ │ │ │ ├── Lato-Black.ttf │ │ │ │ │ │ ├── Lato-BlackItalic.ttf │ │ │ │ │ │ ├── Lato-Bold.ttf │ │ │ │ │ │ ├── Lato-BoldItalic.ttf │ │ │ │ │ │ ├── Lato-Italic.ttf │ │ │ │ │ │ ├── Lato-Light.ttf │ │ │ │ │ │ ├── Lato-LightItalic.ttf │ │ │ │ │ │ ├── Lato-Regular.ttf │ │ │ │ │ │ ├── Lato-Thin.ttf │ │ │ │ │ │ ├── Lato-ThinItalic.ttf │ │ │ │ │ │ └── OFL.txt │ │ │ │ │ ├── fonts.css │ │ │ │ │ └── index.ts │ │ │ │ ├── icons │ │ │ │ │ ├── constants.ts │ │ │ │ │ ├── selection.js │ │ │ │ │ └── selection.json │ │ │ │ ├── images │ │ │ │ │ ├── Arthur_Logo_PBW.svg │ │ │ │ │ ├── Arthur_Logo_Symbol_low_margin.svg │ │ │ │ │ └── welcome.svg │ │ │ │ ├── package.json │ │ │ │ ├── style-constants │ │ │ │ │ └── index.ts │ │ │ │ └── theme │ │ │ │ │ ├── button │ │ │ │ │ └── index.ts │ │ │ │ │ ├── light.ts │ │ │ │ │ ├── tagSelector │ │ │ │ │ └── index.ts │ │ │ │ │ └── types.ts │ │ │ ├── translations │ │ │ │ ├── constants.ts │ │ │ │ ├── en │ │ │ │ │ ├── chat.json │ │ │ │ │ ├── common.json │ │ │ │ │ ├── element.json │ │ │ │ │ ├── overview.json │ │ │ │ │ ├── roundtable.json │ │ │ │ │ └── shield.json │ │ │ │ ├── es │ │ │ │ │ ├── chatEs.json │ │ │ │ │ ├── commonEs.json │ │ │ │ │ ├── elementEs.json │ │ │ │ │ ├── roundtableEs.json │ │ │ │ │ └── shieldEs.json │ │ │ │ ├── hooks.ts │ │ │ │ ├── jp │ │ │ │ │ ├── chatJp.json │ │ │ │ │ ├── common.json │ │ │ │ │ ├── commonJp.json │ │ │ │ │ ├── elementJp.json │ │ │ │ │ ├── roundtableJp.json │ │ │ │ │ └── shieldJp.json │ │ │ │ └── types.ts │ │ │ ├── tsconfig │ │ │ │ ├── README.md │ │ │ │ ├── base.json │ │ │ │ ├── nextjs.json │ │ │ │ ├── package.json │ │ │ │ └── react-library.json │ │ │ └── utils │ │ │ │ ├── .eslintrc.js │ │ │ │ ├── README.md │ │ │ │ ├── babel.config.js │ │ │ │ ├── capitalize-first-letter │ │ │ │ ├── capitalizeFirstLetter.test.js │ │ │ │ └── index.ts │ │ │ │ ├── chunk │ │ │ │ └── index.ts │ │ │ │ ├── format-test-runs │ │ │ │ └── format-test-runs.tsx │ │ │ │ ├── format │ │ │ │ ├── abbreviateNumber │ │ │ │ │ ├── abbreviateNumber.test.js │ │ │ │ │ └── index.ts │ │ │ │ └── date-label-formatter │ │ │ │ │ └── index.ts │ │ │ │ ├── helpers │ │ │ │ └── index.ts │ │ │ │ ├── hex-to-rgba │ │ │ │ └── hex-to-rgba.tsx │ │ │ │ ├── is-nil │ │ │ │ ├── index.ts │ │ │ │ └── isNil.test.js │ │ │ │ ├── jest.config.ts │ │ │ │ ├── keypress-enter │ │ │ │ └── index.ts │ │ │ │ ├── node_modules │ │ │ │ └── .bin │ │ │ │ │ ├── jest │ │ │ │ │ └── ts-jest │ │ │ │ ├── package.json │ │ │ │ ├── parse-int │ │ │ │ ├── index.ts │ │ │ │ └── parseInt.test.js │ │ │ │ ├── range │ │ │ │ └── index.ts │ │ │ │ ├── scroll-to-bottom │ │ │ │ └── scroll-to-bottom.tsx │ │ │ │ └── tsconfig.json │ │ ├── public │ │ │ ├── arthur-triangle-favicon.png │ │ │ └── index.html │ │ ├── src │ │ │ ├── App.tsx │ │ │ ├── Bench │ │ │ │ ├── CompareTestRuns.tsx │ │ │ │ ├── InputsOutputs.tsx │ │ │ │ ├── TestRun.tsx │ │ │ │ ├── TestRuns.tsx │ │ │ │ ├── TestSuiteHeader.tsx │ │ │ │ ├── TestSuites.tsx │ │ │ │ ├── index.tsx │ │ │ │ ├── styles.ts │ │ │ │ └── useTestSuites.ts │ │ │ ├── Skin │ │ │ │ └── skin.tsx │ │ │ ├── copy │ │ │ │ └── initCopy.ts │ │ │ ├── index.html │ │ │ ├── renderer.ts │ │ │ └── routes │ │ │ │ └── index.tsx │ │ ├── tsconfig.json │ │ └── webpack-config │ │ │ └── config.js │ ├── run_server.py │ └── spa_static_files.py ├── telemetry │ ├── __init__.py │ ├── config.py │ └── telemetry.py ├── utils │ ├── __init__.py │ └── loaders.py └── version.py ├── docs ├── Makefile ├── make.bat └── source │ ├── _static │ ├── custom_styles_20221207.css │ └── img │ │ ├── Bench_UI_Screenshot.png │ │ ├── Reference_df.png │ │ ├── dark-mode-logo.svg │ │ └── test_suite_run.png │ ├── _templates │ ├── module.rst_t │ ├── package.rst_t │ └── toc.rst_t │ ├── add_scorer_config.md │ ├── code_evaluation.md │ ├── compare_generation_settings.md │ ├── compare_llm_providers.md │ ├── compare_prompts.md │ ├── concepts.md │ ├── conf.py │ ├── contributing.md │ ├── creating_test_suites.md │ ├── custom_scoring.md │ ├── guides.rst │ ├── index.rst │ ├── quickstart.md │ ├── requirements.txt │ ├── scorers.rst │ ├── scoring.md │ ├── sdk_reference.rst │ ├── setup.md │ └── telemetry.md ├── examples ├── bedrock_client │ ├── __init__.py │ └── client.py ├── bench_runs │ ├── news_summary │ │ ├── corrupt │ │ │ └── run.json │ │ ├── longt5books │ │ │ └── run.json │ │ ├── rephrase │ │ │ └── run.json │ │ ├── run_id_to_name.json │ │ └── suite.json │ ├── stack_dist_to_golden │ │ ├── prompt1 │ │ │ └── run.json │ │ ├── prompt39842 │ │ │ └── run.json │ │ ├── run_id_to_name.json │ │ └── suite.json │ └── suite_id_to_name.json ├── data │ ├── news_summary │ │ └── example_summaries.csv │ └── stackoverflow_qa │ │ ├── gpt35_base.csv │ │ ├── gpt35_engineered.csv │ │ └── stack_overflow_golden.csv ├── demo_specificity.ipynb ├── elo.py ├── specificity │ ├── eli5_25.csv │ └── llm_scores_specificity.csv └── summarize_scoring_using_bedrock.ipynb ├── pyproject.toml ├── requirements.txt └── tests ├── __init__.py ├── client ├── __init__.py ├── test_local_client.py └── test_rest_client.py ├── conftest.py ├── fixtures ├── __init__.py ├── mock_data.py ├── mock_file_system │ ├── suite_id_to_name.json │ ├── test_suite │ │ ├── run_id_to_name.json │ │ └── suite.json │ └── test_suite_custom │ │ ├── run_id_to_name.json │ │ └── suite.json ├── mock_file_system_with_runs │ ├── suite_id_to_name.json │ ├── test_suite │ │ ├── run_id_to_name.json │ │ ├── suite.json │ │ └── test_run │ │ │ └── run.json │ └── test_suite_custom │ │ ├── run_id_to_name.json │ │ └── suite.json ├── mock_python_unit_tests │ ├── mock_unit_test_0.py │ └── mock_unit_test_1.py ├── mock_requests.py ├── mock_responses.py ├── mock_suite.csv ├── mock_suite.json ├── mock_suite_mixed_null_refs.json ├── mock_suite_with_typed_score.json └── mock_suite_without_optional.json ├── helpers.py ├── mock_suite.csv ├── requirements.txt ├── scorers ├── __init__.py ├── test_bertscore.py ├── test_hallucination.py ├── test_hedging_language.py ├── test_helpfulness.py ├── test_python_unit_testing.py ├── test_scoring_method.py └── test_summary_quality.py ├── test_e2e.py ├── test_server.py ├── test_testrun.py ├── test_testsuite.py └── test_utils.py /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | omit = 3 | arthur_bench/client/auth/* 4 | arthur_bench/client/http/* 5 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 88 3 | extend-ignore = E203 4 | -------------------------------------------------------------------------------- /.github/workflows/black.yml: -------------------------------------------------------------------------------- 1 | name: Black formatting 2 | 3 | on: [pull_request] 4 | 5 | jobs: 6 | lint: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v3 10 | - uses: psf/black@stable 11 | with: 12 | src: arthur_bench -------------------------------------------------------------------------------- /.github/workflows/lint.yml: -------------------------------------------------------------------------------- 1 | name: lint 2 | 3 | on: [pull_request] 4 | 5 | 6 | permissions: 7 | contents: read 8 | 9 | jobs: 10 | build: 11 | 12 | runs-on: ubuntu-latest 13 | 14 | steps: 15 | - uses: actions/checkout@v3 16 | - name: Set up Python 3.10 17 | uses: actions/setup-python@v3 18 | with: 19 | python-version: "3.10" 20 | - name: Install dependencies 21 | run: | 22 | python -m pip install --upgrade pip 23 | pip install flake8 24 | if [ -f requirements.txt ]; then pip install -r requirements.txt; fi 25 | - name: lint 26 | run: | 27 | flake8 arthur_bench -------------------------------------------------------------------------------- /.github/workflows/mypy.yml: -------------------------------------------------------------------------------- 1 | name: mypy 2 | 3 | on: [pull_request] 4 | 5 | 6 | permissions: 7 | contents: read 8 | 9 | jobs: 10 | build: 11 | 12 | runs-on: ubuntu-latest 13 | 14 | strategy: 15 | matrix: 16 | python-version: ["3.9", "3.10", "3.11"] 17 | 18 | steps: 19 | - uses: actions/checkout@v3 20 | - name: Set up Python ${{ matrix.python-version }} 21 | uses: actions/setup-python@v3 22 | with: 23 | python-version: ${{ matrix.python-version }} 24 | - name: Install dependencies 25 | run: | 26 | python -m pip install --upgrade pip 27 | pip install mypy types-protobuf==4.24.0.4 28 | if [ -f requirements.txt ]; then pip install -r requirements.txt; fi 29 | - name: mypy 30 | run: | 31 | mkdir .mypy_cache -p 32 | mypy arthur_bench --ignore-missing-imports --install-types --non-interactive -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | # This workflow builds the arthur_bench/server/js directory and then builds the python 2 | # package and publishes to PyPi 3 | 4 | name: Upload Python Package 5 | 6 | on: 7 | push: 8 | tags: 9 | - '*' 10 | 11 | jobs: 12 | 13 | deploy: 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/checkout@v3 17 | - name: Use Node.js 18 | uses: actions/setup-node@v3 19 | with: 20 | node-version: '18.x' 21 | - name: Install dependencies 22 | working-directory: arthur_bench/server/js 23 | run: npm i 24 | - name: Build 25 | working-directory: arthur_bench/server/js 26 | run: npm run build 27 | - name: Set up Python 28 | uses: actions/setup-python@v4 29 | with: 30 | python-version: '3.x' 31 | - name: Install dependencies 32 | run: | 33 | python -m pip install --upgrade pip 34 | pip install pyyaml twine 35 | pip install --upgrade build 36 | - name: Build package 37 | run: | 38 | python -m build --sdist --wheel --outdir dist 39 | - name: Publish package 40 | uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 41 | with: 42 | user: __token__ 43 | password: ${{ secrets.PYPI_API_TOKEN }} 44 | -------------------------------------------------------------------------------- /.github/workflows/test-js.yml: -------------------------------------------------------------------------------- 1 | name: test-frontend 2 | 3 | on: 4 | push: 5 | paths: 6 | - arthur_bench/server/js/** 7 | - .github/workflows/test-js.yml 8 | 9 | pull_request: 10 | paths: 11 | - arthur_bench/server/js/** 12 | - .github/workflows/test-js.yml 13 | 14 | jobs: 15 | build: 16 | 17 | runs-on: ubuntu-latest 18 | 19 | steps: 20 | - uses: actions/checkout@v3 21 | - name: Use Node.js 22 | uses: actions/setup-node@v3 23 | with: 24 | node-version: '18.x' 25 | - name: Install dependencies 26 | working-directory: arthur_bench/server/js 27 | run: npm i 28 | - name: Build 29 | working-directory: arthur_bench/server/js 30 | run: npm run build 31 | -------------------------------------------------------------------------------- /.github/workflows/unit-test.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a single version of Python 2 | # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python 3 | 4 | name: Unit Test 5 | 6 | on: [pull_request] 7 | 8 | 9 | permissions: 10 | contents: read 11 | 12 | jobs: 13 | build: 14 | strategy: 15 | matrix: 16 | os: [ubuntu-latest, windows-latest] 17 | python-version: ["3.9", "3.10", "3.11"] 18 | 19 | runs-on: ${{ matrix.os }} 20 | steps: 21 | - uses: actions/checkout@v3 22 | - name: Set up Python ${{ matrix.python-version }} 23 | uses: actions/setup-python@v3 24 | with: 25 | python-version: ${{ matrix.python-version }} 26 | - name: Install dependencies 27 | run: | 28 | python -m pip install --upgrade pip 29 | pip install pytest 30 | pip install -r requirements.txt 31 | 32 | - name: Test with pytest 33 | run: | 34 | pip install -r tests/requirements.txt 35 | pytest --cov=arthur_bench tests/ 36 | 37 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | ipynb_checkpoints/ 3 | __pycache__/ 4 | .idea 5 | scratch/ 6 | .vscode/ 7 | 8 | # Distribution / packaging 9 | .Python 10 | #build/ 11 | develop-eggs/ 12 | #dist/ 13 | downloads/ 14 | eggs/ 15 | .eggs/ 16 | lib/ 17 | lib64/ 18 | parts/ 19 | sdist/ 20 | var/ 21 | wheels/ 22 | bin/ 23 | lib/ 24 | lib64/ 25 | share/ 26 | include/ 27 | *.egg-info/ 28 | .installed.cfg 29 | *.egg 30 | arthur_bench/server/js/dist/* 31 | arthur_bench/server/js/node_modules/* 32 | arthur_bench/server/js/packages/*/node_modules/ 33 | 34 | bench_runs 35 | *package-lock.json 36 | 37 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # Read the Docs configuration file for Sphinx projects 2 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 3 | 4 | # Required 5 | version: 2 6 | 7 | # Set the OS, Python version and other tools you might need 8 | build: 9 | os: ubuntu-22.04 10 | tools: 11 | python: "3.11" 12 | # You can also specify other tool versions: 13 | # nodejs: "20" 14 | # rust: "1.70" 15 | # golang: "1.20" 16 | jobs: 17 | pre_build: 18 | - sphinx-apidoc -o docs/source/sdk arthur_bench "arthur_bench/logger/*" "arthur_bench/models/client.py" -MET --templatedir docs/source/_templates 19 | 20 | 21 | # Build documentation in the "docs/" directory with Sphinx 22 | sphinx: 23 | configuration: docs/source/conf.py 24 | # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs 25 | # builder: "dirhtml" 26 | # Fail on all warnings to avoid broken references 27 | # fail_on_warning: true 28 | 29 | # Optionally build your docs in additional formats such as PDF and ePub 30 | # formats: 31 | # - pdf 32 | # - epub 33 | 34 | # Optional but recommended, declare the Python requirements required 35 | # to build your documentation 36 | # See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html 37 | python: 38 | install: 39 | - requirements: docs/source/requirements.txt -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Arthur 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include arthur_bench/server/js/dist * 2 | -------------------------------------------------------------------------------- /arthur_bench/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | from .version import __version__ 3 | -------------------------------------------------------------------------------- /arthur_bench/client/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | from .bench_client import BenchClient 3 | from .utils import _get_bench_client 4 | -------------------------------------------------------------------------------- /arthur_bench/client/auth/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arthur-ai/bench/6b714496b0b73179908a3b7e713010911cd8f1f7/arthur_bench/client/auth/__init__.py -------------------------------------------------------------------------------- /arthur_bench/client/http/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arthur-ai/bench/6b714496b0b73179908a3b7e713010911cd8f1f7/arthur_bench/client/http/__init__.py -------------------------------------------------------------------------------- /arthur_bench/client/local/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | from .client import LocalBenchClient 3 | -------------------------------------------------------------------------------- /arthur_bench/client/rest/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | from .client import ArthurClient 3 | 4 | # import subpackages 5 | from . import admin 6 | from . import bench 7 | 8 | # end import subpackages 9 | -------------------------------------------------------------------------------- /arthur_bench/client/rest/admin/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | from . import models 3 | from . import client 4 | -------------------------------------------------------------------------------- /arthur_bench/client/rest/bench/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | 3 | from . import client 4 | -------------------------------------------------------------------------------- /arthur_bench/client/utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from arthur_bench.exceptions import UserValueError, MissingParameterError 4 | from arthur_bench.client.bench_client import BenchClient 5 | from arthur_bench.client.local import LocalBenchClient 6 | from arthur_bench.client.rest import ArthurClient 7 | 8 | 9 | def _get_bench_client() -> BenchClient: 10 | client: BenchClient 11 | use_remote = os.getenv("ARTHUR_BENCH_AUTOLOG", "false").lower() == "true" 12 | if use_remote: # if remote url is specified use remote client 13 | try: 14 | client = ArthurClient().bench 15 | except (UserValueError, MissingParameterError) as e: 16 | raise UserValueError( 17 | f"You must provide authentication when using remote url: {e}" 18 | ) 19 | else: 20 | client = LocalBenchClient() 21 | return client 22 | -------------------------------------------------------------------------------- /arthur_bench/exceptions/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | from .exceptions import ( 3 | ArthurError, 4 | ArthurUserError, 5 | ArthurInternalError, 6 | MissingParameterError, 7 | UserValueError, 8 | UserTypeError, 9 | MethodNotApplicableError, 10 | ResponseClientError, 11 | UnauthorizedError, 12 | PaymentRequiredError, 13 | ForbiddenError, 14 | NotFoundError, 15 | ExpectedParameterNotFoundError, 16 | InternalValueError, 17 | InternalTypeError, 18 | ResponseServerError, 19 | ResponseRedirectError, 20 | ) 21 | -------------------------------------------------------------------------------- /arthur_bench/logger/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | from . import logger 3 | -------------------------------------------------------------------------------- /arthur_bench/logger/logger.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | logger = logging.getLogger("uvicorn.info") 4 | -------------------------------------------------------------------------------- /arthur_bench/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arthur-ai/bench/6b714496b0b73179908a3b7e713010911cd8f1f7/arthur_bench/models/__init__.py -------------------------------------------------------------------------------- /arthur_bench/models/client.py: -------------------------------------------------------------------------------- 1 | # generated by datamodel-codegen: 2 | # filename: common.yaml 3 | # timestamp: 2023-06-21T19:01:59+00:00 4 | 5 | from __future__ import annotations 6 | 7 | from typing import Dict, Optional, Union 8 | 9 | from pydantic import BaseModel, Extra, Field 10 | 11 | 12 | class KeyValueObject(BaseModel): 13 | class Config: 14 | extra = Extra.allow 15 | 16 | __root__: Optional[Dict[str, Union[float, str, bool]]] = None 17 | 18 | 19 | class Page(BaseModel): 20 | __root__: int 21 | 22 | 23 | class PageSize(BaseModel): 24 | __root__: int = Field(..., example=100) 25 | 26 | 27 | class Sort(BaseModel): 28 | __root__: str 29 | """ 30 | Must be supplied in the format [column_name] to denote asc sort by this column OR 31 | -[column_name] to denote desc sort by this column 32 | """ 33 | 34 | 35 | class StatusCounts(BaseModel): 36 | """ 37 | Counts of the results by status 38 | """ 39 | 40 | failure: float 41 | """ 42 | The number of failed inferences in the request 43 | """ 44 | success: float 45 | """ 46 | The number of successful inferences in the request 47 | """ 48 | total: float 49 | """ 50 | The total number of inferences in the request 51 | """ 52 | 53 | 54 | class TotalCount(BaseModel): 55 | __root__: int = Field(..., example=2) 56 | """ 57 | The total number of results the query returned 58 | """ 59 | 60 | 61 | class TotalPages(BaseModel): 62 | __root__: int = Field(..., example=1) 63 | """ 64 | The number of result pages that this query generated, the ceiling of the total 65 | number of results divided by the page size 66 | """ 67 | -------------------------------------------------------------------------------- /arthur_bench/models/scoring.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel 2 | 3 | 4 | class HallucinationScoreRequest(BaseModel): 5 | """ 6 | Request for hallucination classification 7 | """ 8 | 9 | response: str 10 | """ 11 | Model generated response 12 | """ 13 | context: str 14 | """ 15 | Context with which to determine if the model generated response is supported 16 | """ 17 | 18 | 19 | class HallucinationScoreResponse(BaseModel): 20 | """ 21 | Hallucination classification 22 | """ 23 | 24 | hallucination: bool 25 | """ 26 | True if hallucination, false otherwise 27 | """ 28 | reason: str 29 | """ 30 | Justification for the hallucination classification 31 | """ 32 | -------------------------------------------------------------------------------- /arthur_bench/run/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arthur-ai/bench/6b714496b0b73179908a3b7e713010911cd8f1f7/arthur_bench/run/__init__.py -------------------------------------------------------------------------------- /arthur_bench/scoring/readability.py: -------------------------------------------------------------------------------- 1 | from textstat import flesch_reading_ease 2 | from typing import List, Optional 3 | from arthur_bench.scoring import Scorer 4 | from arthur_bench.models.models import ScoreResult 5 | 6 | 7 | class Readability(Scorer): 8 | """ 9 | Flesch Reading Ease Score: the higher the score, the easier to read. 10 | Scores of 100-90 correlate to a 5th grade reading level, while scores <10 are 11 | classified as "Extremely difficult to read, and best understood by university 12 | graduates." 13 | 14 | https://en.wikipedia.org/wiki/Flesch%E2%80%93Kincaid_readability_tests 15 | """ 16 | 17 | @staticmethod 18 | def name() -> str: 19 | return "readability" 20 | 21 | @staticmethod 22 | def requires_reference() -> bool: 23 | return False 24 | 25 | def run_batch( 26 | self, 27 | candidate_batch: List[str], 28 | reference_batch: Optional[List[str]] = None, 29 | input_text_batch: Optional[List[str]] = None, 30 | context_batch: Optional[List[str]] = None, 31 | ) -> List[ScoreResult]: 32 | return [ScoreResult(score=flesch_reading_ease(i)) for i in candidate_batch] 33 | -------------------------------------------------------------------------------- /arthur_bench/scoring/utils.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | 4 | class suppress_warnings: 5 | """ 6 | A context-manager class to temporarily set the logging level for a logger to ERROR 7 | before returning it to its previous state. 8 | """ 9 | 10 | def __init__(self, logger_name: str): 11 | self.logger = logging.getLogger(logger_name) 12 | self.original_state = ( 13 | self.logger.level 14 | ) # note: we get actual level not effective level, could be UNSET 15 | 16 | def __enter__(self): 17 | self.logger.setLevel(logging.ERROR) 18 | 19 | def __exit__(self, exc_type, exc_val, exc_tb): 20 | self.logger.setLevel( 21 | self.original_state 22 | ) # it's ok to use setLevel() to set to anything, even NOTSET 23 | -------------------------------------------------------------------------------- /arthur_bench/server/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arthur-ai/bench/6b714496b0b73179908a3b7e713010911cd8f1f7/arthur_bench/server/__init__.py -------------------------------------------------------------------------------- /arthur_bench/server/js/index.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { createRoot } from 'react-dom/client' 3 | import { Provider as StateProvider } from 'react-redux'; 4 | import { store } from 'arthur-redux'; 5 | import 'resources/icons/selection'; 6 | import 'resources/fonts/fonts.css'; 7 | import Skin from './src/Skin/skin'; 8 | 9 | 10 | const element = document.getElementById('root') as HTMLElement; 11 | const root = createRoot(element); 12 | 13 | root.render( 14 | 15 | 16 | 17 | ); 18 | -------------------------------------------------------------------------------- /arthur_bench/server/js/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "arthur-bench", 3 | "version": "0.1.0", 4 | "private": true, 5 | "overrides": { 6 | "@babel/preset-typescript": "7.21.4", 7 | "@types/react": "^18.0.17", 8 | "@types/react-dom": "^18.0.6", 9 | "react": "18.2.0", 10 | "react-dom": "18.2.0", 11 | "react-fela": "12.2.0", 12 | "react-i18next": "11.18.0", 13 | "react-redux": "8.0.2", 14 | "react-router-dom": "6.3.0", 15 | "ts-loader": "9.3.1", 16 | "ts-node": "10.9.1", 17 | "typescript": "4.7.4" 18 | }, 19 | "dependencies": { 20 | "api": "file:packages/api", 21 | "arthur-axios": "file:packages/arthur-axios", 22 | "arthur-redux": "file:packages/arthur-redux", 23 | "components": "*", 24 | "fela": "^12.1.2", 25 | "i18next": "^21.8.13", 26 | "react": "18.2.0", 27 | "react-dom": "18.2.0", 28 | "react-fela": "12.2.0", 29 | "react-i18next": "11.18.0", 30 | "react-redux": "8.0.2", 31 | "react-router-dom": "6.3.0", 32 | "resources": "file:packages/resources", 33 | "ts-node": "10.9.1", 34 | "tsconfig": "file:packages/tsconfig", 35 | "ui": "file:packages/components", 36 | "utils": "file:packages/utils" 37 | }, 38 | "scripts": { 39 | "build": "NODE_ENV=production webpack --config=webpack-config/config.js" 40 | }, 41 | "devDependencies": { 42 | "@babel/core": "7.20.2", 43 | "html-webpack-plugin": "^5.6.0", 44 | "ts-loader": "9.3.1", 45 | "webpack": "^5.73.0", 46 | "webpack-cli": "^5.1.4" 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /arthur_bench/server/js/packages/api/comparators.types.ts: -------------------------------------------------------------------------------- 1 | export enum ComparatorType { 2 | GREATER_THAN_EQUAL = 'gte', 3 | GREATER_THAN = 'gt', 4 | LESS_THAN = 'lt', 5 | LESS_THAN_EQUAL = 'lte', 6 | EQUAL = 'eq', 7 | NOT_EQUAL = 'ne', 8 | LIKE = 'like', 9 | IN = 'in', 10 | NOT_NULL = 'NotNull', 11 | } 12 | 13 | export const comparatorLangMap: Record = { 14 | gt: 'greater than', 15 | gte: 'greater than or equal to', 16 | lt: 'less than', 17 | lte: 'less than or equal to', 18 | eq: 'equal to', 19 | ne: 'not equal to', 20 | like: 'like', 21 | in: 'in', 22 | NotNull: 'not null', 23 | }; 24 | 25 | export const ComparatorTypeMap = { 26 | gt: ComparatorType.GREATER_THAN, 27 | gte: ComparatorType.GREATER_THAN_EQUAL, 28 | lt: ComparatorType.LESS_THAN, 29 | lte: ComparatorType.LESS_THAN_EQUAL, 30 | eq: ComparatorType.EQUAL, 31 | ne: ComparatorType.NOT_EQUAL, 32 | like: ComparatorType.LIKE, 33 | in: ComparatorType.IN, 34 | NotNull: ComparatorType.NOT_NULL, 35 | }; 36 | -------------------------------------------------------------------------------- /arthur_bench/server/js/packages/api/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "api", 3 | "version": "1.0.0", 4 | "description": "Arthur axios instance", 5 | "main": "index.ts", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "author": "", 10 | "license": "ISC", 11 | "dependencies": { 12 | "react": "18.2.0" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /arthur_bench/server/js/packages/api/useOnClickOutside.ts: -------------------------------------------------------------------------------- 1 | import React, { useEffect } from 'react'; 2 | 3 | type TOnClickOutsideHandler = (event: Event) => void; 4 | 5 | const useOnClickOutside = ( 6 | ref: React.RefObject, 7 | handler: TOnClickOutsideHandler 8 | ) => { 9 | useEffect(() => { 10 | const listener: EventListener = (event: Event) => { 11 | if (!ref.current || ref.current.contains(event.target)) { 12 | return; 13 | } 14 | handler(event); 15 | }; 16 | 17 | document.addEventListener('mousedown', listener); 18 | document.addEventListener('touchstart', listener); 19 | 20 | return () => { 21 | document.removeEventListener('mousedown', listener); 22 | document.removeEventListener('touchstart', listener); 23 | }; 24 | }, [ref.current]); 25 | }; 26 | 27 | export default useOnClickOutside; 28 | -------------------------------------------------------------------------------- /arthur_bench/server/js/packages/arthur-axios/index.ts: -------------------------------------------------------------------------------- 1 | import axios from 'axios'; 2 | 3 | /** 4 | * The default 5 | * @type {AxiosInstance} 6 | */ 7 | const arthurAxios = axios.create({ 8 | baseURL: 'http://localhost:8000/', 9 | headers: { 'Content-Type': 'application/json' }, 10 | timeout: 1000 * 60, //60 seconds 11 | }); 12 | 13 | 14 | export default arthurAxios; 15 | -------------------------------------------------------------------------------- /arthur_bench/server/js/packages/arthur-axios/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "arthur-axios", 3 | "version": "1.0.0", 4 | "description": "Arthur axios instance", 5 | "main": "index.ts", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "author": "", 10 | "license": "ISC", 11 | "dependencies": { 12 | "axios": "^0.27.2" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /arthur_bench/server/js/packages/arthur-redux/config/configureStore.prod.ts: -------------------------------------------------------------------------------- 1 | import { 2 | applyMiddleware, 3 | compose, 4 | legacy_createStore as createStore, 5 | } from 'redux'; 6 | import { apiMiddleware } from 'redux-api-middleware'; 7 | import rootReducer from './rootReducer'; 8 | 9 | export default function configureStore(preloadedState: any) { 10 | const store = createStore( 11 | rootReducer, 12 | preloadedState, 13 | compose(applyMiddleware(apiMiddleware)) 14 | ); 15 | 16 | // expose store when run in Cypress 17 | if ((window as any).Cypress) { 18 | (window as any).store = store; 19 | } 20 | 21 | return store; 22 | } 23 | -------------------------------------------------------------------------------- /arthur_bench/server/js/packages/arthur-redux/config/configureStore.ts: -------------------------------------------------------------------------------- 1 | import configureProductionStore from './configureStore.prod'; 2 | const currentEnvironment = process.env.NODE_ENV; 3 | 4 | const environmentConfigs: Record = { 5 | production: configureProductionStore, 6 | }; 7 | 8 | if (currentEnvironment && !(currentEnvironment in environmentConfigs)) { 9 | throw new Error('Unrecognized node environment'); 10 | } 11 | 12 | export default currentEnvironment 13 | ? environmentConfigs[currentEnvironment] 14 | : null; 15 | -------------------------------------------------------------------------------- /arthur_bench/server/js/packages/arthur-redux/config/rootReducer.ts: -------------------------------------------------------------------------------- 1 | import { combineReducers } from 'redux'; 2 | import Skin from '../slices/skin/skin.reducers'; 3 | import { testSuitesReducer } from '../slices/testSuites/reducers'; 4 | 5 | export default combineReducers({ 6 | testSuites: testSuitesReducer, 7 | skin: Skin, 8 | }); 9 | -------------------------------------------------------------------------------- /arthur_bench/server/js/packages/arthur-redux/config/state.type.ts: -------------------------------------------------------------------------------- 1 | 2 | import {TTestSuitesState} from "../slices/testSuites/types"; 3 | 4 | export type State = { 5 | testSuites: TTestSuitesState; 6 | }; 7 | -------------------------------------------------------------------------------- /arthur_bench/server/js/packages/arthur-redux/index.ts: -------------------------------------------------------------------------------- 1 | import configureStore from './config/configureStore' 2 | import type { State } from './config/state.type'; 3 | 4 | const store = configureStore(); 5 | 6 | export { State, store }; 7 | -------------------------------------------------------------------------------- /arthur_bench/server/js/packages/arthur-redux/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "arthur-redux", 3 | "version": "1.0.0", 4 | "description": "Shared Redux store for Arthur Applications", 5 | "main": "index.ts", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "author": "", 10 | "license": "ISC", 11 | "dependencies": { 12 | "@reduxjs/toolkit": "^1.8.3", 13 | "luxon": "^3.0.1", 14 | "redux": "^4.2.0", 15 | "redux-api-middleware": "^3.2.1", 16 | "redux-logger": "^3.0.6" 17 | }, 18 | "devDependencies": { 19 | "@types/redux-api-middleware": "^3.2.3", 20 | "@types/redux-logger": "^3.0.9" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /arthur_bench/server/js/packages/arthur-redux/slices/skin/skin.constants.ts: -------------------------------------------------------------------------------- 1 | export const FETCH_SKIN_REQUEST = 'FETCH_SKIN_REQUEST'; 2 | export const FETCH_SKIN_RECEIVE = 'FETCH_SKIN_RECEIVE'; 3 | export const FETCH_SKIN_ERROR = 'FETCH_SKIN_ERROR'; 4 | -------------------------------------------------------------------------------- /arthur_bench/server/js/packages/arthur-redux/slices/skin/skin.reducers.ts: -------------------------------------------------------------------------------- 1 | import { SkinState } from './skin.type'; 2 | import { 3 | FETCH_SKIN_REQUEST, 4 | FETCH_SKIN_RECEIVE, 5 | FETCH_SKIN_ERROR, 6 | } from './skin.constants'; 7 | 8 | const defaultState: SkinState = { 9 | skin: {}, 10 | }; 11 | 12 | type Action = { 13 | type: string; 14 | payload?: any; 15 | error?: any; 16 | redirectUrl?: string; 17 | }; 18 | 19 | const Skin = (state: SkinState = defaultState, action: Action): SkinState => { 20 | switch (action.type) { 21 | case FETCH_SKIN_REQUEST: 22 | return { 23 | ...state, 24 | }; 25 | case FETCH_SKIN_RECEIVE: 26 | return { 27 | ...state, 28 | skin: action.payload, 29 | }; 30 | 31 | case FETCH_SKIN_ERROR: 32 | return { 33 | ...state, 34 | }; 35 | 36 | default: 37 | return state; 38 | } 39 | }; 40 | 41 | export default Skin; 42 | -------------------------------------------------------------------------------- /arthur_bench/server/js/packages/arthur-redux/slices/skin/skin.type.ts: -------------------------------------------------------------------------------- 1 | export type SkinState = { 2 | skin: any; 3 | }; 4 | -------------------------------------------------------------------------------- /arthur_bench/server/js/packages/arthur-redux/slices/testSuites/actions.ts: -------------------------------------------------------------------------------- 1 | import {createAction} from "@reduxjs/toolkit"; 2 | import * as constants from "../testSuites/constants"; 3 | 4 | 5 | 6 | export const fetchTestSuitesRequest = createAction( 7 | constants.FETCH_TEST_SUITES_REQUEST 8 | ); 9 | export const fetchTestSuitesReceive = createAction( 10 | constants.FETCH_TEST_SUITES_RECEIVE 11 | ); 12 | export const fetchTestRunsRequest = createAction( 13 | constants.FETCH_TEST_RUNS_REQUEST 14 | ); 15 | export const fetchTestRunsReceive = createAction( 16 | constants.FETCH_TEST_RUNS_RECEIVE 17 | ); 18 | export const fetchTestSuiteDataRequest = createAction( 19 | constants.FETCH_TEST_SUITE_DATA_REQUEST 20 | ); 21 | export const fetchTestSuiteDataReceive = createAction( 22 | constants.FETCH_TEST_SUITE_DATA_RECEIVE 23 | ); 24 | export const fetchTestRunSummaryRequest = createAction( 25 | constants.FETCH_TEST_RUN_SUMMARY_REQUEST 26 | ); 27 | export const fetchTestRunSummaryReceive = createAction( 28 | constants.FETCH_TEST_RUN_SUMMARY_RECEIVE 29 | ); 30 | export const fetchTestRunDetailsRequest = createAction( 31 | constants.FETCH_TEST_RUN_DETAILS_REQUEST 32 | ); 33 | export const fetchTestRunDetailsReceive = createAction( 34 | constants.FETCH_TEST_RUN_DETAILS_RECEIVE 35 | ); 36 | -------------------------------------------------------------------------------- /arthur_bench/server/js/packages/arthur-redux/slices/testSuites/constants.ts: -------------------------------------------------------------------------------- 1 | export const FETCH_TEST_SUITES_REQUEST = 2 | 'FETCH_TEST_SUITES_REQUEST'; 3 | export const FETCH_TEST_SUITES_RECEIVE = 4 | 'FETCH_TEST_SUITES_RECEIVE'; 5 | export const FETCH_TEST_RUNS_REQUEST = 6 | 'FETCH_TEST_RUNS_REQUEST'; 7 | export const FETCH_TEST_RUNS_RECEIVE = 8 | 'FETCH_TEST_RUNS_RECEIVE'; 9 | export const FETCH_TEST_SUITE_DATA_REQUEST = 10 | 'FETCH_TEST_SUITE_DATA_REQUEST'; 11 | export const FETCH_TEST_SUITE_DATA_RECEIVE = 12 | 'FETCH_TEST_SUITE_DATA_RECEIVE'; 13 | export const FETCH_TEST_RUN_SUMMARY_REQUEST = 14 | 'FETCH_TEST_RUN_SUMMARY_REQUEST' 15 | export const FETCH_TEST_RUN_SUMMARY_RECEIVE = 16 | 'FETCH_TEST_RUN_SUMMARY_RECEIVE' 17 | export const FETCH_TEST_RUN_DETAILS_REQUEST = 18 | 'FETCH_TEST_RUN_DETAILS_RECEIVE' 19 | export const FETCH_TEST_RUN_DETAILS_RECEIVE = 20 | 'FETCH_TEST_RUN_DETAILS_RECEIVE' 21 | -------------------------------------------------------------------------------- /arthur_bench/server/js/packages/components/compound/Chip/index.ts: -------------------------------------------------------------------------------- 1 | import Chip from './Chip'; 2 | import { TChip, TChipProps, EChipTheme } from './types'; 3 | 4 | export type { TChip, TChipProps }; 5 | export { EChipTheme }; 6 | export default Chip; 7 | -------------------------------------------------------------------------------- /arthur_bench/server/js/packages/components/compound/Chip/styles.ts: -------------------------------------------------------------------------------- 1 | import { GRAPHIK } from 'resources/fonts'; 2 | import primary from 'resources/colors/Arthur/primary'; 3 | 4 | import { EChipTheme } from './types'; 5 | 6 | const chipThemeColors: Record = { 7 | [EChipTheme.ARTHUR]: primary.purple, 8 | } 9 | 10 | const styles = (chipTheme: EChipTheme , isMinimal?: boolean) => ({ 11 | root: { 12 | whiteSpace: 'nowrap', 13 | fontFamily: GRAPHIK, 14 | fontSize: '12px', 15 | backgroundColor: chipTheme !== EChipTheme.DEFAULT ? 'transparent' : primary.ashGrey, 16 | ...(chipTheme !== EChipTheme.DEFAULT && { 17 | border: `1px solid ${primary.ashGrey}`, 18 | }), 19 | borderRadius: '2px', 20 | display: 'inline-flex', 21 | alignItems: 'center', 22 | justifyContent: 'center', 23 | padding: isMinimal ? '0 4px' : '4px 8px', 24 | color: chipThemeColors[chipTheme] || primary.eggplant, 25 | }, 26 | button: { 27 | width: 'auto', 28 | padding: '0 !important', 29 | }, 30 | icon: { 31 | marginLeft: '4px', 32 | 33 | '& path': { 34 | fill: chipThemeColors[chipTheme] || 'inherit !important', 35 | }, 36 | }, 37 | name: { 38 | overflow: 'hidden', 39 | textOverflow: 'ellipsis', 40 | }, 41 | iconStart: { 42 | marginLeft: '0px', 43 | marginRight: '4px', 44 | }, 45 | }); 46 | 47 | export default styles; 48 | -------------------------------------------------------------------------------- /arthur_bench/server/js/packages/components/compound/Chip/types.ts: -------------------------------------------------------------------------------- 1 | import { EIconType } from '../../core/Icon'; 2 | import { FelaStyle } from 'react-fela'; 3 | 4 | export enum EChipTheme { 5 | ARTHUR = 'arthur', 6 | DEFAULT = 'default', 7 | OUTLINED = 'outlined', 8 | PRODUCTION = 'production' 9 | } 10 | 11 | export type TChip = { 12 | id: string; 13 | name: string; 14 | }; 15 | 16 | export type TChipProps = { 17 | onIconClick?: (tag: TChip) => void; 18 | chip?: TChip; 19 | chipName?: string | number; 20 | minimal?: boolean; 21 | iconStart?: EIconType; 22 | iconEnd?: EIconType; 23 | theme?: EChipTheme 24 | overrideStyles?: FelaStyle<{}>; 25 | iconStartColor?: string 26 | }; 27 | -------------------------------------------------------------------------------- /arthur_bench/server/js/packages/components/compound/FilterButton/FilterButton.tsx: -------------------------------------------------------------------------------- 1 | import React, { ForwardedRef } from 'react'; 2 | import { Button, EButtonSize } from '@core/Button'; 3 | import { useFela } from 'react-fela'; 4 | import styles from './styles'; 5 | import { EIconType } from '@core/Icon/types'; 6 | 7 | export type FilterButtonProps = { 8 | text: string; 9 | icon: EIconType; 10 | isActive?: boolean; 11 | onClick: () => void; 12 | }; 13 | 14 | const FilterButton = React.forwardRef( 15 | (props: FilterButtonProps, ref: ForwardedRef) => { 16 | const { css } = useFela(); 17 | const classNames = styles(props.isActive); 18 | 19 | return ( 20 |