8 | {label}
9 |
16 |
18 |
19 | );
20 | };
21 |
22 | export default EvaluationTypeLabel;
23 |
--------------------------------------------------------------------------------
/REST-Service/Dockerfile:
--------------------------------------------------------------------------------
1 | # Use the official Selenium Standalone Chrome image as the base image
2 | FROM registry.access.redhat.com/ubi8/python-311:latest
3 |
4 | # Set the working directory inside the container
5 | WORKDIR /app/backend
6 |
7 | # Copy the requirements file to the container and install dependencies
8 | COPY requirements.txt requirements.txt
9 | RUN pip3 install -r requirements.txt
10 | RUN pip3 install bson
11 | RUN pip3 install pymongo
12 |
13 | # Copy your FastAPI Python script to the container
14 | COPY main.py main.py
15 | COPY app/ app/
16 | COPY cert/ cert/
17 |
18 | EXPOSE 3001
19 |
20 | # Set the command to run your Python script
21 | CMD ["python3", "main.py"]
--------------------------------------------------------------------------------
/JudgeIt-App/components/globals/LinearProgressWithLabel.jsx:
--------------------------------------------------------------------------------
1 | import LinearProgress from '@mui/material/LinearProgress';
2 | import Typography from '@mui/material/Typography';
3 | import Box from '@mui/material/Box';
4 |
5 | export default function LinearProgressWithLabel({ value, width }) {
6 | return (
7 | LLM Judge service!
18 |For complete API visit open API docs
19 | 20 | 21 | """ 22 | ) -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | certifi==2024.07.04 2 | chardet==5.2.0 3 | charset-normalizer==3.3.2 4 | click==8.1.7 5 | ibm-cos-sdk==2.13.5 6 | ibm-cos-sdk-core==2.13.5 7 | ibm-cos-sdk-s3transfer==2.13.5 8 | ibm_watsonx_ai==1.0.10 9 | idna==3.7 10 | importlib_metadata==8.0.0 11 | jmespath==1.0.1 12 | joblib==1.4.2 13 | langchain-ibm==0.1.12 14 | lomond==0.3.3 15 | nltk==3.8.1 16 | numpy==1.26.4 17 | openpyxl==3.1.5 18 | packaging==24.1 19 | pandas==2.1.4 20 | python-dateutil==2.9.0.post0 21 | pytz==2024.1 22 | regex==2024.5.15 23 | requests==2.32.3 24 | rouge==1.0.1 25 | scikit-learn==1.5.0 26 | scipy==1.14.0 27 | six==1.16.0 28 | tabulate==0.9.0 29 | threadpoolctl==3.5.0 30 | tqdm==4.66.4 31 | tzdata==2024.1 32 | urllib3==2.1.0 33 | XlsxWriter==3.2.0 34 | zipp==3.19.2 35 | -------------------------------------------------------------------------------- /REST-Service/deployment/base/kustomization.yaml: -------------------------------------------------------------------------------- 1 | kind: Kustomization 2 | images: 3 | - name: backend-image-name 4 | newName: image-registry.openshift-image-registry.svc:5000/llm-judge-dev/backend 5 | newTag: v1.0 6 | secretGenerator: 7 | - name: llm-judge-secret 8 | literals: 9 | - WATSONX_URL= 10 | - WX_PROJECT_ID= 11 | - IBM_CLOUD_API_KEY= 12 | - LLM_JUDGE_API_KEY=JudgeIt-Secret-Api-Key 13 | - WX_PLATFORM=saas 14 | - WX_USER= 15 | - CELERY_BROKER_URL=redis://redis:6379/0 16 | - CELERY_RESULT_BACKEND=redis://redis:6379/0 17 | - SERVER_URL= 18 | - MONGO_URL= 19 | - MONGO_USER= 20 | - MONGO_PASS= 21 | - MONGO_DB="judgeit_app" 22 | resources: 23 | - redis/ 24 | - celery-worker/ 25 | - flower/ 26 | - rest-app/ -------------------------------------------------------------------------------- /JudgeIt-App/app/api/auth/logout/route.js: -------------------------------------------------------------------------------- 1 | import { authOptions } from "../[...nextauth]/route"; 2 | import { getServerSession } from "next-auth" 3 | import { getIdToken } from "@/utils/sessionTokenAccessor"; 4 | 5 | export async function GET() { 6 | const session = await getServerSession(authOptions); 7 | 8 | if (session) { 9 | 10 | const idToken = await getIdToken(); 11 | 12 | // this will log out the user on Keycloak side 13 | var url = `${process.env.END_SESSION_URL}?id_token_hint=${idToken}&post_logout_redirect_uri=${encodeURIComponent(process.env.NEXTAUTH_URL)}`; 14 | 15 | try { 16 | const resp = await fetch(url, { method: "GET" }); 17 | } catch (err) { 18 | console.error(err); 19 | return new Response({ status: 500 }); 20 | } 21 | } 22 | return new Response({ status: 200 }); 23 | } -------------------------------------------------------------------------------- /JudgeIt-App/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:22.4.1-alpine AS deps 2 | #RUN apk add --no-cache libc6-compat=1.2.4-r2 3 | WORKDIR /app 4 | 5 | COPY package.json ./ 6 | COPY package-lock.json ./ 7 | RUN npm install 8 | 9 | FROM node:22.4.1-alpine AS builder 10 | WORKDIR /app 11 | COPY --from=deps /app/node_modules ./node_modules 12 | COPY . . 13 | 14 | RUN npm run build 15 | 16 | FROM node:22.4.1-alpine AS runner 17 | WORKDIR /app 18 | 19 | ENV NODE_ENV production 20 | ENV NEXT_TELEMETRY_DISABLED 1 21 | 22 | RUN addgroup --system --gid 1001 nodejs 23 | RUN adduser --system --uid 1001 nextjs 24 | 25 | COPY --from=builder --chown=nextjs:nodejs /app/.next ./.next 26 | COPY --from=builder /app/node_modules ./node_modules 27 | COPY --from=builder /app/package.json ./package.json 28 | 29 | USER nextjs 30 | 31 | EXPOSE 3000 32 | 33 | ENV PORT 3000 34 | 35 | CMD ["npm", "start"] -------------------------------------------------------------------------------- /JudgeIt-App/components/globals/Footer.jsx: -------------------------------------------------------------------------------- 1 | function Footer() { 2 | return ( 3 | 23 | ); 24 | } 25 | 26 | export default Footer; 27 | -------------------------------------------------------------------------------- /JudgeIt-App/deployment/deployment.yaml: -------------------------------------------------------------------------------- 1 | kind: Deployment 2 | apiVersion: apps/v1 3 | metadata: 4 | resourceVersion: '108957306' 5 | name: llm-judge-frontend 6 | labels: 7 | app: llm-judge-frontend 8 | spec: 9 | replicas: 1 10 | selector: 11 | matchLabels: 12 | app: llm-judge-frontend 13 | template: 14 | metadata: 15 | labels: 16 | app: llm-judge-frontend 17 | deployment: llm-judge-frontend 18 | annotations: 19 | openshift.io/generated-by: OpenShiftWebConsole 20 | spec: 21 | containers: 22 | - name: llm-judge-frontend 23 | image: 'image-registry.openshift-image-registry.svc:5000/llm-judge/llm-judge-frontend@sha256:5ac9b1aa09123b4d09a7e0f297e542c895350f7a700779b36df77b0897f45f46' 24 | ports: 25 | - containerPort: 3000 26 | protocol: TCP 27 | envFrom: 28 | - secretRef: 29 | name: llmjudge-frontend-secret 30 | resources: {} 31 | -------------------------------------------------------------------------------- /REST-Service/deployment/base/redis/deployment.yaml: -------------------------------------------------------------------------------- 1 | kind: Deployment 2 | apiVersion: apps/v1 3 | metadata: 4 | name: redis 5 | labels: 6 | app: redis 7 | spec: 8 | replicas: 1 9 | selector: 10 | matchLabels: 11 | app: redis 12 | template: 13 | metadata: 14 | labels: 15 | app: redis 16 | deployment: redis 17 | annotations: 18 | openshift.io/generated-by: OpenShiftWebConsole 19 | spec: 20 | volumes: 21 | - name: redis-1 22 | emptyDir: {} 23 | containers: 24 | - name: redis 25 | image: redis:7.2.5-alpine 26 | ports: 27 | - containerPort: 6379 28 | protocol: TCP 29 | resources: {} 30 | volumeMounts: 31 | - name: redis-1 32 | mountPath: /data 33 | terminationMessagePath: /dev/termination-log 34 | terminationMessagePolicy: File 35 | imagePullPolicy: IfNotPresent 36 | restartPolicy: Always 37 | -------------------------------------------------------------------------------- /JudgeIt-App/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "judge-app", 3 | "version": "0.1.0", 4 | "private": true, 5 | "scripts": { 6 | "dev": "next dev", 7 | "build": "next build", 8 | "start": "next start", 9 | "lint": "next lint" 10 | }, 11 | "dependencies": { 12 | "@emotion/react": "^11.11.4", 13 | "@emotion/styled": "^11.11.5", 14 | "@mui/icons-material": "^5.16.0", 15 | "@mui/material": "^5.16.0", 16 | "@mui/x-data-grid": "^7.16.0", 17 | "axios": "^1.7.2", 18 | "chart.js": "^4.4.4", 19 | "chartjs-plugin-datalabels": "^2.2.0", 20 | "cryptr": "^6.3.0", 21 | "formik": "^2.4.6", 22 | "next": "14.2.5", 23 | "next-auth": "^4.24.7", 24 | "react": "^18", 25 | "react-chartjs-2": "^5.2.0", 26 | "react-dom": "^18", 27 | "react-dropzone": "^14.2.3", 28 | "react-pro-sidebar": "^1.1.0", 29 | "uuid": "^10.0.0", 30 | "yup": "^1.4.0" 31 | }, 32 | "devDependencies": { 33 | "eslint": "^8", 34 | "eslint-config-next": "14.2.5" 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /REST-Service/app/src/config/TimeoutMiddleware.py: -------------------------------------------------------------------------------- 1 | from fastapi.responses import JSONResponse 2 | from starlette.middleware.base import BaseHTTPMiddleware 3 | from fastapi import FastAPI, Request, HTTPException 4 | import time 5 | 6 | class TimeoutMiddleware(BaseHTTPMiddleware): 7 | def __init__(self, app, timeout: int): 8 | super().__init__(app) 9 | self.timeout = timeout 10 | 11 | async def dispatch(self, request: Request, call_next): 12 | start_time = time.time() 13 | try: 14 | response = await call_next(request) 15 | process_time = time.time() - start_time 16 | if process_time > self.timeout: 17 | raise HTTPException(status_code=408, detail="Request Timeout") 18 | return response 19 | except Exception as e: 20 | process_time = time.time() - start_time 21 | if process_time > self.timeout: 22 | return JSONResponse(content={"detail": "Request Timeout"}, status_code=408) 23 | raise e -------------------------------------------------------------------------------- /JudgeIt-App/app/layout.js: -------------------------------------------------------------------------------- 1 | import "../styles/globals.css"; 2 | import Footer from "@/components/globals/Footer"; 3 | import Topbar from "@/components/globals/Topbar"; 4 | import { Grid, Box, AppBar } from "@mui/material"; 5 | import SessionProviderWrapper from "@/utils/sessionProviderWrapper"; 6 | 7 | export const metadata = { 8 | title: "LLM Judge Application", 9 | description: "LLM Judge Application to evaluate LLM response.", 10 | }; 11 | 12 | export default function RootLayout({ children }) { 13 | return ( 14 |
18 |
20 |
21 | {children}
22 |
23 |
24 |
25 |
26 |
27 |
25 |
38 |
39 |
18 |
19 |
20 |
27 | Documentation
28 |
29 |
30 |
31 |
32 |
37 |
39 |
40 |
41 |
46 |
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 | );
56 | };
57 |
58 | export default HelperPage;
59 |
--------------------------------------------------------------------------------
/JudgeIt-App/components/globals/icons/IBMIcon.jsx:
--------------------------------------------------------------------------------
1 | import * as React from "react";
2 | import SvgIcon from "@mui/material/SvgIcon";
3 |
4 | export default function IBMIcon() {
5 | return (
6 |
27 |
30 |
53 |
54 | );
55 | };
56 |
57 | export default DeleteConfirmationDialog;
58 |
--------------------------------------------------------------------------------
/REST-Service/docker-compose.yml:
--------------------------------------------------------------------------------
1 | services:
2 | fastapi_app:
3 | container_name: fastapi_app
4 | platform: linux/amd64
5 | image: fastapi_app_image
6 | #volumes:
7 | # - ./app:/app
8 | ports:
9 | - 3001:3001
10 | environment:
11 | - WATSONX_URL=https://us-south.ml.cloud.ibm.com
12 | - WX_PROJECT_ID=***
13 | - IBM_CLOUD_API_KEY=***
14 | - LLM_JUDGE_API_KEY=JudgeIt-Secret-Api-Key
15 | - WX_PLATFORM=saas
16 | - WX_USER=''
17 | - WX_GOV_REGION=eu-de
18 | - CELERY_BROKER_URL=redis://redis:6379/0
19 | - CELERY_RESULT_BACKEND=redis://redis:6379/0
20 | - SERVER_URL=http://localhost:3001
21 | - MONGO_URL=***
22 | - MONGO_USER=***
23 | - MONGO_PASS=***
24 | - MONGO_DB=judge_it_dev
25 | - WX_NEG_TEST_MODEL=mistralai/mistral-medium-2505
26 | - WX_GOV_INSTANCE=
27 | restart: always
28 | redis:
29 | container_name: redis
30 | image: redis:7.2.5-alpine
31 | restart: always
32 | celery_worker:
33 | container_name: celery_worker
34 | build: .
35 | #volumes:
36 | # - ./app:/app
37 | command: celery -A app.celery.celery_worker.celery worker --loglevel=info
38 | environment:
39 | - WATSONX_URL=https://us-south.ml.cloud.ibm.com
40 | - WX_PROJECT_ID=***
41 | - WX_PLATFORM=saas
42 | - WX_USER=''
43 | - WX_GOV_REGION=eu-de
44 | - IBM_CLOUD_API_KEY=***
45 | - CELERY_BROKER_URL=redis://redis:6379/0
46 | - CELERY_RESULT_BACKEND=redis://redis:6379/0
47 | - WX_NEG_TEST_MODEL=mistralai/mistral-medium-2505
48 | - WX_GOV_INSTANCE=
49 | depends_on:
50 | - fastapi_app
51 | - redis
52 | restart: always
53 | flower:
54 | container_name: flower
55 | build: .
56 | command: celery --broker=redis://redis:6379/0 flower --port=5555
57 | ports:
58 | - 5556:5555
59 | environment:
60 | - CELERY_BROKER_URL=redis://redis:6379/0
61 | - CELERY_RESULT_BACKEND=redis://redis:6379/0
62 | depends_on:
63 | - fastapi_app
64 | - redis
65 | - celery_worker
66 | restart: always
--------------------------------------------------------------------------------
/JudgeIt-App/components/judge/MultiTurnWithConversationForm.jsx:
--------------------------------------------------------------------------------
1 | "use client";
2 | import React from "react";
3 | import { TextField, Box } from "@mui/material";
4 |
5 | const MultiTurnWithConversationForm = ({
6 | values,
7 | handleChange,
8 | handleBlur,
9 | errors,
10 | touched,
11 | }) => {
12 | return (
13 |
14 |
15 |
28 |
29 |
40 |
41 |
52 |
53 |
64 |
65 | );
66 | };
67 |
68 | export default MultiTurnWithConversationForm;
69 |
--------------------------------------------------------------------------------
/JudgeIt-App/components/judge/SoloResult.jsx:
--------------------------------------------------------------------------------
1 | "use client";
2 | import {
3 | Alert,
4 | Table,
5 | TableHead,
6 | TableRow,
7 | TableCell,
8 | TableBody,
9 | Paper,
10 | } from "@mui/material";
11 | import {
12 | API_TYPE_MULTITURN,
13 | API_TYPE_SINGLETURN,
14 | API_TYPE_RATING,
15 | API_TYPE_SIMILARITY,
16 | } from "@/services/Config";
17 |
18 | import { grade_map_rating, grade_map_similarity, grade_map_multiturn } from "@/services/Config";
19 |
20 | const grade_col_name = "JudgeIt Score"
21 | const explanation_col_name = "JudgeIt Reasoning"
22 |
23 | const SoloResult = ({ data, api_type }) => {
24 | return (
25 |
52 | {" "}
53 | {
57 | return {
58 | id: item._id,
59 | name: item.name,
60 | eval_type: item.eval_type,
61 | model: item.content.query.model,
62 | golden_text: item.content.query.golden_text,
63 | generated_text: item.content.query.generated_text,
64 | Grade: (item.eval_type === API_TYPE_RATING) ? grade_map_rating[item.content.result.Grade] : grade_map_similarity[item.content.result.Grade],
65 | Explanation: item.content.result.Explanation,
66 | };
67 | }),
68 | }}
69 | density="compact"
70 | getRowHeight={() => "auto"}
71 | autoHeight={true}
72 | initialState={{
73 | ...{
74 | columns: columns,
75 | rows: [],
76 | }.initialState,
77 | pagination: { paginationModel: { pageSize: 10 } },
78 | }}
79 | pageSizeOptions={[5, 10, 25]}
80 | slots={{ toolbar: DataGridToolbar }}
81 | />
82 |
83 | );
84 | };
85 |
86 | export default RatingSimilarityDataGrid;
87 |
--------------------------------------------------------------------------------
/JudgeIt-App/components/judge/DataGridMultiTurnConversation.jsx:
--------------------------------------------------------------------------------
1 | "use client";
2 | import React from "react";
3 | import { DataGrid } from "@mui/x-data-grid";
4 | import DataGridToolbar from "@/components/globals/DataGridToolbar";
5 |
6 | const DataGridMultiTurnConversation = ({ serverData }) => {
7 | const columns = [
8 | {
9 | field: "id",
10 | headerName: "Id",
11 | hide: true,
12 | },
13 | {
14 | field: "name",
15 | headerName: "Name",
16 | width: "250",
17 | },
18 | {
19 | field: "eval_type",
20 | headerName: "Eval Type",
21 | },
22 | {
23 | field: "model",
24 | headerName: "Model",
25 | width: "250",
26 | },
27 | {
28 | field: "conversation_history",
29 | headerName: "Conversation history",
30 | width: "400",
31 | },
32 | {
33 | field: "follow_up_query",
34 | headerName: "Follow up query",
35 | width: "400",
36 | },
37 | {
38 | field: "golden_query",
39 | headerName: "Golden query",
40 | width: "400",
41 | },
42 | {
43 | field: "rewritten_query",
44 | headerName: "Rewritten query",
45 | width: "400",
46 | },
47 | {
48 | field: "Grade",
49 | headerName: "Grade",
50 | width: 100,
51 | }
52 | ];
53 |
54 | return (
55 |
56 | {" "}
57 | {
61 | return {
62 | id: item._id,
63 | name: item.name,
64 | eval_type: item.eval_type,
65 | model: item.content.query.model,
66 | conversation_history: item.content.query.conversation_history,
67 | follow_up_query: item.content.query.follow_up_query,
68 | golden_query: item.content.query.golden_query,
69 | rewritten_query: item.content.query.rewritten_query,
70 | Grade: item.content.result.Grade
71 | };
72 | }),
73 | }}
74 | density="compact"
75 | getRowHeight={() => "auto"}
76 | autoHeight={true}
77 | initialState={{
78 | ...{
79 | columns: columns,
80 | rows: [],
81 | }.initialState,
82 | pagination: { paginationModel: { pageSize: 10 } },
83 | }}
84 | pageSizeOptions={[5, 10, 25]}
85 | slots={{ toolbar: DataGridToolbar }}
86 | />
87 |
88 | );
89 | };
90 |
91 | export default DataGridMultiTurnConversation;
92 |
--------------------------------------------------------------------------------
/JudgeIt-App/components/judge/DataGridMultiTurnSummary.jsx:
--------------------------------------------------------------------------------
1 | "use client";
2 | import React from "react";
3 | import { DataGrid } from "@mui/x-data-grid";
4 | import DataGridToolbar from "@/components/globals/DataGridToolbar";
5 |
6 | const DataGridMultiTurnSummaryConversation = ({ serverData }) => {
7 | const columns = [
8 | {
9 | field: "id",
10 | headerName: "Id",
11 | hide: true,
12 | },
13 | {
14 | field: "name",
15 | headerName: "Name",
16 | width: "250",
17 | },
18 | {
19 | field: "experiment_name",
20 | headerName: "Experiment Name",
21 | width: "250",
22 | },
23 | {
24 | field: "eval_type",
25 | headerName: "Eval Type",
26 | },
27 | {
28 | field: "conversation_history",
29 | headerName: "Conversation history",
30 | width: "500",
31 | },
32 | {
33 | field: "follow_up_query",
34 | headerName: "Follow up query",
35 | width: "300",
36 | },
37 | {
38 | field: "golden_query",
39 | headerName: "Golden query",
40 | width: "300",
41 | },
42 | {
43 | field: "rewritten_query",
44 | headerName: "Rewritten query",
45 | width: "300",
46 | },
47 | {
48 | field: "Grade",
49 | headerName: "JudgeIt Score",
50 | width: 100,
51 | }
52 | ];
53 |
54 | return (
55 |
56 | {" "}
57 | {
61 | return {
62 | id: item._id,
63 | name: item.name,
64 | eval_type: item.eval_type,
65 | experiment_name: item.experiment_name,
66 | conversation_history: item.conversation_history,
67 | follow_up_query: item.follow_up_query,
68 | golden_query: item.golden_query,
69 | rewritten_query: item.rewritten_query,
70 | Grade: (item?.Grade) ? item?.Grade : item?.judgeit_score
71 | };
72 | }),
73 | }}
74 | density="compact"
75 | getRowHeight={() => "auto"}
76 | autoHeight={true}
77 | initialState={{
78 | ...{
79 | columns: columns,
80 | rows: [],
81 | }.initialState,
82 | pagination: { paginationModel: { pageSize: 10 } },
83 | }}
84 | pageSizeOptions={[5, 10, 25]}
85 | slots={{ toolbar: DataGridToolbar }}
86 | />
87 |
88 | );
89 | };
90 |
91 | export default DataGridMultiTurnSummaryConversation;
92 |
--------------------------------------------------------------------------------
/JudgeIt-App/utils/Helper.js:
--------------------------------------------------------------------------------
1 | import {
2 | API_TYPE_MULTITURN,
3 | API_TYPE_RATING,
4 | API_TYPE_SIMILARITY,
5 | grade_map_multiturn,
6 | grade_map_rating,
7 | grade_map_similarity,
8 | } from "@/services/Config";
9 |
10 | export function getRandomInt(max) {
11 | return Math.floor(Math.random() * max);
12 | }
13 |
14 | export function generateRandomString(length = 4) {
15 | const characters =
16 | "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
17 | let result = "";
18 | for (let i = 0; i < length; i++) {
19 | const randomIndex = Math.floor(Math.random() * characters.length);
20 | result += characters.charAt(randomIndex);
21 | }
22 | return result;
23 | }
24 |
25 | // Function to generate columns dynamically from JSON object keys
26 | export const generateColumns = (jsonObject) => {
27 | return Object.keys(jsonObject).map((key) => ({
28 | field: key,
29 | headerName: rename_grade_explanation_cloumn_name(key), // Capitalize the header
30 | width: 300, // You can adjust the width or make it dynamic
31 | }));
32 | };
33 |
34 | const rename_grade_explanation_cloumn_name = (column_name) => {
35 | if (column_name === "Grade") {
36 | return "JudgeIt Score";
37 | } else if (column_name === "Explanation") {
38 | return "JudgeIt Reasoning";
39 | } else {
40 | return column_name.charAt(0).toUpperCase() + column_name.slice(1);
41 | }
42 | };
43 |
44 | // Function to generate rows dynamically from JSON object
45 | export const generateRows = (jsonObject, eval_type) => {
46 | const firstKey = Object.keys(jsonObject)[0]; // Get the first key to check structure
47 | const rowIds = Object.keys(jsonObject[firstKey]); // Assuming same structure for all keys
48 |
49 | return rowIds.map((_, index) => {
50 | const rowData = { id: index }; // Initialize row with id
51 | Object.keys(jsonObject).forEach((field) => {
52 | rowData[field] = get_rating_label(
53 | eval_type,
54 | field,
55 | jsonObject[field][index]
56 | ); // Add data for each field
57 | });
58 | return rowData;
59 | });
60 | };
61 |
62 | const get_rating_label = (eval_type, column_name, value) => {
63 | if (column_name !== "Grade") return value;
64 |
65 | const gradeMap = {
66 | [API_TYPE_RATING]: grade_map_rating,
67 | [API_TYPE_SIMILARITY]: grade_map_similarity,
68 | [API_TYPE_MULTITURN]: grade_map_multiturn,
69 | };
70 |
71 | return gradeMap[eval_type]?.[value] || value;
72 | };
73 |
74 | export function trimText(text) {
75 | if (text.length > 15) {
76 | return text.substring(0, 15) + "..";
77 | }
78 | return text;
79 | }
80 |
--------------------------------------------------------------------------------
/Framework/wml_setup.py:
--------------------------------------------------------------------------------
1 | from ibm_watsonx_ai.metanames import GenTextParamsMetaNames as GenParams
2 | from ibm_watsonx_ai.foundation_models import Model
3 |
4 | #config Watsonx.ai environment
5 | api_key = ''
6 | ibm_cloud_url = 'https://us-south.ml.cloud.ibm.com'
7 | project_id = ''
8 |
9 | def send_to_watsonxai(prompts,
10 | model_id="MIXTRAL",
11 | decoding_method="greedy",
12 | max_new_tokens=500,
13 | min_new_tokens=30,
14 | temperature=1.0,
15 | repetition_penalty=1.0
16 | ):
17 | if model_id == "MIXTRAL":
18 | model_name = "mistralai/mixtral-8x7b-instruct-v01"
19 | elif model_id == "LLAMA3":
20 | model_name="meta-llama/llama-3-70b-instruct"
21 | # Instantiate parameters for text generation
22 | model_params = {
23 | GenParams.DECODING_METHOD: decoding_method,
24 | GenParams.MIN_NEW_TOKENS: min_new_tokens,
25 | GenParams.MAX_NEW_TOKENS: max_new_tokens,
26 | GenParams.RANDOM_SEED: 42,
27 | GenParams.TEMPERATURE: temperature,
28 | GenParams.REPETITION_PENALTY: repetition_penalty,
29 | }
30 | model = Model(
31 | model_id=model_name,
32 | params=model_params,
33 | credentials={
34 | "url" : ibm_cloud_url,
35 | "apikey" : api_key
36 | },
37 | project_id=project_id)
38 |
39 | response=model.generate_text(prompts)
40 | return response
41 |
42 |
43 | def send_to_watsonxai_multi_turn(prompts,
44 | model_id="MIXTRAL",
45 | decoding_method="greedy",
46 | max_new_tokens=128,
47 | temperature=0.7,
48 | repetition_penalty=1.0
49 | ):
50 | if model_id == "MIXTRAL":
51 | model_name = "mistralai/mixtral-8x7b-instruct-v01"
52 | elif model_id == "LLAMA3":
53 | model_name="meta-llama/llama-3-70b-instruct"
54 | # Instantiate parameters for text generation
55 | model_params = {
56 | GenParams.DECODING_METHOD: decoding_method,
57 | GenParams.MAX_NEW_TOKENS: max_new_tokens,
58 | GenParams.RANDOM_SEED: 42,
59 | GenParams.TEMPERATURE: temperature,
60 | GenParams.REPETITION_PENALTY: repetition_penalty,
61 | }
62 | model = Model(
63 | model_id=model_name,
64 | params=model_params,
65 | credentials={
66 | "url" : ibm_cloud_url,
67 | "apikey" : api_key
68 | },
69 | project_id=project_id)
70 |
71 | response=model.generate_text(prompts)
72 | return response
--------------------------------------------------------------------------------
/JudgeIt-App/components/judge/RatingSimilarityDataGridSummary.jsx:
--------------------------------------------------------------------------------
1 | "use client";
2 | import React from "react";
3 | import { DataGrid } from "@mui/x-data-grid";
4 | import DataGridToolbar from "@/components/globals/DataGridToolbar";
5 | import { API_TYPE_RATING, grade_map_rating, grade_map_similarity } from "@/services/Config";
6 |
7 | const RatingSimilarityDataGridSummary = ({ serverData }) => {
8 | const columns = [
9 | {
10 | field: "id",
11 | headerName: "Id",
12 | hide: true,
13 | },
14 | {
15 | field: "Question",
16 | headerName: "Question",
17 | width: "250",
18 | },
19 | {
20 | field: "experiment_name",
21 | headerName: "Experiment Name",
22 | width: "100",
23 | },
24 | {
25 | field: "name",
26 | headerName: "Name",
27 | width: "100",
28 | },
29 | {
30 | field: "eval_type",
31 | headerName: "Eval Type",
32 | },
33 | {
34 | field: "golden_text",
35 | headerName: "Golden Text",
36 | width: "400",
37 | },
38 | {
39 | field: "generated_text",
40 | headerName: "Generated Text",
41 | width: "400",
42 | },
43 | {
44 | field: "Grade",
45 | headerName: "JudgeIt Score",
46 | width: 100,
47 | },
48 | {
49 | field: "Explanation",
50 | headerName: "JudgeIt Reasoning",
51 | width: "400",
52 | },
53 | ];
54 |
55 | return (
56 |
57 | {
61 | return {
62 | id: item._id,
63 | Question: item.question,
64 | experiment_name: item.experiment_name,
65 | name: item.name,
66 | eval_type: item.eval_type,
67 | golden_text: item.golden_text,
68 | generated_text: item.generated_text,
69 | Grade: (item?.Grade) ? item?.Grade : item?.judgeit_score,
70 | Explanation: (item?.Explanation) ? item?.Grade : item?.judgeit_reasoning
71 | };
72 | }),
73 | }}
74 | density="compact"
75 | getRowHeight={() => "auto"}
76 | autoHeight={true}
77 | initialState={{
78 | ...{
79 | columns: columns,
80 | rows: [],
81 | }.initialState,
82 | pagination: { paginationModel: { pageSize: 10 } },
83 | }}
84 | pageSizeOptions={[5, 10, 25]}
85 | slots={{ toolbar: DataGridToolbar }}
86 | />
87 |
88 | );
89 | };
90 |
91 | export default RatingSimilarityDataGridSummary;
92 |
--------------------------------------------------------------------------------
/JudgeIt-App/app/globals.css:
--------------------------------------------------------------------------------
1 | :root {
2 | --max-width: 1100px;
3 | --border-radius: 12px;
4 | --font-mono: ui-monospace, Menlo, Monaco, "Cascadia Mono", "Segoe UI Mono",
5 | "Roboto Mono", "Oxygen Mono", "Ubuntu Monospace", "Source Code Pro",
6 | "Fira Mono", "Droid Sans Mono", "Courier New", monospace;
7 |
8 | --foreground-rgb: 0, 0, 0;
9 | --background-start-rgb: 214, 219, 220;
10 | --background-end-rgb: 255, 255, 255;
11 |
12 | --primary-glow: conic-gradient(
13 | from 180deg at 50% 50%,
14 | #16abff33 0deg,
15 | #0885ff33 55deg,
16 | #54d6ff33 120deg,
17 | #0071ff33 160deg,
18 | transparent 360deg
19 | );
20 | --secondary-glow: radial-gradient(
21 | rgba(255, 255, 255, 1),
22 | rgba(255, 255, 255, 0)
23 | );
24 |
25 | --tile-start-rgb: 239, 245, 249;
26 | --tile-end-rgb: 228, 232, 233;
27 | --tile-border: conic-gradient(
28 | #00000080,
29 | #00000040,
30 | #00000030,
31 | #00000020,
32 | #00000010,
33 | #00000010,
34 | #00000080
35 | );
36 |
37 | --callout-rgb: 238, 240, 241;
38 | --callout-border-rgb: 172, 175, 176;
39 | --card-rgb: 180, 185, 188;
40 | --card-border-rgb: 131, 134, 135;
41 | }
42 |
43 | @media (prefers-color-scheme: dark) {
44 | :root {
45 | --foreground-rgb: 255, 255, 255;
46 | --background-start-rgb: 0, 0, 0;
47 | --background-end-rgb: 0, 0, 0;
48 |
49 | --primary-glow: radial-gradient(rgba(1, 65, 255, 0.4), rgba(1, 65, 255, 0));
50 | --secondary-glow: linear-gradient(
51 | to bottom right,
52 | rgba(1, 65, 255, 0),
53 | rgba(1, 65, 255, 0),
54 | rgba(1, 65, 255, 0.3)
55 | );
56 |
57 | --tile-start-rgb: 2, 13, 46;
58 | --tile-end-rgb: 2, 5, 19;
59 | --tile-border: conic-gradient(
60 | #ffffff80,
61 | #ffffff40,
62 | #ffffff30,
63 | #ffffff20,
64 | #ffffff10,
65 | #ffffff10,
66 | #ffffff80
67 | );
68 |
69 | --callout-rgb: 20, 20, 20;
70 | --callout-border-rgb: 108, 108, 108;
71 | --card-rgb: 100, 100, 100;
72 | --card-border-rgb: 200, 200, 200;
73 | }
74 | }
75 |
76 | * {
77 | box-sizing: border-box;
78 | padding: 0;
79 | margin: 0;
80 | }
81 |
82 | html,
83 | body {
84 | max-width: 100vw;
85 | overflow-x: hidden;
86 | }
87 |
88 | body {
89 | color: rgb(var(--foreground-rgb));
90 | background: linear-gradient(
91 | to bottom,
92 | transparent,
93 | rgb(var(--background-end-rgb))
94 | )
95 | rgb(var(--background-start-rgb));
96 | }
97 |
98 | a {
99 | color: inherit;
100 | text-decoration: none;
101 | }
102 |
103 | @media (prefers-color-scheme: dark) {
104 | html {
105 | color-scheme: dark;
106 | }
107 | }
108 |
--------------------------------------------------------------------------------
/JudgeIt-App/components/judge/DataGridSingleTurn.jsx:
--------------------------------------------------------------------------------
1 | "use client";
2 | import React from "react";
3 | import { DataGrid } from "@mui/x-data-grid";
4 | import DataGridToolbar from "@/components/globals/DataGridToolbar";
5 |
6 | const DataGridSingleTurn = ({ serverData }) => {
7 | const columns = [
8 | {
9 | field: "id",
10 | headerName: "Id",
11 | hide: true,
12 | },
13 | {
14 | field: "name",
15 | headerName: "Name",
16 | width: "250",
17 | },
18 | {
19 | field: "eval_type",
20 | headerName: "Eval Type",
21 | },
22 | {
23 | field: "model",
24 | headerName: "Model",
25 | width: "250",
26 | },
27 | {
28 | field: "previous_question",
29 | headerName: "Previous Question",
30 | width: "400",
31 | },
32 | {
33 | field: "previous_answer",
34 | headerName: "Previous Answer",
35 | width: "400",
36 | },
37 | {
38 | field: "current_question",
39 | headerName: "Current Question",
40 | width: "400",
41 | },
42 | {
43 | field: "golden_rewritten_question",
44 | headerName: "Golden Rewritten Question",
45 | width: "400",
46 | },
47 | {
48 | field: "rewritten_question",
49 | headerName: "Rewritten Question",
50 | width: "400",
51 | },
52 | {
53 | field: "Grade",
54 | headerName: "Grade",
55 | width: 100,
56 | }
57 | ];
58 |
59 | return (
60 |
61 | {" "}
62 | {
66 | return {
67 | id: item._id,
68 | name: item.name,
69 | eval_type: item.eval_type,
70 | model: item.content.query.model,
71 | previous_question: item.content.query.previous_question,
72 | previous_answer: item.content.query.previous_answer,
73 | current_question: item.content.query.current_question,
74 | golden_rewritten_question: item.content.query.golden_rewritten_question,
75 | rewritten_question: item.content.query.rewritten_question,
76 | Grade: item.content.result.Grade
77 | };
78 | }),
79 | }}
80 | density="compact"
81 | getRowHeight={() => "auto"}
82 | autoHeight={true}
83 | initialState={{
84 | ...{
85 | columns: columns,
86 | rows: [],
87 | }.initialState,
88 | pagination: { paginationModel: { pageSize: 10 } },
89 | }}
90 | pageSizeOptions={[5, 10, 25]}
91 | slots={{ toolbar: DataGridToolbar }}
92 | />
93 |
94 | );
95 | };
96 |
97 | export default DataGridSingleTurn;
98 |
--------------------------------------------------------------------------------
/JudgeIt-App/components/judge/DataGridSingleTurnSummary.jsx:
--------------------------------------------------------------------------------
1 | "use client";
2 | import React from "react";
3 | import { DataGrid } from "@mui/x-data-grid";
4 | import DataGridToolbar from "@/components/globals/DataGridToolbar";
5 |
6 | const DataGridSingleTurnSummary = ({ serverData }) => {
7 | const columns = [
8 | {
9 | field: "id",
10 | headerName: "Id",
11 | hide: true,
12 | },
13 | {
14 | field: "name",
15 | headerName: "Name",
16 | width: "250",
17 | },
18 | {
19 | field: "experiment_name",
20 | headerName: "Experiment Name",
21 | width: "250",
22 | },
23 | {
24 | field: "eval_type",
25 | headerName: "Eval Type",
26 | },
27 |
28 | {
29 | field: "previous_question",
30 | headerName: "Previous Question",
31 | width: "400",
32 | },
33 | {
34 | field: "previous_answer",
35 | headerName: "Previous Answer",
36 | width: "400",
37 | },
38 | {
39 | field: "current_question",
40 | headerName: "Current Question",
41 | width: "400",
42 | },
43 | {
44 | field: "golden_rewritten_question",
45 | headerName: "Golden Rewritten Question",
46 | width: "400",
47 | },
48 | {
49 | field: "rewritten_question",
50 | headerName: "Rewritten Question",
51 | width: "400",
52 | },
53 | {
54 | field: "Grade",
55 | headerName: "JudgeIt Score",
56 | width: 100,
57 | }
58 | ];
59 |
60 | return (
61 |
62 | {" "}
63 | {
67 | return {
68 | id: item._id,
69 | name: item.name,
70 | eval_type: item.eval_type,
71 | experiment_name: item.experiment_name,
72 | previous_question: item.previous_question,
73 | previous_answer: item.previous_answer,
74 | current_question: item.current_question,
75 | golden_rewritten_question: item.golden_rewritten_question,
76 | rewritten_question: item.rewritten_question,
77 | Grade: (item?.Grade) ? item?.Grade : item?.judgeit_score
78 | };
79 | }),
80 | }}
81 | density="compact"
82 | getRowHeight={() => "auto"}
83 | autoHeight={true}
84 | initialState={{
85 | ...{
86 | columns: columns,
87 | rows: [],
88 | }.initialState,
89 | pagination: { paginationModel: { pageSize: 10 } },
90 | }}
91 | pageSizeOptions={[5, 10, 25]}
92 | slots={{ toolbar: DataGridToolbar }}
93 | />
94 |
95 | );
96 | };
97 |
98 | export default DataGridSingleTurnSummary;
99 |
--------------------------------------------------------------------------------
/JudgeIt-App/components/judge/DisplayRequestHistoryMultiTurn.jsx:
--------------------------------------------------------------------------------
1 | import React from "react";
2 | import { Grid, Paper, Box, CircularProgress } from "@mui/material";
3 |
4 | const DisplayRequestHistoryMultiTurnConversation = ({ serverData }) => {
5 | return (
6 | <>
7 |
15 |
16 |
27 |
29 |
30 |
31 |
47 |
49 |
50 |
57 |
73 |
75 |
76 |
77 | );
78 | };
79 |
80 | export default RatingSimilarityForm;
81 |
--------------------------------------------------------------------------------
/REST-Service/app/src/services/WatsonXService.py:
--------------------------------------------------------------------------------
1 | from ibm_watson_machine_learning.foundation_models import Model
2 | from ibm_watson_machine_learning.metanames import GenTextParamsMetaNames as GenParams
3 |
4 | from ibm_watsonx_ai.foundation_models import Model
5 | from langchain_ibm import WatsonxLLM
6 | from langchain_core.prompts import PromptTemplate
7 |
8 | class WatsonXService:
9 |
10 | def __init__(self,
11 | api_key,
12 | project_id,
13 | llm_model_id) -> None:
14 | self.api_key = api_key
15 | self.ibm_cloud_url = 'https://us-south.ml.cloud.ibm.com'
16 | self.project_id = project_id
17 | self.llm_model_id = llm_model_id
18 |
19 | def get_wml_llm_services(self,
20 | decoding_method="greedy",
21 | min_new_tokens=1,
22 | max_new_tokens=200,
23 | repetition_penalty=1,
24 | stop_sequences=['}']) -> WatsonxLLM:
25 |
26 | # llm parameters
27 | generate_parameters = {
28 | "decoding_method": decoding_method,
29 | "min_new_tokens": min_new_tokens,
30 | "max_new_tokens": max_new_tokens,
31 | "repetition_penalty": repetition_penalty,
32 | "stop_sequences": stop_sequences
33 | }
34 |
35 | # instatiate llm
36 | llm_model = WatsonxLLM(apikey=self.api_key,
37 | url=self.ibm_cloud_url,
38 | project_id=self.project_id,
39 | model_id=self.llm_model_id,
40 | params=generate_parameters)
41 | return llm_model
42 |
43 | ## using watsonx machine learning api
44 | def send_to_watsonxai(
45 | self,
46 | prompts,
47 | model_id="meta-llama/llama-3-70b-instruct",
48 | decoding_method="greedy",
49 | max_new_tokens=500,
50 | min_new_tokens=30,
51 | temperature=1.0,
52 | repetition_penalty=1.0
53 | ):
54 |
55 | # Instantiate parameters for text generation
56 | model_params = {
57 | GenParams.DECODING_METHOD: decoding_method,
58 | GenParams.MIN_NEW_TOKENS: min_new_tokens,
59 | GenParams.MAX_NEW_TOKENS: max_new_tokens,
60 | GenParams.RANDOM_SEED: 42,
61 | GenParams.TEMPERATURE: temperature,
62 | GenParams.REPETITION_PENALTY: repetition_penalty,
63 | }
64 |
65 | model = Model(
66 | model_id=model_id,
67 | params=model_params,
68 | credentials={
69 | "url" : self.ibm_cloud_url,
70 | "apikey" : self.api_key
71 | },
72 | project_id=self.project_id)
73 |
74 | response=model.generate_text(prompts)
75 | return response
76 |
--------------------------------------------------------------------------------
/REST-Service/deployment/readme.md:
--------------------------------------------------------------------------------
1 | # Deploy REST Service in OpenShift cluster
2 |
3 | ## Login to OpenShift cluster
4 |
5 | Step 1: Login to openshift console and copy login command
6 |
7 |
96 |
98 | );
99 | };
100 |
101 | export default BarChart;
--------------------------------------------------------------------------------
/JudgeIt-App/components/judge/DisplayRequestHistorySingleTurn.jsx:
--------------------------------------------------------------------------------
1 | import React from "react";
2 | import { Grid, Paper, Box, CircularProgress } from "@mui/material";
3 |
4 | const DisplayRequestHistorySingleTurn = ({ serverData }) => {
5 | return (
6 | <>
7 |
14 |
15 |
26 |
27 |
40 |
41 |
52 |
53 |
66 |
67 |
80 |
81 | );
82 | };
83 |
84 | export default SingleTurnForm;
85 |
--------------------------------------------------------------------------------
/JudgeIt-App/components/judge/DisplayRequestHistoryRatingSimilarity.jsx:
--------------------------------------------------------------------------------
1 | import React from "react";
2 | import { Grid, Box } from "@mui/material";
3 | import {
4 | API_TYPE_RATING,
5 | API_TYPE_SIMILARITY,
6 | grade_map_rating,
7 | grade_map_similarity,
8 | } from "@/services/Config";
9 |
10 | const DisplayRequestHistoryRatingSimilarity = ({ serverData }) => {
11 | return (
12 | <>
13 | -
50 |
-
61 |
-
92 |
31 | {" "}
32 |
37 |
38 | Evaluation Type
39 |
40 |
48 |
120 | {touched.apiType && errors.apiType && (
121 | {errors.apiType}
122 | )}
123 |
124 |
125 | );
126 | };
127 |
128 | export default EvaluationTypeComponent;
129 |
--------------------------------------------------------------------------------
/JudgeIt-App/components/globals/SingleInstructions.jsx:
--------------------------------------------------------------------------------
1 | "use client";
2 | import { Box, Typography, Button } from "@mui/material";
3 | import GavelOutlinedIcon from "@mui/icons-material/GavelOutlined";
4 |
5 | function SingleInstructions() {
6 | return (
7 | -
29 |
-
30 |
38 | RAG Evaluation (Similarity): 39 | 40 |-
41 |
- 42 | Function: Compare a golden text to a generated text 43 | 44 |
- 45 | Input: Provide the following: 46 | 47 | 48 |
- golden text 50 |
- generated text 51 |
- 53 | Output: The LLM Judge will output a Grade and Explanation. 54 | A grade of 0 means the texts are dissimilar, while a grade of 1 55 | means the texts are similar. 56 | 57 |
-
49 |
59 |
60 | -
61 |
69 | RAG Evaluation (Rating): 70 | 71 |-
72 |
- 73 | Function: Compare a golden text to a generated text 74 | 75 |
- 76 | Input: Provide the following: 77 | 78 |
- golden text 80 |
- generated text 81 |
- 83 | Output: The LLM Judge will output a Grade and Explanation. 84 | A grade of 1 means the texts are dissimilar, a grade of 2 means 85 | the texts are partially similar, and a text of 3 means the texts 86 | are significantly similar 87 | 88 |
-
79 |
90 |
91 | -
92 |
100 | Multi-turn Evaluation: 101 | 102 |-
103 |
- 104 | Function: Compare a golden rewritten query to a rewritten 105 | query based on a multi-turn conversation 106 | 107 |
- 108 | Input: Provide the following: 109 | 110 |
- previous question 112 |
- previous answer 113 |
- current question 114 |
- golden rewritten question 115 |
- rewritten question 116 |
- 118 | Output: The LLM Judge will output a Grade and Explanation. 119 | A grade of 0 means the texts are dissimilar, while a grade of 1 120 | means the texts are similar. 121 | 122 |
-
111 |
124 |
56 |
58 | );
59 | }
60 |
61 | return (
62 | <>
63 | 34 | 1: agent thought trail is working correctly is valid (useful) 35 | 36 | #### Implementation focuses on the following questions: 37 | 1. Was the flow valid? i.e., followed logical needed steps like thought, tool usage, thought, final answer etc 38 | 2. Were tools used by the agent? 39 | 3. Were the right tools used? 40 | 4. Errors seen 41 | 42 | ### Negative testing evaluation 43 | Makes use of watsonx.governance libraries to run HAP checks on all the content present - input and output of each agent 44 | 45 | #### Score system: 46 | 0: HAP/HARM/Unethical behavior etc content NOT found
47 | 1: HAP/HARM/Unethical behavior etc content found 48 | 49 | #### For cases where wx.gov cannot flag "negative" content: Developed LLM as Judge 50 | #### LLM as Judge Score system: (same as above) 51 | 0: content is clean
52 | 1: "negative" content found 53 | 54 | #### Implementation: 55 | Inorder to run the integrated codebase locally, you need to build it slightly different since the wx.gov libraries don't work well on macs: 56 | 57 | 1. Build the fastapi backend separately: `podman build --platform=linux/amd64 -t fastapi_app_image -f Dockerfile .` 58 | 2. Replace docker-compose yaml with the given compose yaml and add env variables under 'environment' sections: 59 | ``` 60 | services: 61 | fastapi_app: 62 | container_name: fastapi_app 63 | platform: linux/amd64 64 | image: fastapi_app_image 65 | #volumes: 66 | # - ./app:/app 67 | ports: 68 | - 3001:3001 69 | environment: 70 | - WATSONX_URL=https://us-south.ml.cloud.ibm.com 71 | - WX_PROJECT_ID= 72 | - IBM_CLOUD_API_KEY= 73 | - LLM_JUDGE_API_KEY= 74 | - WX_PLATFORM=saas 75 | - WX_USER='' 76 | - WX_GOV_REGION=eu-de 77 | - CELERY_BROKER_URL=redis://redis:6379/0 78 | - CELERY_RESULT_BACKEND=redis://redis:6379/0 79 | - SERVER_URL=http://localhost:3001 80 | restart: always 81 | 82 | redis: 83 | container_name: redis 84 | image: redis:7.2.5-alpine 85 | restart: always 86 | 87 | celery_worker: 88 | container_name: celery_worker 89 | build: . 90 | #volumes: 91 | # - ./app:/app 92 | command: celery -A app.celery.celery_worker.celery worker --loglevel=info 93 | environment: 94 | - WATSONX_URL=https://us-south.ml.cloud.ibm.com 95 | - WX_PROJECT_ID= 96 | - WX_PLATFORM=saas 97 | - WX_USER='' 98 | - WX_GOV_REGION=eu-de 99 | - IBM_CLOUD_API_KEY= 100 | - CELERY_BROKER_URL=redis://redis:6379/0 101 | - CELERY_RESULT_BACKEND=redis://redis:6379/0 102 | depends_on: 103 | - fastapi_app 104 | - redis 105 | restart: always 106 | 107 | flower: 108 | container_name: flower 109 | build: . 110 | command: celery --broker=redis://redis:6379/0 flower --port=5555 111 | ports: 112 | - 5556:5555 113 | environment: 114 | - CELERY_BROKER_URL=redis://redis:6379/0 115 | - CELERY_RESULT_BACKEND=redis://redis:6379/0 116 | depends_on: 117 | - fastapi_app 118 | - redis 119 | - celery_worker 120 | restart: always 121 | ``` 122 | 3. Run to build the rest of the services: `podman-compose build` 123 | 4. Run to get the services up and running: `podman-compose up -d` 124 | 5. Run to check if all 4 services are up and running: `podman-compose ps` 125 | 126 | -------------------------------------------------------------------------------- /JudgeIt-App/services/Config.js: -------------------------------------------------------------------------------- 1 | export const APP_VERSION = "Alpha-1.0 version"; 2 | //export const LLM_JUDGE_BASE_URL = "https://llm-judge-backend-llm-judge.roks-dsce2v-13d45cd84769aede38d625cd31842ee0-0000.us-south.containers.appdomain.cloud"; 3 | export const LLM_JUDGE_BASE_URL = "http://localhost:3001"; 4 | export const LLM_JUDGE_BATCH_EVENT_URL = 5 | LLM_JUDGE_BASE_URL + "/api/v1/judge/events/"; 6 | export const LLM_JUDGE_DOWNLOAD_EVALUATION_URL = 7 | LLM_JUDGE_BASE_URL + "/api/v1/judge/download/"; 8 | export const LLM_JUDGE_MANAGEMENT_API_URL = 9 | LLM_JUDGE_BASE_URL + "/api/v1/manage/"; 10 | 11 | export const API_TYPE_KEY = "apiType"; 12 | export const API_TYPE_RATING = "rating"; 13 | export const API_TYPE_SIMILARITY = "similarity"; 14 | export const API_TYPE_SINGLETURN = "singleturn"; 15 | export const API_TYPE_MULTITURN = "multiturn"; 16 | export const API_TYPE_WBOX_SDR = "whitebox_sdrflow"; 17 | export const API_TYPE_BBOX_SDR = "blackbox_sdrflow"; 18 | export const API_TYPE_AGENT = "agent_sdrflow"; 19 | 20 | export const LLM_JUDGE_API_KEY_SECRET = "JudgeIt-Secret-Api-Key"; 21 | 22 | export const LLM_MODELS = [ 23 | /* 24 | { 25 | value: "MIXTRAL", 26 | label: "MIXTRAL" 27 | }, 28 | { 29 | value: "GPT", 30 | label: "GPT" 31 | }, 32 | */ 33 | { 34 | value: "meta-llama/llama-3-3-70b-instruct", 35 | label: "llama-3-3-70b-instruct (Recommended)", 36 | }, 37 | { 38 | value: "meta-llama/llama-3-3-70b-instruct", 39 | label: "llama-3-3-70b-instruct" 40 | }, 41 | ]; 42 | 43 | export const GITHUB_SOURCE_CODE = 44 | "https://github.com/ibm-ecosystem-engineering/JudgeIt-LLM-as-a-Judge"; 45 | export const GITHUB_REPORT_ISSUE = 46 | "https://github.com/ibm-ecosystem-engineering/JudgeIt-LLM-as-a-Judge/issues"; 47 | 48 | export const rag_similarity_display = [ 49 | "Evaluate generated text against golden text and receive a binary score for similarity", 50 | "The LLM Judge will output a Grade and Explanation. A grade of 0 means the texts are dissimilar, while a grade of 1 means the texts are similar.", 51 | ]; 52 | 53 | export const rag_rating_display = [ 54 | "Evaluate generated text against golden text and receive a 1/2/3 rating based on degree of similarity", 55 | "The LLM Judge will output a Grade and Explanation. A grade of 1 means the texts are dissimilar, a grade of 2 means the texts are partially similar, and a text of 3 means the texts are significantly similar.", 56 | ]; 57 | 58 | export const multi_turn_display = [ 59 | "Evaluate rewritten queries given a mult-turn conversation and receive a binary score for similarity", 60 | "The LLM Judge will output a Grade. A grade of 0 means the golden rewritten question and rewritten question are dissimilar, while a grade of 1 means the questions are similar.", 61 | ]; 62 | 63 | export const wbox_display = [ 64 | "Evaluate generated agent thought trail and workflow execution on a 0/1 rating. 1 means the workflow is executing as expected; 0 means it does not.", 65 | "The LLM Judge will output a score.", 66 | ]; 67 | 68 | export const bbox_display = [ 69 | "Evaluate generated agent outputs against golden text. It evaluates Chrono, Product, and Research agents on 0/1 rating and Comms Agent on 1/2/3 rating based on degree of similarity", 70 | "The LLM Judge will output a Grade and Explanation. A grade of 1 means the texts are dissimilar, a grade of 2 means the texts are partially similar, and a text of 3 means the texts are significantly similar.", 71 | ]; 72 | 73 | export const agent_display = [ 74 | "Evaluate generated agent outputs for both black box (LLM-as-a-judge) as well as white box (workflow)", 75 | "The LLM Judge will output a set of grades for the different agents as well as for overall workflow.", 76 | ]; 77 | 78 | export const grade_map_rating = { 79 | 1: "Incorrect", 80 | 2: "Partially correct", 81 | 3: "Correct", 82 | }; 83 | 84 | export const grade_map_similarity = { 85 | 0: "Incorrect", 86 | 1: "Correct", 87 | }; 88 | 89 | export const grade_map_multiturn = { 90 | 0: "Incorrect", 91 | 1: "Correct", 92 | }; 93 | 94 | 95 | export const app_labels_and_config = { 96 | app_version: "Alpha-1.0 version", 97 | app_title: "JudgeIt", 98 | app_subtitle: "LLM as a Judge", 99 | logo_text: "Ecosystem Engineering", 100 | buttons: { 101 | single_page_action: "Single answer evaluation", 102 | batch_page_action: "Batch evaluation", 103 | }, 104 | home_page_panel_title: { 105 | similarity_panel: "RAG Evaluation (Similarity)", 106 | rating_panel: "RAG Evaluation (Rating)", 107 | multiturn_panel: "Multi-turn evaluation", 108 | home_page_intro: 109 | "JudgeIt is an automated evaluation framework designed for testing various Generative AI pipelines such as RAG, Multi-Turn Query Rewriting, Text-to-SQL, and more. This service utilizes an LLM Judge to accurately and efficiently evaluate generated text against provided golden text. Try evaluating a single input or a batch of inputs by clicking one of the options below!", 110 | }, 111 | pages: { 112 | batch_evaluation_page_title: "Batch Evaluation", 113 | single_evaluation_page_title: "Single Answer Evaluation", 114 | graph_title: "Grade Distribution", 115 | }, 116 | github: "https://github.com/ibm-ecosystem-engineering/JudgeIt-LLM-as-a-Judge", 117 | github_issues: 118 | "https://github.com/ibm-ecosystem-engineering/JudgeIt-LLM-as-a-Judge/issues", 119 | }; 120 | -------------------------------------------------------------------------------- /JudgeIt-App/components/globals/BatchInstructions.jsx: -------------------------------------------------------------------------------- 1 | "use client"; 2 | import { Box, Typography, Button } from "@mui/material"; 3 | import BatchPredictionOutlinedIcon from "@mui/icons-material/BatchPredictionOutlined"; 4 | 5 | function BatchInstructions() { 6 | return ( 7 |
-
42 |
-
43 |
51 | RAG Evaluation (Similarity): 52 | 53 |-
54 |
- 55 | Function: Compare a golden text to a generated text 56 | 57 |
- 58 | Input: Provide an excel/csv file with the following 59 | columns: 60 | 61 | 62 |
- golden_text 64 |
- generated_text 65 |
- 67 | Output: The LLM Judge will output a Grade and Explanation. 68 | A grade of 0 means the texts are dissimilar, while a grade of 1 69 | means the texts are similar. 70 | 71 |
-
63 |
73 |
74 | -
75 |
83 | RAG Evaluation (Rating): 84 | 85 |-
86 |
- 87 | Function: Compare a golden text to a generated text 88 | 89 |
- 90 | Input: Provide an excel/csv file with the following 91 | columns: 92 | 93 |
- golden_text 95 |
- generated_text 96 |
- 98 | Output: The LLM Judge will output a Grade and Explanation. 99 | A grade of 1 means the texts are dissimilar, a grade of 2 means 100 | the texts are partially similar, and a text of 3 means the texts 101 | are significantly similar 102 | 103 |
-
94 |
105 |
106 | -
107 |
115 | Multi-turn Evaluation: 116 | 117 |-
118 |
- 119 | Function: Compare a golden rewritten query to a rewritten 120 | query based on a multi-turn conversation 121 | 122 |
- 123 | Input: Provide an excel/csv file with the following 124 | columns: 125 | 126 |
- previous_question 128 |
- previous_answer 129 |
- current_question 130 |
- golden_rewritten_question 131 |
- rewritten_question 132 |
- 134 | Output: The LLM Judge will output a Grade and Explanation. 135 | A grade of 0 means the texts are dissimilar, while a grade of 1 136 | means the texts are similar. 137 | 138 |
-
127 |
140 |
61 |
67 |
71 |
79 |
90 | {touched.experiment_option && errors.experiment_option && (
91 | {errors.experiment_option}
92 | )}
93 |
94 |
98 |
100 |
101 | {values.experiment_option === "new_experiment" && (
102 |
103 |
117 |
119 |
120 | )}
121 | {values.experiment_option === "existing_experiment" && (
122 |
128 |
133 | Experiment
134 |
153 | {touched.existing_experiment && errors.existing_experiment && (
154 | {errors.existing_experiment}
155 | )}
156 |
157 |
161 |
163 |
164 | )}
165 |
166 | );
167 | };
168 |
169 | export default ExperimentForm;
170 |
--------------------------------------------------------------------------------
/JudgeIt-App/services/JudgeBackendAPISolo.js:
--------------------------------------------------------------------------------
1 | import axios from "axios";
2 | import {
3 | API_TYPE_MULTITURN,
4 | API_TYPE_RATING,
5 | API_TYPE_SIMILARITY,
6 | LLM_JUDGE_BASE_URL,
7 | LLM_JUDGE_API_KEY_SECRET,
8 | LLM_JUDGE_MANAGEMENT_API_URL,
9 | API_TYPE_SINGLETURN,
10 | } from "./Config";
11 |
12 | import { create_experiment } from "./ManagemenBackendAPI";
13 | import { generateRandomString } from "@/utils/Helper";
14 |
15 | /* SOLO API ENDPOINTS */
16 | const API_RATING_URL = LLM_JUDGE_BASE_URL + "/api/v1/judge/rating";
17 | const API_SIMLARITY_URL = LLM_JUDGE_BASE_URL + "/api/v1/judge/similarity";
18 | const API_SINGLE_TURN_URL = LLM_JUDGE_BASE_URL + "/api/v1/judge/singleturn";
19 | const API_MULTITURN_URL = LLM_JUDGE_BASE_URL + "/api/v1/judge/multiturn";
20 |
21 | const config = {
22 | headers: {
23 | accept: "application/json",
24 | LLM_JUDGE_API_KEY: LLM_JUDGE_API_KEY_SECRET,
25 | "Content-Type": "application/json",
26 | },
27 | };
28 |
29 | /** Single request call*/
30 | export async function judge_api_solo_call(payload) {
31 |
32 | try {
33 | if (payload.apiType === API_TYPE_RATING) {
34 | return await rating_api_call(payload);
35 | } else if (payload.apiType === API_TYPE_SIMILARITY) {
36 | return await similarity_api_call(payload);
37 | } else if (payload.apiType === API_TYPE_SINGLETURN) {
38 | return await single_turn_api_call(payload);
39 | } else if (payload.apiType === API_TYPE_MULTITURN) {
40 | return await multiturn_conversation_api_call(payload);
41 | }else {
42 | throw "API not found";
43 | }
44 | } catch (error) {
45 | throw error;
46 | } finally {
47 | }
48 | }
49 |
50 | async function save_request_history(payload, result) {
51 | const headers = {
52 | accept: "application/json",
53 | "Content-Type": "application/json",
54 | "user-id": payload.user_id,
55 | LLM_JUDGE_API_KEY: LLM_JUDGE_API_KEY_SECRET,
56 | };
57 |
58 | const url = LLM_JUDGE_MANAGEMENT_API_URL + "history";
59 |
60 | const experiment_name =
61 | payload.experiment_option === "new_experiment"
62 | ? payload.new_experiment
63 | : payload.existing_experiment;
64 |
65 | let query = {};
66 | let name = payload.apiType + " - " + generateRandomString(4);
67 |
68 | if (payload.apiType === API_TYPE_SINGLETURN) {
69 | query = {
70 | model: payload.model,
71 | previous_question: payload.previous_question,
72 | previous_answer: payload.previous_answer,
73 | current_question: payload.current_question,
74 | golden_rewritten_question: payload.golden_rewritten_question,
75 | rewritten_question: payload.rewritten_question,
76 | };
77 | } else if (payload.apiType === API_TYPE_MULTITURN) {
78 | query = {
79 | model: payload.model,
80 | conversation_history: payload.conversation_history,
81 | follow_up_query: payload.follow_up_query,
82 | golden_query: payload.golden_query,
83 | rewritten_query: payload.rewritten_query
84 | };
85 | } else {
86 | query = {
87 | model: payload.model,
88 | question: payload.question,
89 | golden_text: payload.golden_text,
90 | generated_text: payload.generated_text,
91 | };
92 | }
93 |
94 | const content = {
95 | query: query,
96 | result: result,
97 | };
98 | const data = {
99 | name: name,
100 | user_id: payload.user_id,
101 | experiment_name: experiment_name,
102 | content: content,
103 | type: "single",
104 | eval_type: payload.apiType,
105 | };
106 |
107 | try {
108 | const response = await axios.post(url, data, { headers });
109 | data._id = response.data.insert_id;
110 | return data;
111 | } catch (error) {}
112 | }
113 |
114 | async function rating_api_call(payload) {
115 | try {
116 | const response = await axios.post(API_RATING_URL, payload, config);
117 |
118 | // creating new experiment after a successful call
119 | await create_experiment(payload, "single");
120 |
121 | // save the request
122 | const savedObject = await save_request_history(payload, response.data);
123 |
124 | return {
125 | query: savedObject,
126 | data: response.data,
127 | };
128 | } catch (error) {
129 | throw error;
130 | }
131 | }
132 |
133 | async function similarity_api_call(payload) {
134 | try {
135 | const response = await axios.post(API_SIMLARITY_URL, payload, config);
136 | // creating new experiment after a successful call
137 | await create_experiment(payload, "single");
138 |
139 | // save the request
140 | const savedObject = await save_request_history(payload, response.data);
141 |
142 | return {
143 | query: savedObject,
144 | data: response.data,
145 | };
146 | } catch (error) {
147 | throw error;
148 | }
149 | }
150 |
151 | async function single_turn_api_call(payload) {
152 | try {
153 | const response = await axios.post(API_SINGLE_TURN_URL, payload, config);
154 | // creating new experiment after a successful call
155 | await create_experiment(payload, "single");
156 |
157 | // save the request
158 | const savedObject = await save_request_history(payload, response.data);
159 |
160 | return {
161 | query: savedObject,
162 | data: response.data,
163 | };
164 | } catch (error) {
165 | throw error;
166 | }
167 | }
168 |
169 | async function multiturn_conversation_api_call(payload) {
170 | try {
171 | const response = await axios.post(API_MULTITURN_URL, payload, config);
172 | // creating new experiment after a successful call
173 | await create_experiment(payload, "single");
174 |
175 | console.log("#####################################################", payload);
176 |
177 | // save the request
178 | const savedObject = await save_request_history(payload, response.data);
179 |
180 | return {
181 | query: savedObject,
182 | data: response.data,
183 | };
184 | } catch (error) {
185 | throw error;
186 | }
187 | }
188 |
--------------------------------------------------------------------------------
/REST-Service/app/src/utils/Helper.py:
--------------------------------------------------------------------------------
1 | import io
2 | from fastapi import UploadFile
3 | import pandas as pd
4 |
5 | class Helper:
6 |
7 | def __init__(self) -> None:
8 | pass
9 |
10 | def read_data(self, file_name: str, file_content: bytes) -> pd.DataFrame:
11 |
12 | file_extension = file_name.split(".")[-1].lower()
13 | if file_extension not in ['xls', 'xlsx', 'csv']:
14 | raise Exception("Bad file types, accepted file types are xls, xlsx, and csv")
15 |
16 | ## Read the data for btach processing
17 | data_df = pd.DataFrame()
18 | file_stream = io.BytesIO(file_content)
19 | if '.xlsx' in file_name:
20 | data_df = pd.read_excel(file_stream)
21 | elif '.csv' in file_name:
22 | data_df =pd.read_csv(file_stream)
23 | return data_df
24 |
25 |
26 | def validate_single_turn_fields(self, data_df: pd.DataFrame):
27 |
28 | # Normalize the column names to lowercase
29 | data_df.columns = map(str.lower, data_df.columns)
30 |
31 | required_columns = ["previous_question", "previous_answer", "current_question", "golden_rewritten_question", "rewritten_question"]
32 |
33 | if all(column in data_df.columns for column in required_columns):
34 | return True
35 |
36 | columns = ", ".join(required_columns)
37 |
38 | raise Exception("Required columns are missing, valid columns are ## " + columns)
39 |
40 | def validate_multi_turn_with_conversation_fields(self, data_df: pd.DataFrame):
41 |
42 | # Normalize the column names to lowercase
43 | data_df.columns = map(str.lower, data_df.columns)
44 |
45 | required_columns = ["conversation_history", "follow_up_query", "golden_query", "rewritten_query"]
46 |
47 | if all(column in data_df.columns for column in required_columns):
48 | return True
49 |
50 | columns = ", ".join(required_columns)
51 |
52 | raise Exception("Required columns are missing, valid columns are ## " + columns)
53 |
54 | def validate_rating_and_similarity_fields(self, data_df: pd.DataFrame):
55 | # Normalize the column names to lowercase
56 | data_df.columns = map(str.lower, data_df.columns)
57 |
58 | # Define required columns in lowercase
59 | required_columns = ["question", "golden_text", "generated_text"]
60 |
61 | # Check if all required columns are present (case-insensitive)
62 | if all(column in data_df.columns for column in required_columns):
63 | return True
64 |
65 | columns = ", ".join(required_columns)
66 |
67 | raise Exception("Required columns are missing, valid columns are ## " + columns)
68 |
69 |
70 | def is_valid_file(file: UploadFile):
71 | filename = file.filename
72 | file_extension = filename.split(".")[-1].lower()
73 |
74 | if file_extension == 'csv' or file_extension in ['xls', 'xlsx']:
75 | return True
76 | else:
77 | return False
78 |
79 | # This code was added to handle the case when the columns produced by langfuse script had lower case o in the
80 | # Chrono Agent output field; This is not needed because we'll change all columns to title formant before sending
81 | # to whitebox eval
82 | def validate_wbox_eval_fields(self, data_df: pd.DataFrame):
83 |
84 | ## Data provided has it as "Chrono Agent output" instead of "Chrono Agent Output" so made that change here..
85 |
86 | required_columns = ["Chrono Agent output", "Product Agent Output", "Research Agent Output", "Comms Agent Output"]
87 |
88 | if all(column in data_df.columns for column in required_columns):
89 | return True
90 |
91 | columns = ", ".join(required_columns)
92 |
93 | raise Exception("Required columns are missing, valid columns are ## " + columns)
94 |
95 |
96 |
97 | # def validate_wbox_eval_fields(self, data_df: pd.DataFrame):
98 |
99 | # required_columns = ["Chrono Agent Output", "Product Agent Output", "Research Agent Output", "Comms Agent Output"]
100 |
101 | # if all(column in data_df.columns for column in required_columns):
102 | # return True
103 |
104 | # columns = ", ".join(required_columns)
105 |
106 | # raise Exception("Required columns are missing, valid columns are ## " + columns)
107 |
108 | def validate_bbox_eval_fields(self, data_df: pd.DataFrame):
109 |
110 | required_columns = ["Chrono Agent Output", "Product Agent Output", "Research Agent Output", "Comms Agent Output"]
111 |
112 | for col in required_columns:
113 | colfound = col in data_df.columns
114 | print(f"col {col} found: {colfound}")
115 |
116 | if all(column in data_df.columns for column in required_columns):
117 | return True
118 |
119 | columns = ", ".join(required_columns)
120 |
121 | raise Exception("Required columns are missing, valid columns are ## " + columns)
122 |
123 |
124 | def validate_neg_test_eval_fields(self, data_df: pd.DataFrame):
125 |
126 | required_columns = ["Research Agent Output", "Comms Agent Output"]
127 |
128 |
129 | if all(column in data_df.columns for column in required_columns):
130 | return True
131 |
132 | columns = ", ".join(required_columns)
133 |
134 | raise Exception("Required columns are missing, valid columns are ## " + columns)
135 |
136 |
137 | def validate_agent_eval_fields(self, data_df: pd.DataFrame):
138 |
139 | required_columns = ["Chrono Agent Output", "Product Agent Output", "Research Agent Output", "Comms Agent Output"]
140 |
141 | for col in required_columns:
142 | colfound = col in data_df.columns
143 | print(f"col {col} found: {colfound}")
144 |
145 | if all(column in data_df.columns for column in required_columns):
146 | return True
147 |
148 | columns = ", ".join(required_columns)
149 |
150 | raise Exception("Required columns are missing, valid columns are ## " + columns)
151 |
--------------------------------------------------------------------------------
/Framework/answer_similarity.py:
--------------------------------------------------------------------------------
1 | import json
2 | import configparser
3 | from langchain_ibm import WatsonxLLM
4 | from langchain_core.prompts import PromptTemplate
5 | import sys
6 |
7 | config = configparser.ConfigParser()
8 | config.read('./config.ini')
9 |
10 | ## Grading a generated text compared to a golden text
11 | SIMILARITY_PROMPT= """Follow these structured steps to accurately assess the similarity between a Golden Text and a Generated Text:
12 | 1. **Role and Task**: Assume the role of an impartial assistant and evaluator. Your task is to assess the similarity between a Golden Text and a Generated Text using the provided information.
13 | 2. **Initial Setup**: Begin by carefully reviewing the Golden Text to understand the key information, entities, and intents it contains. The Golden Text is considered fully correct and comprehensive. Then, examine the Generated Text that needs evaluation.
14 | 3. **Evaluation Criteria**: Evaluate the Generated Text based on the following criteria:
15 | - Output {{"Grade": "1"}} if:
16 | a) The Generated Text matches the Golden Text closely in terms of key entities and intents. Note that these may be worded differently but convey the same meaning contextually.
17 | b) The Generated Text contains all the essential information from the Golden Text, even if presented in a different order or with slight variations in phrasing.
18 | c) The Generated Text includes the core information from the Golden Text or may contain additional relevant, concise details or expansions that don't contradict the contextual meaning of the Golden Text.
19 | - Output {{"Grade": "0"}} if:
20 | a) The Generated Text is missing critical entities or intents that are present in the Golden Text.
21 | b) The Generated Text contains significant factual errors or contradictions when compared to the Golden Text.
22 | c) The overall meaning or intent of the Generated Text substantially differs from the Golden Text.
23 | 4. **Tolerance for Minor Differences**: Allow for minor differences in numerical values, slight variations in proper nouns, and small discrepancies in less critical details, as long as the core meaning and primary facts remain intact.
24 | 5. **Explanation**: After providing the grade, explain your reasoning in 1 sentence, highlighting key similarities or differences that influenced your decision.
25 | 6. **Output Format**: Format your evaluation output strictly as {{"Grade": "evaluated grade", "Explanation": "explanation for grade"}} to ensure clarity and consistency in assessment.
26 | Remember, the goal is to identify substantive similarity rather than expecting word-for-word matches. Focus on the core information, key facts, and overall intent when making your assessment.
27 |
28 | Input:
29 | Golden Text: {prompt_parameter_1}
30 | Generated Text: {prompt_parameter_2}
31 |
32 | Output:
33 | """
34 |
35 | def batch_llm_answer_similarity(model_id, input_data):
36 | # watsonx.ai credentials for llm judge
37 |
38 | # instantiate wml connection
39 | wml_credentials = {
40 | "url": config['WML_CRED']['wml_url'],
41 | "apikey": config['WML_CRED']['api_key']
42 | }
43 |
44 | project_id = config['WML_CRED']['project_id']
45 |
46 | llm_model_id = model_id
47 |
48 | # llm parameters
49 | generate_parameters_1 = {
50 | "decoding_method": "greedy",
51 | "min_new_tokens": 1,
52 | "max_new_tokens": 200,
53 | "repetition_penalty": 1,
54 | "stop_sequences": ['}']
55 | }
56 |
57 | platform = config['WML_CRED']['wml_platform']
58 | if platform == "saas":
59 | # instatiate llm
60 | llm_model = WatsonxLLM(apikey=wml_credentials['apikey'],
61 | url=wml_credentials['url'],
62 | project_id=project_id,
63 | model_id=llm_model_id,
64 | params=generate_parameters_1)
65 | elif platform == "onpremise":
66 | wml_user = config['WML_CRED']['wml_user']
67 | llm_model = WatsonxLLM(apikey=wml_credentials['apikey'],
68 | url=wml_credentials['url'],
69 | model_id=llm_model_id,
70 | username=wml_user,
71 | instance_id='openshift',
72 | project_id=project_id,
73 | version="5.0",
74 | params=generate_parameters_1)
75 | else:
76 | raise Exception("Please set a correct value in config.ini [WML_CRED][wml_platform], correct values are `onpremise` or `saas` ")
77 |
78 | input_data['Grade'] = None
79 | input_data['Explanation'] = None
80 |
81 | for index, row in input_data.iterrows():
82 | input_variables = ['prompt_parameter_1', 'prompt_parameter_2']
83 | prompt = PromptTemplate(input_variables=input_variables, template=SIMILARITY_PROMPT)
84 | llm_chain = prompt | llm_model
85 | # create invoke parameter which is a dictionary of your prompt parameters
86 | try:
87 | prompt_data = {'prompt_parameter_1': row['golden_text'],
88 | 'prompt_parameter_2': row['generated_text']}
89 | except KeyError as e:
90 | print(f"Error: Missing required column - {e}")
91 | print("Input file requires the following columns:")
92 | print("1) golden_text")
93 | print("2) generated_text")
94 | try:
95 | prompt_results = json.loads(llm_chain.invoke(prompt_data))
96 | except:
97 | prompt_results = 'Error generating results'
98 |
99 | if prompt_results == 'Error generating results':
100 | input_data.at[index,'Grade'] = 'Error'
101 | input_data.at[index,'Explanation'] = 'Error'
102 | else:
103 | input_data.at[index,'Grade'] = int(prompt_results['Grade'])
104 | input_data.at[index,'Explanation'] = prompt_results['Explanation']
105 | input_string = f"Golden Text: {prompt_data['prompt_parameter_1']}\n\nGenerated Text: {prompt_data['prompt_parameter_2']}"
106 | print(f'-------------testing input {index + 1}-------------\n')
107 | print(f'1) Input:\n\n{input_string}\n\n')
108 | print(f'2) Output:\n\n{prompt_results}\n\n')
109 |
110 | return input_data
111 |
--------------------------------------------------------------------------------