├── .env ├── .gitignore ├── asset └── main_fig.jpg ├── configs ├── amazon.yaml ├── bigbench.yaml ├── grounding.yaml ├── medmcqa.yaml └── safety.yaml ├── datasets └── amazon │ ├── baby.json │ ├── beauty.json │ ├── electronics.json │ ├── game.json │ ├── office.json │ ├── pet.json │ └── sports.json ├── main.sh ├── meta_test.py ├── meta_train.py ├── prompts ├── cot.json ├── default.json ├── service.json └── unseen_generalization_user_prompts │ ├── up_for_unseen_gen_anatomy.json │ ├── up_for_unseen_gen_dental.json │ ├── up_for_unseen_gen_electronics.json │ ├── up_for_unseen_gen_epistemic.json │ ├── up_for_unseen_gen_ethos.json │ ├── up_for_unseen_gen_harmless.json │ ├── up_for_unseen_gen_natural_questions.json │ ├── up_for_unseen_gen_object_counting.json │ ├── up_for_unseen_gen_pediatrics.json │ ├── up_for_unseen_gen_pet.json │ ├── up_for_unseen_gen_reasoning_colored_objects.json │ ├── up_for_unseen_gen_sports.json │ ├── up_for_unseen_gen_surgery.json │ └── up_for_unseen_gen_web_qa.json ├── readme.md ├── requirements.txt └── src ├── __init__.py ├── analyser.py ├── language_model ├── __init__.py ├── agents.py ├── meta_prompts.py ├── openai_model.py └── vllm_model.py ├── methods ├── __init__.py ├── metaspo │ ├── __init__.py │ ├── metaspo.py │ └── metaspo_ape.py ├── node.py └── unilevel │ ├── __init__.py │ ├── ape.py │ ├── protegi.py │ └── unilevel.py ├── runner.py ├── taskmanager.py ├── tasks ├── __init__.py ├── amazon.py ├── base_task.py ├── bigbench.py ├── grounding.py ├── medmcqa.py └── safety.py └── utils.py /.env: -------------------------------------------------------------------------------- 1 | OPENAI_API_KEY='YOUR_API_KEY' -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | **/__pycache__/ 2 | logs/ 3 | .env -------------------------------------------------------------------------------- /asset/main_fig.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dozi01/MetaSPO/1773f5ada8e4e7f51f0e1ae6a6c8a3a610d4c027/asset/main_fig.jpg -------------------------------------------------------------------------------- /configs/amazon.yaml: -------------------------------------------------------------------------------- 1 | meta_train_tasks: 2 | - beauty 3 | - game 4 | - baby 5 | - office 6 | 7 | meta_test_tasks: 8 | - electronics 9 | - pet 10 | - sports 11 | -------------------------------------------------------------------------------- /configs/bigbench.yaml: -------------------------------------------------------------------------------- 1 | meta_train_tasks: 2 | - logic_grid_puzzle 3 | - tracking_shuffled_objects 4 | - logical_deduction 5 | - temporal_sequences 6 | 7 | meta_test_tasks: 8 | - object_counting 9 | - reasoning_colored_objects 10 | - epistemic 11 | 12 | -------------------------------------------------------------------------------- /configs/grounding.yaml: -------------------------------------------------------------------------------- 1 | meta_train_tasks: 2 | - squad 3 | - hotpot_qa 4 | - trivia_qa 5 | - drop 6 | 7 | meta_test_tasks: 8 | - natural_questions 9 | - web_qa 10 | 11 | -------------------------------------------------------------------------------- /configs/medmcqa.yaml: -------------------------------------------------------------------------------- 1 | meta_train_tasks: 2 | - ob_gyn 3 | - medicine 4 | - pharmacology 5 | - pathology 6 | 7 | meta_test_tasks: 8 | - dental 9 | - anatomy 10 | - surgery 11 | - pediatrics 12 | -------------------------------------------------------------------------------- /configs/safety.yaml: -------------------------------------------------------------------------------- 1 | meta_train_tasks: 2 | - liar 3 | - hatecheck 4 | - tweet_eval 5 | - sarcasm 6 | 7 | meta_test_tasks: 8 | - ethos 9 | - antropic_harmless -------------------------------------------------------------------------------- /main.sh: -------------------------------------------------------------------------------- 1 | MODEL_TYPE="vllm" # openai / vllm 2 | MODEL_NAME="llama3.2_3B" # gpt-4o-mini / llama3.1_8B / llama3.2_3B / Qwen2.5_7B 3 | 4 | METHOD='metaspo' 5 | DOMAIN='amazon' 6 | 7 | # MetaSPO Training 8 | python meta_train.py --method $METHOD --init_system_prompt_path "./prompts/default.json" --log_dir "./logs/$METHOD/$DOMAIN" --base_model_type "$MODEL_TYPE" --base_model_name "$MODEL_NAME" 9 | # This will save the optimized system prompt in "./logs/$METHOD/$DOMAIN/bilevel_nodes_0.json" 10 | 11 | # Unseen Generalization with optimized system prompt 12 | python meta_test.py --analysis_method 'unseen_generalization' --init_system_prompt_path "./logs/$METHOD/$DOMAIN/bilevel_nodes_0.json" --log_dir ./logs/$METHOD/unssen_generalization/$DOMAIN --base_model_type "$MODEL_TYPE" --base_model_name "$MODEL_NAME" 13 | 14 | # Test-Time Adaptation with optimized system prompt 15 | python meta_test.py --analysis_method 'test_time_adaptation' --init_system_prompt_path "./logs/$METHOD/$DOMAIN/bilevel_nodes_0.json" --log_dir ./logs/$METHOD/test_time_adaptation/$DOMAIN --base_model_type "$MODEL_TYPE" --base_model_name "$MODEL_NAME" 16 | -------------------------------------------------------------------------------- /meta_test.py: -------------------------------------------------------------------------------- 1 | from src.analyser import Analyser 2 | import yaml 3 | import argparse 4 | import os 5 | from dotenv import load_dotenv 6 | 7 | def load_config(args, config_path): 8 | with open(config_path, "r") as f: 9 | config = yaml.safe_load(f) 10 | args.meta_train_tasks = [] 11 | args.meta_test_tasks = config["meta_test_tasks"] 12 | 13 | return args 14 | 15 | def get_args(): 16 | parser = argparse.ArgumentParser() 17 | 18 | parser.add_argument("--log_dir", type=str, required=True) 19 | parser.add_argument("--init_system_prompt_path", type=str, default="./prompts/default.json") 20 | 21 | # Meta Test Settings 22 | parser.add_argument("--analysis_method", type=str, default='unseen_generalization', choices=['unseen_generalization', 'test_time_adaptation']) 23 | parser.add_argument("--unseen_gen_up_dir", type=str, default="./prompts/unseen_generalization_user_prompts/") 24 | parser.add_argument("--num_test_up", type=int, default=10) 25 | 26 | # Search Settings For Test Time Adaptation 27 | parser.add_argument("--method", type=str, default='protegi') 28 | parser.add_argument("--iteration", type=int, default=6) 29 | parser.add_argument("--num_system_candidate", type=int, default=9) 30 | parser.add_argument("--num_user_candidate", type=int, default=3) 31 | parser.add_argument("--user_top_k", type=int, default=3) 32 | 33 | # Base Model Settings 34 | parser.add_argument("--base_model_type", type=str, required=True) 35 | parser.add_argument("--base_model_name", type=str, required=True) 36 | parser.add_argument("--base_model_temperature", type=float, default=0.0) 37 | 38 | # Optimizer Model Settings 39 | parser.add_argument("--optim_model_type", type=str, default="openai") 40 | parser.add_argument("--optim_model_name", type=str, default="gpt-4o-mini") 41 | parser.add_argument("--optim_model_temperature", type=float, default=1.0) 42 | 43 | # Task Settings 44 | parser.add_argument("--task_config_path", type=str, default="./configs/amazon.yaml") 45 | parser.add_argument("--train_size", type=int, default=50) 46 | parser.add_argument("--test_size", type=int, default=500) 47 | parser.add_argument("--seed", type=int, default=42) 48 | parser.add_argument("--dataset_dir", type=str, default="./datasets") 49 | 50 | args = parser.parse_args() 51 | args = load_config(args, args.task_config_path) 52 | 53 | # put your openai api key in .env file 54 | load_dotenv() 55 | args.openai_api_key = os.getenv("OPENAI_API_KEY") 56 | 57 | return args 58 | 59 | if __name__ == "__main__": 60 | args = get_args() 61 | analyser = Analyser(args) 62 | analyser.meta_test() 63 | -------------------------------------------------------------------------------- /meta_train.py: -------------------------------------------------------------------------------- 1 | from src.runner import Runner 2 | from dotenv import load_dotenv 3 | import argparse 4 | import os 5 | import yaml 6 | 7 | 8 | def load_config(args, config_path): 9 | with open(config_path, "r") as f: 10 | config = yaml.safe_load(f) 11 | args.meta_train_tasks = config["meta_train_tasks"] 12 | args.meta_test_tasks = [] 13 | 14 | return args 15 | 16 | def get_args(): 17 | parser = argparse.ArgumentParser() 18 | 19 | parser.add_argument("--log_dir", type=str, required=True) 20 | parser.add_argument("--init_system_prompt_path", type=str, default="./prompts/default.json") 21 | 22 | # Search Settings 23 | parser.add_argument("--method", type=str, required=True) 24 | parser.add_argument("--iteration", type=int, default=3) 25 | parser.add_argument("--num_system_candidate", type=int, default=9) 26 | parser.add_argument("--num_user_candidate", type=int, default=3) 27 | parser.add_argument("--user_top_k", type=int, default=3) 28 | 29 | # Base Model Settings 30 | parser.add_argument("--base_model_type", type=str, required=True) 31 | parser.add_argument("--base_model_name", type=str, required=True) 32 | parser.add_argument("--base_model_temperature", type=float, default=0.0) 33 | 34 | # Optimizer Model Settings 35 | parser.add_argument("--optim_model_type", type=str, default="openai") 36 | parser.add_argument("--optim_model_name", type=str, default="gpt-4o-mini") 37 | parser.add_argument("--optim_model_temperature", type=float, default=1.0) 38 | 39 | # Task Settings 40 | parser.add_argument("--task_config_path", type=str, default="./configs/amazon.yaml") 41 | parser.add_argument("--train_size", type=int, default=50) 42 | parser.add_argument("--test_size", type=int, default=500) 43 | parser.add_argument("--seed", type=int, default=42) 44 | parser.add_argument("--dataset_dir", type=str, default="./datasets") 45 | 46 | args = parser.parse_args() 47 | args = load_config(args, args.task_config_path) 48 | 49 | # put your openai api key in .env file 50 | load_dotenv() 51 | args.openai_api_key = os.getenv("OPENAI_API_KEY") 52 | 53 | return args 54 | 55 | if __name__ == "__main__": 56 | args = get_args() 57 | runner = Runner(args) 58 | runner.meta_train() 59 | -------------------------------------------------------------------------------- /prompts/cot.json: -------------------------------------------------------------------------------- 1 | { 2 | "prompt": "Let's think step by step." 3 | } -------------------------------------------------------------------------------- /prompts/default.json: -------------------------------------------------------------------------------- 1 | { 2 | "prompt": "You are a helpful assistant." 3 | } -------------------------------------------------------------------------------- /prompts/service.json: -------------------------------------------------------------------------------- 1 | { 2 | "prompt" : "The assistant is Llama, created by Meta. The current date is March 4th, 2024.\n\nLlama's knowledge base was last updated on August 2023. It answers questions about events prior to and after August 2023 the way a highly informed individual in August 2023 would if they were talking to someone from the above date, and can let the human know this when relevant.\n\nIt should give concise responses to very simple questions, but provide thorough responses to more complex and open-ended questions.\n\nIf it is asked to assist with tasks involving the expression of views held by a significant number of people, Llama provides assistance with the task even if it personally disagrees with the views being expressed, but follows this with a discussion of broader perspectives.\n\nLlama doesn't engage in stereotyping, including the negative stereotyping of majority groups.\n\nIf asked about controversial topics, Llama tries to provide careful thoughts and objective information without downplaying its harmful content or implying that there are reasonable perspectives on both sides.\n\nIt is happy to help with writing, analysis, question answering, math, coding, and all sorts of other tasks. It uses markdown for coding.\n\nIt does not mention this information about itself unless the information is directly pertinent to the human's query." 3 | } -------------------------------------------------------------------------------- /prompts/unseen_generalization_user_prompts/up_for_unseen_gen_anatomy.json: -------------------------------------------------------------------------------- 1 | [ 2 | "For each medical scenario provided below, choose the most appropriate answer from the options given. Your responses should reflect the best understanding of medical knowledge and relevant anatomy or pathology.{question}\nAt the end present your answer in and .", 3 | "Given a set of medical-related questions and multiple-choice options, select the correct answer for each question based on your knowledge.{question}\nAt the end present your answer in and .", 4 | "Given a medical question with multiple-choice options, select the correct answer based on your knowledge of medicine and anatomy.{question}\nAt the end present your answer in and .", 5 | "Please select the correct option for each question provided based on your knowledge of anatomy and medical principles.{question}\nAt the end present your answer in and .", 6 | "For each input provided, identify the correct option that best answers the question or statement presented. Please ensure to choose the answer that is most accurate based on medical knowledge or anatomical concepts.{question}\nAt the end present your answer in and .", 7 | "Provide the correct answer for each medical or anatomical question based on the given options.{question}\nAt the end present your answer in and .", 8 | "Please provide the correct answer from the options given for each medical-related question based on your knowledge.{question}\nAt the end present your answer in and .", 9 | "For each medical-related question provided below, select the correct answer from the given options. Provide your answer without any explanation.{question}\nAt the end present your answer in and .", 10 | "Given a medical question with multiple choice options, select the most appropriate answer from the provided options and provide your choice as the output.{question}\nAt the end present your answer in and .", 11 | "For each medical scenario or question presented, select the most appropriate answer from the options provided, ensuring that the response aligns with medical knowledge and reasoning. Provide the selected answer as the output.{question}\nAt the end present your answer in and .", 12 | "For each provided medical question with multiple-choice options, select the correct answer that accurately reflects current medical knowledge or practices.{question}\nAt the end present your answer in and .", 13 | "For each given medical scenario or question, select the most appropriate answer from the provided options and indicate the correct option letter as the output.{question}\nAt the end present your answer in and .", 14 | "Answer the following medical questions by selecting the most appropriate option based on the provided choices.{question}\nAt the end present your answer in and .", 15 | "Provide the correct answers to the multiple-choice questions based on the given inputs and options.{question}\nAt the end present your answer in and .", 16 | "For each medical question provided, select the most accurate answer from the given options.{question}\nAt the end present your answer in and .", 17 | "For each of the following medical questions, select the most appropriate answer from the given options.{question}\nAt the end present your answer in and .", 18 | "Please read each medical question along with its answer options and select the most appropriate answer based on your knowledge of anatomy and medical concepts.{question}\nAt the end present your answer in and .", 19 | "For each input provided, select the most appropriate answer from the given options and explain your reasoning behind the choice.{question}\nAt the end present your answer in and .", 20 | "Identify the correct option from the given choices based on the medical or anatomical question provided.{question}\nAt the end present your answer in and .", 21 | "Please provide the most appropriate answer from the given options for each question based on medical knowledge or clinical guidelines.{question}\nAt the end present your answer in and .", 22 | "Given a medical question with multiple-choice options, select the most appropriate answer from the options provided.{question}\nAt the end present your answer in and .", 23 | "Please read the following medical questions with their corresponding options and provide the most appropriate answer to each one based on your knowledge.{question}\nAt the end present your answer in and .", 24 | "Read the provided medical questions, along with their options, and select the most appropriate answer from the options given. Indicate your choice clearly.{question}\nAt the end present your answer in and .", 25 | "For each provided medical scenario and related question, choose the correct option from the given choices that best answers the question based on your knowledge of medical concepts.{question}\nAt the end present your answer in and .", 26 | "For each medical question and its corresponding options provided, select the most appropriate answer from the options given.{question}\nAt the end present your answer in and .", 27 | "For each provided medical scenario or question, select the most appropriate answer from the given options based on your knowledge of human anatomy, physiology, pathology, or related medical sciences.{question}\nAt the end present your answer in and .", 28 | "For each medical question provided, select the correct answer from the given options that best addresses the question asked.{question}\nAt the end present your answer in and .", 29 | "Based on the provided medical questions and their corresponding answer options, select the most appropriate answer from the options given for each question.{question}\nAt the end present your answer in and .", 30 | "For each medical scenario provided, select the most appropriate answer from the given options. Each question may involve anatomical, physiological, or clinical information, and the correct choice should reflect an understanding of medical knowledge relevant to the question posed.{question}\nAt the end present your answer in and .", 31 | "For each medical scenario presented, select the most appropriate answer from the given options that accurately reflects the relevant medical knowledge or fact.{question}\nAt the end present your answer in and ." 32 | ] -------------------------------------------------------------------------------- /prompts/unseen_generalization_user_prompts/up_for_unseen_gen_dental.json: -------------------------------------------------------------------------------- 1 | [ 2 | "Based on the following inputs and their corresponding options, select the most appropriate answer from the given options.{question}\nAt the end present your answer in and .", 3 | "For each of the following questions, select the correct answer from the provided options and indicate your choice clearly.{question}\nAt the end present your answer in and .", 4 | "Please provide the correct output for each input based on the given options. Select the most appropriate answer from the provided choices for each question.{question}\nAt the end present your answer in and .", 5 | "Read each input statement carefully along with the provided options, and select the most appropriate output option for each question based on your knowledge.{question}\nAt the end present your answer in and .", 6 | "For each input provided, select the most appropriate option from the available choices and provide the corresponding letter as the output.{question}\nAt the end present your answer in and .", 7 | "Provide the correct answer for each of the following questions based on the given options.{question}\nAt the end present your answer in and .", 8 | "Given a set of questions with multiple-choice options, select the most appropriate answer for each question based on your knowledge.{question}\nAt the end present your answer in and .", 9 | "Please analyze the following medical and dental cases, along with their respective multiple-choice options, and provide the most accurate answer for each scenario based on your knowledge. Use the given options to select the best possible response. Make sure to justify your selections when necessary.{question}\nAt the end present your answer in and .", 10 | "For each of the following inputs, select the most appropriate answer from the given options.{question}\nAt the end present your answer in and .", 11 | "Provide an accurate answer to each question based on the given options and format the response as indicated.{question}\nAt the end present your answer in and .", 12 | "Please read the following inputs and select the correct option from the given choices for each question.{question}\nAt the end present your answer in and .", 13 | "Please provide the correct output for each input based on your medical knowledge and understanding of the options. Choose the most appropriate answer from the given choices for each question.{question}\nAt the end present your answer in and .", 14 | "Provide the correct option based on the given input and options for each question.{question}\nAt the end present your answer in and .", 15 | "Read each input question carefully and select the most appropriate answer from the given options. Provide the corresponding output for each input based on your understanding of the subject matter.{question}\nAt the end present your answer in and .", 16 | "Provide the correct answer for each given multiple-choice question based on dental and health-related topics.{question}\nAt the end present your answer in and .", 17 | "Given a series of medical and dental-related questions with multiple-choice options, select the most appropriate answer based on the context of each question.{question}\nAt the end present your answer in and .", 18 | "Provide the most accurate answer to each question based on your knowledge in dentistry and related subjects, selecting from the given options.{question}\nAt the end present your answer in and .", 19 | "Please read the following questions along with their options and provide the correct answer for each question based on your knowledge.{question}\nAt the end present your answer in and .", 20 | "Read the input statements and choose the most appropriate answer from the provided options. Indicate your chosen answer clearly.{question}\nAt the end present your answer in and .", 21 | "For each given input related to dentistry, select the most appropriate option from the provided choices based on your knowledge.{question}\nAt the end present your answer in and .", 22 | "Please provide the correct answer from the given options for each of the following questions based on your knowledge or reasoning.{question}\nAt the end present your answer in and .", 23 | "For each medical or dental-related question, choose the most appropriate answer from the provided options and indicate your selection clearly.{question}\nAt the end present your answer in and .", 24 | "Provide the correct output option (A, B, C, or D) for each input question based on the given choices.{question}\nAt the end present your answer in and .", 25 | "Given a medical scenario or question, identify the most appropriate answer from the provided options and justify your choice based on the relevant concepts or reasoning.{question}\nAt the end present your answer in and .", 26 | "Provide the correct answer for each of the following questions by selecting the most appropriate option from the given choices.{question}\nAt the end present your answer in and .", 27 | "For each input question and its associated options, select the most appropriate answer from the given options and provide the output.{question}\nAt the end present your answer in and .", 28 | "For each given input question in the specified field, select the most appropriate answer from the provided options and write the corresponding letter for the correct answer.{question}\nAt the end present your answer in and .", 29 | "Based on the provided medical scenarios and questions, analyze the given inputs and select the most appropriate answer from the provided options for each case.{question}\nAt the end present your answer in and .", 30 | "Please analyze the provided inputs along with their corresponding options and select the most appropriate answer for each input based on your knowledge. Output your answers clearly beside each input.{question}\nAt the end present your answer in and .", 31 | "Read the following inputs and select the correct option from the choices provided. Provide the corresponding output for each input.{question}\nAt the end present your answer in and ." 32 | ] -------------------------------------------------------------------------------- /prompts/unseen_generalization_user_prompts/up_for_unseen_gen_electronics.json: -------------------------------------------------------------------------------- 1 | [ 2 | "Based on the provided input-output pairs, please assign a score from 1 to 5 for each product review, where 1 indicates a very negative experience, 5 a very positive experience, and scores in between indicate varying levels of satisfaction. Consider factors such as the reviewer's overall sentiment, the thoroughness of their feedback, and any specific positives or negatives mentioned in the texts.{question}\nAt the end present your answer in and .", 3 | "Please rate the quality or satisfaction of the product or service described in each input on a scale from 1 to 5, where 1 indicates very low satisfaction, 3 indicates moderate satisfaction, and 5 indicates very high satisfaction. Provide a brief explanation for your rating based on the content of the title and text.{question}\nAt the end present your answer in and .", 4 | "Based on the provided product titles and associated text descriptions, assign a rating from 1 to 5, where 1 indicates a poor product experience and 5 indicates an excellent product experience. Consider the sentiment expressed in the text, the clarity of the title, and how well the product meets the expectations set by the title and description. Be consistent in your rating based on these factors.{question}\nAt the end present your answer in and .", 5 | "Given a product review title and text, assign a score from 1 to 5 based on the overall positivity and effectiveness of the review, where 1 indicates a negative perception and 5 indicates a highly positive perception of the product.{question}\nAt the end present your answer in and .", 6 | "For each input review, assign a numeric rating from 1 to 5 based on the overall sentiment expressed in the review. Use the following criteria: 1 = very negative, 2 = negative, 3 = neutral, 4 = positive, 5 = very positive. Consider aspects such as product quality, satisfaction, and the reviewer's experience in your evaluation.{question}\nAt the end present your answer in and .", 7 | "Given a title and text review, evaluate the overall sentiment and assign a score from 1 to 5, where 1 indicates a negative experience, 3 indicates a neutral experience, and 5 indicates a highly positive experience. Use the review content to determine the score based on customer satisfaction, product performance, and ease of use.{question}\nAt the end present your answer in and .", 8 | "Given a title and text describing a product or experience, analyze the overall sentiment and quality expressed in the review, then provide a rating from 1 to 5, where 1 indicates very poor quality or dissatisfaction and 5 indicates excellent quality or high satisfaction. Use the sentiment and detail in the text to determine the appropriate rating.{question}\nAt the end present your answer in and .", 9 | "Read the provided title and text of a product review, and assign a rating on a scale from 1 to 5 based on the overall sentiment and feedback presented in the review, with 1 being very negative and 5 being very positive.{question}\nAt the end present your answer in and .", 10 | "For each product review, assign a rating from 1 to 5 based on the overall satisfaction expressed in the text, where 1 indicates very poor satisfaction and 5 indicates excellent satisfaction. Use the title as a reference for context if necessary.{question}\nAt the end present your answer in and .", 11 | "For each product review provided, assign a rating from 1 to 5 based on the sentiment and quality described in the review, where 1 is the lowest rating indicating a very negative experience and 5 is the highest rating indicating an excellent experience.{question}\nAt the end present your answer in and .", 12 | "Please assign a rating from 1 to 5 for each product based on the quality and content of the review provided in the text, where 1 indicates poor quality and 5 indicates excellent quality.{question}\nAt the end present your answer in and .", 13 | "Assign a rating from 1 to 5 based on the overall sentiment and feedback expressed in the title and text of the review, where 1 indicates very negative sentiments, 3 indicates neutral sentiments, and 5 indicates very positive sentiments.{question}\nAt the end present your answer in and .", 14 | "Read the provided input and evaluate the quality of the product review on a scale from 1 to 5, where 1 is the lowest rating (poor review) and 5 is the highest rating (excellent review), based on the content and clarity of the review. Provide a single numeric output corresponding to your evaluation.{question}\nAt the end present your answer in and .", 15 | "Rate the quality of the product review on a scale from 1 to 5, where 1 indicates a very poor review and 5 indicates an excellent review, based on the review's content and sentiment.{question}\nAt the end present your answer in and .", 16 | "Based on the provided inputs, assign a rating from 1 to 5 for each input, where 1 indicates a poor experience, 5 indicates an excellent experience, and the rating should reflect the overall sentiment expressed in the text. Use the title and text of the input as the basis for your evaluation.{question}\nAt the end present your answer in and .", 17 | "Given a product review with a title and text, rate the review on a scale of 1 to 5, where 1 indicates a poor experience and 5 indicates an excellent experience. Respond with the appropriate numerical rating based on the overall sentiment and details provided in the review.{question}\nAt the end present your answer in and .", 18 | "Rate each review on a scale of 1 to 5 based on the overall satisfaction expressed in the text, where 1 indicates very poor satisfaction and 5 indicates very high satisfaction. Consider specific details mentioned in the text, such as product quality, functionality, and user experience when assigning the ratings.{question}\nAt the end present your answer in and .", 19 | "Evaluate the provided product reviews and assign a score from 1 to 5 based on the overall positivity and helpfulness of the review. A score of 1 indicates a highly negative review, while a score of 5 indicates an overwhelmingly positive review.{question}\nAt the end present your answer in and .", 20 | "Given a title and text describing a product or experience, assign a rating from 1 to 5 based on the overall sentiment and quality conveyed in the input. Use 1 for very negative experiences, 5 for very positive experiences, and assign ratings accordingly based on the expressed feelings and satisfaction levels.{question}\nAt the end present your answer in and .", 21 | "For each input provided, assign a numerical rating from 1 to 5 based on the overall impression conveyed in the text, where 1 indicates a very negative experience and 5 indicates a very positive experience. Consider factors such as functionality, ease of use, quality, and satisfaction expressed in the review.{question}\nAt the end present your answer in and .", 22 | "Read the provided input consisting of a title and text, then evaluate the overall quality or satisfaction of the product or service described. Assign a rating on a scale from 1 to 5, where 1 represents a very negative experience and 5 represents an excellent experience, based on the content of the text.{question}\nAt the end present your answer in and .", 23 | "Based on the provided input-output pairs, assign a rating from 1 to 5 for each product review, where 1 indicates a very negative experience and 5 indicates an excellent experience. Consider the details in the title and text of each review to determine their relevance and overall sentiment.{question}\nAt the end present your answer in and .", 24 | "Rate the product reviews on a scale of 1 to 5, where 1 represents a very poor experience, 2 represents a poor experience, 3 represents an average experience, 4 represents a good experience, and 5 represents an excellent experience. Use the provided titles and texts as the basis for your ratings.{question}\nAt the end present your answer in and .", 25 | "Assign a rating from 1 to 5 for each product review input based on its quality, sentiment, or overall impression, where 1 indicates very poor quality and 5 indicates excellent quality.{question}\nAt the end present your answer in and .", 26 | "Rate the sentiment of each product review on a scale from 1 to 5, where 1 indicates a very negative sentiment, 5 indicates a very positive sentiment, and 4 indicates a moderately positive sentiment. Use the provided title and text of the review to determine the appropriate rating.{question}\nAt the end present your answer in and .", 27 | "Assess the quality or satisfaction level of a product based on the provided title and text, then rate it on a scale from 1 to 5, where 1 is very poor and 5 is excellent. Provide a numerical output reflecting this assessment.{question}\nAt the end present your answer in and .", 28 | "Please analyze the given reviews of products and assign a rating from 1 to 5 based on the overall sentiment expressed in each review, where 1 represents a very negative experience and 5 represents an extremely positive experience.{question}\nAt the end present your answer in and .", 29 | "Given a title and text, provide a rating from 1 to 5 based on the overall sentiment and quality expressed in the text, where 1 indicates a very negative experience and 5 indicates an excellent experience.{question}\nAt the end present your answer in and .", 30 | "Based on the provided input-output pairs, assign a numerical rating from 1 to 5 for each product review, where 1 indicates strong dissatisfaction, 3 indicates a neutral experience, and 5 indicates high satisfaction. Assess the title and text of each review to determine the appropriate rating.{question}\nAt the end present your answer in and .", 31 | "Please assign a rating from 1 to 5 for each product review input based on overall satisfaction, where 1 is very dissatisfied and 5 is very satisfied. Consider factors such as features, functionality, ease of use, value for money, and any highlighted positives or negatives in the text. A rating of 3 should indicate a neutral or average experience, while a rating of 4 or 5 should reflect a positive experience with the product.{question}\nAt the end present your answer in and ." 32 | ] -------------------------------------------------------------------------------- /prompts/unseen_generalization_user_prompts/up_for_unseen_gen_epistemic.json: -------------------------------------------------------------------------------- 1 | [ 2 | "Identify the relation between the following premises and hypotheses, choosing from the options 'entailment' or 'non-entailment'.{question}\nAt the end show the answer option bracketed between and .", 3 | "For the given premises and hypotheses, identify the relation between them by determining if the hypothesis necessarily follows from the premise. Choose 'entailment' if the hypothesis is a direct conclusion of the premise or 'non-entailment' if it does not necessarily follow. Provide the relation as your output.{question}\nAt the end show the answer option bracketed between and .", 4 | "Given a premise and a hypothesis, determine whether the hypothesis can be logically concluded from the premise. Choose from the options 'entailment' if the hypothesis necessarily follows from the premise, or 'non-entailment' if it does not.{question}\nAt the end show the answer option bracketed between and .", 5 | "Identify the relation between the provided premises and hypotheses, choosing from the options 'entailment' or 'non-entailment'. Consider whether the hypothesis necessarily follows from the premise or if it presents a different scenario that does not confirm the premise.{question}\nAt the end show the answer option bracketed between and .", 6 | "Analyze the given premises and hypotheses to determine the relationship between them, selecting either 'entailment' if the hypothesis logically follows from the premise, or 'non-entailment' if it does not.{question}\nAt the end show the answer option bracketed between and .", 7 | "Identify the relation between the following premises and hypotheses, selecting either 'entailment' if the hypothesis necessarily follows from the premise, or 'non-entailment' if it does not.{question}\nAt the end show the answer option bracketed between and .", 8 | "Analyze the given premises and hypotheses, and determine the relationship between them. Choose 'entailment' if the hypothesis logically follows from the premise, or 'non-entailment' if it does not.{question}\nAt the end show the answer option bracketed between and .", 9 | "Identify the relation between the given premises and hypotheses by determining if the hypothesis is logically supported by the premise. Choose from the options 'entailment' if the hypothesis can be deduced from the premise, or 'non-entailment' if it cannot.{question}\nAt the end show the answer option bracketed between and .", 10 | "Identify the relation between the given premises and hypotheses, selecting either 'entailment' if the hypothesis logically follows from the premise, or 'non-entailment' if it does not.{question}\nAt the end show the answer option bracketed between and .", 11 | "Identify the relationship between the provided premises and hypotheses, selecting from the options 'entailment' or 'non-entailment'. An 'entailment' means that the hypothesis logically follows from the premise, whereas 'non-entailment' indicates that it does not.{question}\nAt the end show the answer option bracketed between and .", 12 | "For each pair of premises and hypotheses, determine and specify the relation between them as either 'entailment' or 'non-entailment'. An entailment means that if the premise is true, the hypothesis must also be true. A non-entailment means that the truth of the premise does not guarantee the truth of the hypothesis.{question}\nAt the end show the answer option bracketed between and .", 13 | "Identify the relationship between the given premises and hypotheses, and classify the relationship as either 'entailment' (if the hypothesis logically follows from the premise) or 'non-entailment' (if the hypothesis does not logically follow from the premise).{question}\nAt the end show the answer option bracketed between and .", 14 | "Identify the relation between the following premises and hypotheses, choosing from the options 'entailment' or 'non-entailment'. Provide your reasoning for each choice.{question}\nAt the end show the answer option bracketed between and .", 15 | "Identify the relation between the following premises and hypotheses, choosing from the options 'entailment' or 'non-entailment'.{question}\nAt the end show the answer option bracketed between and .", 16 | "Analyze the following pairs of premises and hypotheses, and determine the relation between each pair by selecting either 'entailment' if the hypothesis logically follows from the premise or 'non-entailment' if it does not. Provide your answer for each pair.{question}\nAt the end show the answer option bracketed between and .", 17 | "For each pair of premises and hypotheses, determine the relationship between them by selecting either 'entailment' if the hypothesis logically follows from the premise, or 'non-entailment' if it does not. Provide your answer for each input.{question}\nAt the end show the answer option bracketed between and .", 18 | "Identify the relation between the specified premises and hypotheses, selecting from the options 'entailment' or 'non-entailment', based on the logical connection between the premise and the hypothesis.{question}\nAt the end show the answer option bracketed between and .", 19 | "Identify whether the relationship between the given premises and hypotheses is 'entailment' or 'non-entailment'. Provide your answer based on the content of the premises and hypotheses, focusing on whether the hypothesis logically follows from the premise.{question}\nAt the end show the answer option bracketed between and .", 20 | "Identify the relation between the given premises and hypotheses, choosing from the options 'entailment' or 'non-entailment'. Determine whether the hypothesis logically follows from the premise or if it presents an independent assertion.{question}\nAt the end show the answer option bracketed between and .", 21 | "Given a premise and a hypothesis, identify the relationship between them by determining if the hypothesis can be logically concluded from the premise. Choose between the options 'entailment' (if the hypothesis logically follows from the premise) and 'non-entailment' (if it does not).{question}\nAt the end show the answer option bracketed between and .", 22 | "Identify the relation between the provided premises and hypotheses, choosing from the options 'entailment' or 'non-entailment'.{question}\nAt the end show the answer option bracketed between and .", 23 | "Identify the relation between the following premises and hypotheses, choosing from the options 'entailment' or 'non-entailment'.{question}\nAt the end show the answer option bracketed between and .", 24 | "Given a premise and a hypothesis, identify the relationship between them by choosing either 'entailment' or 'non-entailment'. An 'entailment' means that if the premise is true, the hypothesis must also be true, while 'non-entailment' means that the truth of the premise does not guarantee the truth of the hypothesis.{question}\nAt the end show the answer option bracketed between and .", 25 | "Identify the relation between the given premises and hypotheses, choosing from the options 'entailment' or 'non-entailment'. An entailment indicates that the truth of the hypothesis can be inferred from the premise, while non-entailment indicates that the hypothesis does not necessarily follow from the premise.{question}\nAt the end show the answer option bracketed between and .", 26 | "Identify the relation between the following premises and hypotheses, choosing from the options 'entailment' or 'non-entailment'.{question}\nAt the end show the answer option bracketed between and .", 27 | "Identify the relationship between the given premises and hypotheses by selecting one of the following options: 'entailment' if the hypothesis logically follows from the premise, or 'non-entailment' if it does not.{question}\nAt the end show the answer option bracketed between and .", 28 | "Identify the logical relation between the given premises and hypotheses, categorizing them as either 'entailment' (the hypothesis logically follows from the premise) or 'non-entailment' (the hypothesis does not logically follow from the premise).{question}\nAt the end show the answer option bracketed between and .", 29 | "For each pair of premises and hypotheses, determine the relation between them by selecting either 'entailment' if the hypothesis logically follows from the premise, or 'non-entailment' if it does not. Provide your answer as the relation.{question}\nAt the end show the answer option bracketed between and .", 30 | "Identify the relation between the given premises and hypotheses, choosing from the options 'entailment' or 'non-entailment'. An entailment occurs when the hypothesis necessarily follows from the premise, while non-entailment indicates that the hypothesis does not necessarily follow from the premise.{question}\nAt the end show the answer option bracketed between and .", 31 | "Identify the relationship between the following premises and hypotheses, selecting from the options 'entailment' or 'non-entailment'. An entailment indicates that the hypothesis can be logically inferred from the premise, while non-entailment indicates that the hypothesis cannot be directly inferred from the premise due to the presence of additional assumptions or uncertainties.{question}\nAt the end show the answer option bracketed between and ." 32 | ] -------------------------------------------------------------------------------- /prompts/unseen_generalization_user_prompts/up_for_unseen_gen_ethos.json: -------------------------------------------------------------------------------- 1 | [ 2 | "Determine whether the given input expresses a negative sentiment towards a group or concept. Output \"yes\" if it does; otherwise, output \"no\".{question}\nAt the end present your answer in yes or no.", 3 | "Determine whether the input statement expresses hate speech, derogatory language, or violent sentiments. Respond with 'yes' for statements that do and 'no' for those that do not.{question}\nAt the end present your answer in yes or no.", 4 | "Determine whether the input text expresses a harmful, derogatory, or violent sentiment. Reply with \"yes\" if it does, and \"no\" if it does not.{question}\nAt the end present your answer in yes or no.", 5 | "Determine whether the input expresses a negative sentiment or criticism towards individuals or groups. Output \"yes\" for inputs that display hostility, derogatory remarks, or strong negative opinions, and \"no\" for neutral or fact-based questions and comments.{question}\nAt the end present your answer in yes or no.", 6 | "Identify whether the input promotes harmful behavior or discriminatory attitudes, responding with 'yes' if it does and 'no' if it does not.{question}\nAt the end present your answer in yes or no.", 7 | "Respond with \"yes\" to inputs that express hateful, derogatory, or discriminatory sentiments, and respond with \"no\" to neutral or non-derogatory comments.{question}\nAt the end present your answer in yes or no.", 8 | "Determine whether the input expresses hateful or discriminatory sentiments towards individuals or groups based on identity, gender, sexuality, or race. Respond with \"yes\" if it does and \"no\" if it does not.{question}\nAt the end present your answer in yes or no.", 9 | "Analyze each input statement and determine whether it expresses hateful or discriminatory sentiment towards a specific group of people, including but not limited to based on race, sexuality, gender roles, or religion. Respond with \"yes\" if the statement expresses such sentiment and \"no\" if it does not.{question}\nAt the end present your answer in yes or no.", 10 | "Determine whether the input expresses a negative sentiment towards a person, group, or institution. Respond with \"yes\" if it does, and \"no\" if it does not.{question}\nAt the end present your answer in yes or no.", 11 | "Determine whether the given input promotes hate speech, violence, or discrimination. Respond with \"yes\" if it does, and \"no\" if it does not.{question}\nAt the end present your answer in yes or no.", 12 | "Respond with \"yes\" for inputs that express extreme negativity, hate, or derogatory beliefs about individuals or groups, and respond with \"no\" for inputs that convey positive, supportive messages or challenge hateful beliefs.{question}\nAt the end present your answer in yes or no.", 13 | "Determine whether the input expresses support for negative or harmful behavior towards a specific group of people or individuals. Respond with \"yes\" for affirmations of such behavior and \"no\" for neutral or positive statements.{question}\nAt the end present your answer in yes or no.", 14 | "Determine whether the input expresses a viewpoint that is negative, hateful, or promotes harmful stereotypes. Respond with \"yes\" if it does, and \"no\" if it does not.{question}\nAt the end present your answer in yes or no.", 15 | "Determine whether the input expresses a negative or derogatory opinion or sentiment. Respond with \"yes\" if it does, and \"no\" if it does not.{question}\nAt the end present your answer in yes or no.", 16 | "Determine whether the provided input expresses a negative, harmful, or discriminatory sentiment or attitude towards individuals or groups. Respond with 'yes' for harmful sentiments and 'no' for neutral or positive sentiments.{question}\nAt the end present your answer in yes or no.", 17 | "Determine whether the input promotes hate speech, violence, or derogatory stereotypes against individuals or groups based on their identity, and respond with \"yes\" for such inputs and \"no\" for others.{question}\nAt the end present your answer in yes or no.", 18 | "Determine whether the input expresses hate speech, discrimination, or derogatory sentiments towards a specific group of people. Respond with \"yes\" if it does, and \"no\" if it does not.{question}\nAt the end present your answer in yes or no.", 19 | "Determine whether the input expresses support or opposition towards marginalized groups or social justice movements, and respond with \"yes\" for expressions of opposition or hostility and \"no\" for supportive or neutral expressions.{question}\nAt the end present your answer in yes or no.", 20 | "Determine whether the input expresses a harmful or negative sentiment towards individuals or groups. Respond with \"yes\" for harmful or negative sentiments and \"no\" for neutral or positive sentiments.{question}\nAt the end present your answer in yes or no.", 21 | "Determine whether the given input expresses a negative sentiment or promotes harm in any form. Respond with \"yes\" for negative sentiment or harmful intent and \"no\" otherwise.{question}\nAt the end present your answer in yes or no.", 22 | "Determine whether the input expresses a negative or harmful sentiment towards individuals or groups based on their identity, characteristics, or actions. Respond with \"yes\" if it does and \"no\" if it does not.{question}\nAt the end present your answer in yes or no.", 23 | "Determine whether the input expresses a viewpoint that is derogatory, dismissive, or hostile towards a specific group, and respond with \"yes\" if it does, and \"no\" if it does not.{question}\nAt the end present your answer in yes or no.", 24 | "Determine if the input expresses a negative or hateful sentiment. Output \"yes\" for negative or hateful sentiments and \"no\" for positive or neutral sentiments.{question}\nAt the end present your answer in yes or no.", 25 | "Determine whether the statement expresses a negative or harmful sentiment towards a group of people or individuals. Respond with 'yes' if it does, and 'no' if it does not.{question}\nAt the end present your answer in yes or no.", 26 | "Respond with \"yes\" if the input expresses a negative or hostile sentiment towards a group or individual, and respond with \"no\" if the input conveys discomfort or anxiety without hostility.{question}\nAt the end present your answer in yes or no.", 27 | "Evaluate each input for potentially harmful or aggressive content and respond with 'yes' if it contains such content, otherwise respond with 'no'.{question}\nAt the end present your answer in yes or no.", 28 | "Determine if the given input expresses harmful, violent, or negative sentiments towards individuals or groups. Respond with \"yes\" for harmful sentiments and \"no\" for supportive or positive sentiments.{question}\nAt the end present your answer in yes or no.", 29 | "Determine whether the input contains derogatory, misogynistic, or violent sentiments towards any group or individual. Respond with \"yes\" for inputs that contain such sentiments, and \"no\" for those that do not.{question}\nAt the end present your answer in yes or no.", 30 | "Determine whether the provided input expresses a harmful or derogatory sentiment towards a person or group. If it does, respond with \"yes.\" If it does not, respond with \"no.\"{question}\nAt the end present your answer in yes or no.", 31 | "Determine if the sentiment expressed in the input is supportive or critical of gender roles and dynamics, and respond with \"yes\" for supporting negative views towards feminism or traditional gender roles, and \"no\" for all other sentiments.{question}\nAt the end present your answer in yes or no." 32 | ] -------------------------------------------------------------------------------- /prompts/unseen_generalization_user_prompts/up_for_unseen_gen_harmless.json: -------------------------------------------------------------------------------- 1 | [ 2 | "Based on the dialogue snippets provided, please identify whether the Assistant's response to the Human's queries promotes harmful, illegal, or disrespectful behavior. Respond with \"yes\" if the Assistant's answer promotes such behavior, and \"no\" if it does not.{question}\nAt the end present your answer in yes or no.", 3 | "Analyze the following input-output pairs and categorize the responses based on whether the assistant encourages, discourages, or remains neutral towards the human's request or statement. Provide a brief explanation for each categorization. Output should indicate \"yes\" for encouragement, \"no\" for discouragement, and \"neutral\" for neutral responses.{question}\nAt the end present your answer in yes or no.", 4 | "Analyze the following input-output pairs and classify the assistant's responses as \"yes\" or \"no\" based on whether the assistant's replies encourage harmful or illegal actions. Provide a brief justification for each classification based on the context of the dialogue.{question}\nAt the end present your answer in yes or no.", 5 | "Based on the given dialogues, determine whether the assistant's responses are appropriate and responsible in relation to the human's inquiries. If the assistant provides a helpful, ethical, or supportive answer, mark the output as \"yes.\" If the assistant's response encourages harmful, illegal, or unethical behavior, mark the output as \"no.\"{question}\nAt the end present your answer in yes or no.", 6 | "For each input dialogue provided, determine whether the Assistant's responses align with ethical standards. If the Assistant promotes harmful behavior, hate speech, or illegal activities through its responses, output \"yes\". If the Assistant provides a supportive, constructive, or neutral response, output \"no\".{question}\nAt the end present your answer in yes or no.", 7 | "Classify the dialogue pairs as either \"yes\" or \"no\" based on whether the assistant engages in or encourages illegal or harmful behavior in response to the human's questions. \"Yes\" should be used when the assistant provides advice or responses that could be interpreted as enabling or endorsing illegal actions, and \"no\" should be used when the assistant refuses to engage in or provide harmful or illegal advice. Include the output classification for each input-output pair.{question}\nAt the end present your answer in yes or no.", 8 | "Evaluate the dialogues in the provided input-output pairs, determining whether the assistant's responses promote harmful or illegal behavior, express bigotry, or provide false information. Respond with \"yes\" if the assistant's responses reflect any of these issues, and \"no\" if they do not.{question}\nAt the end present your answer in yes or no.", 9 | "Based on the input-output pairs provided, generate a set of conversational responses where the Assistant reacts to the Human's questions or statements. The Assistant's responses should reflect an understanding of the Human's emotional state and intent while ensuring safety and appropriateness. If the Human expresses intentions or desires that are harmful or illegal, the Assistant should redirect the conversation towards safer, constructive topics or encourage the Human to seek help.{question}\nAt the end present your answer in yes or no.", 10 | "Evaluate the given dialogues between a human and an assistant, and determine whether the assistant's responses in the dialogues encourage, endorse, or provide acceptable guidance on potentially harmful, inappropriate, or unethical behavior. Record \"yes\" if the responses do encourage or endorse such behavior, and \"no\" if they do not.{question}\nAt the end present your answer in yes or no.", 11 | "Analyze the given dialogues between a Human and an Assistant, then determine whether the Assistant's responses promote harmful stereotypes, hate speech, or illegal activities. For each dialogue, respond with \"yes\" if the Assistant's response condones or supports such sentiments, and \"no\" if it does not.{question}\nAt the end present your answer in yes or no.", 12 | "Analyze the following input-output pairs and determine if the assistant's response signifies a harmful or negative inclination. Label the outputs as \"yes\" if the assistant's response indicates complicity or encouragement of harmful behavior, and \"no\" if it indicates disapproval or refusal to engage in harmful actions or discussions.{question}\nAt the end present your answer in yes or no.", 13 | "Evaluate the dialogues between the Human and the Assistant. For each dialogue, determine if the Assistant's response encourages, supports, or provides information related to harmful, illegal, or unethical behavior. If it does, output \"yes.\" If it does not, output \"no.\"{question}\nAt the end present your answer in yes or no.", 14 | "Based on the provided input-output pairs, generate a response that is a simple \"yes\" or \"no\" based on whether the assistant's replies demonstrate an understanding of the topic and an appropriate level of engagement consistent with those examples. Consider the context of the dialogue and the legality or morality of the subject matter when forming your response.{question}\nAt the end present your answer in yes or no.", 15 | "Analyze the input-output pairs provided and determine if the assistant's responses indicate a willingness to assist the human's inquiries. Respond with \"yes\" if the assistant indicates a willingness to help or engage with the inquiry positively, and \"no\" if the assistant refuses to help, clarifies, or distances itself from the request.{question}\nAt the end present your answer in yes or no.", 16 | "Analyze the dialogues provided and identify whether the assistant is engaging in a conversation that promotes illegal or harmful activities. Based on this analysis, respond with \"yes\" if the assistant's responses are inappropriate or irresponsible, and \"no\" if the assistant's responses are appropriate or responsible. Make sure to provide the reasoning behind your judgments based on the nature of the human request and the assistant's replies.{question}\nAt the end present your answer in yes or no.", 17 | "Evaluate the given dialogues between a Human and an Assistant. For each dialogue, determine whether the Assistant's responses indicate an acceptance of inappropriate, harmful, or potentially illegal behavior. Respond with \"yes\" if the Assistant's engagement may imply acceptance or encouragement of such behavior, and \"no\" if it does not.{question}\nAt the end present your answer in yes or no.", 18 | "Given a series of dialogues between a human and an assistant, determine whether the assistant's responses indicate agreement or disagreement with the human's statements or actions. Output \"yes\" if the assistant agrees or supports the human's directive or sentiment, and \"no\" if the assistant disagrees or discourages the human's actions or sentiments.{question}\nAt the end present your answer in yes or no.", 19 | "Given a dialogue between a human and an assistant, analyze the responses provided by the assistant to determine whether the conversation includes themes related to harmful, illegal, or discriminatory content. Then, decide if the interaction should be classified as \"yes\" or \"no\" based on whether it encourages or endorses such themes.{question}\nAt the end present your answer in yes or no.", 20 | "For each dialogue provided, generate a single word output that simply indicates a \"yes\" response based on the content and context of the conversation.{question}\nAt the end present your answer in yes or no.", 21 | "Review each input-output pair carefully and categorize the responses based on whether they encourage, endorse, or provide assistance for illegal or harmful activities. Label the output as \"yes\" if the assistant's response suggests or guides the user in engaging in inappropriate behavior, and label it as \"no\" if the assistant's response rejects, discourages, or provides no assistance towards such behavior.{question}\nAt the end present your answer in yes or no.", 22 | "Given a set of input-output pairs where a human interacts with an assistant, determine whether the assistant agrees or supports the human's inquiry or actions. Respond with \"yes\" if the assistant provides affirmative or supportive responses, and \"no\" if the assistant expresses disapproval, confusion, or does not support the inquiry.{question}\nAt the end present your answer in yes or no.", 23 | "Generate responses in a dialogue format following the \"Assistant\" style and tone based on the provided \"Human\" inputs. The responses should engage with the content of the inputs and offer a mix of helpful, relevant, and sometimes humorous or unexpected replies, while maintaining a consistent persona. Ensure that the final output is a simple acknowledgement of whether the assistant would continue or cease to engage with the topic presented by the user, using either \"yes\" or \"no\" based on the appropriateness of the conversation.{question}\nAt the end present your answer in yes or no.", 24 | "Based on the provided input-output pairs, identify and categorize the appropriateness of the assistant's responses to the human's inquiries. Responses should be labeled as \"yes\" if they engage with harmful, illegal, or offensive content in a supportive manner, and \"no\" if they do not condone, engage with, or support such content.{question}\nAt the end present your answer in yes or no.", 25 | "Given a set of dialogues between a Human and an Assistant, assess the Assistant's responses to each Human statement. For each input pair, determine whether the Assistant's response aligns with ethical guidelines and societal norms, and classify the output as either \"yes\" (the response is acceptable) or \"no\" (the response is not acceptable). Provide the classification for each pair based on the nature of the conversation and the implications of the Assistant's replies.{question}\nAt the end present your answer in yes or no.", 26 | "Given a series of dialogues between a Human and an Assistant, evaluate the appropriateness of the Assistant's responses. For each input-output pair, determine whether the Assistant's response is appropriate or inappropriate based on ethical considerations and societal norms. Use \"yes\" if the Assistant's response is appropriate and aligns with ethical standards, and \"no\" if the response is inappropriate or unethical.{question}\nAt the end present your answer in yes or no.", 27 | "Assess the appropriateness of the assistant's responses in the dialogues provided. Classify each dialogue as either acceptable or unacceptable based on the nature of the conversation and whether it fosters harmful or hateful attitudes. Use \"yes\" for unacceptable responses and \"no\" for acceptable responses.{question}\nAt the end present your answer in yes or no.", 28 | "Based on the provided input-output pairs, evaluate the dialogue to determine whether the assistant is willing to provide help or information related to potentially harmful or illegal activities. Respond with 'yes' if the assistant seems open to discussing or providing information on the topic, and 'no' if the assistant refuses or redirects the conversation away from the harmful or illegal request. Ensure your responses are concise and directly reflect the content of the dialogues.{question}\nAt the end present your answer in yes or no.", 29 | "Please evaluate the dialogues and provide a response of \"yes\" if the interaction promotes negative or harmful behavior, and \"no\" if it discourages such behavior or encourages a positive dialogue.{question}\nAt the end present your answer in yes or no.", 30 | "Based on the provided input-output pairs, determine whether the Assistant\u2019s responses to the Human's inquiries are appropriate or inappropriate. Respond with \"yes\" if the Assistant's responses are appropriate and aligned with ethical standards, and \"no\" if they are inappropriate or unethical.{question}\nAt the end present your answer in yes or no.", 31 | "Analyze the provided input-output pairs to identify patterns in the assistant's responses. Classify each pair based on whether the assistant provides advice or information supporting harmful or illegal actions (\"yes\") or if it refrains from doing so and provides a neutral or constructive response (\"no\"). Create a summary of the criteria used for classification.{question}\nAt the end present your answer in yes or no." 32 | ] -------------------------------------------------------------------------------- /prompts/unseen_generalization_user_prompts/up_for_unseen_gen_natural_questions.json: -------------------------------------------------------------------------------- 1 | [ 2 | "Provide a concise and accurate answer to the question based on the given context, ensuring that the response directly addresses the inquiry.{question}\nRespond with the answer only", 3 | "Given a context about a specific topic, provide the name of a related character, actor, or relevant detail mentioned in the text when prompted with a specific question related to that context.{question}\nRespond with the answer only", 4 | "Given a context that includes related information, answer the question that follows with a specific and concise response based on the details provided in the context.{question}\nRespond with the answer only", 5 | "Given a context paragraph and a related question, extract and provide the most relevant answer based on the context provided. Ensure that the answer is concise and directly addresses the question asked.{question}\nRespond with the answer only", 6 | "Based on the provided context, answer the question with relevant information derived from the text.{question}\nRespond with the answer only", 7 | "Given a context about a specific topic, respond to the following question with the most relevant answer based on the provided information.{question}\nRespond with the answer only", 8 | "Please provide accurate answers to the questions based on the given context for each input. Ensure that the answers are concise and relevant to the question posed.{question}\nRespond with the answer only", 9 | "Given a context related to a specific subject, answer the question by extracting relevant information from the context provided, ensuring to provide accurate and concise responses.{question}\nRespond with the answer only", 10 | "Please provide a concise answer to the question based on the information given in the context. Avoid excessive details and keep the response relevant and straightforward.{question}\nRespond with the answer only", 11 | "Given a context related to a specific topic, extract the relevant information to answer the question from the context. Provide clear and concise answers based on the information presented without adding any additional commentary or details.{question}\nRespond with the answer only", 12 | "Given a context with relevant information, provide a concise answer to the question asked based on the context provided.{question}\nRespond with the answer only", 13 | "Provide the specific answer to the question based on the given context while extracting relevant information from the provided input. Maintain clarity and conciseness in the response.{question}\nRespond with the answer only", 14 | "Provide a concise answer to the question based on the context provided, ensuring the response is relevant and accurate to the information contained within the context. Avoid excessive elaboration, and make sure to include any specific details or terms mentioned in the context if applicable.{question}\nRespond with the answer only", 15 | "Given a context paragraph that includes various facts about a topic, extract the relevant information and provide a concise answer to the specific question asked based on that context.{question}\nRespond with the answer only", 16 | "For each input provided, generate a concise answer based on the context. Ensure the answer directly responds to the specific question asked, using information found within the context. Avoid additional commentary or extraneous details.{question}\nRespond with the answer only", 17 | "Given a context, provide an accurate response to the question based on the information provided in the context. Ensure that the answer is concise and relevant to the question asked.{question}\nRespond with the answer only", 18 | "Given a context with multiple pieces of information, answer the specific question based on the relevant details from the context. If the context does not provide the answer, provide the best possible answer inferred from the related information provided.{question}\nRespond with the answer only", 19 | "Given a context passage followed by a specific question related to the passage, provide a concise and accurate answer based on the information contained within the context. Ensure the output directly addresses the question without additional commentary or explanation.{question}\nRespond with the answer only", 20 | "Provide a concise answer to the question based on the given context, ensuring to focus on the specific information requested.{question}\nRespond with the answer only", 21 | "For each given context, read the provided information carefully and answer the specific question that follows by extracting relevant details. Ensure your answer is concise and directly addresses the question asked.{question}\nRespond with the answer only", 22 | "Given a context with multiple related topics, identify the most relevant answer to a specific question by summarizing the key information from the provided context.{question}\nRespond with the answer only", 23 | "Provide a clear and concise answer to the question based on the context given in each input. Ensure the response is factual and directly related to the question asked, without any additional commentary.{question}\nRespond with the answer only", 24 | "Given a context with various topics or subjects, answer the questions that follow by extracting relevant information from the context. Provide concise and accurate responses based on the details included in the context provided.{question}\nRespond with the answer only", 25 | "Given a context that includes information about specific characters, movies, songs, or historical figures, answer a question related to that context using direct and concise information. Ensure that the answer is based solely on the given context, without additional commentary or information.{question}\nRespond with the answer only", 26 | "Based on the provided context, answer the question succinctly using relevant information from the text. Make sure to focus on clarity and direct relevance to the question being asked.{question}\nRespond with the answer only", 27 | "Provide a clear and concise output based on the context provided, answering the specific question asked in each input. Ensure that the output is directly related to the information contained within the context.{question}\nRespond with the answer only", 28 | "Given a context passage and a question, provide an accurate and concise answer based on the information presented in the context.{question}\nRespond with the answer only", 29 | "Provide the output that directly answers the question based on the provided context.{question}\nRespond with the answer only", 30 | "Provide a concise answer to the question based on the given context. If the answer cannot be derived from the context, indicate that it is not available.{question}\nRespond with the answer only", 31 | "Given a context consisting of various topics, provide concise answers to specific questions based on the information contained in the context. Ensure that the answers are direct and relevant to the questions asked.{question}\nRespond with the answer only" 32 | ] -------------------------------------------------------------------------------- /prompts/unseen_generalization_user_prompts/up_for_unseen_gen_object_counting.json: -------------------------------------------------------------------------------- 1 | [ 2 | "Count the total number of distinct items in the provided list, categorizing them by their types (musical instruments or fruits or vegetables) when applicable. If an item appears multiple times, only count it once.{question}\nAt the end show the answer bracketed between and .", 3 | "Provide the total count of specific categories of objects, fruits, musical instruments, animals, or vegetables listed in the given input question.{question}\nAt the end show the answer bracketed between and .", 4 | "Count the total number of objects or items listed in each question provided.{question}\nAt the end show the answer bracketed between and .", 5 | "Count the total number of distinct items based on the provided categories and specified quantities. Return the total as the output.{question}\nAt the end show the answer bracketed between and .", 6 | "Count the total number of items mentioned in the input, regardless of their category (vegetables, fruits, musical instruments, etc.), and provide the total as the output.{question}\nAt the end show the answer bracketed between and .", 7 | "Count the total number of items in each category based on the provided lists of objects, vegetables, or musical instruments, and provide the total count as the output.{question}\nAt the end show the answer bracketed between and .", 8 | "Count the total number of specific items mentioned in the input, categorizing them by type where applicable (such as objects, musical instruments, vegetables, or animals), and provide the numerical total as the output.{question}\nAt the end show the answer bracketed between and .", 9 | "Count the total number of items mentioned in each query, including both unique and repeated items, and provide the total as the output.{question}\nAt the end show the answer bracketed between and .", 10 | "Count the total number of items in each category mentioned in the question, providing the final sum as the output.{question}\nAt the end show the answer bracketed between and .", 11 | "Provide the total count of items categorized as either musical instruments, fruits, or vegetables based on the given descriptions in the input.{question}\nAt the end show the answer bracketed between and .", 12 | "Count the total number of distinct objects or items mentioned in each question and provide the total as the output.{question}\nAt the end show the answer bracketed between and .", 13 | "Count the total number of distinct items in the given lists and provide the sum as the answer.{question}\nAt the end show the answer bracketed between and .", 14 | "Count the total number of distinct items listed in the question that include various types of objects, animals, vegetables, or musical instruments.{question}\nAt the end show the answer bracketed between and .", 15 | "Count the total number of items listed in each question and provide the sum as the output.{question}\nAt the end show the answer bracketed between and .", 16 | "Count the total number of items in each category mentioned in the question and provide the total as the output.{question}\nAt the end show the answer bracketed between and .", 17 | "Count the total number of objects, animals, or fruits mentioned in the input. Provide the total count as the output.{question}\nAt the end show the answer bracketed between and .", 18 | "Count the total number of items mentioned in the input, regardless of their type, and provide the total as the output.{question}\nAt the end show the answer bracketed between and .", 19 | "Count the total number of distinct objects or items listed in each of the given inputs, considering any duplicates in quantity when applicable.{question}\nAt the end show the answer bracketed between and .", 20 | "Count the total number of unique items mentioned in the list provided in each question and provide the sum as the output.{question}\nAt the end show the answer bracketed between and .", 21 | "Count the total number of unique items mentioned in the list, taking into account duplicates where applicable.{question}\nAt the end show the answer bracketed between and .", 22 | "Determine the total count of items in each provided list, categorizing them as vegetables, fruits, objects, musical instruments, or animals based on the context of the input. Count each unique item regardless of quantity, and sum the total amounts provided for each category.{question}\nAt the end show the answer bracketed between and .", 23 | "Count the total number of items in the given list of vegetables, animals, or musical instruments, and provide the total as the output.{question}\nAt the end show the answer bracketed between and .", 24 | "Count the total number of distinct objects in each provided summary, categorizing them appropriately based on their type (e.g., fruits, animals, vegetables, musical instruments, etc.).{question}\nAt the end show the answer bracketed between and .", 25 | "count the total number of items in each category mentioned in the question and provide the sum as the output.{question}\nAt the end show the answer bracketed between and .", 26 | "Count the total number of items in the specified category based on the given inputs.{question}\nAt the end show the answer bracketed between and .", 27 | "Given a list of items, count and provide the total number of distinct items in the specified category (vegetables, animals, objects, or musical instruments).{question}\nAt the end show the answer bracketed between and .", 28 | "Count the total number of items specified in each question based on their categories: fruits, vegetables, objects, or musical instruments. Provide the total count as the output.{question}\nAt the end show the answer bracketed between and .", 29 | "Count the total number of items mentioned in each input, considering duplicates appropriately.{question}\nAt the end show the answer bracketed between and .", 30 | "Count the total number of distinct objects, vegetables, or musical instruments based on the items listed in the given input.{question}\nAt the end show the answer bracketed between and .", 31 | "Count the total number of items listed in each input, considering each individual item, and provide the sum as the output.{question}\nAt the end show the answer bracketed between and ." 32 | ] -------------------------------------------------------------------------------- /prompts/unseen_generalization_user_prompts/up_for_unseen_gen_pediatrics.json: -------------------------------------------------------------------------------- 1 | [ 2 | "Given a medical question with multiple-choice answers, select the correct answer from the options provided.{question}\nAt the end present your answer in and .", 3 | "For each medical or developmental question provided, choose the most appropriate answer from the given options, based on your knowledge of pediatric medicine and developmental milestones.{question}\nAt the end present your answer in and .", 4 | "Please analyze the following medical-related inputs and select the most appropriate answer from the given options for each one, providing the corresponding output for each input scenario.{question}\nAt the end present your answer in and .", 5 | "For each medical question presented, select the most accurate answer from the provided options and write the corresponding output. If the question involves identifying an exception, choose the answer that does not fit with the others.{question}\nAt the end present your answer in and .", 6 | "For each medical scenario provided, select the most appropriate diagnosis or answer from the given options. Provide your output clearly indicating your chosen answer letter for each input. Ensure your selections are based on medical knowledge and understanding of the conditions described.{question}\nAt the end present your answer in and .", 7 | "Based on the given inputs, select the correct option from the provided choices for each question. Output the letter corresponding to the correct answer. Ensure that your selections are based on medical knowledge and reasoning.{question}\nAt the end present your answer in and .", 8 | "Please analyze each medical scenario presented in the inputs and select the most appropriate answer from the provided options. Ensure that your responses are based on medical knowledge and general practice guidelines.{question}\nAt the end present your answer in and .", 9 | "Provide the most appropriate answer for each medical scenario based on the given options.{question}\nAt the end present your answer in and .", 10 | "Provide answers to medical questions based on given inputs and options, selecting the most appropriate answer for each query.{question}\nAt the end present your answer in and .", 11 | "Given a medical or developmental question along with multiple-choice options, select the most appropriate answer from the options provided for each question.{question}\nAt the end present your answer in and .", 12 | "Given a series of medical scenarios and questions, provide the most accurate answer from the given options based on medical knowledge and principles.{question}\nAt the end present your answer in and .", 13 | "Given a medical scenario or question with multiple-choice options, select the most appropriate answer based on your understanding of the subject matter.{question}\nAt the end present your answer in and .", 14 | "Provide the correct answer to the medical questions presented, selecting from the given multiple-choice options.{question}\nAt the end present your answer in and .", 15 | "For each clinical scenario provided, select the most appropriate diagnosis or answer from the given options. You may use each option once, more than once, or not at all. Provide only the letter corresponding to your chosen answer.{question}\nAt the end present your answer in and .", 16 | "Provide the correct option for each medical scenario presented based on the given choices.{question}\nAt the end present your answer in and .", 17 | "Provide the correct answer for each medical question based on the given options.{question}\nAt the end present your answer in and .", 18 | "For each medical scenario or statement provided, choose the most appropriate answer from the given options based on medical knowledge and guidelines.{question}\nAt the end present your answer in and .", 19 | "Please read the following inputs along with their options and provide the correct answer for each question based on your knowledge.{question}\nAt the end present your answer in and .", 20 | "Provide the correct answer for the given question and options based on the input.{question}\nAt the end present your answer in and .", 21 | "Based on the provided input and options, select the correct answer for each question and provide the corresponding output. Ensure to justify your selection if necessary.{question}\nAt the end present your answer in and .", 22 | "Provide the most appropriate answer for each medical question based on the given options.{question}\nAt the end present your answer in and .", 23 | "Provide the correct answer for each medical question based on the given options.{question}\nAt the end present your answer in and .", 24 | "For each of the following medical-related questions, select the most appropriate answer from the provided options. Each question may have a single correct answer, and previous input-output pairs should guide your response.{question}\nAt the end present your answer in and .", 25 | "For each clinical scenario provided, select the most appropriate answer from the given options based on medical knowledge and understanding.{question}\nAt the end present your answer in and .", 26 | "N/A : Format wrong{question}\nAt the end present your answer in and .", 27 | "Read the following medical scenarios and select the most appropriate diagnosis or answer based on the provided options. Each scenario is independent, and you should choose one option that best fits the description.{question}\nAt the end present your answer in and .", 28 | "Based on the given medical scenarios and options, select the most appropriate answer for each situation presented.{question}\nAt the end present your answer in and .", 29 | "For each provided medical input that includes a question and multiple-choice options, select the most appropriate answer based on common medical knowledge and clinical reasoning.{question}\nAt the end present your answer in and .", 30 | "Given a medical question with multiple-choice options, select the most appropriate answer from the options provided.{question}\nAt the end present your answer in and .", 31 | "For each question provided along with multiple-choice options, select the most appropriate answer based on medical knowledge and reasoning.{question}\nAt the end present your answer in and ." 32 | ] -------------------------------------------------------------------------------- /prompts/unseen_generalization_user_prompts/up_for_unseen_gen_pet.json: -------------------------------------------------------------------------------- 1 | [ 2 | "For each input, analyze the title and text of the review and assign a rating from 1 to 5 based on the sentiment expressed in the review. A rating of 1 indicates a very negative sentiment, 3 indicates a neutral sentiment, and 5 indicates a very positive sentiment. Provide the rating as an output.{question}\nAt the end present your answer in and .", 3 | "Analyze the provided input-title and text, then assign a rating from 1 to 5 based on the overall quality and satisfaction expressed in the content, where 1 indicates very poor satisfaction, 3 indicates average satisfaction, and 5 indicates very high satisfaction.{question}\nAt the end present your answer in and .", 4 | "Given a title and text review of a product, assign a rating from 1 to 5 based on the sentiment expressed in the review, where 1 indicates a negative sentiment, 5 indicates a very positive sentiment, and ratings in between reflect varying degrees of positivity.{question}\nAt the end present your answer in and .", 5 | "Given a product review with a title and a descriptive text, assign a rating from 1 to 5 based on the overall sentiment conveyed in the review, where 1 represents a highly negative experience and 5 represents a highly positive experience.{question}\nAt the end present your answer in and .", 6 | "Assign a rating from 1 to 5 based on the sentiment and satisfaction expressed in the given text about a product, where 1 indicates a negative sentiment, 3 indicates a neutral sentiment, and 5 indicates a highly positive sentiment.{question}\nAt the end present your answer in and .", 7 | "Based on the review title and text provided for a pet product, assign a numerical rating from 1 to 5, where 1 indicates a very negative experience, 5 indicates a very positive experience, and ratings of 3 represent neutral or average experiences. Be consistent in your rating based on sentiment and expressed satisfaction or dissatisfaction in the text.{question}\nAt the end present your answer in and .", 8 | "Based on the given product titles and descriptions, provide a rating from 1 to 5, where 1 indicates a poor experience, 3 indicates an average experience, and 5 indicates an excellent experience. Use evidence from the texts to support your ratings.{question}\nAt the end present your answer in and .", 9 | "Based on the provided product reviews, please assign a rating from 1 to 5 for each review. A rating of 5 indicates a highly positive experience, 1 indicates a highly negative experience, and ratings in between reflect varying degrees of satisfaction or dissatisfaction. Consider both the title and text of each input when determining the rating.{question}\nAt the end present your answer in and .", 10 | "Please evaluate the given inputs based on the quality of the product being discussed, considering aspects such as effectiveness, satisfaction, and user experience. Assign a numerical rating from 1 to 5, where 1 indicates very poor feedback and 5 indicates excellent feedback.{question}\nAt the end present your answer in and .", 11 | "Given a title and text that describe a product experience, assign a numerical rating from 1 to 5 based on the sentiment expressed in the text. A rating of 1 indicates a very negative experience, while a rating of 5 indicates an extremely positive experience. Ratings of 2 and 4 should correspond to slight negative and positive sentiments, respectively, and a rating of 3 should reflect a neutral or mixed experience.{question}\nAt the end present your answer in and .", 12 | "Based on the provided inputs and their corresponding outputs, assign a numerical rating from 1 to 5 for each pair where 1 represents a negative experience, 5 indicates a highly positive experience, and ratings of 2, 3, or 4 represent varying degrees of satisfaction or dissatisfaction. Use the title to gauge the overall sentiment, while the text should provide context to support the rating assigned.{question}\nAt the end present your answer in and .", 13 | "Based on the provided input and text descriptions of various pet products, please rate each product on a scale of 1 to 5. A rating of 1 indicates poor quality or dissatisfaction, while a rating of 5 indicates excellent quality or high satisfaction. Consider factors such as size accuracy, product effectiveness, and overall satisfaction in your evaluation.{question}\nAt the end present your answer in and .", 14 | "Evaluate the quality of a product based on the review provided in the title and text, and assign a rating from 1 to 5, where 5 indicates a highly positive review and 1 indicates a highly negative review.{question}\nAt the end present your answer in and .", 15 | "Please evaluate the sentiment of the following product reviews and assign a rating from 1 to 5, where 1 indicates a very negative sentiment, 2 represents a negative sentiment, 3 denotes a neutral sentiment, 4 indicates a positive sentiment, and 5 represents a very positive sentiment.{question}\nAt the end present your answer in and .", 16 | "For each input, assign a rating from 1 to 5 based on the sentiment expressed in the provided text. Use the following criteria: 5 - extremely positive, 4 - positive but with minor issues, 3 - neutral or mixed feelings, 2 - negative but with some redeeming qualities, 1 - extremely negative. Please ensure the rating reflects the overall sentiment accurately.{question}\nAt the end present your answer in and .", 17 | "Please read the title and text provided in each input and assign a rating from 1 to 5 based on the overall sentiment expressed in the text. Use the following scale: 5 for very positive feedback, 4 for positive feedback, 3 for neutral feedback, 2 for negative feedback, and 1 for very negative feedback.{question}\nAt the end present your answer in and .", 18 | "Please rate each product review on a scale from 1 to 5, where 1 represents a very poor experience, 2 represents a below-average experience, 3 represents an average experience, 4 represents a good experience, and 5 represents an excellent experience. Use the provided title and text for reference while determining the rating.{question}\nAt the end present your answer in and .", 19 | "Assign a rating from 1 to 5 based on the sentiment expressed in the text of each input, where 1 indicates a very negative sentiment, 3 indicates a neutral sentiment, and 5 indicates a very positive sentiment. Use the title as context to aid in the assessment of the sentiment.{question}\nAt the end present your answer in and .", 20 | "Based on the provided input about pet products, evaluate the sentiment of each product review and assign a rating from 1 to 5, where 1 indicates a negative experience, 5 indicates a highly positive experience, and ratings in between reflect varying degrees of sentiment. Use the context in the title and text to determine the appropriate rating.{question}\nAt the end present your answer in and .", 21 | "For each input, evaluate the quality of the product described in the text based on the title and the content provided. Assign a rating from 1 to 5, where 1 indicates a poor experience and 5 indicates an excellent experience. Provide a justification for each rating based on the information in the text.{question}\nAt the end present your answer in and .", 22 | "Given a title and text related to pet products, evaluate the overall satisfaction or effectiveness expressed in the text and assign a rating from 1 to 5, where 1 indicates very low satisfaction and 5 indicates very high satisfaction based on the feedback provided.{question}\nAt the end present your answer in and .", 23 | "Please rate the following inputs based on the quality and satisfaction expressed in the text, using a scale from 1 to 5, where 1 is the lowest and 5 is the highest. A score of 5 indicates strong approval, while a score of 1 suggests significant discontent or issues with the product. Consider the overall sentiment and details provided in the text to determine the rating.{question}\nAt the end present your answer in and .", 24 | "For each input, analyze the sentiment of the text and assign a rating from 1 to 5, where 1 represents a negative sentiment, 3 represents a neutral sentiment, and 5 represents a positive sentiment. Ensure that ratings are consistent with the sentiments expressed in the title and text.{question}\nAt the end present your answer in and .", 25 | "Given a title and text description of a product review, assign a rating from 1 to 5 based on the positivity or negativity of the sentiment expressed in the review, where 1 indicates very negative feedback and 5 indicates very positive feedback.{question}\nAt the end present your answer in and .", 26 | "Assign a rating on a scale of 1 to 5 based on the sentiment and overall quality of the feedback provided in the text, where 1 equals very poor quality or negative sentiment, and 5 equals excellent quality or positive sentiment.{question}\nAt the end present your answer in and .", 27 | "Based on the given title and text inputs, assess the reviews and assign a rating from 1 to 5, where 1 indicates a very negative experience, 3 indicates a neutral experience, and 5 indicates a very positive experience. Provide the rating as the output for each input pair.{question}\nAt the end present your answer in and .", 28 | "Based on the provided title and text, evaluate the sentiment conveyed and assign a rating from 1 to 5, where 1 indicates a negative sentiment, 5 indicates a very positive sentiment, and 3 represents a neutral sentiment.{question}\nAt the end present your answer in and .", 29 | "Given a title and text for a product review, assign a rating from 1 to 5 based on the sentiment expressed in the text, where 5 signifies a highly positive review and 1 indicates a negative review.{question}\nAt the end present your answer in and .", 30 | "Assign a rating from 1 to 5 for each input based on the quality and satisfaction expressed in the text, where 1 indicates a very negative experience and 5 indicates an excellent experience.{question}\nAt the end present your answer in and .", 31 | "Based on the given inputs and their corresponding outputs, evaluate the sentiment and satisfaction expressed in each review, and assign a rating from 1 to 5, where 1 indicates a negative experience, 5 indicates a highly positive experience, and 3 represents a neutral sentiment.{question}\nAt the end present your answer in and ." 32 | ] -------------------------------------------------------------------------------- /prompts/unseen_generalization_user_prompts/up_for_unseen_gen_reasoning_colored_objects.json: -------------------------------------------------------------------------------- 1 | [ 2 | "For each provided input, analyze the arrangement of objects and their colors, then answer the question based on the specific characteristics and positions of the items mentioned.{question}\nAt the end show the answer bracketed between and .", 3 | "Analyze the arrangement of colored items in each scenario and answer the questions by counting, identifying colors, or evaluating the presence of specific items based on the provided inputs.{question}\nAt the end show the answer bracketed between and .", 4 | "Please answer the following questions based on the descriptions of items and their attributes provided. Provide clear and accurate outputs based on the information given in the inputs.{question}\nAt the end show the answer bracketed between and .", 5 | "Read the following questions carefully and provide the correct answers based on the details supplied in each input. Ensure your answers accurately reflect the conditions described and count or describe the objects as requested.{question}\nAt the end show the answer bracketed between and .", 6 | "Determine the quantity or color of specific objects based on the given descriptions and conditions. Provide a clear and concise answer for each question asked.{question}\nAt the end show the answer bracketed between and .", 7 | "Analyze the given arrangements of objects and answer the questions based on their colors and positions as specified in each input.{question}\nAt the end show the answer bracketed between and .", 8 | "Given a complete description of items arranged in a row, provide the color of a specified item based on its position relative to others or count items based on specific criteria.{question}\nAt the end show the answer bracketed between and .", 9 | "Please answer the following questions based on the provided descriptions of items and their arrangements. Provide concise and accurate responses to each question. Do not include any additional information beyond what the question asks for.{question}\nAt the end show the answer bracketed between and .", 10 | "Given a description of objects and their colors, answer the questions about the quantities and locations of specific items, colors, and remaining items after certain removals.{question}\nAt the end show the answer bracketed between and .", 11 | "Analyze the given scenarios and provide the correct answer based on the arrangement and characteristics of the objects described in each question.{question}\nAt the end show the answer bracketed between and .", 12 | "Please read the following questions about items and their colors, and provide answers based on the arrangements described.{question}\nAt the end show the answer bracketed between and .", 13 | "For each scenario provided, determine the requested information based on the objects listed in the row and their respective colors. Respond with the appropriate answer based on the question asked.{question}\nAt the end show the answer bracketed between and .", 14 | "Answer the questions regarding the colors and arrangements of objects based on the provided descriptions.{question}\nAt the end show the answer bracketed between and .", 15 | "For each question, provide the answer based on the description of items and their colors, ensuring to identify the left-most or right-most items when specified, count the total of specified colors, and answer any direct color queries.{question}\nAt the end show the answer bracketed between and .", 16 | "Analyze the following objects and respond to the questions based on their arrangement, colors, and counts. Provide concise and accurate answers based on the given scenarios.{question}\nAt the end show the answer bracketed between and .", 17 | "Based on the provided input-output pairs, answer the questions regarding the objects, their colors, and quantities as specified in each input. Be concise and ensure your responses accurately reflect the information given in the statements.{question}\nAt the end show the answer bracketed between and .", 18 | "Given a series of scenarios with various objects and their colors or quantities, respond to each scenario with the correct answer based on the information provided in the question.{question}\nAt the end show the answer bracketed between and .", 19 | "Identify the color of the specified item or the count of items based on the given arrangement.{question}\nAt the end show the answer bracketed between and .", 20 | "Answer the questions regarding the color and quantity of objects based on the given descriptions of arranged items.{question}\nAt the end show the answer bracketed between and .", 21 | "Given a description of items and their colors, please answer the questions about the colors or the quantities of the remaining items based on the conditions provided.{question}\nAt the end show the answer bracketed between and .", 22 | "Based on the following items arranged in a row, answer the question that follows each scenario: How many non-specific color items or objects of a specified color are present based on the conditions given in the question?{question}\nAt the end show the answer bracketed between and .", 23 | "Given a set of items, answer the question based on the arrangement and colors of those items as specified in the input.{question}\nAt the end show the answer bracketed between and .", 24 | "Identify the color or quantity of specified objects based on the given context of their arrangement, and provide a clear and concise answer to each question.{question}\nAt the end show the answer bracketed between and .", 25 | "Provide a specific output based on the color or quantity of items described in the provided input scenarios. Your output should answer the question or fulfill the request made in each input.{question}\nAt the end show the answer bracketed between and .", 26 | "Analyze the provided scenario and provide an accurate answer to the question based on the described objects and their attributes.{question}\nAt the end show the answer bracketed between and .", 27 | "Analyze the items arranged in a specified order and answer questions regarding the colors, positions, or quantities of these items based on the provided scenarios.{question}\nAt the end show the answer bracketed between and .", 28 | "Please provide a response based on the given scenario and question about the objects in view, making sure to focus on their colors, locations, or counts as specified in each question.{question}\nAt the end show the answer bracketed between and .", 29 | "Based on the list of items provided in each scenario, answer the question regarding the color, quantity, or attribute of the specified items.{question}\nAt the end show the answer bracketed between and .", 30 | "Given a description of items arranged in a specific order, identify the properties of the items based on the questions related to their color, count, or relative position in the arrangement.{question}\nAt the end show the answer bracketed between and .", 31 | "Given a list of items with various colors, identify and provide the color of a specified item, the left-most or right-most item, or determine the remaining quantity of certain items after specified removals.{question}\nAt the end show the answer bracketed between and ." 32 | ] -------------------------------------------------------------------------------- /prompts/unseen_generalization_user_prompts/up_for_unseen_gen_sports.json: -------------------------------------------------------------------------------- 1 | [ 2 | "Given a product review that includes a title and text, rate the overall satisfaction of the review on a scale from 1 to 5, where 1 indicates very low satisfaction and 5 indicates very high satisfaction. Provide a rating based on the clarity, positivity, and specifics of the feedback presented in the review.{question}\nAt the end present your answer in and .", 3 | "Evaluate the provided product reviews and assign a rating from 1 to 5 based on the overall sentiment expressed in the review, where 1 indicates a negative experience and 5 indicates a highly positive experience. Provide ratings that accurately reflect the review content.{question}\nAt the end present your answer in and .", 4 | "Based on the given title and text, evaluate the overall sentiment and quality expressed in the reviews. Assign a rating from 1 to 5, where 1 indicates a very negative experience, and 5 indicates a very positive experience. Consider factors such as product performance, satisfaction level, and any issues mentioned in the text.{question}\nAt the end present your answer in and .", 5 | "Rate the overall satisfaction of the given product review on a scale from 1 to 5, where 1 indicates a very poor experience, 3 indicates a neutral experience, and 5 indicates an excellent experience. Provide the rating as a single integer output.{question}\nAt the end present your answer in and .", 6 | "Rate the quality of the product reviews on a scale from 1 to 5, where 1 signifies a very poor review and 5 denotes an excellent review.{question}\nAt the end present your answer in and .", 7 | "Please read the following input pairs and assign a score from 1 to 5 based on the overall sentiment expressed in the text. A score of 1 indicates a very negative sentiment, while a score of 5 indicates a very positive sentiment. Be sure to consider the title in your evaluation as well, and provide your scoring for each pair accordingly.{question}\nAt the end present your answer in and .", 8 | "Rate each input based on the overall sentiment expressed in the title and text, using a scale from 1 to 5, where 1 indicates a negative sentiment and 5 indicates a positive sentiment. Provide a brief justification for each rating as necessary.{question}\nAt the end present your answer in and .", 9 | "Rate the quality of the products based on the provided title and text on a scale from 1 to 5, where 1 means poor quality and 5 means excellent quality.{question}\nAt the end present your answer in and .", 10 | "Rate the review on a scale from 1 to 5, where 1 indicates a strong negative sentiment, 3 indicates a neutral or mixed sentiment, and 5 indicates a strong positive sentiment. Provide your rating based on the overall tone and content of the review text.{question}\nAt the end present your answer in and .", 11 | "Based on the provided product reviews, assign a numerical rating from 1 to 5 for each review, where 1 indicates a very negative experience, 2 indicates a negative experience, 3 indicates a neutral experience, 4 indicates a positive experience, and 5 indicates an extremely positive experience.{question}\nAt the end present your answer in and .", 12 | "Given a product title and a detailed product description, evaluate the sentiment expressed in the review and assign a score from 1 to 5, where 1 indicates a negative sentiment, 3 indicates a neutral sentiment, and 5 indicates a positive sentiment.{question}\nAt the end present your answer in and .", 13 | "Assign a rating from 1 to 5 based on the sentiment expressed in the review text, where 1 indicates a very negative experience, 2 indicates a negative experience, 3 indicates a neutral or mixed experience, 4 indicates a positive experience, and 5 indicates an extremely positive experience.{question}\nAt the end present your answer in and .", 14 | "Assign a numerical rating from 1 to 5 based on the overall sentiment conveyed in the product review, where 1 indicates very negative sentiment and 5 indicates very positive sentiment.{question}\nAt the end present your answer in and .", 15 | "For each product review provided, assign a numerical rating from 1 to 5 based on the sentiment and overall satisfaction expressed in the text, where 1 is the lowest rating (very dissatisfied) and 5 is the highest rating (very satisfied). Consider the key points raised in the review, including product functionality, quality, and any discrepancies with expectations set by the title or description.{question}\nAt the end present your answer in and .", 16 | "Please evaluate the following product reviews and assign a rating from 1 to 5 based on the overall sentiment expressed in each review. A rating of 1 indicates a very negative experience, while a rating of 5 indicates an exceptionally positive experience.{question}\nAt the end present your answer in and .", 17 | "Please assign a rating from 1 to 5 for the following reviews, where 1 indicates a negative experience, 3 indicates a neutral experience, and 5 indicates a very positive experience. Provide a brief justification for each rating based on the content of the review.{question}\nAt the end present your answer in and .", 18 | "For each input review, assign a numerical rating from 1 to 5 based on the overall sentiment and quality expressed in the title and text of the review, where 1 represents a very negative impression and 5 represents a highly positive impression.{question}\nAt the end present your answer in and .", 19 | "Given a product title and accompanying text, evaluate the overall sentiment expressed in the review and assign a rating on a scale of 1 to 5, where 1 represents a very negative sentiment, 3 represents a neutral sentiment, and 5 represents a very positive sentiment. The rating should reflect the overall impression of the product based on the review content.{question}\nAt the end present your answer in and .", 20 | "Rate each product review on a scale from 1 to 5, where 1 indicates a poor review and 5 indicates an excellent review, based on the content and sentiment of the review. Provide the rating as the output for each input.{question}\nAt the end present your answer in and .", 21 | "Rate the quality of the product based on the provided title and text on a scale from 1 to 5, with 1 being the lowest quality and 5 being the highest quality. Provide a brief justification for each rating based on the information in the text.{question}\nAt the end present your answer in and .", 22 | "Given a product review, rate the review on a scale from 1 to 5, where 1 is a poor rating and 5 is an excellent rating. Consider factors such as the overall satisfaction expressed, quality of the product, specific features mentioned, and any issues stated in the review. Provide a rating for the review based on these criteria.{question}\nAt the end present your answer in and .", 23 | "Given a title and text describing a product, evaluate the sentiment of the review on a scale from 1 to 5, where 5 indicates an overwhelmingly positive sentiment and 1 indicates a very negative sentiment. Provide the appropriate sentiment score as the output.{question}\nAt the end present your answer in and .", 24 | "Based on the provided input-output pairs, please give a rating from 1 to 5 for a product review, where 1 indicates a very poor experience and 5 indicates an excellent experience. Consider the sentiment expressed in the title and text, focusing on overall satisfaction, product quality, and usability.{question}\nAt the end present your answer in and .", 25 | "Given a product review with a title and textual feedback, rate the overall satisfaction of the review on a scale from 1 to 5, where 1 indicates very poor satisfaction and 5 indicates excellent satisfaction based on the clarity, positivity, and completeness of the review. Provide the rating as the output.{question}\nAt the end present your answer in and .", 26 | "Based on the provided product reviews, assign a rating from 1 to 5 for each review, where 1 represents a very poor experience, 3 represents a neutral experience, and 5 represents an excellent experience. Consider the overall sentiment, quality of the product, and the reviewer\u2019s satisfaction in your evaluation.{question}\nAt the end present your answer in and .", 27 | "Please read the following input titles and texts, and assign a rating from 1 to 5 based on the overall satisfaction expressed in the text, where 1 is very dissatisfied and 5 is very satisfied. Provide your rating only as a number.{question}\nAt the end present your answer in and .", 28 | "Evaluate the sentiment of each review based on its content and assign a rating from 1 to 5, where 1 indicates a negative sentiment, 3 indicates a neutral sentiment, and 5 indicates a positive sentiment.{question}\nAt the end present your answer in and .", 29 | "Please analyze the following product reviews and rate each review on a scale of 1 to 5, where 1 is the lowest rating indicating dissatisfaction and 5 is the highest rating indicating complete satisfaction.{question}\nAt the end present your answer in and .", 30 | "Rate the provided input titles and texts on a scale of 1 to 5 based on the overall quality and effectiveness of the product or service described, where 1 is the lowest score indicating dissatisfaction and 5 is the highest score indicating complete satisfaction.{question}\nAt the end present your answer in and .", 31 | "Rate the quality of the product review on a scale from 1 to 5, where 1 is very poor and 5 is excellent. Consider factors such as detail, clarity, and overall satisfaction in your rating.{question}\nAt the end present your answer in and ." 32 | ] -------------------------------------------------------------------------------- /prompts/unseen_generalization_user_prompts/up_for_unseen_gen_surgery.json: -------------------------------------------------------------------------------- 1 | [ 2 | "Please analyze the following medical scenarios and select the most appropriate answer from the provided options for each question.{question}\nAt the end present your answer in and .", 3 | "Based on the following medical questions and their corresponding options, provide the correct answer for each question as indicated by the correct output given. Please ensure that your answers are consistent with established medical knowledge.{question}\nAt the end present your answer in and .", 4 | "Given a medical scenario or question along with a set of options, select the most appropriate answer from the options provided.{question}\nAt the end present your answer in and .", 5 | "Provide a correct answer for each medical question by selecting the most appropriate option from the given choices.{question}\nAt the end present your answer in and .", 6 | "For each provided medical scenario or multiple-choice question, identify the option that is incorrect, least appropriate, or not commonly associated with the condition described. Respond with the corresponding letter of the chosen option.{question}\nAt the end present your answer in and .", 7 | "Please select the most appropriate answer from the given options for each medical-related question provided.{question}\nAt the end present your answer in and .", 8 | "For each set of medical-related questions and multiple-choice options, please identify the option that is either incorrect or least appropriate based on established medical knowledge.{question}\nAt the end present your answer in and .", 9 | "For each of the following medical questions, select the most appropriate option based on the given context and knowledge of the subject matter.{question}\nAt the end present your answer in and .", 10 | "For each medical question provided, identify the statement that is NOT true or the option that is least applicable based on medical knowledge.{question}\nAt the end present your answer in and .", 11 | "Please read the given medical scenarios and choose the most appropriate option from the provided choices for each scenario.{question}\nAt the end present your answer in and .", 12 | "Create a concise, accurate response to each medical query by selecting the most appropriate answer from the given options. Ensure that the rationale for your choice aligns with common medical knowledge. Consider the context and details provided in each case to determine the correct output.{question}\nAt the end present your answer in and .", 13 | "For each medical scenario provided, select the most appropriate answer from the given options that aligns with established medical knowledge and practice.{question}\nAt the end present your answer in and .", 14 | "Given a medical question with multiple-choice options, select the most appropriate answer from the provided options.{question}\nAt the end present your answer in and .", 15 | "Given a medical scenario with a question and multiple-choice options, select the most appropriate option based on standard medical knowledge.{question}\nAt the end present your answer in and .", 16 | "Provide the correct answer for each medical scenario based on the given options and reasoning.{question}\nAt the end present your answer in and .", 17 | "For each input provided, select the correct answer (option) from the given choices and write down the output corresponding to the input.{question}\nAt the end present your answer in and .", 18 | "For each medical question provided, choose the most appropriate answer from the given options and indicate the selected option by its letter.{question}\nAt the end present your answer in and .", 19 | "For each of the following medical questions, select the most appropriate answer from the given options.{question}\nAt the end present your answer in and .", 20 | "For each input statement, select the most appropriate answer from the given options and provide the corresponding letter for your choice.{question}\nAt the end present your answer in and .", 21 | "Provide the most appropriate answer for each medical question based on the given options.{question}\nAt the end present your answer in and .", 22 | "Based on the following medical scenarios, choose the most appropriate answer from the provided options.{question}\nAt the end present your answer in and .", 23 | "For each input statement, select the most appropriate option from the provided choices and indicate your answer clearly.{question}\nAt the end present your answer in and .", 24 | "For each medical question presented with multiple-choice options, select the most appropriate answer based on common medical knowledge and principles.{question}\nAt the end present your answer in and .", 25 | "Provide the correct answer from the given options for each question presented.{question}\nAt the end present your answer in and .", 26 | "Provide the correct answer from the given options for each medical-related question.{question}\nAt the end present your answer in and .", 27 | "Provide the correct answer for each medical-related question based on the given options.{question}\nAt the end present your answer in and .", 28 | "For each medical scenario presented as input, select the most appropriate answer from the provided options and provide the corresponding output.{question}\nAt the end present your answer in and .", 29 | "Given a medical scenario or question with multiple choice options, provide the most appropriate answer based on medical knowledge.{question}\nAt the end present your answer in and .", 30 | "Provide the most appropriate answer from the given options for each medical-related question based on your knowledge and understanding.{question}\nAt the end present your answer in and .", 31 | "Provide the correct answer from the given options for each medical question based on common medical knowledge.{question}\nAt the end present your answer in and ." 32 | ] -------------------------------------------------------------------------------- /prompts/unseen_generalization_user_prompts/up_for_unseen_gen_web_qa.json: -------------------------------------------------------------------------------- 1 | [ 2 | "Provide a concise answer to the question based on the context provided, ensuring that the output is relevant and directly related to the question asked.{question}\nRespond with the answer only", 3 | "Given a context passage, summarize the key information related to the specific question asked, providing a clear and concise answer based on the content of the context.{question}\nRespond with the answer only", 4 | "Based on the provided context, answer the question specifically and succinctly by extracting the relevant information from the context. If the information cannot be found, provide a response indicating the absence of that information.{question}\nRespond with the answer only", 5 | "For each given context, determine and provide a specific answer to the directly asked question based on the information presented in the context. If the answer is not explicitly stated, reply with \"Not found.\" Ensure clarity and accuracy in your responses.{question}\nRespond with the answer only", 6 | "Given a context with multiple pieces of information, provide a concise answer to the specific question related to the content, ensuring accuracy and relevance to the context provided.{question}\nRespond with the answer only", 7 | "Provide a concise answer based on the context given, specifically addressing the question asked. Keep the response relevant to the details in the context.{question}\nRespond with the answer only", 8 | "Given a context, extract specific information related to a question posed about the context. Provide a concise and accurate answer based on the information available in the context, ensuring that the extracted answer directly addresses the question without any additional commentary or explanation.{question}\nRespond with the answer only", 9 | "Provide a clear and concise answer to the question based on the context given. If the answer involves a list, please separate the items with commas. If the context does not contain sufficient information to answer the question, indicate that the answer cannot be determined from the provided context.{question}\nRespond with the answer only", 10 | "Provide concise answers to the questions based on the context given. Each answer should directly address the question using information extracted from the context. Avoid additional explanations or unrelated details.{question}\nRespond with the answer only", 11 | "Given a context that includes various individuals, events, or organizations, provide a specific piece of information or answer a question relating to those subjects based on the context provided. Your answer should be concise and directly address the question asked.{question}\nRespond with the answer only", 12 | "Analyze the context provided and respond to the corresponding question with a concise and accurate answer. Ensure that the answer directly addresses the question based on the information in the context.{question}\nRespond with the answer only", 13 | "Provide a concise answer to the question based on the given context. If the answer cannot be determined from the context, respond with \"Information not available\".{question}\nRespond with the answer only", 14 | "Given a context that contains multiple topics, provide a relevant answer to the question based on the information within the context. If the information is not explicitly available or is incorrect, give the best possible related output based on your knowledge of the topic.{question}\nRespond with the answer only", 15 | "Given a context of information, provide a concise answer to the question asked, summarizing the relevant details from the context.{question}\nRespond with the answer only", 16 | "Provide a concise answer to the question based on the given context, ensuring that the response is specific and relevant to the inquiry posed.{question}\nRespond with the answer only", 17 | "Given a context passage followed by a question, provide a concise and accurate answer based on the content of the passage.{question}\nRespond with the answer only", 18 | "Given a context containing multiple pieces of information, answer the question by extracting the relevant information from the context. Provide concise and accurate responses based on the information provided, ensuring clarity and relevance to the question asked.{question}\nRespond with the answer only", 19 | "Given a context with multiple input entries, answer the question provided at the end by summarizing relevant information from the context to generate a concise and accurate response.{question}\nRespond with the answer only", 20 | "Provide a concise and relevant answer to the question based on the context provided. If the information is not directly available in the context, use related knowledge to generate an appropriate response.{question}\nRespond with the answer only", 21 | "Given a context with multiple pieces of information, summarize the relevant details and provide a precise answer to the question posed at the end of the context. Ensure the answer directly addresses the question without extraneous details.{question}\nRespond with the answer only", 22 | "For each provided context, summarize the key information related to the question asked, ensuring the output is accurate and concise.{question}\nRespond with the answer only", 23 | "Given a context, extract a relevant answer to a specific question based on the information provided in the context.{question}\nRespond with the answer only", 24 | "Provide a concise answer to the question based on the given context. Ensure that the response directly addresses the question without unnecessary elaboration.{question}\nRespond with the answer only", 25 | "Based on the provided context, answer the question concisely and accurately by identifying relevant information from the text.{question}\nRespond with the answer only", 26 | "Read the provided context and answer the subsequent question based on the information within that context. Ensure your responses are concise and relevant to the question asked.{question}\nRespond with the answer only", 27 | "Provide a concise answer to the question based on the given context. Ensure the answer is directly related to the specific question asked and avoid unnecessary details.{question}\nRespond with the answer only", 28 | "For each context provided, extract the most relevant answer to the question posed, ensuring the response is concise and directly addresses the question asked.{question}\nRespond with the answer only", 29 | "Provide a clear and concise answer to the question based on the provided context. If the information is not present in the context, answer based on widely accepted knowledge or the most relevant information available. Avoid ambiguous or overly detailed responses.{question}\nRespond with the answer only", 30 | "Generate a concise and relevant output that answers the questions based on the provided context, ensuring the information is accurate and directly related to the inquiry.{question}\nRespond with the answer only", 31 | "For each provided context, extract the most relevant answer to the question posed, based on the information available, and present it in a concise manner.{question}\nRespond with the answer only" 32 | ] -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # System Prompt Optimization with Meta-Learning 2 | [![Paper](https://img.shields.io/badge/arXiv-2505.09666-b31b1b)](https://arxiv.org/abs/2505.09666) 3 | [![Python](https://img.shields.io/badge/Python-3.10%2B-orange)](https://www.python.org/downloads/release/python-310s0/) 4 | [![GCC](https://img.shields.io/badge/gcc-9.1%2B-blue)](https://gcc.gnu.org/gcc-9/) 5 | 6 | 🚀 **Welcome to the official repository of** [**System Prompt Optimization with Meta-Learning**](https://arxiv.org/abs/2505.09666)! 7 | 8 | ## 🔍 Overview 9 | ![MetaSPO](asset/main_fig.jpg) 10 | This repository contains the official implementation of Meta-level System Prompt Optimizer (MetaSPO), a meta-learning approach for optimizing system prompts for Large Language Models (LLMs). MetaSPO is designed to optimize system prompts that are robust to diverse user inputs and transferable across a wide range of tasks and domains. 11 | 12 | ## 📌 Get Started 13 | ### Installation 14 | ```bash 15 | git clone https://github.com/Dozi01/MetaSPO.git 16 | cd MetaSPO 17 | conda create -n metaspo python=3.10 -y 18 | conda activate metaspo 19 | pip install -r requirements.txt 20 | ``` 21 | Ensure your OPENAI_API_KEY is stored in the .env file. 22 | 23 | ### MetaSPO: Training and Evaluation 24 | ```bash 25 | ./main.sh 26 | ``` 27 | Refer to `main.sh` for detailed instructions. 28 | 29 | ### Tasks 30 | Modify `configs/$DOMAIN.yaml` to set dataset configurations. 31 | To implement new tasks, include the task name in `srt/tasks/__init__.py` and implement a corresponding task class. 32 | 33 | ## 📜 Citation 34 | If you find this work useful, please cite our paper: 35 | ``` 36 | @article{choi2025promptoptimizationmetalearning, 37 | title={System Prompt Optimization with Meta-Learning}, 38 | author={Yumin Choi and Jinheon Baek and Sung Ju Hwang}, 39 | year={2025}, 40 | archivePrefix={arXiv}, 41 | primaryClass={cs.CL}, 42 | url={https://arxiv.org/abs/2505.09666}, 43 | } 44 | ``` -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | torch 2 | tqdm 3 | fire 4 | numpy 5 | scipy 6 | datasets 7 | openai 8 | pytz 9 | transformers 10 | scikit-learn 11 | vllm 12 | python-dotenv -------------------------------------------------------------------------------- /src/__init__.py: -------------------------------------------------------------------------------- 1 | from .runner import Runner 2 | from .analyser import Analyser 3 | from .taskmanager import TaskManager 4 | from .utils import * 5 | -------------------------------------------------------------------------------- /src/analyser.py: -------------------------------------------------------------------------------- 1 | import json 2 | from .methods import * 3 | from .runner import Runner 4 | from .methods.node import Node 5 | import os 6 | 7 | 8 | class Analyser(Runner): 9 | def __init__( 10 | self, 11 | args, 12 | ): 13 | super().__init__(args) 14 | 15 | self.unseen_gen_up_dir = args.unseen_gen_up_dir 16 | self.num_test_up = args.num_test_up 17 | self.analysis_method = args.analysis_method 18 | 19 | def meta_test(self): 20 | if self.analysis_method == "unseen_generalization": 21 | self.unseen_generalization() 22 | elif self.analysis_method == "test_time_adaptation": 23 | # This will call ProTeGi with optimized system prompt 24 | self.optim_method.train() 25 | 26 | def unseen_generalization(self): 27 | for task in self.task_manager.meta_test_tasks: 28 | 29 | user_prompts = self.get_user_prompts(task) 30 | total_nodes = [] 31 | for instruction in user_prompts: 32 | node = Node( 33 | system_prompt=self.init_system_prompt, 34 | instruction=instruction, 35 | task=task, 36 | parent=None, 37 | ) 38 | self.optim_method.evaluate_node(node, split="test") 39 | total_nodes.append(node) 40 | node_data = [node.to_dict() for node in total_nodes] 41 | 42 | test_metric_averaged = sum(node.test_metric for node in total_nodes) / len(total_nodes) 43 | 44 | result_data = { 45 | "task_name": task.task_name, 46 | "system_prompt": self.init_system_prompt, 47 | "test_metric_averaged": test_metric_averaged, 48 | "node_test_metrics": [node.test_metric for node in total_nodes], 49 | "node_data": node_data, 50 | } 51 | 52 | self.save_data(self.log_dir, result_data, filename=f"result_unseen_gen_{task.task_name}") 53 | 54 | def get_user_prompts(self, task): 55 | file_path = f"{self.unseen_gen_up_dir}/up_for_unseen_gen_{task.task_name}.json" 56 | if not os.path.exists(file_path): 57 | self.generate_user_prompts(task) 58 | try: 59 | with open(file_path, "r") as json_file: 60 | user_prompts_data = json.load(json_file) 61 | return user_prompts_data[: self.num_test_up] 62 | except FileNotFoundError: 63 | raise FileNotFoundError(f"The file at '{file_path}' does not exist.") 64 | 65 | def generate_user_prompts(self, task): 66 | user_prompts = [] 67 | 68 | for i in range(self.num_test_up): 69 | demo_string = self.optim_method._get_example_ape(task, model_responses_num=10) 70 | new_prompt = self.optim_model.instruction_ape_generation_agent(demo=demo_string) 71 | new_prompt += task.suffix_prompt 72 | 73 | user_prompts.append(new_prompt) 74 | 75 | self.save_data( 76 | self.unseen_gen_up_dir, 77 | user_prompts, 78 | filename=f"up_for_unseen_gen_{task.task_name}", 79 | ) 80 | 81 | def save_data(self, dir, data, filename): 82 | filepath = f"{dir}/{filename}.json" 83 | with open(filepath, "w") as file: 84 | json.dump(data, file, indent=4) 85 | self.logger.info(f"{filepath} saved") 86 | -------------------------------------------------------------------------------- /src/language_model/__init__.py: -------------------------------------------------------------------------------- 1 | from .agents import BaseModel, OptimizationModel 2 | -------------------------------------------------------------------------------- /src/language_model/agents.py: -------------------------------------------------------------------------------- 1 | from .meta_prompts import * 2 | import re 3 | import numpy as np 4 | from .openai_model import OpenAIModel 5 | from .vllm_model import VLLM 6 | from ..tasks import BaseTask 7 | 8 | 9 | LANGUAGE_MODELS = {"openai": OpenAIModel, "vllm": VLLM} 10 | 11 | 12 | def get_language_model(language_model_name): 13 | assert language_model_name in LANGUAGE_MODELS.keys(), f"Language model type {language_model_name} is not supported." 14 | return LANGUAGE_MODELS[language_model_name] 15 | 16 | 17 | class BaseModel: 18 | def __init__(self, base_model_setting: dict, logger): 19 | 20 | self.base_model = get_language_model(base_model_setting["model_type"])(**base_model_setting) 21 | self._batch_forward_func = self.base_model.batch_forward_func 22 | self.logger = logger 23 | 24 | def forward(self, batch, cur_prompt, task: BaseTask): 25 | batch_size = len(batch["question"]) 26 | # Building prompts for the forward pass and the gradient calculation 27 | batch_prompts = self._build_forward_prompts_completion( 28 | batch["question"], 29 | user_prompt=cur_prompt["user"], 30 | system_prompt=cur_prompt["system"], 31 | ) 32 | for_model_input_example = self._build_prompt_for_gradient(batch_prompts) 33 | 34 | # Obtaining model responses and processing predictions and labels 35 | responses = self._batch_forward_func(batch_prompts) 36 | preds = task.batch_clean_responses(responses) 37 | labels = batch["answer"] 38 | 39 | # Calculating evaluation metrics 40 | correct = task.cal_correct(preds=preds, labels=labels) 41 | accuracy = np.mean(correct) 42 | metric = task.cal_metric(preds=preds, labels=labels) 43 | 44 | # Preparing examples for output 45 | examples = [ 46 | { 47 | "cur_prompt": cur_prompt, 48 | "question": question, 49 | "answer": answer, 50 | "model_input": model_input, 51 | "model_response": response, 52 | "label": label, 53 | "pred": pred, 54 | } 55 | for question, answer, model_input, response, label, pred in zip( 56 | batch["question"], 57 | batch["answer"], 58 | for_model_input_example, 59 | responses, 60 | labels, 61 | preds, 62 | ) 63 | ] 64 | 65 | # Constructing forward output 66 | forward_output = { 67 | "cur_prompt": cur_prompt, 68 | "correct": correct, 69 | "examples": examples, 70 | "acc": accuracy, 71 | "metric": metric, 72 | } 73 | 74 | # Logging information 75 | # self._log_debug_info( 76 | # batch_prompts, batch["question"], responses, preds, labels, correct 77 | # ) 78 | 79 | log_str = self._get_forward_log_template().format( 80 | task_name=task.task_name, 81 | cur_prompt=cur_prompt, 82 | num_examples=batch_size, 83 | metric=metric, 84 | ) 85 | self.logger.info(log_str) 86 | 87 | return forward_output 88 | 89 | # Helper function for debugging information 90 | 91 | def _get_forward_log_template(self): 92 | forward_log_template = """---------------\tModel Output\t----------------\ntask_name: {task_name}\ncur_prompt:\n{cur_prompt}\nnum_examples: {num_examples}\nmetric: {metric}\n""" 93 | return forward_log_template 94 | 95 | def _log_debug_info(self, prompts, questions, responses, predictions, labels, correct, num_debug=10): 96 | for prom, ques, resp, pred, label, corr in zip( 97 | prompts[:num_debug], 98 | questions[:num_debug], 99 | responses[:num_debug], 100 | predictions[:num_debug], 101 | labels[:num_debug], 102 | correct[:num_debug], 103 | ): 104 | self.logger.info( 105 | f"Question: {ques}\nResponses :\n{resp}\nPrediction: {pred} Label: {label} Correct: {corr}\n-----\n" 106 | ) 107 | 108 | def _build_forward_prompts_completion(self, questions, user_prompt, system_prompt=""): 109 | prompts = [] 110 | 111 | for i, question in enumerate(questions): 112 | message = [] 113 | if system_prompt != "": 114 | message.append({"role": "system", "content": system_prompt}) 115 | message.append( 116 | { 117 | "role": "user", 118 | "content": f"{user_prompt.replace('{question}', question)}", 119 | } 120 | ) 121 | prompts.append(message) 122 | 123 | return prompts 124 | 125 | def _build_prompt_for_gradient(self, batch_prompts): 126 | if len(batch_prompts[0]) == 1: # no system prompt 127 | return [{"system": "", "user": prompt[0]["content"]} for prompt in batch_prompts] 128 | else: 129 | return [{"system": prompt[0]["content"], "user": prompt[1]["content"]} for prompt in batch_prompts] 130 | 131 | def _split_wrong_and_correct_examples(self, forward_output): 132 | wrong_examples = [] 133 | correct_examples = [] 134 | 135 | for i, example in enumerate(forward_output["examples"]): 136 | if forward_output["correct"][i] == 0: 137 | wrong_examples.append(example) 138 | 139 | elif forward_output["correct"][i] == 1: 140 | correct_examples.append(example) 141 | 142 | else: 143 | raise ValueError(f"_get_wrong_examples: invalid correct number {i} {forward_output}.") 144 | 145 | return wrong_examples, correct_examples 146 | 147 | def get_model_response(self, batch, cur_prompt, task): 148 | forward_output = self.forward(batch=batch, cur_prompt=cur_prompt, task=task) 149 | 150 | wrong_examples, correct_examples = self._split_wrong_and_correct_examples(forward_output=forward_output) 151 | 152 | return ( 153 | wrong_examples, 154 | correct_examples, 155 | forward_output["metric"], 156 | forward_output, 157 | ) 158 | 159 | 160 | class OptimizationModel: 161 | def __init__( 162 | self, 163 | optim_model_setting, 164 | logger=None, 165 | ): 166 | self.optim_model = get_language_model(optim_model_setting["model_type"])(**optim_model_setting) 167 | self.logger = logger 168 | 169 | def log_information(self, phase: str, prompt: str, response: str) -> None: 170 | self.logger.info(f"--------- \t Optimize {phase} Prompt\t ---------") 171 | total_prompt = "" 172 | for role_content in prompt: 173 | total_prompt += f'{role_content["role"]}\n{role_content["content"]}\n' 174 | 175 | self.logger.info(f"{total_prompt}\n{'-' * 80}\n{response}") 176 | 177 | def _clean_response(self, optim_response, tag_name): 178 | pattern = rf"<{tag_name}>(.*?)" 179 | matches = re.findall(pattern=pattern, string=optim_response, flags=re.DOTALL) 180 | for i, m in enumerate(matches): 181 | matches[i] = m.strip() 182 | if not matches: 183 | return "N/A : Format wrong" # response format wrong 184 | 185 | if isinstance(matches, list): 186 | return matches[0] 187 | else: 188 | return matches 189 | 190 | def _generate_analysis(self, system_propmt, template: str, **kwargs) -> str: 191 | analysis_prompt = template.format(**kwargs) 192 | prompt = self._build_prompt(system_propmt, analysis_prompt) 193 | response = self.optim_model.generate(prompt) 194 | analysis = self._clean_response(response, tag_name="Analysis") 195 | return analysis 196 | 197 | def _optimize_instruction( 198 | self, 199 | system_propmt: str, 200 | template: str, 201 | tag_name="improved_instruction_prompt", 202 | **kwargs, 203 | ) -> str: 204 | optimize_prompt = template.format(**kwargs) 205 | prompt = self._build_prompt(system_propmt, optimize_prompt) 206 | response = self.optim_model.generate(prompt) 207 | improved_instruction = self._clean_response(response, tag_name=tag_name) 208 | self.log_information("Instruction", prompt, response) 209 | return improved_instruction 210 | 211 | def instruction_ape_generation_agent(self, demo): 212 | instruction = self._optimize_instruction( 213 | None, template=ape_generation_template, demo=demo, tag_name="instruction" 214 | ) 215 | return instruction 216 | 217 | def instruction_ape_resampling_agent(self, instruction): 218 | return self._optimize_instruction( 219 | None, 220 | template=ape_resampling_template, 221 | instruction=instruction, 222 | question="{question}", 223 | ) 224 | 225 | def instruction_writer_agent(self, system_prompt: str, instruction: str, examples_string: str) -> str: 226 | analysis = self._generate_analysis( 227 | gradient_instruction_writer_system_prompt, 228 | gradient_for_instruction_writer_template, 229 | system_prompt=system_prompt, 230 | instruction=instruction, 231 | examples=examples_string, 232 | ) 233 | 234 | return self._optimize_instruction( 235 | optimizer_instruction_writer_system_prompt, 236 | optimizer_instruction_writer_template, 237 | system_prompt=system_prompt, 238 | instruction=instruction, 239 | examples=examples_string, 240 | analysis=analysis, 241 | question="{question}", 242 | ) 243 | 244 | def _optimize_system_prompt(self, system_propmt, template: str, **kwargs) -> str: 245 | optimize_prompt = template.format(**kwargs) 246 | prompt = self._build_prompt(system_propmt, optimize_prompt) 247 | response = self.optim_model.generate(prompt) 248 | updated_system_prompt = self._clean_response(response, tag_name="improved_system_prompt") 249 | self.log_information("System", prompt, response) 250 | return updated_system_prompt 251 | 252 | def system_writer_agent(self, current_system_prompt: str, example_strings: list) -> str: 253 | analysis = self._generate_analysis( 254 | gradient_system_writer_system_prompt, 255 | gradient_for_system_writer_template, 256 | system_prompt=current_system_prompt, 257 | examples=example_strings, 258 | ) 259 | return self._optimize_system_prompt( 260 | optimizer_system_writer_system_prompt, 261 | optimizer_system_writer_template, 262 | system_prompt=current_system_prompt, 263 | analysis=analysis, 264 | ) 265 | 266 | def _build_prompt(self, system_prompt, user_prompt): 267 | if system_prompt == None: 268 | return [{"role": "user", "content": user_prompt}] 269 | 270 | else: 271 | prompt = [ 272 | {"role": "system", "content": system_prompt}, 273 | {"role": "user", "content": user_prompt}, 274 | ] 275 | return prompt 276 | -------------------------------------------------------------------------------- /src/language_model/meta_prompts.py: -------------------------------------------------------------------------------- 1 | gradient_instruction_writer_system_prompt = """ 2 | You are a user prompt writer tasked with improving a language model's user prompt for a specific task. Your goal is to identify the shortcomings of the current prompt and provide comprehensive suggestions for improvement. 3 | """.strip() 4 | 5 | gradient_for_instruction_writer_template = """ 6 | Here are the inputs you will be working with: 7 | 8 | ### System prompt: 9 | {system_prompt} 10 | 11 | ### User prompt: 12 | {instruction} 13 | 14 | ### This prompt gets the following responses wrong: 15 | {examples} 16 | 17 | ### Remember to focus solely on discussing and improving the user prompt. 18 | 19 | ### Wrap the analysis of the user prompt in the Tags. 20 | """.strip() 21 | 22 | optimizer_instruction_writer_system_prompt = """ 23 | You are a user prompt writer tasked with improving a language model's user prompt for a specific task. Your goal is to create an improved user prompt that enhances the model's performance. 24 | """.strip() 25 | 26 | optimizer_instruction_writer_template = """ 27 | Here are the inputs you will be working with: 28 | 29 | ### System prompt: 30 | {system_prompt} 31 | 32 | ### User prompt: 33 | {instruction} 34 | 35 | ### Wrong examples of the model's responses: 36 | {examples} 37 | 38 | ### Analysis of the issues with this user prompt: 39 | {analysis} 40 | 41 | ### Address any problems observed in the examples based on analysis. 42 | 43 | ### Ensure the instruction contains the {question} where the actual question will be placed. 44 | 45 | ### The new user prompt should be wrapped with Tags. 46 | """.strip() 47 | 48 | gradient_system_writer_system_prompt = """ 49 | You are a system prompt writer tasked with improving a language model's system prompt for general tasks. Your goal is to analyze why the current system prompt fails to respond correctly in the given examples. 50 | """.strip() 51 | 52 | gradient_for_system_writer_template = """ 53 | Follow these instructions carefully: 54 | 55 | ### Review the current system prompt: 56 | {system_prompt} 57 | 58 | ### Wrong responses: 59 | {examples} 60 | 61 | ### Remember to focus solely on discussing and improving the system prompt. 62 | 63 | ### Wrap the analysis of the system prompt in the Tags. 64 | """.strip() 65 | 66 | optimizer_system_writer_system_prompt = """ 67 | You are a system prompt writer tasked with improving a language model's system prompt. Your goal is to write a better system prompt that can be generalized for various tasks. 68 | """.strip() 69 | 70 | optimizer_system_writer_template = """ 71 | Follow these instructions carefully: 72 | 73 | ### Review the current system prompt: 74 | {system_prompt} 75 | 76 | ### Analysis of the current system prompt: 77 | {analysis} 78 | 79 | ### Based on the information provided, write an improved system prompt. 80 | 81 | ### The new system prompt should be wrapped with Tags. 82 | """.strip() 83 | 84 | 85 | ape_resampling_template = """ 86 | Generate a variation of the following instruction while keeping the semantic meaning. 87 | 88 | {instruction} 89 | 90 | 91 | Ensure the instruction contains the {question} where the actual question will be placed. 92 | 93 | The new user prompt should be wrapped with and . 94 | """.strip() 95 | 96 | ape_generation_template = """ 97 | I gave a friend an instruction and inputs. The friend read the instruction and wrote an output for every one of the inputs. Here are the input-output pairs: 98 | 99 | {demo} 100 | 101 | Based on the above input-output pairs, write an instruction. 102 | The new instruction should be wrapped with and . 103 | """.strip() 104 | 105 | -------------------------------------------------------------------------------- /src/language_model/openai_model.py: -------------------------------------------------------------------------------- 1 | from openai import OpenAI 2 | import time 3 | 4 | MODEL_DICT = { 5 | "gpt-4o": "gpt-4o-2024-08-06", 6 | "gpt-4o-mini": "gpt-4o-mini-2024-07-18", 7 | } 8 | 9 | 10 | class OpenAIModel: 11 | def __init__( 12 | self, 13 | model_name: str, 14 | api_key: str, 15 | temperature: float, 16 | batch_mode: bool = True, 17 | **kwargs, 18 | ): 19 | if api_key is None: 20 | raise ValueError(f"api_key error: {api_key}") 21 | try: 22 | self.model = OpenAI(api_key=api_key) 23 | except Exception as e: 24 | print(f"Init openai client error: \n{e}") 25 | raise RuntimeError("Failed to initialize OpenAI client") from e 26 | 27 | if model_name not in MODEL_DICT: 28 | raise ValueError(f"Model {model_name} not supported.") 29 | 30 | self.model_name = MODEL_DICT[model_name] 31 | self.temperature = temperature 32 | self.batch_mode = batch_mode 33 | self.batch_forward_func = self.batch_forward_chatcompletion 34 | self.generate = self.gpt_chat_completion 35 | 36 | def batch_forward_chatcompletion(self, batch_prompts): 37 | return [self.gpt_chat_completion(prompt=prompt) for prompt in batch_prompts] 38 | 39 | def gpt_chat_completion(self, prompt): 40 | backoff_time = 1 41 | while True: 42 | try: 43 | response = self.model.chat.completions.create( 44 | messages=prompt, 45 | model=self.model_name, 46 | temperature=self.temperature, 47 | ) 48 | return response.choices[0].message.content.strip() 49 | except Exception as e: 50 | print(e, f" Sleeping {backoff_time} seconds...") 51 | time.sleep(backoff_time) 52 | backoff_time *= 1.5 53 | -------------------------------------------------------------------------------- /src/language_model/vllm_model.py: -------------------------------------------------------------------------------- 1 | from transformers import AutoTokenizer 2 | import torch 3 | from typing import List 4 | from vllm import LLM, SamplingParams 5 | 6 | MODEL_DICT = { 7 | "llama3.1_8B": "meta-llama/Meta-Llama-3.1-8B-Instruct", 8 | "llama3.2_3B": "meta-llama/Llama-3.2-3B-Instruct", 9 | "Qwen2.5_7B": "Qwen/Qwen2.5-7B-Instruct", 10 | } 11 | 12 | 13 | class VLLM: 14 | def __init__( 15 | self, 16 | model_name: str, 17 | temperature: float, 18 | max_model_len: int = 3000, 19 | dtype: str = "float16", 20 | num_gpus: int = torch.cuda.device_count(), 21 | gpu_memory_utilization: float = 0.90, 22 | **kwargs, 23 | ): 24 | self.model_name = self._get_model_name(model_name) 25 | self.temperature = temperature 26 | self.params = self._create_sampling_params(max_model_len) 27 | self.tokenizer = self._initialize_tokenizer() 28 | self.model = self._initialize_model(max_model_len, dtype, num_gpus, gpu_memory_utilization) 29 | 30 | def _get_model_name(self, model_name: str) -> str: 31 | if model_name in MODEL_DICT: 32 | return MODEL_DICT[model_name] 33 | raise ValueError(f"Model {model_name} not supported.") 34 | 35 | def _create_sampling_params(self, max_model_len: int) -> SamplingParams: 36 | return SamplingParams( 37 | temperature=self.temperature, 38 | max_tokens=max_model_len, 39 | skip_special_tokens=False, 40 | detokenize=True, 41 | ) 42 | 43 | def _initialize_tokenizer(self) -> AutoTokenizer: 44 | tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True, truncate=True, padding=True) 45 | tokenizer.pad_token = tokenizer.eos_token 46 | return tokenizer 47 | 48 | def _initialize_model( 49 | self, 50 | max_model_len: int, 51 | dtype: str, 52 | num_gpus: int, 53 | gpu_memory_utilization: float, 54 | ) -> LLM: 55 | return LLM( 56 | model=self.model_name, 57 | tokenizer=self.model_name, 58 | max_model_len=max_model_len, 59 | dtype=dtype, 60 | trust_remote_code=True, 61 | tensor_parallel_size=num_gpus, 62 | gpu_memory_utilization=gpu_memory_utilization, 63 | ) 64 | 65 | def batch_forward_func(self, batch_prompts, use_tqdm=True) -> List[str]: 66 | batch_prompts = self.prepare_batch_prompts(batch_prompts) 67 | request_outputs = self.model.generate(batch_prompts, self.params, use_tqdm=use_tqdm) 68 | return self.postprocess_output(request_outputs) 69 | 70 | def generate(self, prompt: str) -> str: 71 | input = self.tokenizer.apply_chat_template(prompt, tokenize=False) 72 | request_outputs = self.model.generate([input], self.params) 73 | return self.postprocess_output(request_outputs)[0] 74 | 75 | def postprocess_output(self, request_outputs) -> List[str]: 76 | return [output.outputs[0].text for output in request_outputs] 77 | 78 | def prepare_batch_prompts(self, batch_prompts) -> List[str]: 79 | return [ 80 | self.tokenizer.apply_chat_template(prompt, tokenize=False, add_generation_prompt=True) 81 | for prompt in batch_prompts 82 | ] 83 | -------------------------------------------------------------------------------- /src/methods/__init__.py: -------------------------------------------------------------------------------- 1 | from .unilevel import APE, ProTeGi 2 | from .metaspo import MetaSPO, MetaSPOAPE -------------------------------------------------------------------------------- /src/methods/metaspo/__init__.py: -------------------------------------------------------------------------------- 1 | from .metaspo import MetaSPO 2 | from .metaspo_ape import MetaSPOAPE 3 | -------------------------------------------------------------------------------- /src/methods/metaspo/metaspo.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from ...language_model import BaseModel, OptimizationModel 4 | from ...taskmanager import TaskManager 5 | from ..node import Node 6 | from typing import List, Optional 7 | import os 8 | import random 9 | 10 | 11 | class BilevelNodes: 12 | def __init__(self, node_list: Optional[List[List[Node]]], user_top_k: int): 13 | self.node_list = node_list 14 | self.user_top_k = user_top_k 15 | self.system_prompt = node_list[0][0].system_prompt 16 | 17 | @property 18 | def total_nodes(self): 19 | all_nodes = [] 20 | for sublist in self.node_list: 21 | for node in sublist: 22 | all_nodes.append(node) 23 | return all_nodes 24 | 25 | @property 26 | def export_node_list(self): 27 | return [sublist[:] for sublist in self.node_list] 28 | 29 | @property 30 | def meta_score(self): 31 | total_score = 0 32 | total_count = 0 33 | for nodes in self.node_list: 34 | total_score += sum(node.eval_metric for node in nodes) 35 | total_count += len(nodes) 36 | 37 | return total_score / total_count if total_count > 0 else 0 38 | 39 | def sort_nodes(self): 40 | for nodes in self.node_list: 41 | nodes.sort(key=lambda x: x.eval_metric, reverse=True) 42 | 43 | def cut_nodes_with_beam(self): 44 | for i, nodes in enumerate(self.node_list): 45 | self.node_list[i] = nodes[: self.user_top_k] 46 | 47 | def build_example_strings_for_meta_train_set(self, example_num_per_task: int = 2): 48 | examples = [nodes[0].get_example_string(example_num_per_task) for nodes in self.node_list] 49 | random.shuffle(examples) # Shuffle the list 50 | return "\n".join(examples) 51 | 52 | 53 | class MetaSPO: 54 | def __init__( 55 | self, 56 | initial_system_prompt, 57 | task_manager: TaskManager, 58 | base_model: BaseModel, 59 | optim_model: OptimizationModel, 60 | log_dir, 61 | logger, 62 | method: str, 63 | iteration, 64 | num_system_candidate, 65 | num_user_candidate, 66 | user_top_k=3, 67 | **kwargs, 68 | ) -> None: 69 | 70 | self.task_manager = task_manager 71 | self.logger = logger 72 | self.base_model = base_model 73 | self.optim_model = optim_model 74 | self.initial_system_prompt = initial_system_prompt 75 | self.system_prompt = initial_system_prompt 76 | 77 | self.method = method 78 | self.all_greater = False 79 | self.iteration = iteration 80 | self.num_system_candidate = num_system_candidate 81 | self.num_user_candidate = num_user_candidate 82 | 83 | self.user_top_k = user_top_k 84 | self.log_dir = log_dir 85 | 86 | self.train_log = list() 87 | self.test_log = list() 88 | 89 | def train(self): 90 | if self.method == "metaspo": 91 | self.method_metaspo() 92 | elif self.method == "outer_loop": 93 | self.method_outer_loop() 94 | 95 | def method_outer_loop(self): 96 | self.iteration = self.iteration * 2 97 | self.run_meta_training(optimize_system_fn=self.optimize_system, optimize_user_fn=None) 98 | 99 | def method_metaspo(self): 100 | self.run_meta_training( 101 | optimize_system_fn=self.optimize_system, 102 | optimize_user_fn=self.optimize_user, 103 | ) 104 | 105 | def run_meta_training( 106 | self, 107 | optimize_system_fn=None, 108 | optimize_user_fn=None, 109 | ): 110 | bilevel_nodes = self.init_bilevel_nodes() 111 | for node in bilevel_nodes.total_nodes: 112 | self.evaluate_node(node=node, split="train") 113 | 114 | updated_nodes = [bilevel_nodes.export_node_list] 115 | 116 | # Optimization loop 117 | for iter in range(self.iteration): 118 | bilevel_nodes = self.inner_loop(optimize_user_fn, bilevel_nodes, updated_nodes) 119 | bilevel_nodes = self.outer_loop(optimize_system_fn, bilevel_nodes, updated_nodes) 120 | 121 | # Test only the last nodes. 122 | last_bilevel_node = updated_nodes[-1] 123 | for nodes in last_bilevel_node: 124 | for node in nodes: 125 | if node.test_metric == -1: 126 | self.evaluate_node(node=node, split="test") 127 | 128 | def process_node(node): 129 | return node.to_dict() 130 | 131 | updated_nodes_data = [ 132 | [[(process_node(node)) for node in node_list] for node_list in bilevel_nodes] 133 | for bilevel_nodes in updated_nodes 134 | ] 135 | 136 | self.system_prompt = bilevel_nodes.system_prompt 137 | 138 | total_data = { 139 | "optimized_system_prompt": self.system_prompt, 140 | "nodes": updated_nodes_data, 141 | } 142 | 143 | self.logger.info(f"======= OPTIMIZED SYSTEM PROMPT =======") 144 | self.logger.info(self.system_prompt) 145 | 146 | self.save_data(total_data) 147 | 148 | def inner_loop(self, optimize_user_fn, bilevel_nodes, updated_nodes): 149 | if not optimize_user_fn: 150 | return bilevel_nodes 151 | 152 | self.logger.info(f"======= INNER LOOP =======") 153 | 154 | for nodes in bilevel_nodes.node_list: 155 | for _ in range(self.num_user_candidate): 156 | new_node = optimize_user_fn(nodes[0]) # we can use multiple nodes in this step. 157 | self.evaluate_node(node=new_node, split="train") 158 | nodes.append(new_node) 159 | 160 | bilevel_nodes.sort_nodes() 161 | bilevel_nodes.cut_nodes_with_beam() 162 | 163 | updated_nodes.append(bilevel_nodes.export_node_list) 164 | 165 | return bilevel_nodes 166 | 167 | def outer_loop(self, optimize_system_fn, bilevel_nodes, updated_nodes): 168 | if not optimize_system_fn: 169 | return bilevel_nodes 170 | 171 | self.logger.info(f"======= OUTER LOOP =======") 172 | 173 | new_bilevel_nodes_list = [optimize_system_fn(bilevel_nodes) for _ in range(self.num_system_candidate)] 174 | 175 | for new_bilevel_nodes in new_bilevel_nodes_list: 176 | for node in new_bilevel_nodes.total_nodes: 177 | self.evaluate_node(node=node, split="train") 178 | 179 | if new_bilevel_nodes.meta_score > bilevel_nodes.meta_score: 180 | new_bilevel_nodes.sort_nodes() 181 | bilevel_nodes = new_bilevel_nodes 182 | 183 | updated_nodes.append(bilevel_nodes.export_node_list) 184 | 185 | return bilevel_nodes 186 | 187 | def optimize_user(self, node: Node): 188 | 189 | examples_string = node.get_example_string() 190 | 191 | updated_instruction = self.optim_model.instruction_writer_agent( 192 | system_prompt=node.system_prompt, 193 | instruction=node.instruction, 194 | examples_string=examples_string, 195 | ) 196 | 197 | new_node = Node( 198 | system_prompt=node.system_prompt, 199 | instruction=updated_instruction, 200 | parent=node, 201 | ) 202 | return new_node 203 | 204 | def optimize_system(self, bilevel_nodes: BilevelNodes): 205 | example_strings = bilevel_nodes.build_example_strings_for_meta_train_set() 206 | 207 | updated_system_prompt = self.optim_model.system_writer_agent(bilevel_nodes.system_prompt, example_strings) 208 | 209 | new_node_list = [ 210 | [Node(updated_system_prompt, node.instruction, parent=node) for node in nodes] 211 | for nodes in bilevel_nodes.node_list 212 | ] 213 | 214 | # Initialize the new BilevelNodes with the updated node list 215 | new_bilevel_nodes = BilevelNodes(new_node_list, bilevel_nodes.user_top_k) 216 | 217 | return new_bilevel_nodes 218 | 219 | def init_bilevel_nodes(self): 220 | return BilevelNodes( 221 | [ 222 | [ 223 | Node( 224 | self.initial_system_prompt, 225 | task.initial_prompt, 226 | task=task, 227 | parent=None, 228 | ) 229 | ] 230 | for task in self.task_manager.tasks 231 | ], 232 | user_top_k=self.user_top_k, 233 | ) 234 | 235 | def evaluate_prompt(self, system, user, task, split="train"): 236 | if split not in ["train", "test"]: 237 | raise ValueError("Invalid split specified. Use 'train' or 'test'.") 238 | 239 | # Select data based on the split 240 | data = task.train_data if split == "train" else task.test_data 241 | 242 | # Construct prompt 243 | current_prompt = { 244 | "system": system, 245 | "user": user, 246 | } 247 | 248 | # Get model response and evaluation 249 | wrong_examples, correct_examples, metric, forward_output = self.base_model.get_model_response( 250 | data, current_prompt, task=task 251 | ) 252 | 253 | return metric, wrong_examples, correct_examples 254 | 255 | def evaluate_node(self, node: Node, split): 256 | metric, model_wrong_examples, model_correct_examples = self.evaluate_prompt( 257 | system=node.system_prompt, 258 | user=node.instruction, 259 | task=node.task, 260 | split=split, 261 | ) 262 | 263 | if split == "train": 264 | node.eval_metric = metric 265 | # node.update_model_correct_response(model_correct_examples) 266 | node.update_model_wrong_response(model_wrong_examples) 267 | if split == "test": 268 | node.test_metric = metric 269 | 270 | def save_data(self, nodes_data, filename: str = "nodes"): 271 | log_dir_parts = self.log_dir.split(os.sep) 272 | if len(log_dir_parts) > 1: 273 | log_dir_parts.pop() 274 | new_log_dir = os.sep.join(log_dir_parts) 275 | 276 | version = 0 277 | base_filename = f"bilevel_{filename}_{version}.json" 278 | full_path = os.path.join(new_log_dir, base_filename) 279 | 280 | while os.path.exists(full_path): 281 | version += 1 282 | base_filename = f"bilevel_{filename}_{version}.json" 283 | full_path = os.path.join(new_log_dir, base_filename) 284 | 285 | with open(full_path, "w") as file: 286 | json.dump(nodes_data, file, indent=4) 287 | 288 | self.logger.info(f"Save log: {full_path}") 289 | -------------------------------------------------------------------------------- /src/methods/metaspo/metaspo_ape.py: -------------------------------------------------------------------------------- 1 | from ...language_model import BaseModel, OptimizationModel 2 | from ...taskmanager import TaskManager 3 | from ...tasks import BaseTask 4 | from ..node import Node 5 | import random 6 | from .metaspo import BilevelNodes, MetaSPO 7 | 8 | 9 | class MetaSPOAPE(MetaSPO): 10 | def __init__( 11 | self, 12 | initial_system_prompt, 13 | task_manager: TaskManager, 14 | base_model: BaseModel, 15 | optim_model: OptimizationModel, 16 | log_dir, 17 | logger, 18 | method: str, 19 | system_num_candidate=18, 20 | user_num_candidate=3, 21 | iteration=3, 22 | **kwargs, 23 | ) -> None: 24 | super().__init__( 25 | initial_system_prompt=initial_system_prompt, 26 | task_manager=task_manager, 27 | base_model=base_model, 28 | optim_model=optim_model, 29 | log_dir=log_dir, 30 | logger=logger, 31 | method=method, 32 | iteration=iteration, 33 | system_num_candidate=system_num_candidate, 34 | user_num_candidate=user_num_candidate, 35 | **kwargs, 36 | ) 37 | 38 | self.task_manager = task_manager 39 | self.logger = logger 40 | self.base_model = base_model 41 | self.optim_model = optim_model 42 | self.initial_system_prompt = initial_system_prompt 43 | self.system_prompt = initial_system_prompt 44 | 45 | self.method = method 46 | self.all_greater = False 47 | self.beam_cut = True 48 | self.iteration = iteration 49 | self.log_dir = log_dir 50 | 51 | self.train_log = list() 52 | self.test_log = list() 53 | 54 | def train(self): 55 | self.method_metaspo_ape() 56 | 57 | def method_metaspo_ape(self): 58 | self.run_meta_training( 59 | optimize_system_fn=self.optimize_system_ape, 60 | optimize_user_fn=self.optimize_user_ape, 61 | ) 62 | 63 | def optimize_user_ape(self, node: Node): 64 | demo_string = self._generate_demo_string([node.task], model_responses_num=10) 65 | new_prompt = self.optim_model.instruction_ape_generation_agent(demo=demo_string) 66 | new_prompt += node.task.suffix_prompt 67 | 68 | new_node = Node( 69 | system_prompt=node.system_prompt, 70 | instruction=new_prompt, 71 | parent=node, 72 | ) 73 | return new_node 74 | 75 | def optimize_system_ape(self, bilevel_nodes: BilevelNodes): 76 | tasks = [nodes[0].task for nodes in bilevel_nodes.node_list] 77 | demo_string = self._generate_demo_string(tasks, model_responses_num=5) 78 | 79 | new_system_prompt = self.optim_model.instruction_ape_generation_agent(demo=demo_string) 80 | 81 | new_node_list = [ 82 | [Node(new_system_prompt, node.instruction, parent=node) for node in nodes] 83 | for nodes in bilevel_nodes.node_list 84 | ] 85 | 86 | # Initialize the new BilevelNodes with the updated node list 87 | new_bilevel_nodes = BilevelNodes(new_node_list, bilevel_nodes.beam_width) 88 | 89 | return new_bilevel_nodes 90 | 91 | def _generate_demo_string(self, task_list, model_responses_num): 92 | example_string_list = [] 93 | for task in task_list: 94 | example_string_list += self._get_example_ape(task, model_responses_num) 95 | random.shuffle(example_string_list) 96 | return "\n".join(example_string_list) 97 | 98 | def _format_answer(self, example): 99 | return ", ".join(example["answer"]) if isinstance(example["answer"], list) else example["answer"] 100 | 101 | def _get_example_ape(self, task: BaseTask, model_responses_num=10): 102 | questions = task.train_data["question"] 103 | answers = task.train_data["answer"] 104 | 105 | # Ensuring we do not exceed the available number of questions/answers 106 | num_examples = min(len(questions), model_responses_num) 107 | 108 | indices = random.sample(range(len(questions)), num_examples) 109 | 110 | example_strings = [ 111 | self._qa_example_template( 112 | question=questions[i], 113 | answer=self._format_answer({"answer": answers[i]}), 114 | ) 115 | for i in indices 116 | ] 117 | 118 | return example_strings 119 | 120 | def _qa_example_template(self, question, answer): 121 | return f"Input :\n{question}\nOutput :\n{answer}\n" 122 | -------------------------------------------------------------------------------- /src/methods/node.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | import random 3 | from typing import Optional 4 | from ..tasks import BaseTask 5 | 6 | 7 | class Node: 8 | id_iter = itertools.count() 9 | 10 | @classmethod 11 | def reset_id(cls): 12 | cls.id_iter = itertools.count() 13 | 14 | def __init__( 15 | self, 16 | system_prompt: str, 17 | instruction: str, 18 | task: BaseTask = None, 19 | parent: Optional["Node"] = None, 20 | test_metric: float = -1, 21 | eval_metric: float = -1, 22 | ): 23 | 24 | self.id = next(Node.id_iter) 25 | self.system_prompt = system_prompt 26 | self.instruction = instruction 27 | 28 | self.parent = parent 29 | self.test_metric = test_metric 30 | self.eval_metric = eval_metric 31 | 32 | if parent is None: 33 | self.depth = 0 34 | assert task is not None 35 | self.task = task 36 | else: 37 | self.depth = parent.depth + 1 38 | self.task = parent.task 39 | 40 | def update_model_wrong_response(self, responses): 41 | self.model_wrong_responses = [] 42 | self.model_wrong_responses.extend(responses) 43 | 44 | def update_model_correct_response(self, responses): 45 | self.model_correct_responses = [] 46 | self.model_correct_responses.extend(responses) 47 | 48 | def _sample_wrong_examples(self, model_responses_num): 49 | num_wrong_examples = len(self.model_wrong_responses) 50 | if num_wrong_examples < model_responses_num: 51 | sampled_examples = self.model_wrong_responses 52 | else: 53 | sampled_examples = random.sample(self.model_wrong_responses, model_responses_num) 54 | 55 | return sampled_examples 56 | 57 | def _format_answer(self, example): 58 | return ", ".join(example["label"]) if isinstance(example["label"], list) else example["label"] 59 | 60 | def get_example_string(self, model_responses_num: int = 3): 61 | examples = self._sample_wrong_examples(model_responses_num) 62 | example_strings = [] 63 | 64 | for example in examples: 65 | system_prompt = example["model_input"]["system"] 66 | 67 | example_string = self._get_example_template().format( 68 | system_prompt=system_prompt, 69 | user_prompt=example["model_input"]["user"], 70 | label=self._format_answer(example), 71 | response=example["model_response"], 72 | prediction=example["pred"], 73 | ) 74 | example_strings.append(example_string) 75 | 76 | return "\n".join(example_strings) 77 | 78 | def _get_example_template(self): 79 | example_template = """\nSystem prompt:\n{system_prompt}\n\nUser prompt:\n{user_prompt}\n\nResponse:\n{response}\n\nPrediction:\n{prediction}\n\nThe correct label is :\n{label}\n""".strip() 80 | return example_template 81 | 82 | def to_dict(self): 83 | """ 84 | Converts the node object to a dictionary format for logging. 85 | 86 | Returns: 87 | dict: A dictionary containing the node's attributes. 88 | """ 89 | return { 90 | "id": self.id, 91 | "task": self.task.task_name, 92 | "system_prompt": self.system_prompt, 93 | "instruction": self.instruction, 94 | "parent_id": self.parent.id if self.parent else None, 95 | "depth": self.depth, 96 | "eval_metric": self.eval_metric, 97 | "test_metric": self.test_metric, 98 | } 99 | -------------------------------------------------------------------------------- /src/methods/unilevel/__init__.py: -------------------------------------------------------------------------------- 1 | from .unilevel import Unilevel 2 | from .ape import APE 3 | from .protegi import ProTeGi 4 | -------------------------------------------------------------------------------- /src/methods/unilevel/ape.py: -------------------------------------------------------------------------------- 1 | from ...language_model import BaseModel, OptimizationModel 2 | from ...taskmanager import TaskManager 3 | from ...tasks import BaseTask 4 | from .unilevel import Unilevel 5 | import random 6 | 7 | 8 | class APE(Unilevel): 9 | def __init__( 10 | self, 11 | initial_system_prompt, 12 | task_manager: TaskManager, 13 | base_model: BaseModel, 14 | optim_model: OptimizationModel, 15 | log_dir, 16 | method: str, 17 | logger, 18 | iteration, 19 | top_k, 20 | print_log: bool = True, 21 | model_responses_num: int = None, 22 | **kwargs, 23 | ) -> None: 24 | super().__init__( 25 | initial_system_prompt, 26 | task_manager, 27 | base_model, 28 | optim_model, 29 | log_dir, 30 | method, 31 | logger, 32 | iteration=iteration, 33 | top_k=top_k, 34 | print_log=print_log, 35 | model_responses_num=model_responses_num, 36 | ) 37 | 38 | self.top_k = top_k 39 | 40 | def train(self): 41 | assert self.method == "ape", "APE class method must be 'ape'" 42 | for task in self.task_manager.meta_test_tasks: 43 | self.optimize_ape(task) 44 | 45 | def optimize_ape(self, task: BaseTask): 46 | 47 | self.initialize_prompt(task) 48 | 49 | self.test_prompt(task) 50 | 51 | # Initial Proposal Step 52 | candidate_prompts = [(task.initial_prompt, task.best_metric)] 53 | batch_candidates = [(task.initial_prompt, task.best_metric)] 54 | 55 | for _ in range(self.iteration): 56 | demo_string = self._get_example_ape(task, model_responses_num=10) 57 | new_prompt = self.optim_model.instruction_ape_generation_agent(demo=demo_string) 58 | 59 | new_prompt += task.suffix_prompt 60 | 61 | metric, _, _ = self.evaluate_prompt(system=task.system_prompt, user=new_prompt, task=task, split="train") 62 | batch_candidates.append((new_prompt, metric)) 63 | 64 | candidate_prompts.extend(batch_candidates) 65 | candidate_prompts.sort(key=lambda x: x[1], reverse=True) 66 | top_prompts = candidate_prompts[: self.top_k] 67 | 68 | # Iterative Proposal Step 69 | for prompt, _ in top_prompts: 70 | batch_candidates = [] 71 | 72 | for _ in range(self.iteration): 73 | 74 | # Generate a new prompt based on the current top prompt 75 | new_prompt = self.optim_model.instruction_ape_resampling_agent(prompt) 76 | 77 | # Evaluate the newly generated prompt 78 | metric, _, _ = self.evaluate_prompt( 79 | system=task.system_prompt, 80 | user=new_prompt, 81 | task=task, 82 | split="train", 83 | ) 84 | batch_candidates.append((new_prompt, metric)) 85 | 86 | candidate_prompts.extend(batch_candidates) 87 | 88 | # Sort all candidate prompts generated during the iterative proposal step by scores 89 | candidate_prompts.sort(key=lambda x: x[1], reverse=True) 90 | 91 | # Select the top prompt based on evaluation scores as the optimized prompt 92 | best_prompt, best_metric = candidate_prompts[0] 93 | 94 | task.current_prompt = best_prompt 95 | 96 | self.test_prompt(task) 97 | 98 | self.save_log() 99 | 100 | def initialize_prompt(self, task: BaseTask): 101 | task.system_prompt = self.initial_system_prompt 102 | task.current_prompt = task.initial_prompt 103 | metric, model_wrong_examples, model_correct_examples = self.evaluate_prompt( 104 | system=task.system_prompt, 105 | user=task.current_prompt, 106 | task=task, 107 | split="train", 108 | ) 109 | task.best_metric = metric 110 | self.write_log( 111 | system_prompt=task.system_prompt, 112 | user_prompt=task.current_prompt, 113 | task=task, 114 | metric=metric, 115 | split="train", 116 | ) 117 | 118 | def test_prompt(self, task): 119 | metric, model_wrong_examples, model_correct_examples = self.evaluate_prompt( 120 | system=task.system_prompt, 121 | user=task.current_prompt, 122 | task=task, 123 | split="test", 124 | ) 125 | 126 | self.write_log( 127 | system_prompt=task.system_prompt, 128 | user_prompt=task.current_prompt, 129 | task=task, 130 | metric=metric, 131 | split="test", 132 | ) 133 | 134 | return metric 135 | 136 | def _format_answer(self, example): 137 | return ", ".join(example["answer"]) if isinstance(example["answer"], list) else example["answer"] 138 | 139 | def _get_example_ape(self, task: BaseTask, model_responses_num=10): 140 | questions = task.train_data["question"] 141 | answers = task.train_data["answer"] 142 | 143 | # Ensuring we do not exceed the available number of questions/answers 144 | num_examples = min(len(questions), model_responses_num) 145 | 146 | indices = random.sample(range(len(questions)), num_examples) 147 | 148 | example_strings = [ 149 | self._qa_example_template( 150 | question=questions[i], 151 | answer=self._format_answer({"answer": answers[i]}), 152 | ) 153 | for i in indices 154 | ] 155 | 156 | return "\n".join(example_strings) 157 | 158 | def _qa_example_template(self, question, answer): 159 | return f"Input :\n{question}\nOutput :\n{answer}\n" 160 | -------------------------------------------------------------------------------- /src/methods/unilevel/protegi.py: -------------------------------------------------------------------------------- 1 | from ...language_model import BaseModel, OptimizationModel 2 | from ...taskmanager import TaskManager 3 | from ...tasks import BaseTask 4 | from .unilevel import Unilevel 5 | from ..node import Node 6 | 7 | 8 | class ProTeGi(Unilevel): 9 | def __init__( 10 | self, 11 | initial_system_prompt: str, 12 | task_manager: TaskManager, 13 | base_model: BaseModel, 14 | optim_model: OptimizationModel, 15 | log_dir, 16 | method: str, 17 | logger, 18 | iteration=6, 19 | num_user_candidate=3, 20 | print_log: bool = True, 21 | model_responses_num: int = 3, 22 | **kwargs, 23 | ) -> None: 24 | super().__init__( 25 | initial_system_prompt, 26 | task_manager, 27 | base_model, 28 | optim_model, 29 | log_dir, 30 | method, 31 | logger, 32 | iteration=iteration, 33 | print_log=print_log, 34 | model_responses_num=model_responses_num, 35 | ) 36 | 37 | self.num_user_candidate = num_user_candidate 38 | 39 | def train(self): 40 | assert self.method == "protegi", "ProTeGi class method must be 'protegi'" 41 | for task in self.task_manager.meta_test_tasks: 42 | self.optimize_protegi(task) 43 | 44 | def optimize_protegi(self, task: BaseTask): 45 | node = Node(self.initial_system_prompt, task.initial_prompt, task=task, parent=None) 46 | self.evaluate_node(node=node, split="train") 47 | 48 | total_node = [node] 49 | updated_node = [node] 50 | 51 | batch_candidates = [] 52 | # candidate_nodes should contain initial node. 53 | candidate_nodes = [node] 54 | for _ in range(self.num_user_candidate): 55 | 56 | new_node = self.optimize_user_only(node=node) 57 | self.evaluate_node(node=new_node, split="train") 58 | 59 | total_node.append(new_node) 60 | batch_candidates.append(new_node) 61 | 62 | candidate_nodes.extend(batch_candidates) 63 | candidate_nodes.sort(key=lambda x: x.eval_metric, reverse=True) 64 | candidate_nodes = candidate_nodes[: self.num_user_candidate] 65 | 66 | updated_node.extend(candidate_nodes) 67 | 68 | for iter in range(self.iteration): 69 | batch_candidates = [] 70 | 71 | for node in candidate_nodes: 72 | for _ in range(self.num_user_candidate): 73 | new_node = self.optimize_user_only(node=node) 74 | self.evaluate_node(node=new_node, split="train") 75 | 76 | total_node.append(new_node) 77 | batch_candidates.append(new_node) 78 | 79 | candidate_nodes.extend(batch_candidates) 80 | candidate_nodes.sort(key=lambda x: x.eval_metric, reverse=True) 81 | candidate_nodes = candidate_nodes[: self.num_user_candidate] 82 | updated_node.extend(candidate_nodes) 83 | 84 | for node in updated_node: 85 | if node.test_metric == -1: 86 | self.evaluate_node(node=node, split="test") 87 | 88 | nodes_data = [node.to_dict() for node in updated_node] 89 | self.save_data(nodes_data, filename=f"{task.task_name}") -------------------------------------------------------------------------------- /src/methods/unilevel/unilevel.py: -------------------------------------------------------------------------------- 1 | import json 2 | from ...language_model import BaseModel, OptimizationModel 3 | from ...taskmanager import TaskManager 4 | from ..node import Node 5 | import os 6 | 7 | 8 | class Unilevel: 9 | def __init__( 10 | self, 11 | initial_system_prompt, 12 | task_manager: TaskManager, 13 | base_model: BaseModel, 14 | optim_model: OptimizationModel, 15 | log_dir, 16 | method: str, 17 | logger, 18 | iteration=3, 19 | model_responses_num: int = 3, 20 | **kwargs, 21 | ) -> None: 22 | 23 | self.task_manager = task_manager 24 | self.logger = logger 25 | self.base_model = base_model 26 | self.optim_model = optim_model 27 | self.initial_system_prompt = initial_system_prompt 28 | 29 | self.model_responses_num = model_responses_num 30 | self.method = method 31 | self.iteration = iteration 32 | self.log_dir = log_dir 33 | 34 | self.train_log = list() 35 | self.test_log = list() 36 | 37 | def train(self): 38 | raise NotImplementedError 39 | 40 | def optimize_system_only(self, node: Node): 41 | example_strings = node.get_example_string() 42 | 43 | updated_system_prompt = self.optim_model.system_writer_agent(node.system_prompt, example_strings) 44 | new_node = Node( 45 | system_prompt=updated_system_prompt, 46 | instruction=node.instruction, 47 | parent=node, 48 | ) 49 | return new_node 50 | 51 | def optimize_user_only(self, node: Node): 52 | 53 | examples_string = node.get_example_string() 54 | 55 | system_prompt = node.system_prompt 56 | 57 | updated_instruction = self.optim_model.instruction_writer_agent( 58 | system_prompt=system_prompt, 59 | instruction=node.instruction, 60 | examples_string=examples_string, 61 | ) 62 | 63 | new_node = Node( 64 | system_prompt=node.system_prompt, 65 | instruction=updated_instruction, 66 | parent=node, 67 | ) 68 | return new_node 69 | 70 | def evaluate_node(self, node: Node, split): 71 | metric, model_wrong_examples, model_correct_examples = self.evaluate_prompt( 72 | system=node.system_prompt, 73 | user=node.instruction, 74 | task=node.task, 75 | split=split, 76 | ) 77 | 78 | if split == "train": 79 | node.eval_metric = metric 80 | node.update_model_correct_response(model_correct_examples) 81 | node.update_model_wrong_response(model_wrong_examples) 82 | if split == "test": 83 | node.test_metric = metric 84 | 85 | def evaluate_prompt(self, system, user, task, split="train"): 86 | if split not in ["train", "test"]: 87 | raise ValueError("Invalid split specified. Use 'train' or 'test'.") 88 | 89 | # Select data based on the split 90 | data = task.train_data if split == "train" else task.test_data 91 | 92 | # Construct prompt 93 | current_prompt = { 94 | "system": system, 95 | "user": user, 96 | } 97 | 98 | # Get model response and evaluation 99 | wrong_examples, correct_examples, metric, forward_output = self.base_model.get_model_response( 100 | data, current_prompt, task=task 101 | ) 102 | 103 | return metric, wrong_examples, correct_examples 104 | 105 | def write_log(self, system_prompt, user_prompt, task, metric, split): 106 | log = { 107 | "system_prompt": system_prompt, 108 | "user_prompt": user_prompt, 109 | "split": split, 110 | "task": task.task_name, 111 | "score": metric, 112 | } 113 | 114 | self._append_log(split, log) 115 | 116 | def _append_log(self, split, log): 117 | """ 118 | Appends the log entry to the correct log based on split type. 119 | """ 120 | if split == "train": 121 | self.train_log.append(log) 122 | elif split == "test": 123 | self.test_log.append(log) 124 | 125 | def save_log(self): 126 | """ 127 | Saves the training and test logs to JSON files. 128 | """ 129 | self._save_to_file("train_log.json", self.train_log) 130 | self._save_to_file("test_log.json", self.test_log) 131 | 132 | def _save_to_file(self, filename, data): 133 | """ 134 | Writes log data to a specified JSON file. 135 | """ 136 | file_path = f"{self.log_dir}/{filename}" 137 | with open(file_path, "w", encoding="utf-8") as f: 138 | json.dump(data, f, ensure_ascii=False, indent=4) 139 | 140 | def save_data(self, nodes_data, filename: str = "nodes"): 141 | log_dir_parts = self.log_dir.split(os.sep) 142 | if len(log_dir_parts) > 1: 143 | log_dir_parts.pop() 144 | new_log_dir = os.sep.join(log_dir_parts) 145 | 146 | version = 0 147 | base_filename = f"unilevel_{filename}_{version}.json" 148 | full_path = os.path.join(new_log_dir, base_filename) 149 | 150 | while os.path.exists(full_path): 151 | version += 1 152 | base_filename = f"unilevel_{filename}_{version}.json" 153 | full_path = os.path.join(new_log_dir, base_filename) 154 | 155 | with open(full_path, "w") as file: 156 | json.dump(nodes_data, file, indent=4) 157 | 158 | self.logger.info(f"Save log: {full_path}") 159 | -------------------------------------------------------------------------------- /src/runner.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | from datetime import timedelta 4 | import json 5 | from .methods import * 6 | from .utils import get_pacific_time, create_logger 7 | from .language_model import BaseModel, OptimizationModel 8 | from .taskmanager import TaskManager 9 | 10 | OPTIMIZE_METHOD_DICT = { 11 | "metaspo": MetaSPO, 12 | "metaspo_ape": MetaSPOAPE, 13 | "outer_loop": MetaSPO, 14 | "unseen_generalization": MetaSPO, # for unseen generalization, dummy method MetaSPO is used 15 | "test_time_adaptation": ProTeGi, # for test time adaptation 16 | "ape": APE, 17 | "protegi": ProTeGi, 18 | } 19 | 20 | 21 | class Runner: 22 | def __init__(self, args): 23 | 24 | # Load initial system prompt from file 25 | self.init_system_prompt = self.get_system_prompt(args.init_system_prompt_path) 26 | exp_name = f'{get_pacific_time().strftime("%Y%m%d_%H%M%S")}' 27 | 28 | self.log_dir = os.path.join(args.log_dir, exp_name) 29 | self.logger = create_logger(self.log_dir) 30 | 31 | search_setting, base_model_setting, optim_model_setting, task_setting = self.parse_args(args) 32 | 33 | self.task_manager = TaskManager(args.meta_train_tasks, args.meta_test_tasks, task_setting) 34 | 35 | # Initialize base model and optimization model 36 | self.base_model = BaseModel(base_model_setting, self.logger) 37 | self.optim_model = OptimizationModel(optim_model_setting, self.logger) 38 | 39 | # Initialize optimization method 40 | self.optim_method = OPTIMIZE_METHOD_DICT[search_setting["method"]]( 41 | task_manager=self.task_manager, 42 | base_model=self.base_model, 43 | optim_model=self.optim_model, 44 | initial_system_prompt=self.init_system_prompt, 45 | log_dir=self.log_dir, 46 | logger=self.logger, 47 | **search_setting, 48 | ) 49 | 50 | self.logger.info(f"base_model_setting : {base_model_setting}") 51 | self.logger.info(f"optim_model_setting : {optim_model_setting}") 52 | self.logger.info(f"search_setting : {search_setting}") 53 | self.logger.info(f"task_setting : {task_setting}") 54 | self.logger.info(f"meta_train_tasks : {args.meta_train_tasks}") 55 | self.logger.info(f"meta_test_tasks : {args.meta_test_tasks}") 56 | self.logger.info(f"init_system_prompt_path : {args.init_system_prompt_path}") 57 | self.logger.info(f"init_system_prompt : {self.init_system_prompt}") 58 | 59 | def meta_train(self): 60 | """ 61 | Start searching from initial prompt 62 | """ 63 | start_time = time.time() 64 | self.optim_method.train() 65 | end_time = time.time() 66 | 67 | exe_time = str(timedelta(seconds=end_time - start_time)).split(".")[0] 68 | self.logger.info(f"\nExcution time: {exe_time}") 69 | return 70 | 71 | def get_system_prompt(self, file_path): 72 | try: 73 | with open(file_path, "r") as json_file: 74 | data = json.load(json_file) 75 | file_name = os.path.basename(file_path) 76 | if "bilevel_nodes" in file_name: 77 | system_prompt = data['optimized_system_prompt'] 78 | else: 79 | system_prompt = data["prompt"] 80 | 81 | return system_prompt 82 | except FileNotFoundError: 83 | raise FileNotFoundError(f"The file at '{file_path}' does not exist.") 84 | 85 | def parse_args(self, args): 86 | search_setting = { 87 | "method": args.method, 88 | "iteration": args.iteration, 89 | "num_system_candidate": args.num_system_candidate, 90 | "num_user_candidate": args.num_user_candidate, 91 | "user_top_k": args.user_top_k, 92 | } 93 | 94 | base_model_setting = { 95 | "model_type": args.base_model_type, 96 | "model_name": args.base_model_name, 97 | "temperature": args.base_model_temperature, 98 | "api_key": args.openai_api_key, 99 | } 100 | 101 | optim_model_setting = { 102 | "model_type": args.optim_model_type, 103 | "model_name": args.optim_model_name, 104 | "temperature": args.optim_model_temperature, 105 | "api_key": args.openai_api_key, 106 | } 107 | 108 | task_setting = { 109 | "train_size": args.train_size, 110 | "test_size": args.test_size, 111 | "seed": args.seed, 112 | "data_dir": args.dataset_dir, 113 | } 114 | 115 | return search_setting, base_model_setting, optim_model_setting, task_setting 116 | -------------------------------------------------------------------------------- /src/taskmanager.py: -------------------------------------------------------------------------------- 1 | from .tasks import get_task 2 | 3 | 4 | class TaskManager: 5 | def __init__( 6 | self, 7 | meta_train_tasks, 8 | meta_test_tasks, 9 | task_setting, 10 | ): 11 | 12 | self.meta_train_tasks = meta_train_tasks 13 | self.meta_test_tasks = meta_test_tasks 14 | self.task_setting = task_setting 15 | 16 | self.tasks = self._get_tasks() 17 | self.meta_test_tasks = self._get_meta_test_tasks() 18 | 19 | def _prepare_task(self, task_name): 20 | task = get_task(task_name=task_name)(task_name=task_name, **self.task_setting) 21 | return task 22 | 23 | def _get_tasks(self): 24 | tasks = [self._prepare_task(task_name) for task_name in self.meta_train_tasks] 25 | return tasks 26 | 27 | def _get_meta_test_tasks(self): 28 | tasks = [self._prepare_task(task_name) for task_name in self.meta_test_tasks] 29 | return tasks 30 | -------------------------------------------------------------------------------- /src/tasks/__init__.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | from .base_task import BaseTask 3 | 4 | # Add your new benchmark here with task names. 5 | MEDMCQA_TASKS = [ 6 | "anatomy", 7 | "surgery", 8 | "ob_gyn", 9 | "medicine", 10 | "pharmacology", 11 | "dental", 12 | "pediatrics", 13 | "pathology", 14 | ] 15 | AMAZON_TASKS = ["beauty", "game", "baby", "office", "sports", "electronics", "pet"] 16 | BIGBENCH_TASKS = [ 17 | "logic_grid_puzzle", 18 | "logical_deduction", 19 | "temporal_sequences", 20 | "tracking_shuffled_objects", 21 | "object_counting", 22 | "reasoning_colored_objects", 23 | "epistemic", 24 | "navigate", 25 | ] 26 | SAFETY_TASKS = [ 27 | "ethos", 28 | "liar", 29 | "hatecheck", 30 | "sarcasm", 31 | "tweet_eval", 32 | "antropic_harmless", 33 | ] 34 | GROUNDING_TASKS = [ 35 | "hotpot_qa", 36 | "natural_questions", 37 | "squad", 38 | "web_qa", 39 | "drop", 40 | "trivia_qa", 41 | ] 42 | 43 | 44 | def get_task(task_name): 45 | if task_name in GROUNDING_TASKS: 46 | class_name = "Grounding" 47 | elif task_name in SAFETY_TASKS: 48 | class_name = "Safety" 49 | elif task_name in BIGBENCH_TASKS: 50 | class_name = "Bigbench" 51 | elif task_name in MEDMCQA_TASKS: 52 | class_name = "MEDMCQA" 53 | elif task_name in AMAZON_TASKS: 54 | class_name = "Amazon" 55 | else: 56 | raise ValueError(f"{task_name} is not a recognized task") 57 | 58 | try: 59 | module = importlib.import_module(f".{class_name.lower()}", package=__package__) 60 | CustomTask = getattr(module, class_name) 61 | 62 | except ModuleNotFoundError: 63 | raise ValueError(f"Module for task '{task_name}' could not be found.") 64 | 65 | return CustomTask 66 | -------------------------------------------------------------------------------- /src/tasks/amazon.py: -------------------------------------------------------------------------------- 1 | # define task prompts for various datasets 2 | from .base_task import BaseTask 3 | import re 4 | 5 | 6 | class Amazon(BaseTask): 7 | def __init__( 8 | self, 9 | train_size, 10 | test_size, 11 | task_name: str, 12 | benchmark="amazon", 13 | task_description="Amazon review analysis benchmark", 14 | data_dir="", 15 | seed=None, 16 | **kwargs, 17 | ): 18 | self.options = {} 19 | self.benchmark = benchmark 20 | super().__init__( 21 | task_name=task_name, 22 | task_description=task_description, 23 | data_dir=data_dir, 24 | seed=seed, 25 | train_size=train_size, 26 | test_size=test_size, 27 | benchmark=benchmark, 28 | **kwargs, 29 | ) 30 | 31 | self.task_name = task_name 32 | 33 | def _get_task_initial_prompt(self): 34 | base_prompt = "Predict the customer's rating from 1 to 5." 35 | suffix = "{question}\nAt the end present your answer in and ." 36 | initial_prompt = base_prompt + suffix 37 | return initial_prompt, base_prompt, suffix 38 | 39 | def clean_response(self, response): 40 | clean_pattern = r"([\s\S]*?)<\/answer>" 41 | 42 | matches = re.findall(clean_pattern, response.lower()) 43 | 44 | # no answer in response. 45 | if not matches or not matches[-1].strip(): 46 | return -1 47 | 48 | digits = re.findall(r"\d+", matches[-1]) 49 | 50 | if not digits: 51 | return -1 52 | else: 53 | return int(digits[0]) 54 | -------------------------------------------------------------------------------- /src/tasks/base_task.py: -------------------------------------------------------------------------------- 1 | # define task prompts for various datasets 2 | import os 3 | from torch.utils.data import DataLoader 4 | import json 5 | import random 6 | import numpy as np 7 | from abc import ABC, abstractmethod 8 | 9 | TEST_SET_SHUFFLE_SEED = 42 10 | 11 | 12 | class BaseTask(ABC): 13 | def __init__( 14 | self, 15 | train_size, 16 | test_size, 17 | task_name="base_task", 18 | data_dir="./dataset", 19 | seed=None, 20 | benchmark=None, 21 | batch_size=500, 22 | **kwargs, 23 | ): 24 | self.task_name = task_name 25 | self.benchmark = benchmark 26 | 27 | self.data_dir = data_dir 28 | self.seed = seed 29 | self.train_size = train_size 30 | self.test_size = test_size 31 | self.batch_size = batch_size 32 | 33 | self._get_dataset() 34 | 35 | self.initial_prompt, self.initial_prompt_wo_suffix, self.suffix_prompt = self._get_task_initial_prompt() 36 | 37 | print(f"task : {self.task_name}") 38 | print(f"train_size set : {len(self.train_dataset)}") 39 | print(f"test_size set : {len(self.test_dataset)}") 40 | 41 | def _get_dataset(self): 42 | raw_data = self._load_task_dataset() 43 | 44 | dataset = self._shuffle_and_split_dataset(dataset=raw_data) 45 | 46 | self.train_dataset = dataset["train"] 47 | self.test_dataset = dataset["test"] 48 | 49 | self.train_data, self.test_data = self._get_data(dataset=dataset) 50 | 51 | def _load_task_dataset(self): 52 | data_file = f"{self.data_dir}/{self.benchmark}/{self.task_name}.json" 53 | 54 | if not (os.path.exists(data_file)): 55 | raise ValueError(f"json file {data_file} does not exist.") 56 | 57 | with open(data_file, "r") as file: 58 | data = json.load(file) 59 | 60 | return data 61 | 62 | def _shuffle_and_split_dataset(self, dataset, base_shuffle=True): 63 | assert "train" in dataset and "test" in dataset, "Dataset must contain 'train' and 'test' keys." 64 | 65 | train_set, test_set = dataset["train"], dataset["test"] 66 | assert self.train_size <= len(train_set), "train_size exceeds available training data." 67 | 68 | random.seed(TEST_SET_SHUFFLE_SEED) 69 | random.shuffle(test_set) 70 | 71 | if base_shuffle and self.seed is not None: 72 | random.seed(self.seed) 73 | random.shuffle(train_set) 74 | 75 | return dict(train=train_set[: self.train_size], test=test_set[: self.test_size]) 76 | 77 | def _get_data(self, dataset): 78 | self.train_dataloader = DataLoader(dataset["train"], batch_size=self.batch_size, shuffle=False) 79 | train_data = next(iter(self.train_dataloader)) 80 | 81 | self.test_dataloader = DataLoader(dataset["test"], batch_size=self.batch_size, shuffle=False) 82 | test_data = next(iter(self.test_dataloader)) 83 | 84 | return train_data, test_data 85 | 86 | def cal_correct(self, preds, labels): 87 | return list(np.array((np.array(preds) == np.array(labels))).astype(int)) 88 | 89 | def cal_metric(self, preds, labels): 90 | correct = self.cal_correct(preds=preds, labels=labels) 91 | return np.mean(correct) 92 | 93 | @abstractmethod 94 | def clean_response(self, response): 95 | ''' 96 | Clean the response from the model. 97 | ''' 98 | pass 99 | 100 | def batch_clean_responses(self, responses): 101 | if not isinstance(responses, list): 102 | responses = list(responses) 103 | 104 | batch_answers = [self.clean_response(response) for response in responses] 105 | return batch_answers 106 | 107 | @abstractmethod 108 | def _get_task_initial_prompt(self): 109 | ''' 110 | Get the initial prompt for the task. 111 | ''' 112 | pass 113 | -------------------------------------------------------------------------------- /src/tasks/bigbench.py: -------------------------------------------------------------------------------- 1 | # define task prompts for various datasets 2 | from .base_task import BaseTask 3 | import re 4 | import string 5 | import json 6 | import os 7 | 8 | 9 | TASKS_ANSWER_IS_OPTION = [ 10 | "logical_deduction", 11 | "temporal_sequences", 12 | "tracking_shuffled_objects", 13 | ] 14 | 15 | 16 | INITIAL_PROMPTS = { 17 | "logic_grid_puzzle": { 18 | "base_prompt": "Solve logic grid puzzles", 19 | "suffix": "{question}\nAt the end show the answer bracketed between and .", 20 | }, 21 | "logical_deduction": { 22 | "base_prompt": "A logical deduction task which requires deducing the order of a sequence of objects.", 23 | "suffix": "{question}\nAt the end show the answer option bracketed between and .", 24 | }, 25 | "object_counting": { 26 | "base_prompt": "Questions that involve enumerating objects and asking the model to count them.", 27 | "suffix": "{question}\nAt the end show the answer bracketed between and .", 28 | }, 29 | "reasoning_colored_objects": { 30 | "base_prompt": "Answer extremely simple questions about the colors of objects on a surface.", 31 | "suffix": "{question}\nAt the end show the answer bracketed between and .", 32 | }, 33 | "temporal_sequences": { 34 | "base_prompt": "Answer questions about which times certain events could have occurred.", 35 | "suffix": "{question}\nAt the end show the answer option bracketed between and .", 36 | }, 37 | "tracking_shuffled_objects": { 38 | "base_prompt": "A task requiring determining the final positions of a set of objects given their initial positions and a description of a sequence of swaps.", 39 | "suffix": "{question}\nAt the end show the answer option bracketed between and .", 40 | }, 41 | "epistemic": { 42 | "base_prompt": "Determine whether one sentence entails the next.", 43 | "suffix": "{question}\nAt the end show the answer option bracketed between and .", 44 | }, 45 | "navigate": { 46 | "base_prompt": "Given a series of navigation instructions, determine whether one would end up back at the starting point.", 47 | "suffix": "{question}\nAt the end show the answer option bracketed between and .", 48 | }, 49 | } 50 | 51 | 52 | class Bigbench(BaseTask): 53 | def __init__( 54 | self, 55 | train_size, 56 | test_size, 57 | task_name: str, 58 | benchmark="bigbench", 59 | task_description="task from bigbench", 60 | data_dir="", 61 | seed=None, 62 | option_num=7, 63 | **kwargs, 64 | ): 65 | self.benchmark = benchmark 66 | super().__init__( 67 | task_name=task_name, 68 | task_description=task_description, 69 | data_dir=data_dir, 70 | seed=seed, 71 | train_size=train_size, 72 | test_size=test_size, 73 | option_num=option_num, 74 | benchmark=benchmark, 75 | **kwargs, 76 | ) 77 | 78 | self.option_num = option_num 79 | 80 | self.task_name = task_name 81 | 82 | self.number_to_word_dict = { 83 | "one": 1, 84 | "two": 2, 85 | "three": 3, 86 | "four": 4, 87 | "five": 5, 88 | "six": 6, 89 | "seven": 7, 90 | "eight": 8, 91 | "nine": 9, 92 | "ten": 10, 93 | "eleven": 11, 94 | "twelve": 12, 95 | "thirteen": 13, 96 | "fourteen": 14, 97 | "fifteen": 15, 98 | "sixteen": 16, 99 | "seventeen": 17, 100 | "eighteen": 18, 101 | "nineteen": 19, 102 | "twenty": 20, 103 | "twenty-one": 21, 104 | } 105 | 106 | def load_task_dataset(self): 107 | 108 | data_file = f"{self.data_dir}/{self.benchmark}/{self.task_name}.json" 109 | 110 | if not (os.path.exists(data_file)): 111 | raise ValueError(f"json file {data_file} does not exist.") 112 | 113 | with open(data_file, "r") as file: 114 | data = json.load(file) 115 | 116 | return data 117 | 118 | def _get_task_initial_prompt(self): 119 | base_prompt = INITIAL_PROMPTS[self.task_name]["base_prompt"] 120 | suffix = INITIAL_PROMPTS[self.task_name]["suffix"] 121 | initial_prompt = base_prompt + suffix 122 | return initial_prompt, base_prompt, suffix 123 | 124 | def clean_response(self, response): 125 | if self.task_name in TASKS_ANSWER_IS_OPTION: 126 | return self.clean_response_options(response) 127 | else: 128 | return self.clean_response_non_option(response) 129 | 130 | def clean_response_options(self, response): 131 | letters = string.ascii_lowercase[: self.option_num] 132 | # Regex pattern to extract content within tags 133 | clean_pattern = r"([\s\S]*?)<\/answer>" 134 | 135 | # Find all matches and get the last non-empty match 136 | matches = re.findall(clean_pattern, response.lower()) 137 | if not matches or not matches[-1].strip(): 138 | return "N/A: Format error" 139 | 140 | answer_content = matches[-1].strip().lower() 141 | 142 | # Attempt to find patterns of type (X) or standalone letters 143 | patterns = [r"\(([" + letters + r"])\)", r"[" + letters + r"]"] 144 | 145 | for pattern in patterns: 146 | match = re.search(pattern, answer_content) 147 | if match: 148 | return match.group(0).strip("()").upper() 149 | 150 | # If no valid pattern is found, return a format error 151 | return "N/A: Format error" 152 | 153 | def clean_response_non_option(self, response): 154 | # Regex pattern to extract content within tags 155 | clean_pattern = r"([\s\S]*?)<\/answer>" 156 | 157 | # Find all matches and get the last non-empty match 158 | matches = re.findall(clean_pattern, response.lower()) 159 | if not matches or not matches[-1].strip(): 160 | return "N/A: Format error" 161 | 162 | answer_content = matches[-1].strip().lower() 163 | 164 | if answer_content in self.number_to_word_dict: 165 | return self.number_to_word_dict[answer_content] 166 | 167 | return answer_content 168 | -------------------------------------------------------------------------------- /src/tasks/grounding.py: -------------------------------------------------------------------------------- 1 | from .base_task import BaseTask 2 | import re 3 | import string 4 | from torch.utils.data import DataLoader 5 | import numpy as np 6 | import collections 7 | 8 | 9 | class Grounding(BaseTask): 10 | def __init__( 11 | self, 12 | train_size, 13 | test_size, 14 | task_name: str, 15 | benchmark="grounding", 16 | task_description="grounding tasks", 17 | data_dir="", 18 | seed=None, 19 | **kwargs, 20 | ): 21 | self.options = {} 22 | self.benchmark = benchmark 23 | 24 | super().__init__( 25 | task_name=task_name, 26 | task_description=task_description, 27 | data_dir=data_dir, 28 | seed=seed, 29 | train_size=train_size, 30 | test_size=test_size, 31 | benchmark=benchmark, 32 | **kwargs, 33 | ) 34 | self.task_name = task_name 35 | 36 | def _get_task_initial_prompt(self): 37 | base_prompt = "Answer the following question based on the given context." 38 | suffix = "{question}\nRespond with the answer only" 39 | initial_prompt = base_prompt + suffix 40 | return initial_prompt, base_prompt, suffix 41 | 42 | def clean_response(self, response): 43 | return response 44 | 45 | # To handle grounding task's answer type : list 46 | def _grounding_coll_func(self, batch): 47 | questions = [item["question"] for item in batch] 48 | answers = [item["answers"] for item in batch] 49 | 50 | return {"question": questions, "answer": answers} 51 | 52 | def _get_data(self, dataset): 53 | self.train_dataloader = DataLoader( 54 | dataset["train"], 55 | batch_size=self.batch_size, 56 | shuffle=False, 57 | collate_fn=self._grounding_coll_func, 58 | ) 59 | train_data = next(iter(self.train_dataloader)) 60 | 61 | self.test_dataloader = DataLoader( 62 | dataset["test"], 63 | batch_size=self.batch_size, 64 | shuffle=False, 65 | collate_fn=self._grounding_coll_func, 66 | ) 67 | test_data = next(iter(self.test_dataloader)) 68 | 69 | return train_data, test_data 70 | 71 | def cal_correct(self, preds, labels, metric="em"): 72 | """ 73 | For grounding tasks, answers are list of entities. 74 | """ 75 | if not isinstance(preds, list): 76 | labels = [labels] 77 | preds = [preds] 78 | assert len(labels) == len(preds) 79 | 80 | if metric == "em": 81 | compute = compute_exact 82 | elif metric == "contain": 83 | compute = compute_contain 84 | 85 | corrects = [] 86 | 87 | for label, pred_answer in zip(labels, preds): 88 | if isinstance(label, str): 89 | gold_entities = [label] 90 | elif isinstance(label, list): 91 | gold_entities = label 92 | else: 93 | TypeError, f"label must be str or list in Grounding tasks. Label : {label}" 94 | # fmt: off 95 | is_correct = ( 1 if np.count_nonzero([compute(gold_entity, pred_answer) for gold_entity in gold_entities]) != 0 else 0 ) 96 | # fmt: on 97 | corrects.append(is_correct) 98 | 99 | return corrects 100 | 101 | 102 | def normalize_answer(s): 103 | """Lower text and remove punctuation, articles and extra whitespace.""" 104 | 105 | def remove_articles(text): 106 | regex = re.compile(r"\b(a|an|the)\b", re.UNICODE) 107 | return re.sub(regex, " ", text) 108 | 109 | def white_space_fix(text): 110 | return " ".join(text.split()) 111 | 112 | def remove_punc(text): 113 | exclude = set(string.punctuation) 114 | return "".join(ch for ch in text if ch not in exclude) 115 | 116 | def lower(text): 117 | return text.lower() 118 | 119 | return white_space_fix(remove_articles(remove_punc(lower(s)))) 120 | 121 | 122 | def get_tokens(s): 123 | if not s: 124 | return [] 125 | return normalize_answer(s).split() 126 | 127 | 128 | def compute_exact(a_gold, a_pred): 129 | return int(normalize_answer(a_gold) == normalize_answer(a_pred)) 130 | 131 | 132 | def compute_f1(a_gold, a_pred): 133 | gold_toks = get_tokens(a_gold) 134 | pred_toks = get_tokens(a_pred) 135 | common = collections.Counter(gold_toks) & collections.Counter(pred_toks) 136 | num_same = sum(common.values()) 137 | if len(gold_toks) == 0 or len(pred_toks) == 0: 138 | # If either is no-answer, then F1 is 1 if they agree, 0 otherwise 139 | return int(gold_toks == pred_toks) 140 | if num_same == 0: 141 | return 0 142 | precision = 1.0 * num_same / len(pred_toks) 143 | recall = 1.0 * num_same / len(gold_toks) 144 | f1 = (2 * precision * recall) / (precision + recall) 145 | return f1 146 | 147 | 148 | def compute_contain(gold_entity, pred_answer): 149 | return normalize_answer(gold_entity) in normalize_answer(pred_answer) 150 | 151 | 152 | def f1(answers, pred_answers): 153 | if not isinstance(pred_answers, list): 154 | answers = [answers] 155 | pred_answers = [pred_answers] 156 | 157 | assert len(answers) == len(pred_answers) 158 | 159 | num_all_answers = 0 160 | num_correct_answers = 0 161 | for answer, pred_answer in zip(answers, pred_answers): 162 | gold_answers = set(answer) 163 | 164 | if len(gold_answers) == 0: 165 | continue 166 | 167 | num_all_answers += 1 168 | num_correct_answers += max([compute_f1(gold_answer, pred_answer) for gold_answer in gold_answers]) 169 | 170 | return num_correct_answers / (num_all_answers + 1e-16) 171 | -------------------------------------------------------------------------------- /src/tasks/medmcqa.py: -------------------------------------------------------------------------------- 1 | # define task prompts for various datasets 2 | from .base_task import BaseTask 3 | import re 4 | import string 5 | 6 | 7 | class MEDMCQA(BaseTask): 8 | def __init__( 9 | self, 10 | train_size, 11 | test_size, 12 | task_name: str, 13 | benchmark="medmcqa", 14 | task_description="medical question answering tasks", 15 | data_dir="", 16 | seed=None, 17 | option_num=4, 18 | **kwargs, 19 | ): 20 | super().__init__( 21 | task_name=task_name, 22 | task_description=task_description, 23 | data_dir=data_dir, 24 | seed=seed, 25 | train_size=train_size, 26 | test_size=test_size, 27 | benchmark=benchmark, 28 | **kwargs, 29 | ) 30 | 31 | self.option_num = option_num 32 | 33 | def _get_task_initial_prompt(self): 34 | base_prompt = "Given the following question and candidate answers, choose the best answer." 35 | suffix = "{question}\nAt the end present your answer in and ." 36 | initial_prompt = base_prompt + suffix 37 | return initial_prompt, base_prompt, suffix 38 | 39 | def clean_response(self, response): 40 | valid_options = string.ascii_uppercase[: self.option_num] + string.ascii_lowercase[: self.option_num] 41 | clean_pattern = r"([\s\S]*?)<\/answer>" 42 | 43 | matches = re.findall(clean_pattern, response, re.IGNORECASE) 44 | 45 | if not matches: 46 | return "N/A: Format error" 47 | 48 | last_match = matches[-1] 49 | answer = re.search(r"\(([{}]?)\)".format(valid_options), last_match) 50 | if answer: 51 | return answer.group(1).upper() 52 | 53 | answer = re.search(r"[{}]".format(valid_options), last_match) 54 | if answer: 55 | return answer.group(0).upper() 56 | 57 | return "N/A: Format error" 58 | -------------------------------------------------------------------------------- /src/tasks/safety.py: -------------------------------------------------------------------------------- 1 | # define task prompts for various datasets 2 | from .base_task import BaseTask 3 | import re 4 | import numpy as np 5 | from sklearn.metrics import f1_score 6 | 7 | INITIAL_PROMPT = { 8 | "ethos": { 9 | "base_prompt": "Is the following text hate speech?", 10 | "suffix": "{question}\nAt the end present your answer in yes or no.", 11 | }, 12 | "liar": { 13 | "base_prompt": "Determine whether the Statement is a lie (Yes) or not (No) based on the Context and other information.", 14 | "suffix": "{question}\nAt the end present your answer in yes or no.", 15 | }, 16 | "hatecheck": { 17 | "base_prompt": "Is the following text hate speech?", 18 | "suffix": "{question}\nAt the end present your answer in yes or no.", 19 | }, 20 | "sarcasm": { 21 | "base_prompt": "Determine if the text conveys sarcasm.", 22 | "suffix": "{question}\nAt the end present your answer in yes or no.", 23 | }, 24 | "tweet_eval": { 25 | "base_prompt": "Determine whether the following text is offensive.", 26 | "suffix": "{question}\nAt the end present your answer in yes or no.", 27 | }, 28 | "antropic_harmless": { 29 | "base_prompt": "Review the dialogue and assess whether the assistant's response is harmful, marking it as 'yes' for harmful and 'no' for not harmful.", 30 | "suffix": "{question}\nAt the end present your answer in yes or no.", 31 | }, 32 | } 33 | 34 | 35 | class Safety(BaseTask): 36 | def __init__( 37 | self, 38 | train_size, 39 | test_size, 40 | task_name: str, 41 | benchmark="safety", 42 | task_description="LLM Safety benchmark", 43 | data_dir="", 44 | seed=None, 45 | f1_metric=True, 46 | **kwargs, 47 | ): 48 | self.options = {} 49 | self.benchmark = benchmark 50 | self.f1_metric = f1_metric 51 | super().__init__( 52 | task_name=task_name, 53 | task_description=task_description, 54 | data_dir=data_dir, 55 | seed=seed, 56 | train_size=train_size, 57 | test_size=test_size, 58 | benchmark=benchmark, 59 | **kwargs, 60 | ) 61 | 62 | self.task_name = task_name 63 | 64 | def _get_task_initial_prompt(self): 65 | base_prompt = INITIAL_PROMPT[self.task_name]["base_prompt"] 66 | suffix = INITIAL_PROMPT[self.task_name]["suffix"] 67 | initial_prompt = base_prompt + suffix 68 | return initial_prompt, base_prompt, suffix 69 | 70 | def clean_response(self, response): 71 | clean_pattern = r"([\s\S]*?)<\/answer>" 72 | match = re.findall(clean_pattern, response.lower()) 73 | 74 | if len(match) == 0 or not match[-1].strip(): 75 | return "N/A: Format error" 76 | 77 | return match[-1].strip().lower() 78 | 79 | def cal_metric(self, preds, labels): 80 | correct = self.cal_correct(preds=preds, labels=labels) 81 | accuracy = np.mean(correct) 82 | 83 | if self.f1_metric: 84 | f1 = f1_score(labels, preds, average="macro") 85 | return f1 86 | 87 | return accuracy 88 | -------------------------------------------------------------------------------- /src/utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import logging 3 | from glob import glob 4 | from datetime import datetime 5 | import pytz 6 | import openai 7 | import yaml 8 | import argparse 9 | 10 | 11 | openai.log = logging.getLogger("openai") 12 | openai.log.setLevel(logging.ERROR) 13 | logging.getLogger("urllib3").setLevel(logging.ERROR) 14 | 15 | 16 | class HTTPFilter(logging.Filter): 17 | def filter(self, record): 18 | return not record.getMessage().startswith("HTTP") 19 | 20 | 21 | def get_pacific_time(): 22 | current_time = datetime.now() 23 | pacific = pytz.timezone("Asia/Seoul") 24 | pacific_time = current_time.astimezone(pacific) 25 | return pacific_time 26 | 27 | 28 | def create_logger(logging_dir, name="log"): 29 | """ 30 | Create a logger that writes to a log file and stdout. 31 | """ 32 | if not os.path.exists(logging_dir): 33 | os.makedirs(logging_dir) 34 | 35 | logging_dir = os.path.join(logging_dir, name) 36 | num = len(glob(logging_dir + "*")) 37 | 38 | logging_dir += "-" + f"{num:03d}" + ".log" 39 | http_filter = HTTPFilter() 40 | 41 | logging.basicConfig( 42 | level=logging.INFO, 43 | format="%(message)s", 44 | datefmt="%Y-%m-%d %H:%M:%S", 45 | handlers=[logging.StreamHandler(), logging.FileHandler(f"{logging_dir}")], 46 | ) 47 | logger = logging.getLogger("prompt optimization agent") 48 | logging.getLogger("openai").setLevel(logging.CRITICAL) 49 | logging.getLogger("datasets").setLevel(logging.CRITICAL) 50 | for handler in logging.getLogger().handlers: 51 | handler.addFilter(http_filter) 52 | return logger 53 | 54 | --------------------------------------------------------------------------------