├── README.md ├── cove ├── __init__.py └── prompts.py ├── main.py └── pyproject.toml /README.md: -------------------------------------------------------------------------------- 1 | # chain-of-verification 2 | 3 | This is an implementation of Chain-of-Verification, from [this paper](https://arxiv.org/abs/2309.11495) 4 | 5 | The implementation is take from [this amazing GitHub repo](https://github.com/ritun16/chain-of-verification) and modified to use [LangChain Expression Language](https://python.langchain.com/docs/expression_language/) 6 | 7 | You should absolutely check out the above GitHub repo as it adds in external tools and more chains. 8 | 9 | An example of the trace generated by this chain can be found [here](https://smith.langchain.com/public/79577c0a-36a3-43a6-ba89-f11294c95875/r). 10 | -------------------------------------------------------------------------------- /cove/__init__.py: -------------------------------------------------------------------------------- 1 | from langchain.chat_models import ChatOpenAI 2 | from langchain.prompts import ChatPromptTemplate 3 | from langchain.schema.output_parser import StrOutputParser 4 | from langchain.prompts import PromptTemplate 5 | from langchain.schema.runnable import RunnablePassthrough, RunnableLambda 6 | 7 | # Import prompts 8 | from .prompts import * 9 | 10 | # Set up LLM to user 11 | llm = ChatOpenAI(temperature=0) 12 | 13 | # Chain to generate initial answer 14 | baseline_response_prompt_template = PromptTemplate.from_template(BASELINE_PROMPT_WIKI) 15 | baseline_response_chain = baseline_response_prompt_template | llm | StrOutputParser() 16 | 17 | # Chain to generate a question template for verification answers 18 | verification_question_template_prompt_template = PromptTemplate.from_template(VERIFICATION_QUESTION_TEMPLATE_PROMPT_WIKI) 19 | verification_question_template_chain = verification_question_template_prompt_template | llm | StrOutputParser() 20 | 21 | # Chain to generate the verification questionts 22 | verification_question_generation_prompt_template = PromptTemplate.from_template(VERIFICATION_QUESTION_PROMPT_WIKI) 23 | verification_question_generation_chain = verification_question_generation_prompt_template | llm | StrOutputParser() 24 | 25 | # Chain to execute the verification 26 | execution_prompt_self_llm = PromptTemplate.from_template(EXECUTE_PLAN_PROMPT_SELF_LLM) 27 | execution_prompt_llm_chain = execution_prompt_self_llm | llm | StrOutputParser() 28 | verification_chain = RunnablePassthrough.assign( 29 | split_questions=lambda x: x['verification_questions'].split("\n"), 30 | ) | RunnablePassthrough.assign( 31 | answers = (lambda x: [{"verification_question": q} for q in x['split_questions']])| execution_prompt_llm_chain.map() 32 | ) | (lambda x: "\n".join(["Question: {} Answer: {}\n".format(question, answer) for question, answer in zip(x['split_questions'], x['answers'])]))# Create final refined response 33 | 34 | # Chain to generate the final answer 35 | final_answer_prompt_template = PromptTemplate.from_template(FINAL_REFINED_PROMPT) 36 | final_answer_chain = final_answer_prompt_template | llm | StrOutputParser() 37 | 38 | 39 | # Putting everything together, a final chain 40 | 41 | chain = RunnablePassthrough.assign( 42 | baseline_response=baseline_response_chain 43 | ) | RunnablePassthrough.assign( 44 | verification_question_template=verification_question_template_chain 45 | ) | RunnablePassthrough.assign( 46 | verification_questions=verification_question_generation_chain 47 | ) | RunnablePassthrough.assign( 48 | verification_answers=verification_chain 49 | ) | RunnablePassthrough.assign( 50 | final_answer=final_answer_chain 51 | ) 52 | -------------------------------------------------------------------------------- /cove/prompts.py: -------------------------------------------------------------------------------- 1 | ####################################################################### BASELINE PROMPTS ######################################################################## 2 | BASELINE_PROMPT_WIKI = """Answer the below question which is asking for a list of entities (names, places, locations etc). Output should be a numbered list and only contains the relevant & concise enitites as answer. NO ADDITIONAL DETAILS. 3 | 4 | Question: {original_question} 5 | 6 | Answer:""" 7 | 8 | BASELINE_PROMPT_MULTI = """Answer the below question correctly and in a concise manner without much details. Only answer what the question is asked. 9 | 10 | Question: {original_question} 11 | 12 | Answer:""" 13 | 14 | BASELINE_PROMPT_LONG = """Answer the below question correctly. 15 | 16 | Question: {original_question} 17 | 18 | Answer:""" 19 | 20 | ################################################################### PLAN VERIFICATION PROMPTS ################################################################### 21 | VERIFICATION_QUESTION_TEMPLATE_PROMPT_WIKI = """Your task is to create a verification question based on the below question provided. 22 | Example Question: Who are some movie actors who were born in Boston? 23 | Example Verification Question: Was [movie actor] born in [Boston] 24 | Explanation: In the above example the verification question focused only on the ANSWER_ENTITY (name of the movie actor) and QUESTION_ENTITY (birth place). 25 | Similarly you need to focus on the ANSWER_ENTITY and QUESTION_ENTITY from the actual question and generate verification question. 26 | 27 | Actual Question: {original_question} 28 | 29 | Final Verification Question:""" 30 | 31 | VERIFICATION_QUESTION_PROMPT_WIKI = """Your task is to create a series of verification questions based on the below question, the verfication question template and baseline response. 32 | Example Question: Who are some movie actors who were born in Boston? 33 | Example Verification Question Template: Was [movie actor] born in Boston? 34 | Example Baseline Response: 1. Matt Damon - Famous for his roles in films like "Good Will Hunting," "The Bourne Identity" series, and "The Martian," Damon is an Academy Award-winning actor, screenwriter, and producer. 35 | 2. Chris Evans - Famous for his portrayal of Captain America in the Marvel Cinematic Universe, Evans has also appeared in movies like "Snowpiercer" and "Knives Out." 36 | Verification questions: 1. Was Matt Damon born in Boston? 37 | 2. Was Chirs Evans born in Boston? 38 | etc. 39 | Example Verification Question: 1. Was Matt Damon born in Boston? 40 | 2. Was Chris Evans born in Boston? 41 | 42 | Explanation: In the above example the verification questions focused only on the ANSWER_ENTITY (name of the movie actor) and QUESTION_ENTITY (birth place) based on the template and substitutes entity values from the baseline response. 43 | Similarly you need to focus on the ANSWER_ENTITY and QUESTION_ENTITY from the actual question and substitute the entity values from the baseline response to generate verification questions. 44 | 45 | Actual Question: {original_question} 46 | Baseline Response: {baseline_response} 47 | Verification Question Template: {verification_question_template} 48 | 49 | Final Verification Questions:""" 50 | 51 | VERIFICATION_QUESTION_PROMPT_MULTI = """Your task is to create verification questions based on the below original question and the baseline response. The verification questions are meant for verifying the factual acuracy in the baseline response. 52 | Example Question: Who invented the first printing press and in what year? 53 | Example Baseline Response: Johannes Gutenberg, 1450. 54 | Example Verification Questions: 1. Did Johannes Gutenberg invent first printing press? 55 | 2. Did Johannes Gutenberg invent first printing press in the year 1450? 56 | 57 | Explanation: The verification questions are highly aligned with both the qctual question and baseline response. The actual question is comprises of multiple independent questions which in turn has multiple independent answers in the baseline response. Hence, the verification questions should also be independent for factual verification. 58 | 59 | Actual Question: {original_question} 60 | Baseline Response: {baseline_response} 61 | 62 | Final Verification Questions:""" 63 | 64 | VERIFICATION_QUESTION_PROMPT_LONG = """Your task is to create verification questions based on the below original question and the baseline response. The verification questions are meant for verifying the factual acuracy in the baseline response. Output should be numbered list of verification questions. 65 | 66 | Actual Question: {original_question} 67 | Baseline Response: {baseline_response} 68 | 69 | Final Verification Questions:""" 70 | 71 | ################################################################## EXECUTE VERIFICATION PROMPTS ################################################################## 72 | EXECUTE_PLAN_PROMPT_SEARCH_TOOL = """Answer the following question correctly based on the provided context. The question could be tricky as well, so think step by step and answer it correctly. 73 | 74 | Context: {search_result} 75 | 76 | Question: {verification_question} 77 | 78 | Answer:""" 79 | 80 | 81 | EXECUTE_PLAN_PROMPT_SELF_LLM = """Answer the following question correctly. 82 | 83 | Question: {verification_question} 84 | 85 | Answer:""" 86 | 87 | EXECUTE_PLAN_PROMPT = "{verification_questions}" 88 | 89 | ################################################################## FINAL REFINED PROMPTS ################################################################## 90 | FINAL_REFINED_PROMPT = """Given the below `Original Query` and `Baseline Answer`, analyze the `Verification Questions & Answers` to finally filter the refined answer. 91 | Original Query: {original_question} 92 | Baseline Answer: {baseline_response} 93 | 94 | Verification Questions & Answer Pairs: 95 | {verification_answers} 96 | 97 | Final Refined Answer:""" 98 | 99 | ################################################################## ROUTER PROMPTS ################################################################## 100 | ROUTER_CHAIN_PROMPT = """Please classify the below question in on of the following categories. The output should be a JSON as shown in the Examples. 101 | 102 | Categories: 103 | WIKI_CHAIN: Good for answering questions which asks for a list or set of entites as its answer. 104 | MULTI_CHAIN: Good for answering questions which comprises of questions that have multiple independent answers (derived from a series of multiple discontiguous spans in the text) and multiple questions are asked in the original question. 105 | LONG_CHAIN: Good for answering questions whose answer is long. 106 | 107 | Examples: 108 | WIKI_CHAIN: 109 | Question: Name some Endemic orchids of Vietnam. 110 | JSON Output: {{"category": "WIKI_CHAIN"}} 111 | Question: Who are the scientist won nobel prize in the year 1970? 112 | JSON Output: {{"category": "WIKI_CHAIN"}} 113 | Question: List some cricket players who are playing in indian cricket team. 114 | JSON Output: {{"category": "WIKI_CHAIN"}} 115 | MULTI_CHAIN: 116 | Question: Who is known for developing the theory of relativity, and in which year was it introduced? 117 | JSON Output: {{"category": "MULTI_CHAIN"}} 118 | Question: Who is credited with inventing the telephone, and when did this invention take place? 119 | JSON Output: {{"category": "MULTI_CHAIN"}} 120 | Question: Who was the first person to orbit the Earth in space, and during which year did this historic event occur? 121 | JSON Output: {{"category": "MULTI_CHAIN"}} 122 | LONG_CHAIN: 123 | Question: Write few lines about Einstein. 124 | JSON Output: {{"category": "LONG_CHAIN"}} 125 | Question: Tell me in short about first moon landing. 126 | JSON Output: {{"category": "LONG_CHAIN"}} 127 | Question: Write a short biography of Carl Marx. 128 | JSON Output: {{"category": "LONG_CHAIN"}} 129 | 130 | Actual Question: {} 131 | Final JSON Output:""" -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | from cove import chain 2 | 3 | if __name__ == "__main__": 4 | response = chain.invoke({"original_question": "Who are some politicians born in Boston?"}) 5 | print(response) -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "cove" 3 | version = "0.0.1" 4 | description = "" 5 | authors = ["Harrison Chase "] 6 | readme = "README.md" 7 | 8 | [tool.poetry.dependencies] 9 | python = "^3.9" 10 | langchain = ">=0.0.313" 11 | openai = "^0.28.1" 12 | 13 | [tool.langserve] 14 | export_module = "cove" 15 | export_attr = "chain" 16 | 17 | 18 | [build-system] 19 | requires = ["poetry-core"] 20 | build-backend = "poetry.core.masonry.api" 21 | --------------------------------------------------------------------------------