9 |
18 |
19 |
29 |
57 |
58 |
59 |
--------------------------------------------------------------------------------
/src/yikes/cli.py:
--------------------------------------------------------------------------------
1 | import argparse
2 | import boto3
3 | import json
4 | import os
5 | from x_x.cli import splash
6 |
7 |
8 | from yikes.embed import mess_with_embedding
9 | from yikes.summarize import summarize_url
10 |
11 |
12 | from aiohttp import web
13 |
14 |
15 | HERE = os.path.realpath(os.path.dirname(__file__))
16 |
17 |
18 | def webapp():
19 |
20 | async def root_handler(request):
21 | return web.FileResponse(f"{HERE}/static/index.html")
22 |
23 | async def query_preview_handler(request):
24 | return web.Respose(text="Preview")
25 |
26 | async def query_handler(request):
27 | params = await request.json()
28 | query = params["query"]
29 | return web.Response(text=query_llama(query))
30 |
31 | async def url_preview_handler(request):
32 | """Return a json response with a summary of the url"""
33 |
34 | print("preview handler")
35 |
36 | url = request.query.get("url")
37 |
38 | if not url:
39 | resp = {"error": "No URL provided"}
40 | else:
41 | resp = summarize_url(url)
42 |
43 | print("finished summarizing... responding")
44 |
45 | return web.json_response(resp)
46 |
47 | app = web.Application()
48 |
49 | app.add_routes([
50 | web.get("/", root_handler),
51 | web.get("/scrape", url_preview_handler),
52 | web.post("/query", query_handler),
53 | web.get("/preview", query_preview_handler),
54 | web.static("/", f"{HERE}/static"),
55 | ])
56 |
57 | return app
58 |
59 |
60 |
61 |
62 | def main():
63 | """Serve website to query LLaMa"""
64 |
65 | print(splash("sword"))
66 |
67 | mess_with_embedding()
68 |
69 | return
70 |
71 | parser = argparse.ArgumentParser()
72 | parser.add_argument("--port", type=int, default=8181)
73 | # parser.add_argument("--temperature", type=float, default=0.5)
74 | # parser.add_argument("--top-p", type=float, default=0.9)
75 | # parser.add_argument("--max-gen-len", type=int, default=2**8)
76 | args = parser.parse_args()
77 |
78 | app = webapp()
79 |
80 | web.run_app(
81 | app,
82 | print=None,
83 | port=args.port,
84 | )
85 |
--------------------------------------------------------------------------------
/src/yikes/summarize.py:
--------------------------------------------------------------------------------
1 | import boto3
2 | import json
3 | import logging
4 | import requests
5 |
6 | from bs4 import BeautifulSoup
7 |
8 |
9 | def fetch(url):
10 | """Return the HTML from a URL"""
11 |
12 | resp = requests.get(url)
13 | return resp.text
14 |
15 |
16 | def extract_text_from_html(html):
17 | """Return readable text from HTML"""
18 |
19 | resp = BeautifulSoup(html, "html.parser")
20 |
21 | for script in resp(["script", "style"]):
22 | script.extract()
23 |
24 | text = resp.get_text()
25 |
26 | lines = (
27 | line.strip()
28 | for line in text.splitlines()
29 | )
30 |
31 | chunks = (
32 | phrase.strip()
33 | for line in lines
34 | for phrase in line.split(" ")
35 | )
36 |
37 | return "\n".join(
38 | chunk
39 | for chunk in chunks
40 | if chunk
41 | if len(chunk) > 30
42 | )
43 |
44 |
45 | def query_jurassic(prompt):
46 | """Return generated text from bedrock"""
47 |
48 | client = boto3.client("bedrock-runtime")
49 |
50 | prompt = "\n".join([
51 | # "{text}Summarize this block of text with 5 bullet points:{/text}",
52 | # prompt,
53 | "The following is text from a website:",
54 | prompt,
55 | "Summarize the website with 5 bullet points:",
56 | ])
57 |
58 | model_kwargs = { #AI21
59 | "prompt": prompt,
60 | "maxTokens": 3000,
61 | "temperature": 0.3,
62 | "topP": 0.7,
63 | "stopSequences": [],
64 | "countPenalty": {"scale": 0 },
65 | "presencePenalty": {"scale": 0 },
66 | "frequencyPenalty": {"scale": 0 }
67 | }
68 |
69 | resp = client.invoke_model(
70 | modelId="ai21.j2-ultra-v1",
71 | body=json.dumps(model_kwargs),
72 | )
73 |
74 | resp = resp["body"].read().decode("utf-8")
75 | resp = json.loads(resp)
76 |
77 | result = ""
78 |
79 | for chunk in resp["completions"]:
80 | result += chunk["data"]["text"]
81 |
82 | return result
83 |
84 |
85 | from collections import namedtuple
86 |
87 | Summarization = namedtuple(
88 | "Summarization",
89 | ["url", "title", "image", "summary"],
90 | )
91 |
92 | logging.basicConfig()
93 | logger = logging.getLogger(__name__)
94 | logger.setLevel(logging.INFO)
95 |
96 |
97 | def summarize_url(url):
98 | """Return a text summary of a URL"""
99 |
100 | logger.info("fetching %url", url)
101 | html = fetch(url)
102 |
103 | logger.info("extracting text from HTML")
104 | text = extract_text_from_html(html)
105 |
106 | logger.info("summarizing text with bedrock")
107 | summary = query_jurassic(text[:10_000]).strip()
108 |
109 | soup = BeautifulSoup(html, "html.parser")
110 | title = soup.find("meta", property="og:title")
111 | image = soup.find("meta", property="og:image")
112 | description = soup.find("meta", property="og:description")
113 |
114 | return {
115 | "url": url,
116 | "title": title["content"] if title else "n/a",
117 | "image": image["content"] if image else None,
118 | "description": description["content"] if description else "n/a",
119 | "summary": summary,
120 | }
121 |
--------------------------------------------------------------------------------
/src/yikes/embed.py:
--------------------------------------------------------------------------------
1 | import pdb
2 |
3 | from langchain_core.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
4 |
5 | from langchain.chains import ConversationalRetrievalChain
6 | from langchain.indexes.vectorstore import VectorstoreIndexCreator
7 | from langchain.memory import ConversationBufferWindowMemory
8 | from langchain.text_splitter import RecursiveCharacterTextSplitter
9 | from langchain_community.document_loaders import PyPDFLoader
10 | from langchain_community.embeddings import BedrockEmbeddings
11 | from langchain_community.llms import Bedrock
12 | from langchain_community.vectorstores import FAISS
13 |
14 | import json
15 |
16 | from pathlib import Path
17 |
18 | import unittest.mock as mock
19 | import pdb
20 |
21 | import boto3
22 | import botocore
23 | # import BedrockRuntime.Client.invoke_model(**kwargs)
24 |
25 |
26 | def fullname(o):
27 | klass = o.__class__
28 | module = klass.__module__
29 | if module == 'builtins':
30 | return klass.__qualname__ # avoid outputs like 'builtins.str'
31 | return module + '.' + klass.__qualname__
32 |
33 |
34 | client = boto3.client("bedrock-runtime")
35 |
36 |
37 | HERE = Path(__file__).parent
38 |
39 |
40 | def get_llm():
41 |
42 | model_kwargs = { #AI21
43 | "maxTokens": 1024,
44 | "temperature": 0,
45 | "topP": 0.5,
46 | "stopSequences": ["Human:"],
47 | "countPenalty": {"scale": 0 },
48 | "presencePenalty": {"scale": 0 },
49 | "frequencyPenalty": {"scale": 0 },
50 | }
51 |
52 | model_kwargs = { #Meta
53 | "temperature": 0,
54 | "top_p": 0.5,
55 | "max_gen_len": 2**7,
56 | }
57 |
58 | return Bedrock(
59 | # model_id="ai21.j2-ultra-v1",
60 | # model_id="meta.llama2-70b-v1",
61 | model_id="meta.llama2-70b-chat-v1",
62 | model_kwargs=model_kwargs,
63 | # streaming=True,
64 | # callbacks=[StreamingStdOutCallbackHandler()],
65 | )
66 |
67 |
68 | def get_index():
69 | """Return ..."""
70 |
71 | embeddings = BedrockEmbeddings()
72 |
73 | pdf_path = str(HERE / "borges-and-ai.pdf")
74 | # pdf_path = str(HERE / "manifesto_futurista.pdf")
75 |
76 | loader = PyPDFLoader(file_path=pdf_path)
77 |
78 | text_splitter = RecursiveCharacterTextSplitter(
79 | separators=["\n\n", "\n", ".", " "],
80 | chunk_size=1000,
81 | chunk_overlap=100,
82 | )
83 |
84 | index_creator = VectorstoreIndexCreator(
85 | vectorstore_cls=FAISS,
86 | embedding=embeddings,
87 | text_splitter=text_splitter,
88 | )
89 |
90 | return index_creator.from_loaders([loader])
91 |
92 |
93 | def get_memory():
94 | """Return ..."""
95 |
96 | return ConversationBufferWindowMemory(
97 | memory_key="chat_history",
98 | return_messages=True,
99 | )
100 |
101 |
102 |
103 | def MUTATE_invoke_model(client):
104 | """
105 | Modify the boto3 client to print out the prompt and response.
106 | NOTE: This is a hacky way to do this. It would be better if we coulduse a callback handler passed into Bedrock...
107 | """
108 | def inspect_boto_calls(*args, **kwargs):
109 | body = kwargs["body"]
110 | body = json.loads(body)
111 | print_aside(kwargs["modelId"])
112 | print_aside(body["prompt"])
113 |
114 | resp = client.__invoke_model(*args, **kwargs)
115 | resp_body = resp["body"].read()
116 | resp["body"].read = mock.MagicMock(return_value=resp_body)
117 |
118 | try:
119 | r = json.loads(resp_body.decode("utf8"))["generation"]
120 | # r = json.loads(resp_body.decode("utf-8"))["completions"][0]["data"]["text"]
121 | print_aside(r)
122 | except:
123 | print_aside(resp_body)
124 |
125 | return resp # client.__invoke_model(*args, **kwargs)
126 | client.__invoke_model = client.invoke_model
127 | client.invoke_model = inspect_boto_calls
128 |
129 | TICKER = 0
130 | COLORS = ["94", "96", "92"]
131 |
132 | def print_aside(x):
133 | global TICKER
134 | color = COLORS[TICKER % len(COLORS)]
135 | TICKER += 1
136 | print(f"\n\033[{color}m{x}\033[0m\n")
137 |
138 |
139 | def get_rag_chat_response(input_text, memory, index):
140 |
141 | llm = get_llm()
142 |
143 | def get_rag_chat_response(input_text, memory, index):
144 |
145 | llm = get_llm()
146 |
147 | # Modify the llm's boto3 client to print out the prompt and response
148 | MUTATE_invoke_model(llm.client)
149 |
150 | conversation_with_retrieval = ConversationalRetrievalChain.from_llm(
151 | llm,
152 | index.vectorstore.as_retriever(),
153 | memory=memory,
154 | )
155 |
156 | chat_response = conversation_with_retrieval.invoke({"question": input_text})
157 |
158 | return chat_response
159 |
160 |
161 | def mess_with_embedding():
162 |
163 |
164 | """
165 | bedrock = boto3.client(service_name='bedrock', region_name='us-east-1')
166 | listModels = bedrock.list_foundation_models(byProvider='meta')
167 | print(json.dumps(listModels, indent=2))
168 | return
169 | """
170 |
171 | mem = get_memory()
172 |
173 | ind = get_index()
174 |
175 | try:
176 | while True:
177 | print("\033[31m>\033[0m", end="", flush=True)
178 | line = input(" ").strip()
179 |
180 | if not line:
181 | continue
182 |
183 | res = get_rag_chat_response(line, mem, ind)
184 |
185 | print()
186 | print(res["answer"])
187 | print()
188 |
189 | except (KeyboardInterrupt, EOFError):
190 | print("Exiting...")
191 | raise SystemExit(0)
192 |
--------------------------------------------------------------------------------
Punch in a URL and scrape it (:
20 | 27 | 28 |Results
30 | 31 | 32 |
33 |
34 | ![]()
35 |
36 |
37 |
38 |
56 |