├── autism.png ├── README.md ├── LICENSE ├── load_creds.py ├── gemma_text.py ├── paligemma.py ├── text_to_text.py ├── audio_to_text.py ├── chatlab.py ├── image_to_text.py ├── chatbot.py └── classification.py /autism.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ushareng/AIForAutism_GoogleIOConnect/main/autism.png -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Source Code for AI For Autism at Google IO Connect Bengaluru 2024 2 | 3 | Google Tech Stack - Gemini , Gemma , Paligemma , Mesop and Vertex AI 4 | 5 | link to the Demo - https://drive.google.com/file/d/1T4BIMD7PmfbtQoxpTuAyAQiz0E65n7Pk/view?usp=sharing 6 | 7 | ![autism](https://github.com/user-attachments/assets/1b394ec2-5268-4518-aa6c-2a9a9f5823dd) 8 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Usha Rengaraju 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /load_creds.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | 3 | from google.auth.transport.requests import Request 4 | from google.oauth2.credentials import Credentials 5 | from google_auth_oauthlib.flow import InstalledAppFlow 6 | 7 | SCOPES = ['https://www.googleapis.com/auth/generative-language.retriever'] 8 | 9 | def load_creds(): 10 | """Converts `client_secret.json` to a credential object. 11 | 12 | This function caches the generated tokens to minimize the use of the 13 | consent screen. 14 | """ 15 | creds = None 16 | # The file token.json stores the user's access and refresh tokens, and is 17 | # created automatically when the authorization flow completes for the first 18 | # time. 19 | if os.path.exists('token.json'): 20 | creds = Credentials.from_authorized_user_file('token.json', SCOPES) 21 | # If there are no (valid) credentials available, let the user log in. 22 | if not creds or not creds.valid: 23 | if creds and creds.expired and creds.refresh_token: 24 | creds.refresh(Request()) 25 | else: 26 | flow = InstalledAppFlow.from_client_secrets_file( 27 | 'client_secret.json', SCOPES) 28 | creds = flow.run_local_server(port=0) 29 | # Save the credentials for the next run 30 | with open('token.json', 'w') as token: 31 | token.write(creds.to_json()) 32 | return creds -------------------------------------------------------------------------------- /gemma_text.py: -------------------------------------------------------------------------------- 1 | import os 2 | import jax 3 | 4 | # The Keras 3 distribution API is only implemented for the JAX backend for now 5 | os.environ["KERAS_BACKEND"] = "jax" 6 | # Pre-allocate 100% of TPU memory to minimize memory fragmentation and allocation overhead 7 | os.environ["XLA_PYTHON_CLIENT_MEM_FRACTION"] = "1.0" 8 | 9 | import torch 10 | import numpy as np 11 | #from transformers import AutoTokenizer, GemmaForSequenceClassification 12 | from transformers import ( 13 | AutoTokenizer, 14 | BitsAndBytesConfig, 15 | AutoModelForSequenceClassification, 16 | ) 17 | 18 | import bitsandbytes as bnb 19 | 20 | import numpy as np 21 | 22 | 23 | 24 | os.environ['HF_TOKEN']='PLACE YOUR HF TOKEN HERE' 25 | bnb_config = BitsAndBytesConfig( 26 | load_in_4bit=True, # Enables 4-bit quantization 27 | bnb_4bit_use_double_quant=True, # Use double quantization for potentially higher accuracy (optional) 28 | bnb_4bit_quant_type="nf4", # Quantization type (specifics depend on hardware and library) 29 | bnb_4bit_compute_dtype=torch.bfloat16 # Compute dtype for improved efficiency (optional) 30 | ) 31 | NUM_CLASSES=10 32 | id2label={0: 'Sadness', 33 | 1: 'Neutral', 34 | 2: 'Happiness', 35 | 3: 'Anger', 36 | 4: 'Affection', 37 | 5: 'Fear', 38 | 6: 'Surprise', 39 | 7: 'Disgust', 40 | 8: 'Desire', 41 | 9: 'Optimism'} 42 | 43 | label2id= {'Sadness': 0, 44 | 'Neutral': 1, 45 | 'Happiness': 2, 46 | 'Anger': 3, 47 | 'Affection': 4, 48 | 'Fear': 5, 49 | 'Surprise': 6, 50 | 'Disgust': 7, 51 | 'Desire': 8, 52 | 'Optimism': 9} 53 | 54 | class ModelClass: 55 | _model = None # Class-level variable to store the model 56 | _tokenizer = None 57 | 58 | @classmethod 59 | def load_model(cls): 60 | if cls._model is None: 61 | model_id = 'akshay-k/gemma_2b_cls_10classes' 62 | model = AutoModelForSequenceClassification.from_pretrained( 63 | model_id, # "google/gemma-2b-it" 64 | num_labels=NUM_CLASSES, # Number of output labels (2 for binary sentiment classification) 65 | quantization_config=bnb_config, # configuration for quantization 66 | device_map={"": 0} # Optional dictionary specifying device mapping (single GPU with index 0 here) 67 | ) 68 | tokenizer = AutoTokenizer.from_pretrained(model_id,truncation=True) 69 | print("Loading the model...") 70 | cls._model = model # Simulate loading the model 71 | cls._tokenizer = tokenizer 72 | return (cls._model, cls._tokenizer) 73 | 74 | @classmethod 75 | def predict(cls, text): 76 | (model, tokenizer) = cls.load_model() 77 | inputs = tokenizer(text, return_tensors="pt").to("cuda") # Convert to PyTorch tensors and move to GPU (if available) 78 | with torch.no_grad(): 79 | outputs = model(**inputs).logits # Get the model's output logits 80 | y_prob = torch.sigmoid(outputs).tolist()[0] # Apply sigmoid activation and convert to list 81 | print(y_prob) 82 | print('score : '+str(y_prob[np.argmax(np.round(y_prob, 5))])) 83 | return id2label[np.argmax(np.round(y_prob, 5))] 84 | 85 | 86 | model = None 87 | tokenizer = None 88 | model_id = 'akshay-k/gemma_2b_cls_10classes' 89 | def load_model(): 90 | 91 | model = AutoModelForSequenceClassification.from_pretrained( 92 | model_id, # "google/gemma-2b-it" 93 | num_labels=NUM_CLASSES, # Number of output labels (2 for binary sentiment classification) 94 | quantization_config=bnb_config, # configuration for quantization 95 | device_map={"": 0} # Optional dictionary specifying device mapping (single GPU with index 0 here) 96 | ) 97 | tokenizer = AutoTokenizer.from_pretrained(model_id,truncation=True) 98 | 99 | 100 | -------------------------------------------------------------------------------- /paligemma.py: -------------------------------------------------------------------------------- 1 | from typing import Callable 2 | import mesop as mp 3 | import base64 4 | import paligemma.facial_emotion as fe 5 | 6 | 7 | @mp.stateclass 8 | class State: 9 | name: str 10 | path: str 11 | size: int 12 | mime_type: str 13 | image_data: str 14 | output: str 15 | textarea_key: int 16 | 17 | 18 | def image_classification(data: str): 19 | # Decode base64 string 20 | decoded_data = base64.b64decode(data) 21 | 22 | # Write binary data to a file 23 | with open("image.png", "wb") as image_file: 24 | image_file.write(decoded_data) 25 | 26 | response = fe.predict("image.png") 27 | print(response.text) 28 | 29 | return response.text 30 | 31 | def image_to_text( 32 | transform: Callable[[str], str], 33 | *, 34 | title: str | None = None, 35 | ): 36 | """Creates a simple UI which takes in a text input and returns an image output. 37 | 38 | This function creates event handlers for text input and output operations 39 | using the provided function `transform` to process the input and generate the image 40 | output. 41 | 42 | Args: 43 | transform: Function that takes in a string input and returns a URL to an image or a base64 encoded image. 44 | title: Headline text to display at the top of the UI. 45 | """ 46 | 47 | 48 | def on_image_upload(e: mp.UploadEvent): 49 | state = mp.state(State) 50 | state.image_data = base64.b64encode(e.file.read()).decode() 51 | print("file ", e.file) 52 | state.name = e.file.name 53 | print("name ", e.file.name) 54 | 55 | # Decode base64 string 56 | decoded_data = base64.b64decode(state.image_data) 57 | 58 | # Write binary data to a file 59 | # saving image as a file 60 | with open("decoded_image.png", "wb") as image_file: 61 | image_file.write(decoded_data) 62 | 63 | def on_click_generate(e: mp.ClickEvent): 64 | state = mp.state(State) 65 | state.output = image_classification(state.image_data) 66 | #state.output = transform(state.image_data) 67 | 68 | def on_click_clear(e: mp.ClickEvent): 69 | state = mp.state(State) 70 | state.image_data = "" 71 | state.name = "" 72 | state.output = "" 73 | state.textarea_key += 1 74 | 75 | with mp.box( 76 | style=mp.Style( 77 | background="#f0f4f8", 78 | height="100%", 79 | ) 80 | ): 81 | with mp.box( 82 | style=mp.Style( 83 | background="#f0f4f8", 84 | padding=mp.Padding(top=24, left=24, right=24, bottom=24), 85 | display="flex", 86 | flex_direction="column", 87 | ) 88 | ): 89 | if title: 90 | mp.text(title, type="headline-5") 91 | with mp.box( 92 | style=mp.Style( 93 | margin=mp.Margin(left="auto", right="auto"), 94 | width="min(1024px, 100%)", 95 | gap="24px", 96 | flex_grow=1, 97 | display="flex", 98 | flex_wrap="wrap", 99 | ) 100 | ): 101 | box_style = mp.Style( 102 | flex_basis="max(480px, calc(50% - 48px))", 103 | background="#fff", 104 | border_radius=12, 105 | box_shadow=( 106 | "0 3px 1px -2px #0003, 0 2px 2px #00000024, 0 1px 5px #0000001f" 107 | ), 108 | padding=mp.Padding(top=16, left=16, right=16, bottom=16), 109 | display="flex", 110 | flex_direction="column", 111 | ) 112 | 113 | with mp.box(style=box_style): 114 | mp.text("Input", style=mp.Style(font_weight=500)) 115 | mp.box(style=mp.Style(height=16)) 116 | mp.uploader( 117 | label="Upload Image", 118 | accepted_file_types=["application/pdf"], 119 | on_upload=on_image_upload, 120 | type="flat", 121 | color="primary", 122 | style=mp.Style(font_weight="bold"), 123 | ) 124 | # mp.textarea( 125 | # key=str(mp.state(State).textarea_key), 126 | # on_input=on_input, 127 | # rows=5, 128 | # autosize=True, 129 | # max_rows=15, 130 | # style=mp.Style(width="100%"), 131 | # ) 132 | if mp.state(State).image_data: 133 | with mp.box(style=box_style): 134 | with mp.box( 135 | style=mp.Style( 136 | display="grid", 137 | justify_content="center", 138 | justify_items="center", 139 | ) 140 | ): 141 | mp.image( 142 | src=f"data:image/jpeg;base64,{mp.state(State).image_data}", 143 | style=mp.Style(width="100%", margin=mp.Margin(top=10)), 144 | ) 145 | mp.box(style=mp.Style(height=12)) 146 | with mp.box( 147 | style=mp.Style(display="flex", justify_content="space-between") 148 | ): 149 | mp.button( 150 | "Clear", 151 | color="primary", 152 | type="stroked", 153 | on_click=on_click_clear, 154 | ) 155 | mp.button( 156 | "Generate", 157 | color="primary", 158 | type="flat", 159 | on_click=on_click_generate, 160 | ) 161 | with mp.box(style=box_style): 162 | mp.text("Output", style=mp.Style(font_weight=500)) 163 | mp.markdown(mp.state(State).output) 164 | -------------------------------------------------------------------------------- /text_to_text.py: -------------------------------------------------------------------------------- 1 | import types 2 | from typing import Callable, Generator, Literal, cast 3 | 4 | import mesop as me 5 | 6 | 7 | @me.stateclass 8 | class State: 9 | input: str 10 | output: str 11 | textarea_key: int 12 | 13 | 14 | def text_io( 15 | transform: Callable[[str], Generator[str, None, None] | str], 16 | *, 17 | title: str | None = None, 18 | transform_mode: Literal["append", "replace"] = "replace", 19 | ): 20 | """Deprecated: Use `text_to_text` instead which provides the same functionality 21 | with better default settings. 22 | 23 | This function creates event handlers for text input and output operations 24 | using the provided transform function to process the input and generate the output. 25 | 26 | Args: 27 | transform: Function that takes in a string input and either returns or yields a string output. 28 | title: Headline text to display at the top of the UI 29 | transform_mode: Specifies how the output should be updated when yielding an output using a generator. 30 | - "append": Concatenates each new piece of text to the existing output. 31 | - "replace": Replaces the existing output with each new piece of text. 32 | """ 33 | print( 34 | "\033[93m[warning]\033[0m text_io is deprecated, use text_to_text instead" 35 | ) 36 | text_to_text(transform=transform, title=title, transform_mode=transform_mode) 37 | 38 | 39 | def text_to_text( 40 | transform: Callable[[str], Generator[str, None, None] | str], 41 | *, 42 | title: str | None = None, 43 | transform_mode: Literal["append", "replace"] = "append", 44 | ): 45 | """Creates a simple UI which takes in a text input and returns a text output. 46 | 47 | This function creates event handlers for text input and output operations 48 | using the provided transform function to process the input and generate the output. 49 | 50 | Args: 51 | transform: Function that takes in a string input and either returns or yields a string output. 52 | title: Headline text to display at the top of the UI 53 | transform_mode: Specifies how the output should be updated when yielding an output using a generator. 54 | - "append": Concatenates each new piece of text to the existing output. 55 | - "replace": Replaces the existing output with each new piece of text. 56 | """ 57 | 58 | def on_input(e: me.InputEvent): 59 | state = me.state(State) 60 | state.input = e.value 61 | 62 | def on_click_generate(e: me.ClickEvent): 63 | state = me.state(State) 64 | output = transform(state.input) 65 | if isinstance(output, types.GeneratorType): 66 | for val in output: 67 | if transform_mode == "append": 68 | state.output += val 69 | elif transform_mode == "replace": 70 | state.output = val 71 | else: 72 | raise ValueError(f"Unsupported transform_mode: {transform_mode}") 73 | yield 74 | else: 75 | # `output` is a str, however type inference doesn't 76 | # work w/ generator's unusual ininstance check. 77 | state.output = cast(str, output) 78 | yield 79 | 80 | def on_click_clear(e: me.ClickEvent): 81 | state = me.state(State) 82 | state.input = "" 83 | state.textarea_key += 1 84 | 85 | with me.box( 86 | style=me.Style( 87 | background="#fdfdfd", #lavender 88 | height="100%", 89 | ) 90 | ): 91 | with me.box( 92 | style=me.Style( 93 | margin=me.Margin(left="5%", right="5%"), 94 | background="#dcdcdc", #purple 95 | padding=me.Padding(top=24, left=24, right=24, bottom=24), 96 | display="flex", 97 | flex_direction="column", 98 | ) 99 | ): 100 | if title: 101 | me.text(title,type="headline-5",style=me.Style( 102 | 103 | font_family="Serif" 104 | #padding=mp.Padding(left=5, right=5, bottom=5), 105 | )) 106 | with me.box( 107 | style=me.Style( 108 | justify_content="space-between", 109 | padding=me.Padding(top=24, left=24, right=24, bottom=24), 110 | background="#000", #green 111 | margin=me.Margin(left="auto", right="auto"), 112 | width="min(1024px, 100%)", 113 | gap="24px", 114 | flex_grow=1, 115 | display="flex", 116 | flex_wrap="wrap", 117 | ) 118 | ): 119 | box_style = me.Style( 120 | flex_basis="max(480px, calc(50% - 48px))", 121 | background="#fff", 122 | border_radius=12, 123 | box_shadow="0 3px 1px -2px #0003, 0 2px 2px #00000024, 0 1px 5px #0000001f", 124 | padding=me.Padding(top=16, left=16, right=16, bottom=16), 125 | display="flex", 126 | flex_direction="column", 127 | ) 128 | with me.box(style=me.Style( 129 | flex_basis="max(360px, calc(60% - 48px))", 130 | background="#fff", 131 | border_radius=12, 132 | box_shadow=( 133 | "0 3px 1px -2px #0003, 0 2px 2px #00000024, 0 1px 5px #0000001f" 134 | ), 135 | padding=me.Padding(top=16, left=16, right=16, bottom=16), 136 | display="flex", 137 | flex_direction="column", 138 | )): 139 | me.text("Input", style=me.Style(font_weight=500)) 140 | me.box(style=me.Style(height=16)) 141 | me.textarea( 142 | key=str(me.state(State).textarea_key), 143 | on_input=on_input, 144 | rows=5, 145 | autosize=True, 146 | max_rows=15, 147 | style=me.Style(width="100%"), 148 | ) 149 | me.box(style=me.Style(height=12)) 150 | with me.box( 151 | style=me.Style(display="flex", justify_content="space-between") 152 | ): 153 | me.button( 154 | "Clear", color="primary", type="stroked", on_click=on_click_clear 155 | ) 156 | me.button( 157 | "Detect", 158 | color="primary", 159 | type="flat", 160 | on_click=on_click_generate, 161 | ) 162 | with me.box(style=me.Style( 163 | flex_basis="max(360px, calc(30% - 48px))", 164 | background="#fff", 165 | border_radius=12, 166 | box_shadow=( 167 | "0 3px 1px -2px #0003, 0 2px 2px #00000024, 0 1px 5px #0000001f" 168 | ), 169 | padding=me.Padding(top=16, left=16, right=16, bottom=16), 170 | display="flex", 171 | flex_direction="column", 172 | )): 173 | me.text("Output", style=me.Style(font_weight=500)) 174 | me.markdown(me.state(State).output) -------------------------------------------------------------------------------- /audio_to_text.py: -------------------------------------------------------------------------------- 1 | from typing import Callable 2 | import mesop as mp 3 | import base64 4 | 5 | @mp.stateclass 6 | class State: 7 | name: str 8 | path: str 9 | size: int 10 | mime_type: str 11 | audio_data: str 12 | output: str 13 | textarea_key: int 14 | 15 | import base64 16 | import vertexai 17 | from vertexai.generative_models import GenerativeModel, Part, FinishReason 18 | import vertexai.preview.generative_models as generative_models 19 | 20 | generation_config = { 21 | "max_output_tokens": 8192, 22 | "temperature": 0.3, 23 | "top_p": 0.95, 24 | } 25 | 26 | safety_settings = { 27 | generative_models.HarmCategory.HARM_CATEGORY_HATE_SPEECH: generative_models.HarmBlockThreshold.BLOCK_ONLY_HIGH, 28 | generative_models.HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: generative_models.HarmBlockThreshold.BLOCK_ONLY_HIGH, 29 | generative_models.HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT: generative_models.HarmBlockThreshold.BLOCK_ONLY_HIGH, 30 | generative_models.HarmCategory.HARM_CATEGORY_HARASSMENT: generative_models.HarmBlockThreshold.BLOCK_ONLY_HIGH, 31 | } 32 | 33 | 34 | 35 | def generate(audio): 36 | vertexai.init(project="just-well-429210-n7", location="us-central1") 37 | model = GenerativeModel( 38 | "gemini-1.5-flash-001", 39 | system_instruction=["""From the audio given, analyze the tone of the speaker and content, emphasize tone more than content and answer which emotion does it convey among (neutral, calm, happy, sad, angry, fear, disgust, surprise). Answer in one word"""] 40 | ) 41 | responses = model.generate_content( 42 | [audio, """a"""], 43 | generation_config=generation_config, 44 | safety_settings=safety_settings, 45 | stream=True, 46 | ) 47 | 48 | for response in responses: 49 | print(response.text, end="") 50 | return response.text 51 | 52 | return "" 53 | 54 | def audio_to_text( 55 | transform: Callable[[str], str], 56 | *, 57 | title: str | None = None, 58 | ): 59 | """Creates a simple UI which takes in a text input and returns an image output. 60 | 61 | This function creates event handlers for text input and output operations 62 | using the provided function `transform` to process the input and generate the image 63 | output. 64 | 65 | Args: 66 | transform: Function that takes in a string input and returns a URL to an image or a base64 encoded image. 67 | title: Headline text to display at the top of the UI. 68 | """ 69 | 70 | 71 | def on_audio_upload(e: mp.UploadEvent): 72 | state = mp.state(State) 73 | state.audio_data = base64.b64encode(e.file.read()).decode() 74 | print("file ", e.file) 75 | state.name = e.file.name 76 | print("name ", e.file.name) 77 | 78 | # Decode base64 string 79 | decoded_data = base64.b64decode(state.audio_data) 80 | 81 | # Write binary data to a file 82 | # saving image as a file 83 | with open("decoded_audio.mp3", "wb") as audio_file: 84 | audio_file.write(decoded_data) 85 | 86 | def on_click_generate(e: mp.ClickEvent): 87 | state = mp.state(State) 88 | if state.audio_data: 89 | audio = Part.from_data( 90 | mime_type="audio/wav", 91 | data=base64.b64decode(state.audio_data)) 92 | state.output = generate(audio) 93 | print("output is ", state.output) 94 | #state.output = transform(state.output) 95 | def on_click_clear(e: mp.ClickEvent): 96 | state = mp.state(State) 97 | state.audio_data = "" 98 | state.name = "" 99 | state.output = "" 100 | state.textarea_key += 1 101 | 102 | with mp.box( 103 | style=mp.Style( 104 | background="#fdfdfd", #lavender 105 | height="100%", 106 | ) 107 | ): 108 | with mp.box( 109 | style=mp.Style( 110 | margin=mp.Margin(left="5%", right="5%"), 111 | background="#dcdcdc", #purple 112 | padding=mp.Padding(top=24, left=24, right=24, bottom=24), 113 | display="flex", 114 | flex_direction="column", 115 | ) 116 | ): 117 | if title: 118 | mp.text(title,type="headline-5",style=mp.Style( 119 | 120 | font_family="Serif" 121 | #padding=mp.Padding(left=5, right=5, bottom=5), 122 | )) 123 | with mp.box( 124 | style=mp.Style( 125 | justify_content="space-between", 126 | padding=mp.Padding(top=24, left=24, right=24, bottom=24), 127 | background="#000", #green 128 | margin=mp.Margin(left="auto", right="auto"), 129 | width="min(1024px, 100%)", 130 | gap="24px", 131 | flex_grow=1, 132 | display="flex", 133 | flex_wrap="wrap", 134 | ) 135 | ): 136 | box_style = mp.Style( 137 | flex_basis="max(480px, calc(50% - 48px))", 138 | background="#fff", 139 | border_radius=12, 140 | box_shadow=( 141 | "0 3px 1px -2px #0003, 0 2px 2px #00000024, 0 1px 5px #0000001f" 142 | ), 143 | padding=mp.Padding(top=16, left=16, right=16, bottom=16), 144 | display="flex", 145 | flex_direction="column", 146 | ) 147 | 148 | with mp.box(style=mp.Style( 149 | flex_basis="max(360px, calc(60% - 48px))", 150 | background="#fff", 151 | border_radius=12, 152 | box_shadow=( 153 | "0 3px 1px -2px #0003, 0 2px 2px #00000024, 0 1px 5px #0000001f" 154 | ), 155 | padding=mp.Padding(top=16, left=16, right=16, bottom=16), 156 | display="flex", 157 | flex_direction="column", 158 | )): 159 | mp.text("Input", style=mp.Style(font_weight=500)) 160 | mp.box(style=mp.Style(height=16)) 161 | mp.uploader( 162 | label="Upload Audio", 163 | accepted_file_types=["application/pdf"], 164 | on_upload=on_audio_upload, 165 | type="flat", 166 | color="primary", 167 | style=mp.Style(font_weight="bold"), 168 | ) 169 | if mp.state(State).audio_data: 170 | with mp.box(style=box_style): 171 | with mp.box( 172 | style=mp.Style( 173 | display="grid", 174 | justify_content="center", 175 | justify_items="center", 176 | ) 177 | ): 178 | mp.audio( 179 | src=f"data:audio/wav;base64,{mp.state(State).audio_data}", 180 | ) 181 | mp.box(style=mp.Style(height=12)) 182 | with mp.box( 183 | style=mp.Style(display="flex", justify_content="space-between") 184 | ): 185 | mp.button( 186 | "Clear", 187 | color="primary", 188 | type="stroked", 189 | on_click=on_click_clear, 190 | ) 191 | mp.button( 192 | "Detect", 193 | color="primary", 194 | type="flat", 195 | on_click=on_click_generate, 196 | ) 197 | with mp.box(style=mp.Style( 198 | flex_basis="max(360px, calc(30% - 48px))", 199 | background="#fff", 200 | border_radius=12, 201 | box_shadow=( 202 | "0 3px 1px -2px #0003, 0 2px 2px #00000024, 0 1px 5px #0000001f" 203 | ), 204 | padding=mp.Padding(top=16, left=16, right=16, bottom=16), 205 | display="flex", 206 | flex_direction="column", 207 | )): 208 | mp.text("Output", style=mp.Style(font_weight=500)) 209 | mp.markdown(mp.state(State).output) 210 | 211 | -------------------------------------------------------------------------------- /chatlab.py: -------------------------------------------------------------------------------- 1 | import time 2 | from dataclasses import dataclass 3 | from typing import Callable, Generator, Literal 4 | 5 | import mesop as me 6 | 7 | Role = Literal["user", "assistant"] 8 | 9 | _ROLE_USER = "user" 10 | _ROLE_ASSISTANT = "assistant" 11 | 12 | _BOT_USER_DEFAULT = "mesop-bot" 13 | 14 | _COLOR_BACKGROUND = "#a2a4f8" 15 | _COLOR_CHAT_BUBBLE_YOU = "#b8ceb0" # light green 16 | _COLOR_CHAT_BUBBLE_BOT = "#fff" #pink shade 17 | 18 | _DEFAULT_PADDING = me.Padding.all(20) 19 | _DEFAULT_BORDER_SIDE = me.BorderSide( 20 | width="1px", style="solid", color="#ececec" 21 | ) 22 | 23 | _LABEL_BUTTON = "send" 24 | _LABEL_BUTTON_IN_PROGRESS = "pending" 25 | _LABEL_INPUT = "Enter your prompt" 26 | 27 | _STYLE_APP_CONTAINER = me.Style( 28 | background=_COLOR_BACKGROUND, 29 | display="grid", 30 | height="80vh", 31 | grid_template_columns="repeat(1, 1fr)", 32 | ) 33 | _STYLE_TITLE = me.Style(padding=me.Padding(left=10)) 34 | _STYLE_CHAT_BOX = me.Style( 35 | height="100%", 36 | overflow_y="scroll", 37 | padding=_DEFAULT_PADDING, 38 | margin=me.Margin(bottom=20), 39 | border_radius="10px", 40 | border=me.Border( 41 | left=_DEFAULT_BORDER_SIDE, 42 | right=_DEFAULT_BORDER_SIDE, 43 | top=_DEFAULT_BORDER_SIDE, 44 | bottom=_DEFAULT_BORDER_SIDE, 45 | ), 46 | ) 47 | _STYLE_CHAT_INPUT = me.Style(width="100%") 48 | _STYLE_CHAT_INPUT_BOX = me.Style( 49 | padding=me.Padding(top=30), display="flex", flex_direction="row" 50 | ) 51 | _STYLE_CHAT_BUTTON = me.Style(margin=me.Margin(top=8, left=8)) 52 | _STYLE_CHAT_BUBBLE_NAME = me.Style( 53 | font_weight="bold", 54 | font_size="13px", 55 | padding=me.Padding(left=15, right=15, bottom=5), 56 | ) 57 | _STYLE_CHAT_BUBBLE_PLAINTEXT = me.Style(margin=me.Margin.symmetric(vertical=15)) 58 | 59 | 60 | def _make_style_chat_ui_container(has_title: bool) -> me.Style: 61 | """Generates styles for chat UI container depending on if there is a title or not. 62 | 63 | Args: 64 | has_title: Whether the Chat UI is display a title or not. 65 | """ 66 | return me.Style( 67 | display="grid", 68 | grid_template_columns="repeat(1, 1fr)", 69 | grid_template_rows="1fr 14fr 1fr" if has_title else "5fr 1fr", 70 | margin=me.Margin.symmetric(vertical=0, horizontal="auto"), 71 | width="min(1024px, 100%)", 72 | height="80vh", 73 | background="#f2f2f2 ", #yellow 74 | box_shadow=( 75 | "0 3px 1px -2px #0003, 0 2px 2px #00000024, 0 1px 5px #0000001f" 76 | ), 77 | padding=me.Padding(top=20, left=20, right=20), 78 | ) 79 | 80 | 81 | def _make_style_chat_bubble_wrapper(role: Role) -> me.Style: 82 | """Generates styles for chat bubble position. 83 | 84 | Args: 85 | role: Chat bubble alignment depends on the role 86 | """ 87 | align_items = "end" if role == _ROLE_USER else "start" 88 | return me.Style( 89 | display="flex", 90 | flex_direction="column", 91 | align_items=align_items, 92 | ) 93 | 94 | 95 | def _make_chat_bubble_style(role: Role) -> me.Style: 96 | """Generates styles for chat bubble. 97 | 98 | Args: 99 | role: Chat bubble background color depends on the role 100 | """ 101 | background = ( 102 | _COLOR_CHAT_BUBBLE_YOU if role == _ROLE_USER else _COLOR_CHAT_BUBBLE_BOT 103 | ) 104 | return me.Style( 105 | width="80%", 106 | font_size="16px", 107 | line_height="1.5", 108 | background=background, 109 | border_radius="15px", 110 | padding=me.Padding(right=15, left=15, bottom=3), 111 | margin=me.Margin(bottom=10), 112 | border=me.Border( 113 | left=_DEFAULT_BORDER_SIDE, 114 | right=_DEFAULT_BORDER_SIDE, 115 | top=_DEFAULT_BORDER_SIDE, 116 | bottom=_DEFAULT_BORDER_SIDE, 117 | ), 118 | ) 119 | 120 | 121 | @dataclass(kw_only=True) 122 | class ChatMessage: 123 | """Chat message metadata.""" 124 | 125 | role: Role = "user" 126 | content: str = "" 127 | 128 | 129 | @me.stateclass 130 | class State: 131 | input: str 132 | output: list[ChatMessage] 133 | in_progress: bool = False 134 | 135 | 136 | def on_blur(e: me.InputBlurEvent): 137 | state = me.state(State) 138 | state.input = e.value 139 | 140 | 141 | def chat( 142 | transform: Callable[ 143 | [str, list[ChatMessage]], Generator[str, None, None] | str 144 | ], 145 | *, 146 | title: str | None = None, 147 | bot_user: str = _BOT_USER_DEFAULT, 148 | ): 149 | """Creates a simple chat UI which takes in a prompt and chat history and returns a 150 | response to the prompt. 151 | 152 | This function creates event handlers for text input and output operations 153 | using the provided function `transform` to process the input and generate the output. 154 | 155 | Args: 156 | transform: Function that takes in a prompt and chat history and returns a response to the prompt. 157 | title: Headline text to display at the top of the UI. 158 | bot_user: Name of your bot / assistant. 159 | """ 160 | state = me.state(State) 161 | 162 | def on_click_submit(e: me.ClickEvent): 163 | yield from submit() 164 | 165 | def on_input_enter(e: me.InputEnterEvent): 166 | state = me.state(State) 167 | state.input = e.value 168 | yield from submit() 169 | 170 | def submit(): 171 | state = me.state(State) 172 | if state.in_progress or not state.input: 173 | return 174 | input = state.input 175 | state.input = "" 176 | yield 177 | 178 | output = state.output 179 | if output is None: 180 | output = [] 181 | output.append(ChatMessage(role=_ROLE_USER, content=input)) 182 | state.in_progress = True 183 | yield 184 | 185 | me.scroll_into_view(key="scroll-to") 186 | time.sleep(0.15) 187 | yield 188 | 189 | start_time = time.time() 190 | output_message = transform(input, state.output) 191 | assistant_message = ChatMessage(role=_ROLE_ASSISTANT) 192 | output.append(assistant_message) 193 | state.output = output 194 | 195 | for content in output_message: 196 | assistant_message.content += content 197 | # TODO: 0.25 is an abitrary choice. In the future, consider making this adjustable. 198 | if (time.time() - start_time) >= 0.25: 199 | start_time = time.time() 200 | yield 201 | state.in_progress = False 202 | yield 203 | 204 | with me.box(style=_STYLE_APP_CONTAINER): 205 | with me.box(style=_make_style_chat_ui_container(bool(title))): 206 | if title: 207 | me.text(title, type="headline-5", style=_STYLE_TITLE) 208 | with me.box(style=_STYLE_CHAT_BOX): 209 | for msg in state.output: 210 | with me.box(style=_make_style_chat_bubble_wrapper(msg.role)): 211 | if msg.role == _ROLE_ASSISTANT: 212 | me.text(bot_user, style=_STYLE_CHAT_BUBBLE_NAME) 213 | with me.box(style=_make_chat_bubble_style(msg.role)): 214 | if msg.role == _ROLE_USER: 215 | me.text(msg.content, style=_STYLE_CHAT_BUBBLE_PLAINTEXT) 216 | else: 217 | me.markdown(msg.content) 218 | 219 | if state.in_progress: 220 | with me.box(key="scroll-to", style=me.Style(height=300)): 221 | pass 222 | 223 | with me.box(style=_STYLE_CHAT_INPUT_BOX): 224 | with me.box(style=me.Style(flex_grow=1)): 225 | me.input( 226 | label=_LABEL_INPUT, 227 | # Workaround: update key to clear input. 228 | key=f"{len(state.output)}", 229 | on_blur=on_blur, 230 | on_enter=on_input_enter, 231 | style=_STYLE_CHAT_INPUT, 232 | ) 233 | with me.content_button( 234 | color="primary", 235 | type="flat", 236 | disabled=state.in_progress, 237 | on_click=on_click_submit, 238 | style=_STYLE_CHAT_BUTTON, 239 | ): 240 | me.icon( 241 | _LABEL_BUTTON_IN_PROGRESS if state.in_progress else _LABEL_BUTTON 242 | ) -------------------------------------------------------------------------------- /image_to_text.py: -------------------------------------------------------------------------------- 1 | from typing import Callable 2 | import mesop as mp 3 | import base64 4 | 5 | import google.generativeai as genai 6 | genai.configure(api_key='PLACE YOUR GEMINI API KEY HERE') 7 | generation_config = { 8 | "temperature": 0.9, 9 | "top_p": 1, 10 | "top_k": 0, 11 | "max_output_tokens": 8192, 12 | "response_mime_type": "text/plain", 13 | } 14 | 15 | model1 = genai.GenerativeModel( 16 | model_name="gemini-1.5-flash", 17 | generation_config=generation_config, 18 | # safety_settings = Adjust safety settings 19 | # See https://ai.google.dev/gemini-api/docs/safety-settings 20 | ) 21 | 22 | @mp.stateclass 23 | class State: 24 | name: str 25 | path: str 26 | size: int 27 | mime_type: str 28 | image_data: str 29 | output: str 30 | textarea_key: int 31 | 32 | 33 | def image_classification(data: str): 34 | # Decode base64 string 35 | decoded_data = base64.b64decode(data) 36 | 37 | # Write binary data to a file 38 | with open("image.png", "wb") as image_file: 39 | image_file.write(decoded_data) 40 | 41 | file = upload_to_gemini("image.png") 42 | response = model1.generate_content([ 43 | "Analyse the emotion of the image, determine which emotion best describes the image among ('Fear', 'Neutral', 'Surprise', 'Disgust', 'Desire', 'Affection', 'Happiness', 'Anger', 'Sadness', 'Optimism')", 44 | "image: ", 45 | file, 46 | "input 2: Answer in one word", 47 | 48 | ]) 49 | print(response.text) 50 | 51 | return response.text 52 | 53 | def upload_to_gemini(path, mime_type=None): 54 | """Uploads the given file to Gemini. 55 | 56 | See https://ai.google.dev/gemini-api/docs/prompting_with_media 57 | """ 58 | file = genai.upload_file(path, mime_type=mime_type) 59 | print(f"Uploaded file '{file.display_name}' as: {file.uri}") 60 | return file 61 | 62 | 63 | def image_to_text( 64 | transform: Callable[[str], str], 65 | *, 66 | title: str | None = None, 67 | ): 68 | """Creates a simple UI which takes in a text input and returns an image output. 69 | 70 | This function creates event handlers for text input and output operations 71 | using the provided function `transform` to process the input and generate the image 72 | output. 73 | 74 | Args: 75 | transform: Function that takes in a string input and returns a URL to an image or a base64 encoded image. 76 | title: Headline text to display at the top of the UI. 77 | """ 78 | 79 | 80 | def on_image_upload(e: mp.UploadEvent): 81 | state = mp.state(State) 82 | state.image_data = base64.b64encode(e.file.read()).decode() 83 | print("file ", e.file) 84 | state.name = e.file.name 85 | print("name ", e.file.name) 86 | 87 | # Decode base64 string 88 | decoded_data = base64.b64decode(state.image_data) 89 | 90 | # Write binary data to a file 91 | # saving image as a file 92 | with open("decoded_image.png", "wb") as image_file: 93 | image_file.write(decoded_data) 94 | 95 | def on_click_generate(e: mp.ClickEvent): 96 | state = mp.state(State) 97 | state.output = image_classification(state.image_data) 98 | #state.output = transform(state.image_data) 99 | 100 | def on_click_clear(e: mp.ClickEvent): 101 | state = mp.state(State) 102 | state.image_data = "" 103 | state.name = "" 104 | state.output = "" 105 | state.textarea_key += 1 106 | 107 | with mp.box( 108 | style=mp.Style( 109 | background="#fdfdfd", #lavender 110 | height="100%", 111 | ) 112 | ): 113 | with mp.box( 114 | style=mp.Style( 115 | margin=mp.Margin(left="5%", right="5%"), 116 | background="#dcdcdc", #purple 117 | padding=mp.Padding(top=24, left=24, right=24, bottom=24), 118 | display="flex", 119 | flex_direction="column", 120 | ) 121 | ): 122 | if title: 123 | mp.text(title,type="headline-5",style=mp.Style( 124 | 125 | font_family="Serif" 126 | #padding=mp.Padding(left=5, right=5, bottom=5), 127 | )) 128 | with mp.box( 129 | style=mp.Style( 130 | justify_content="space-between", 131 | padding=mp.Padding(top=24, left=24, right=24, bottom=24), 132 | background="#000", #green 133 | margin=mp.Margin(left="auto", right="auto"), 134 | width="min(1024px, 100%)", 135 | gap="24px", 136 | flex_grow=1, 137 | display="flex", 138 | flex_wrap="wrap", 139 | ) 140 | ): 141 | box_style = mp.Style( 142 | flex_basis="max(480px, calc(50% - 48px))", 143 | background="#fff", 144 | border_radius=12, 145 | box_shadow=( 146 | "0 3px 1px -2px #0003, 0 2px 2px #00000024, 0 1px 5px #0000001f" 147 | ), 148 | padding=mp.Padding(top=16, left=16, right=16, bottom=16), 149 | display="flex", 150 | flex_direction="column", 151 | ) 152 | 153 | with mp.box(style=mp.Style( 154 | flex_basis="max(360px, calc(60% - 48px))", 155 | background="#fff", 156 | border_radius=12, 157 | box_shadow=( 158 | "0 3px 1px -2px #0003, 0 2px 2px #00000024, 0 1px 5px #0000001f" 159 | ), 160 | padding=mp.Padding(top=16, left=16, right=16, bottom=16), 161 | display="flex", 162 | flex_direction="column", 163 | )): 164 | mp.text("Input", style=mp.Style(font_weight=500)) 165 | mp.box(style=mp.Style(height=16)) 166 | mp.uploader( 167 | label="Upload Image", 168 | accepted_file_types=["application/pdf"], 169 | on_upload=on_image_upload, 170 | type="flat", 171 | color="primary", 172 | style=mp.Style(font_weight="bold"), 173 | ) 174 | # mp.textarea( 175 | # key=str(mp.state(State).textarea_key), 176 | # on_input=on_input, 177 | # rows=5, 178 | # autosize=True, 179 | # max_rows=15, 180 | # style=mp.Style(width="100%"), 181 | # ) 182 | if mp.state(State).image_data: 183 | with mp.box(style=box_style): 184 | with mp.box( 185 | style=mp.Style( 186 | display="grid", 187 | justify_content="center", 188 | justify_items="center", 189 | ) 190 | ): 191 | mp.image( 192 | src=f"data:image/jpeg;base64,{mp.state(State).image_data}", 193 | style=mp.Style(width="100%", margin=mp.Margin(top=10)), 194 | ) 195 | mp.box(style=mp.Style(height=12)) 196 | with mp.box( 197 | style=mp.Style(display="flex", justify_content="space-between") 198 | ): 199 | mp.button( 200 | "Clear", 201 | color="primary", 202 | type="stroked", 203 | on_click=on_click_clear, 204 | ) 205 | mp.button( 206 | "Detect", 207 | color="primary", 208 | type="flat", 209 | on_click=on_click_generate, 210 | ) 211 | with mp.box(style=mp.Style( 212 | flex_basis="max(360px, calc(30% - 48px))", 213 | background="#fff", 214 | border_radius=12, 215 | box_shadow=( 216 | "0 3px 1px -2px #0003, 0 2px 2px #00000024, 0 1px 5px #0000001f" 217 | ), 218 | padding=mp.Padding(top=16, left=16, right=16, bottom=16), 219 | display="flex", 220 | flex_direction="column", 221 | )): 222 | mp.text("Output", style=mp.Style(font_weight=500)) 223 | mp.markdown(mp.state(State).output) 224 | 225 | -------------------------------------------------------------------------------- /chatbot.py: -------------------------------------------------------------------------------- 1 | from pypdf import PdfReader 2 | from langchain.text_splitter import RecursiveCharacterTextSplitter 3 | import os 4 | from langchain_google_genai import GoogleGenerativeAIEmbeddings 5 | import google.generativeai as genai 6 | from langchain_community.vectorstores import FAISS 7 | from langchain_google_genai import ChatGoogleGenerativeAI 8 | from langchain.chains.question_answering import load_qa_chain 9 | from langchain.prompts import PromptTemplate 10 | from dotenv import load_dotenv 11 | import mesop as mp 12 | import mesop.labs as mel 13 | import chatlab 14 | import io 15 | 16 | from typing import Callable 17 | import base64 18 | 19 | @mp.stateclass 20 | class State: 21 | name: str 22 | path: str 23 | size: int 24 | mime_type: str 25 | pdf_data: str 26 | output: str 27 | textarea_key: int 28 | 29 | load_dotenv() 30 | os.getenv("GOOGLE_API_KEY") 31 | genai.configure(api_key="PLACE YOUR GEMINI API KEY HERE") 32 | 33 | # Reading the text from pdf page by page and storing it into various 34 | def get_pdf_text(pdf): 35 | text="" 36 | with io.BytesIO(pdf) as open_pdf_file: 37 | pdf_reader = PdfReader(open_pdf_file) 38 | #pdf_reader= PdfReader(pdf) 39 | for page in pdf_reader.pages: 40 | text+= page.extract_text() 41 | return text 42 | 43 | #Getting the text into number of chunks as it is helpful in faster processing 44 | def get_text_chunks(text): 45 | text_splitter = RecursiveCharacterTextSplitter(chunk_size=10000, chunk_overlap=1000) 46 | chunks = text_splitter.split_text(text) 47 | return chunks 48 | 49 | #Storing the text chunks into embeddings to retrive the answer for the query outoff it 50 | def get_vector_store(text_chunks): 51 | embeddings = GoogleGenerativeAIEmbeddings(model = "models/embedding-001") 52 | vector_store = FAISS.from_texts(text_chunks, embedding=embeddings) 53 | vector_store.save_local("faiss_index") 54 | 55 | def get_conversational_chain(): 56 | 57 | prompt_template = """ 58 | You are an expert assistance extracting information from context provided. 59 | Answer the question as detailed as possible from the provided context, 60 | make sure to provide all the details, Be concise and do not hallucinate. 61 | Context:\n {context}?\n 62 | Question: \n{question}\n 63 | 64 | Answer: 65 | """ 66 | model = ChatGoogleGenerativeAI(model="gemini-pro", 67 | temperature=0.3) 68 | 69 | prompt = PromptTemplate(template = prompt_template, input_variables = ["context", "question"]) 70 | chain = load_qa_chain(model, chain_type="stuff", prompt=prompt) 71 | 72 | return chain 73 | 74 | def on_pdf_upload(e: mp.UploadEvent): 75 | state = mp.state(State) 76 | state.pdf_data = base64.b64encode(e.file.read()).decode() 77 | print("file ", e.file) 78 | state.name = e.file.name 79 | print("name ", e.file.name) 80 | 81 | # Decode base64 string 82 | decoded_data = base64.b64decode(state.pdf_data) 83 | 84 | # Write binary data to a file 85 | # saving image as a file 86 | with open("document.pdf", "wb") as pdf_file: 87 | pdf_file.write(decoded_data) 88 | 89 | def on_click_generate(e: mp.ClickEvent): 90 | state = mp.state(State) 91 | raw_text = get_pdf_text(base64.b64decode(state.pdf_data)) 92 | print("pdf text ", raw_text) 93 | text_chunks = get_text_chunks(raw_text) 94 | get_vector_store(text_chunks) 95 | 96 | 97 | state.output = "hello ji" 98 | print("output is ", state.output) 99 | 100 | def on_click_clear(e: mp.ClickEvent): 101 | state = mp.state(State) 102 | state.pdf_data = "" 103 | state.name = "" 104 | state.output = "" 105 | state.textarea_key += 1 106 | 107 | def answer(input: str, history: list[mel.ChatMessage]): 108 | embeddings = GoogleGenerativeAIEmbeddings(model = "models/embedding-001") 109 | 110 | new_db = FAISS.load_local("faiss_index", embeddings,allow_dangerous_deserialization=True) 111 | docs = new_db.similarity_search(input) 112 | 113 | chain = get_conversational_chain() 114 | 115 | 116 | response = chain( 117 | {"input_documents":docs, "question": input} 118 | , return_only_outputs=True) 119 | 120 | print(response) 121 | 122 | return response['output_text'] 123 | 124 | 125 | def transform(s: str): 126 | 127 | with mp.box( 128 | style=mp.Style( 129 | background="#fdfdfd", #lavender 130 | height="100%", 131 | ) 132 | ): 133 | with mp.box( 134 | style=mp.Style( 135 | margin=mp.Margin(left="5%", right="5%"), 136 | background="#dcdcdc", #purple 137 | padding=mp.Padding(top=24, left=24, right=24, bottom=24), 138 | display="flex", 139 | flex_direction="column", 140 | ) 141 | ): 142 | if s: 143 | mp.text(s,type="headline-5",style=mp.Style( 144 | 145 | font_family="Serif" 146 | #padding=mp.Padding(left=5, right=5, bottom=5), 147 | )) 148 | with mp.box( 149 | style=mp.Style( 150 | justify_content="space-between", 151 | padding=mp.Padding(top=24, left=24, right=24, bottom=24), 152 | background="#000", #green 153 | margin=mp.Margin(left="auto", right="auto"), 154 | width="min(1024px, 100%)", 155 | gap="24px", 156 | flex_grow=1, 157 | display="flex", 158 | flex_wrap="wrap", 159 | ) 160 | ): 161 | with mp.box(style=mp.Style( 162 | flex_basis="max(360px, calc(30% - 48px))", 163 | background="#fff", 164 | height="20%", 165 | border_radius=12, 166 | box_shadow=( 167 | "0 3px 1px -2px #0003, 0 2px 2px #00000024, 0 1px 5px #0000001f" 168 | ), 169 | padding=mp.Padding(top=16, left=16, right=16, bottom=16), 170 | display="flex", 171 | flex_direction="column", 172 | )): 173 | mp.text("Input", style=mp.Style(font_weight=500)) 174 | mp.box(style=mp.Style( 175 | height=16)) 176 | mp.uploader( 177 | label="Upload Your PDF files", 178 | accepted_file_types=["application/pdf"], 179 | on_upload=on_pdf_upload, 180 | type="flat", 181 | color="primary", 182 | style=mp.Style(font_weight="bold"), 183 | ) 184 | 185 | # if mp.state(State).pdf_data: 186 | # with mp.box(style=box_style): 187 | # with mp.box( 188 | # style=mp.Style( 189 | # display="grid", 190 | # justify_content="center", 191 | # justify_items="center", 192 | # ) 193 | # ): 194 | # # mp.audio( 195 | # # src=f"data:audio/wav;base64,{mp.state(State).pdf_data}", 196 | # # ) 197 | mp.box(style=mp.Style(height=12)) 198 | with mp.box( 199 | style=mp.Style(display="flex", justify_content="space-between") 200 | ): 201 | mp.button( 202 | "Clear", 203 | color="primary", 204 | type="stroked", 205 | on_click=on_click_clear, 206 | ) 207 | mp.button( 208 | "Submit", 209 | color="primary", 210 | type="flat", 211 | on_click=on_click_generate, 212 | ) 213 | with mp.box(style=mp.Style( 214 | flex_basis="max(480px, calc(60% - 48px))", 215 | background="#fff", 216 | border_radius=12, 217 | box_shadow=( 218 | "0 3px 1px -2px #0003, 0 2px 2px #00000024, 0 1px 5px #0000001f" 219 | ), 220 | padding=mp.Padding(top=16, left=16, right=16, bottom=16), 221 | display="flex", 222 | flex_direction="column", 223 | )): 224 | chatlab.chat(answer , title="Get Your Queries Resolved", bot_user="Autism Bot") 225 | 226 | -------------------------------------------------------------------------------- /classification.py: -------------------------------------------------------------------------------- 1 | import mesop as mp 2 | import mesop.labs as mel 3 | import base64 4 | from typing import Callable 5 | import image_to_text 6 | import audio_to_text 7 | import text_to_text 8 | import chatbot 9 | import os 10 | 11 | from dotenv import load_dotenv 12 | from PIL import Image 13 | import pathlib 14 | # from gemma_text import ModelClass 15 | 16 | 17 | import google.generativeai as genai 18 | from load_creds import load_creds 19 | 20 | 21 | creds = load_creds() 22 | genai.configure(api_key='PLACE YOUR GEMINI API KEY HERE') 23 | #genai.configure(credentials=creds) 24 | # Create the model 25 | # See https://ai.google.dev/api/python/google/generativeai/GenerativeModel 26 | generation_config = { 27 | "temperature": 0.9, 28 | "top_p": 1, 29 | "top_k": 0, 30 | "max_output_tokens": 8192, 31 | "response_mime_type": "text/plain", 32 | } 33 | 34 | 35 | 36 | load_dotenv() 37 | model = genai.GenerativeModel( 38 | model_name="gemini-1.5-flash", 39 | #model_name="tunedModels/textemotionclassificationall-ggxdrkt84w7", 40 | generation_config=generation_config, 41 | # safety_settings = Adjust safety settings 42 | # See https://ai.google.dev/gemini-api/docs/safety-settings 43 | ) 44 | def nav_gemini(): 45 | # with mp.box( 46 | # style=mp.Style( 47 | # background="#302b2b", 48 | # height="5%", 49 | # border_radius=12, 50 | # box_shadow=( 51 | # "0 3px 1px -2px #0003, 0 2px 2px #00000024, 0 1px 5px #0000001f") 52 | # ) 53 | # ): 54 | with mp.box( 55 | style=mp.Style(display="flex", justify_content="right", 56 | padding=mp.Padding(top=16, left=16, right=16, bottom=16), 57 | #box_shadow=("0 3px 1px -2px #0003, 0 2px 2px #00000024, 0 1px 5px #0000001f"), 58 | margin=mp.Margin(left="auto", right="auto"), 59 | width="min(1024px, 100%)",) 60 | ): 61 | for example in ['text', 'image', 'speech']: 62 | path = f"/gemini/{example}" 63 | if example =='image': 64 | example = 'Facial' 65 | mp.button( 66 | example.title()+" Emotion", 67 | color="accent", 68 | type="raised", 69 | style=mp.Style(font_family="Serif", cursor="pointer"), 70 | on_click=set_demo, 71 | key=path, 72 | ) 73 | mp.button( 74 | "Autism Chatbot", 75 | color="accent", 76 | type="raised", 77 | 78 | style=mp.Style(font_family="Serif", cursor="pointer"), 79 | on_click=set_demo, 80 | key='/gemini/chatbot', 81 | ) 82 | 83 | 84 | def nav_section(): 85 | with mp.box( 86 | style=mp.Style(justify_content="space-between", 87 | padding=mp.Padding(top=5,left=5, right=2, bottom=5), 88 | background="#123456", #green 89 | 90 | margin=mp.Margin(left="auto", right="auto"), 91 | width="max(1024px, 100%)", 92 | gap="24px", 93 | flex_grow=1, 94 | display="flex", 95 | flex_wrap="wrap", 96 | ) 97 | ): 98 | with mp.box( 99 | style=mp.Style(display="flex", justify_content="right", 100 | background="#f2f2f2", 101 | flex_basis="max(360px, calc(30% - 48px))", 102 | padding=mp.Padding(top=16, left=16, right=6, bottom=16), 103 | #box_shadow=("0 3px 1px -2px #0003, 0 2px 2px #00000024, 0 1px 5px #0000001f"), 104 | #margin=mp.Margin(left="auto", right="200px"), 105 | width="min(360px, 30%)", 106 | border_radius=12, 107 | flex_direction="column", 108 | ) 109 | ): 110 | mp.text("AI for Autism", style=mp.Style( 111 | font_family="Serif", 112 | font_size="30px" 113 | )) 114 | with mp.box( 115 | style=mp.Style(display="flex", 116 | background="#123456", 117 | padding=mp.Padding(top=16, left=16, right=6, bottom=16), 118 | #box_shadow=("0 3px 1px -2px #0003, 0 2px 2px #00000024, 0 1px 5px #0000001f"), 119 | #margin=mp.Margin(left="auto", right="200px"), 120 | width="max(480px, 60%)", 121 | justify_content="right", 122 | gap="24px", 123 | flex_grow=1, 124 | flex_wrap="wrap", 125 | #flex_direction="column", 126 | ) 127 | ): 128 | 129 | for example in ['gemini', 'gemma', 'paligemma']: 130 | path = f"/{example}" 131 | if example is 'gemini': 132 | path = f"/{example}/text" 133 | 134 | with mp.content_button( 135 | 136 | color="warn", 137 | type="raised", 138 | style=mp.Style(color="#000", cursor="pointer", align_self="center", margin=mp.Margin( 139 | left="3px", right="3px" 140 | )), 141 | on_click=set_demo, 142 | key=path, 143 | ): 144 | mp.text(text=example.title(), type="button", style=mp.Style( 145 | font_weight="bold", 146 | font_size="18px", 147 | font_family="Serif" 148 | #padding=mp.Padding(left=5, right=5, bottom=5), 149 | )) 150 | 151 | 152 | def set_demo(e: mp.ClickEvent): 153 | mp.navigate(e.key) 154 | 155 | 156 | ########### HOME ############################### 157 | 158 | @mp.page( 159 | security_policy=mp.SecurityPolicy( 160 | allowed_iframe_parents=["https://google.github.io"] 161 | ), 162 | path="/", 163 | title="AIForAutism", 164 | ) 165 | 166 | def app(): 167 | nav_section() 168 | #pathlib.Path().resolve() current directory path 169 | print("path ", os.path.abspath(__file__)) 170 | image = Image.open("autism.png") 171 | image.save("background.png") 172 | with open("autism.png", "rb") as image: 173 | f = image.read() 174 | with mp.box( 175 | style= mp.Style( 176 | border_radius=12, 177 | box_shadow=( 178 | "0 3px 1px -2px #0003, 0 2px 2px #00000024, 0 1px 5px #0000001f"), 179 | margin=mp.Margin(right=8, left=8) 180 | 181 | ) 182 | ): 183 | 184 | mp.image( 185 | src=f"data:image/png;base64,{base64.b64encode(f).decode()}", 186 | style=mp.Style( 187 | height="100%", 188 | width="100%", 189 | display="flex", 190 | flex_wrap="wrap", 191 | ) 192 | ) 193 | 194 | 195 | 196 | 197 | 198 | ############### FOR TEXT ########################## 199 | 200 | @mp.page( 201 | security_policy=mp.SecurityPolicy( 202 | allowed_iframe_parents=["https://google.github.io"] 203 | ), 204 | path="/gemini/text", 205 | title="AIForAutism", 206 | ) 207 | 208 | def app(): 209 | nav_section() 210 | with mp.box( 211 | style=mp.Style( 212 | background="#302b2b", 213 | height="100%", 214 | border_radius=12, 215 | box_shadow=( 216 | "0 3px 1px -2px #0003, 0 2px 2px #00000024, 0 1px 5px #0000001f") 217 | )): 218 | nav_gemini() 219 | text_to_text.text_to_text( 220 | text_classifier, 221 | title="Detect Emotion of the Sentence", 222 | ) 223 | 224 | 225 | def text_classifier(s: str): 226 | response = model.generate_content([ 227 | "input: Analyse the emotion of the sentence, determine which emotion best describes the statement among ('Fear', 'Neutral', 'Surprise', 'Disgust', 'Desire', 'Affection', 'Happiness', 'Anger', 'Sadness', 'Optimism'),{}".format(s), 228 | "output: Answer in one word", 229 | ]) 230 | return response.text 231 | 232 | ############# FOR IMAGE ########################## 233 | 234 | 235 | @mp.stateclass 236 | class State: 237 | name: str 238 | path: str 239 | size: int 240 | mime_type: str 241 | image_data: str 242 | output: str 243 | textarea_key: int 244 | 245 | 246 | @mp.page(path="/gemini/image") 247 | def app(): 248 | nav_section() 249 | with mp.box( 250 | style=mp.Style( 251 | background="#302b2b", 252 | height="100%", 253 | border_radius=12, 254 | box_shadow=( 255 | "0 3px 1px -2px #0003, 0 2px 2px #00000024, 0 1px 5px #0000001f") 256 | )): 257 | nav_gemini() 258 | #genai.configure(api_key="AIzaSyAGB9ZOLj-mnf0RVSboFHEa4FhbkImOOho") 259 | image_to_text.image_to_text( 260 | image_classification, 261 | title="Detect Facial Emotion from the Image", 262 | ) 263 | 264 | 265 | def image_classification(data: str): 266 | # Decode base64 string 267 | return data 268 | 269 | 270 | def upload_to_gemini(path, mime_type=None): 271 | """Uploads the given file to Gemini. 272 | 273 | See https://ai.google.dev/gemini-api/docs/prompting_with_media 274 | """ 275 | file = genai.upload_file(path, mime_type=mime_type) 276 | print(f"Uploaded file '{file.display_name}' as: {file.uri}") 277 | return file 278 | 279 | 280 | ################ FOR SPEECH ################### 281 | 282 | @mp.page( 283 | security_policy=mp.SecurityPolicy( 284 | allowed_iframe_parents=["https://google.github.io"] 285 | ), 286 | path="/gemini/speech", 287 | title="AIForAutism", 288 | ) 289 | 290 | def app(): 291 | nav_section() 292 | with mp.box( 293 | style=mp.Style( 294 | background="#302b2b", 295 | height="100%", 296 | border_radius=12, 297 | box_shadow=( 298 | "0 3px 1px -2px #0003, 0 2px 2px #00000024, 0 1px 5px #0000001f") 299 | )): 300 | nav_gemini() 301 | audio_to_text.audio_to_text( 302 | audio, 303 | title="Detect Emotion of the Speech", 304 | ) 305 | 306 | def audio(s: str): 307 | return s 308 | 309 | 310 | ############# CHAT BOT ####################### 311 | 312 | @mp.page( 313 | security_policy=mp.SecurityPolicy( 314 | allowed_iframe_parents=["https://google.github.io"] 315 | ), 316 | path="/gemini/chatbot", 317 | title="AIForAutism", 318 | ) 319 | 320 | def app(): 321 | nav_section() 322 | with mp.box( 323 | style=mp.Style( 324 | background="#302b2b", 325 | height="100%", 326 | border_radius=12, 327 | box_shadow=( 328 | "0 3px 1px -2px #0003, 0 2px 2px #00000024, 0 1px 5px #0000001f") 329 | )): 330 | nav_gemini() 331 | chatbot.transform("Autism Chat Bot") 332 | #mel.chat(transform, title="Autism Chat Bot", bot_user="Autism Bot" ) 333 | # audio_to_text.audio_to_text( 334 | # audio, 335 | # title="Get Emotion of the Audio", 336 | # ) 337 | 338 | ########### GEMMA ############################### 339 | @mp.page( 340 | security_policy=mp.SecurityPolicy( 341 | allowed_iframe_parents=["https://google.github.io"] 342 | ), 343 | path="/gemma", 344 | title="AIForAutism", 345 | ) 346 | 347 | def app(): 348 | nav_section() 349 | with mp.box( 350 | style=mp.Style( 351 | background="#302b2b", 352 | height="100%", 353 | border_radius=12, 354 | box_shadow=( 355 | "0 3px 1px -2px #0003, 0 2px 2px #00000024, 0 1px 5px #0000001f") 356 | )): 357 | nav_gemini() 358 | text_to_text.text_to_text( 359 | text_2classifier, 360 | title="Detect Emotion of the Sentence", 361 | ) 362 | 363 | 364 | def text_2classifier(s: str): 365 | return s 366 | return ModelClass.predict(s) 367 | --------------------------------------------------------------------------------