├── runtime.txt ├── .gitignore ├── Procfile ├── .vscode └── settings.json ├── requirements.txt ├── Dockerfile ├── .github └── workflows │ └── flake8.yml ├── README.md └── main.py /runtime.txt: -------------------------------------------------------------------------------- 1 | python-3.10.12 -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | __pycache__/main.cpython-310.pyc 3 | -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | web: uvicorn main:app --host=0.0.0.0 --port=${PORT:-5000} -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "[python]": { 3 | "editor.defaultFormatter": "ms-python.autopep8" 4 | }, 5 | "python.formatting.provider": "none" 6 | } -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | line-bot-sdk==3.14.0 2 | fastapi 3 | uvicorn[standard] 4 | google.generativeai 5 | langchain 6 | langchain-google-vertexai 7 | google-cloud-aiplatform 8 | yfinance 9 | pydantic 10 | tiktoken 11 | Pillow 12 | google-cloud-storage -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.10.12 2 | 3 | # 將專案複製到容器中 4 | COPY . /app 5 | WORKDIR /app 6 | 7 | # 安裝必要的套件 8 | RUN pip install --upgrade pip 9 | COPY requirements.txt . 10 | RUN pip install -r requirements.txt 11 | 12 | EXPOSE 8080 13 | CMD uvicorn main:app --host=0.0.0.0 --port=$PORT -------------------------------------------------------------------------------- /.github/workflows/flake8.yml: -------------------------------------------------------------------------------- 1 | name: Linting 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | flake8: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v2 10 | - name: Set up Python 11 | uses: actions/setup-python@v2 12 | with: 13 | python-version: '3.x' 14 | - name: Install flake8 15 | run: pip install flake8 16 | - name: Run flake8 17 | run: flake8 . -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Gemini Helper with LangChain and Vertex AI 2 | 3 | ## Project Background 4 | 5 | This project is a LINE bot that uses Google's Vertex AI Gemini models through LangChain to generate responses to both text and image inputs. The bot can answer questions in Traditional Chinese and provide detailed descriptions of images. 6 | 7 | ## Screenshot 8 | 9 | ![image](https://github.com/kkdai/linebot-gemini-python/assets/2252691/466fbe7c-e704-45f9-8584-91cfa2c99e48) 10 | 11 | ## Features 12 | 13 | - Text message processing using Gemini AI in Traditional Chinese 14 | - Image analysis with scientific detail in Traditional Chinese 15 | - Integration with LINE Messaging API for easy mobile access 16 | - Built with FastAPI for efficient asynchronous processing 17 | 18 | ## Technologies Used 19 | 20 | - Python 3 21 | - FastAPI 22 | - LINE Messaging API 23 | - Google Vertex AI (Gemini 2.0 Flash) 24 | - LangChain 25 | - Aiohttp 26 | - PIL (Python Imaging Library) 27 | 28 | ## Setup 29 | 30 | 1. Clone the repository to your local machine. 31 | 2. Set up Google Cloud: 32 | - Create a Google Cloud project 33 | - Enable Vertex AI API 34 | - Set up authentication (service account or application default credentials) 35 | 36 | 3. Set the following environment variables: 37 | - `ChannelSecret`: Your LINE channel secret 38 | - `ChannelAccessToken`: Your LINE channel access token 39 | - `GOOGLE_PROJECT_ID`: Your Google Cloud Project ID 40 | - `GOOGLE_LOCATION`: Google Cloud region (default: us-central1) 41 | - Optional: `GOOGLE_APPLICATION_CREDENTIALS`: Path to service account key file (if running locally) 42 | 43 | 4. Install the required dependencies: 44 | 45 | ``` 46 | pip install -r requirements.txt 47 | ``` 48 | 49 | 5. Start the FastAPI server: 50 | 51 | ``` 52 | uvicorn main:app --reload 53 | ``` 54 | 55 | 6. Set up your LINE bot webhook URL to point to your server's endpoint. 56 | 57 | ## Usage 58 | 59 | ### Text Processing 60 | 61 | Send any text message to the LINE bot, and it will use Vertex AI's Gemini model to generate a response in Traditional Chinese. 62 | 63 | ### Image Processing 64 | 65 | Send an image to the bot, and it will analyze and describe the image with scientific detail in Traditional Chinese. 66 | 67 | ## Deployment Options 68 | 69 | ### Local Development 70 | 71 | Use ngrok or similar tools to expose your local server to the internet for webhook access: 72 | 73 | ### Google Cloud Run 74 | 75 | 1. Install the Google Cloud SDK and authenticate with your Google Cloud account. 76 | 2. Build the Docker image: 77 | 78 | ``` 79 | gcloud builds submit --tag gcr.io/$GOOGLE_PROJECT_ID/linebot-gemini 80 | ``` 81 | 82 | 3. Deploy the Docker image to Cloud Run: 83 | 84 | ``` 85 | gcloud run deploy linebot-gemini --image gcr.io/$GOOGLE_PROJECT_ID/linebot-gemini --platform managed --region $GOOGLE_LOCATION --allow-unauthenticated 86 | ``` 87 | 88 | 4. Set up your LINE bot webhook URL to point to the Cloud Run service URL. 89 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | from linebot.models import ( 2 | MessageEvent, TextSendMessage 3 | ) 4 | from linebot.exceptions import ( 5 | InvalidSignatureError 6 | ) 7 | from linebot.aiohttp_async_http_client import AiohttpAsyncHttpClient 8 | from linebot import ( 9 | AsyncLineBotApi, WebhookParser 10 | ) 11 | from fastapi import Request, FastAPI, HTTPException 12 | import os 13 | import sys 14 | from io import BytesIO 15 | import aiohttp 16 | import uuid 17 | from google.cloud import storage 18 | 19 | 20 | # Import LangChain components with Vertex AI 21 | from langchain_google_vertexai import ChatVertexAI 22 | from langchain.schema.messages import HumanMessage, SystemMessage 23 | from langchain_core.prompts import ChatPromptTemplate 24 | 25 | 26 | # get channel_secret and channel_access_token from your environment variable 27 | channel_secret = os.getenv('ChannelSecret', None) 28 | channel_access_token = os.getenv('ChannelAccessToken', None) 29 | imgage_prompt = ''' 30 | Describe this image with scientific detail, reply in zh-TW: 31 | ''' 32 | 33 | # Vertex AI needs a project ID and possibly authentication 34 | google_project_id = os.getenv('GOOGLE_PROJECT_ID') 35 | # Location for Vertex AI resources, e.g., "us-central1" 36 | google_location = os.getenv('GOOGLE_LOCATION', 'us-central1') 37 | # Google Cloud Storage bucket for image uploads 38 | google_storage_bucket = os.getenv('GOOGLE_STORAGE_BUCKET', None) 39 | 40 | 41 | if channel_secret is None: 42 | print('Specify ChannelSecret as environment variable.') 43 | sys.exit(1) 44 | if channel_access_token is None: 45 | print('Specify ChannelAccessToken as environment variable.') 46 | sys.exit(1) 47 | if google_project_id is None: 48 | print('Specify GOOGLE_PROJECT_ID as environment variable.') 49 | sys.exit(1) 50 | if google_storage_bucket is None: 51 | print('Specify GOOGLE_STORAGE_BUCKET as environment variable.') 52 | sys.exit(1) 53 | 54 | 55 | # Initialize the FastAPI app for LINEBot 56 | app = FastAPI() 57 | session = aiohttp.ClientSession() 58 | async_http_client = AiohttpAsyncHttpClient(session) 59 | line_bot_api = AsyncLineBotApi(channel_access_token, async_http_client) 60 | parser = WebhookParser(channel_secret) 61 | 62 | # Using a single, powerful multimodal model for both text and images. 63 | # gemini-2.0-flash is a powerful, cost-effective model for multimodal tasks. 64 | model = ChatVertexAI( 65 | model_name="gemini-2.0-flash", 66 | project=google_project_id, 67 | location=google_location, 68 | # Increased token limit for detailed image descriptions 69 | max_output_tokens=2048 70 | ) 71 | 72 | 73 | def upload_to_gcs(file_stream, file_name, bucket_name): 74 | """Uploads a file to the bucket.""" 75 | try: 76 | storage_client = storage.Client() 77 | bucket = storage_client.bucket(bucket_name) 78 | blob = bucket.blob(file_name) 79 | 80 | blob.upload_from_file(file_stream, content_type='image/jpeg') 81 | 82 | # Return the GCS URI 83 | return f"gs://{bucket_name}/{file_name}" 84 | except Exception as e: 85 | print(f"Error uploading to GCS: {e}") 86 | return None 87 | 88 | 89 | def delete_from_gcs(bucket_name, blob_name): 90 | """Deletes a blob from the bucket.""" 91 | try: 92 | storage_client = storage.Client() 93 | bucket = storage_client.bucket(bucket_name) 94 | blob = bucket.blob(blob_name) 95 | blob.delete() 96 | print(f"Blob {blob_name} deleted from bucket {bucket_name}.") 97 | except Exception as e: 98 | print(f"Error deleting from GCS: {e}") 99 | 100 | 101 | @app.post("/") 102 | async def handle_callback(request: Request): 103 | signature = request.headers['X-Line-Signature'] 104 | 105 | # get request body as text 106 | body = await request.body() 107 | body = body.decode() 108 | 109 | try: 110 | events = parser.parse(body, signature) 111 | except InvalidSignatureError: 112 | raise HTTPException(status_code=400, detail="Invalid signature") 113 | 114 | for event in events: 115 | if not isinstance(event, MessageEvent): 116 | continue 117 | 118 | if (event.message.type == "text"): 119 | # Process text message using LangChain with Vertex AI 120 | msg = event.message.text 121 | user_id = event.source.user_id 122 | print(f"Received message: {msg} from user: {user_id}") 123 | response = generate_text_with_langchain( 124 | f'{msg}, reply in zh-TW:' 125 | ) 126 | reply_msg = TextSendMessage(text=response) 127 | await line_bot_api.reply_message( 128 | event.reply_token, 129 | reply_msg 130 | ) 131 | elif (event.message.type == "image"): 132 | user_id = event.source.user_id 133 | print(f"Received image from user: {user_id}") 134 | 135 | message_content = await line_bot_api.get_message_content( 136 | event.message.id 137 | ) 138 | 139 | # Asynchronously read all content chunks into a byte string 140 | image_bytes = b'' 141 | async for chunk in message_content.iter_content(): 142 | image_bytes += chunk 143 | 144 | # Create an in-memory binary stream from the bytes 145 | image_stream = BytesIO(image_bytes) 146 | # Reset the stream's pointer to the beginning for the upload 147 | image_stream.seek(0) 148 | 149 | file_name = f"{uuid.uuid4()}.jpg" 150 | gcs_uri = None 151 | # Default error message 152 | response = "抱歉，處理您的圖片時發生錯誤。" 153 | 154 | try: 155 | gcs_uri = upload_to_gcs( 156 | image_stream, file_name, google_storage_bucket) 157 | if gcs_uri: 158 | print(f"Image uploaded to {gcs_uri}") 159 | response = generate_image_description(gcs_uri) 160 | finally: 161 | # Clean up the GCS file if it was uploaded 162 | if gcs_uri: 163 | delete_from_gcs(google_storage_bucket, file_name) 164 | 165 | reply_msg = TextSendMessage(text=response) 166 | await line_bot_api.reply_message( 167 | event.reply_token, 168 | reply_msg 169 | ) 170 | else: 171 | continue 172 | 173 | return 'OK' 174 | 175 | 176 | def generate_text_with_langchain(prompt): 177 | """ 178 | Generate a text completion using LangChain with Vertex AI model. 179 | """ 180 | # Create a chat prompt template with system instructions 181 | prompt_template = ChatPromptTemplate.from_messages([ 182 | SystemMessage( 183 | content="You are a helpful assistant that responds in " 184 | "Traditional Chinese (zh-TW)." 185 | ), 186 | HumanMessage(content=prompt) 187 | ]) 188 | 189 | # Format the prompt and call the model 190 | formatted_prompt = prompt_template.format_messages() 191 | response = model.invoke(formatted_prompt) 192 | 193 | return response.content 194 | 195 | 196 | def generate_image_description(image_uri): 197 | """ 198 | 199 | Generate a description for an image using LangChain with Vertex AI. 200 | """ 201 | # The prompt is already defined globally as imgage_prompt 202 | message = HumanMessage( 203 | content=[ 204 | { 205 | "type": "text", 206 | "text": imgage_prompt 207 | }, 208 | { 209 | "type": "image_url", 210 | "image_url": {"url": image_uri} 211 | }, 212 | ] 213 | ) 214 | 215 | response = model.invoke([message]) 216 | return response.content 217 | --------------------------------------------------------------------------------