├── sqlite3 └── .gitignore ├── examples ├── 05.en.twitter.png ├── 01.en.gift.social.png ├── 03.en.billing.sms.png ├── 02.es.banking.social.png ├── 07.fr.operaticket.ham.png ├── 04.en.multifactor.email.png ├── 06.notext.landscape.ham.png └── log.csv ├── ui_tests ├── 01.en.gift.scam.png ├── 02.es.banking.scam.png ├── 03.en.billing.scam.png ├── 05.en.twitter.scam.png ├── 04.en.multifactorscam.png ├── 07.fr.operaticket.ham.png ├── minerva.results.panel.png └── 06.notext.minerva.termination.png ├── static └── minerva.architecture.png ├── evals ├── results │ ├── minerva.scamprediction.accuracy.latency.png │ ├── minerva.scamprediction.accuracy.confidence.png │ ├── minerva.metrics.agents.categories.txt │ └── minerva.metrics.prompt.categories.txt └── experiments │ ├── scam.financial.csv │ ├── scam.authority.csv │ ├── scam.customer_service.csv │ ├── scam.commerce.csv │ ├── scam.opportunity_rewards.csv │ └── scam.all.categories.csv ├── tools ├── __init__.py ├── image_ocr.py ├── formatter.py ├── url_checker.py └── db_connector.py ├── requirements.txt ├── .github └── workflows │ └── sync.yml ├── config └── agents.yaml ├── app.py ├── .gitignore ├── evaluation.py ├── agents.py ├── README.md └── LICENSE /sqlite3/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore -------------------------------------------------------------------------------- /examples/05.en.twitter.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dcarpintero/minerva/HEAD/examples/05.en.twitter.png -------------------------------------------------------------------------------- /ui_tests/01.en.gift.scam.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dcarpintero/minerva/HEAD/ui_tests/01.en.gift.scam.png -------------------------------------------------------------------------------- /examples/01.en.gift.social.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dcarpintero/minerva/HEAD/examples/01.en.gift.social.png -------------------------------------------------------------------------------- /examples/03.en.billing.sms.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dcarpintero/minerva/HEAD/examples/03.en.billing.sms.png -------------------------------------------------------------------------------- /static/minerva.architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dcarpintero/minerva/HEAD/static/minerva.architecture.png -------------------------------------------------------------------------------- /ui_tests/02.es.banking.scam.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dcarpintero/minerva/HEAD/ui_tests/02.es.banking.scam.png -------------------------------------------------------------------------------- /ui_tests/03.en.billing.scam.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dcarpintero/minerva/HEAD/ui_tests/03.en.billing.scam.png -------------------------------------------------------------------------------- /ui_tests/05.en.twitter.scam.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dcarpintero/minerva/HEAD/ui_tests/05.en.twitter.scam.png -------------------------------------------------------------------------------- /examples/02.es.banking.social.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dcarpintero/minerva/HEAD/examples/02.es.banking.social.png -------------------------------------------------------------------------------- /examples/07.fr.operaticket.ham.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dcarpintero/minerva/HEAD/examples/07.fr.operaticket.ham.png -------------------------------------------------------------------------------- /ui_tests/04.en.multifactorscam.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dcarpintero/minerva/HEAD/ui_tests/04.en.multifactorscam.png -------------------------------------------------------------------------------- /ui_tests/07.fr.operaticket.ham.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dcarpintero/minerva/HEAD/ui_tests/07.fr.operaticket.ham.png -------------------------------------------------------------------------------- /ui_tests/minerva.results.panel.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dcarpintero/minerva/HEAD/ui_tests/minerva.results.panel.png -------------------------------------------------------------------------------- /examples/04.en.multifactor.email.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dcarpintero/minerva/HEAD/examples/04.en.multifactor.email.png -------------------------------------------------------------------------------- /examples/06.notext.landscape.ham.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dcarpintero/minerva/HEAD/examples/06.notext.landscape.ham.png -------------------------------------------------------------------------------- /ui_tests/06.notext.minerva.termination.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dcarpintero/minerva/HEAD/ui_tests/06.notext.minerva.termination.png -------------------------------------------------------------------------------- /evals/results/minerva.scamprediction.accuracy.latency.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dcarpintero/minerva/HEAD/evals/results/minerva.scamprediction.accuracy.latency.png -------------------------------------------------------------------------------- /evals/results/minerva.scamprediction.accuracy.confidence.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dcarpintero/minerva/HEAD/evals/results/minerva.scamprediction.accuracy.confidence.png -------------------------------------------------------------------------------- /tools/__init__.py: -------------------------------------------------------------------------------- 1 | from .image_ocr import ImageOCR 2 | from .url_checker import URLChecker 3 | from .db_connector import DatabaseConnector 4 | 5 | __all__ = ['ImageOCR', 'URLChecker', 'DatabaseConnector'] -------------------------------------------------------------------------------- /examples/log.csv: -------------------------------------------------------------------------------- 1 | input_image 2 | 01.en.gift.social.png 3 | 02.es.banking.social.png 4 | 03.en.billing.sms.png 5 | 04.en.multifactor.email.png 6 | 05.en.twitter.png 7 | 06.notext.landscape.ham.png 8 | 07.fr.operaticket.ham.png -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | autogen-agentchat==0.4.0.dev11 2 | autogen-ext[openai]==0.4.0.dev11 3 | pillow==11.0.0 4 | pyyaml==6.0.2 5 | pytesseract==0.3.13 6 | python-dotenv==1.0.1 7 | #gradio==5.9.0 # huggingface spaces handles gradio version -------------------------------------------------------------------------------- /.github/workflows/sync.yml: -------------------------------------------------------------------------------- 1 | name: Sync to Hugging Face hub 2 | on: 3 | push: 4 | branches: [main] 5 | workflow_dispatch: 6 | 7 | jobs: 8 | sync-to-hub: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v3 12 | with: 13 | fetch-depth: 0 14 | lfs: true 15 | - name: Push to hub 16 | env: 17 | HF_TOKEN: ${{ secrets.HF_TOKEN }} 18 | run: git push https://dcarpintero:$HF_TOKEN@huggingface.co/spaces/dcarpintero/minerva main 19 | -------------------------------------------------------------------------------- /tools/image_ocr.py: -------------------------------------------------------------------------------- 1 | from PIL import Image 2 | import pytesseract 3 | 4 | class ImageOCR: 5 | """A class to handle OCR (Optical Character Recognition) operations.""" 6 | 7 | def extract_text(self, image_path: str) -> str: 8 | """Extract text from image using OCR. 9 | """ 10 | try: 11 | image = Image.open(image_path) 12 | text = pytesseract.image_to_string(image) 13 | return text 14 | except Exception as e: 15 | return f"Error in text extraction: {str(e)}" -------------------------------------------------------------------------------- /evals/results/minerva.metrics.agents.categories.txt: -------------------------------------------------------------------------------- 1 | ---------- Commerce ------------ 2 | Evaluation Results (gpt-4o-mini with agents): 3 | Accuracy: 93.33% 4 | Correct Predictions: 28/30 5 | Average Confidence: 4.6 6 | Average Latency: 20.4 seconds 7 | 8 | ---------- Rewards and Opportunities ----------- 9 | Evaluation Results (gpt-4o-mini with agents): 10 | Accuracy: 81.48% 11 | Correct Predictions: 22/27 12 | Average Confidence: 4.5 13 | Average Latency: 21.5 seconds 14 | 15 | ---------- Service ----------- 16 | Evaluation Results (gpt-4o-mini with agents): 17 | Accuracy: 100.00% 18 | Correct Predictions: 27/27 19 | Average Confidence: 4.7 20 | Average Latency: 20.4 seconds 21 | 22 | ---------- Authority ----------- 23 | Evaluation Results (gpt-4o-mini with agents): 24 | Accuracy: 81.48% 25 | Correct Predictions: 22/27 26 | Average Confidence: 4.7 27 | Average Latency: 20.8 seconds 28 | 29 | ---------- Financial ----------- 30 | Evaluation Results (gpt-4o-mini with agents): 31 | Accuracy: 85.19% 32 | Correct Predictions: 23/27 33 | Average Confidence: 4.6 34 | Average Latency: 20.4 seconds -------------------------------------------------------------------------------- /evals/results/minerva.metrics.prompt.categories.txt: -------------------------------------------------------------------------------- 1 | ---------- Commerce ----------- 2 | Evaluation Results (gpt-4o-mini with prompt, temperature = 0.0): 3 | Accuracy: 70.00% 4 | Correct Predictions: 21/30 5 | Average Confidence: 4.7 6 | Average Latency: 0.5 seconds 7 | 8 | ---------- Rewards and Opportunities ----------- 9 | Evaluation Results (gpt-4o-mini with prompt, temperature = 0.0): 10 | Accuracy: 66.67% 11 | Correct Predictions: 18/27 12 | Average Confidence: 4.7 13 | Average Latency: 0.5 seconds 14 | 15 | ---------- Service ----------- 16 | Evaluation Results (gpt-4o-mini with prompt, temperature = 0.0): 17 | Evaluation Results: 18 | Accuracy: 70.37% 19 | Correct Predictions: 19/27 20 | Average Confidence: 4.7 21 | Average Latency: 0.5 seconds 22 | 23 | ---------- Authority ----------- 24 | Evaluation Results (gpt-4o-mini with prompt, temperature = 0.0): 25 | Accuracy: 66.67% 26 | Correct Predictions: 18/27 27 | Average Confidence: 4.8 28 | Average Latency: 0.5 seconds 29 | 30 | ---------- Financial ----------- 31 | Evaluation Results (gpt-4o-mini with prompt, temperature = 0.0): 32 | Accuracy: 74.07% 33 | Correct Predictions: 20/27 34 | Average Confidence: 4.5 35 | Average Latency: 0.5 seconds -------------------------------------------------------------------------------- /tools/formatter.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from autogen_agentchat.base import Response, TaskResult 4 | from autogen_agentchat.messages import AgentMessage, MultiModalMessage 5 | 6 | 7 | class AutoGenFormatter: 8 | def _message_to_str(self, message: AgentMessage) -> str: 9 | if isinstance(message, MultiModalMessage): 10 | result: List[str] = [] 11 | for c in message.content: 12 | if isinstance(c, str): 13 | result.append(c) 14 | else: 15 | result.append("") 16 | return "\n".join(result) 17 | else: 18 | return f"{message.content}" 19 | 20 | async def to_output(self, message: AgentMessage) -> str: 21 | try: 22 | if isinstance(message, TaskResult): 23 | output = ( 24 | f"{'-' * 10} Summary {'-' * 10}\n" 25 | f"Number of messages: {len(message.messages)}\n" 26 | f"Finish reason: {message.stop_reason}\n" 27 | ) 28 | return output 29 | 30 | if isinstance(message, Response): 31 | output = f"{'-' * 10} {message.chat_message.source} {'-' * 10}\n{self._message_to_str(message.chat_message)}\n" 32 | return output 33 | else: 34 | output = f"{'-' * 10} {message.source} {'-' * 10}\n{self._message_to_str(message)}\n" 35 | return output 36 | except Exception as e: 37 | return "" -------------------------------------------------------------------------------- /tools/url_checker.py: -------------------------------------------------------------------------------- 1 | import os 2 | from typing import Dict, List, Tuple 3 | import requests 4 | from dotenv import load_dotenv, find_dotenv 5 | 6 | class URLChecker: 7 | """A class to handle URL safety checks.""" 8 | 9 | # Standard threat types for Google Safe Browsing API 10 | THREAT_TYPES = [ 11 | "MALWARE", 12 | "SOCIAL_ENGINEERING", 13 | "UNWANTED_SOFTWARE", 14 | "POTENTIALLY_HARMFUL_APPLICATION" 15 | ] 16 | 17 | def __init__(self): 18 | load_dotenv(find_dotenv()) 19 | 20 | self.safebrowsing_key = os.getenv("SAFEBROWSING_API_KEY") 21 | if not self.safebrowsing_key: 22 | raise ValueError("SAFEBROWSING_API_KEY is missing in environment variables.") 23 | 24 | self.base_url = "https://safebrowsing.googleapis.com/v4" 25 | self.client_id = "minerva" 26 | self.client_version = "0.1.0" 27 | 28 | def expand_url(self, url: str) -> str: 29 | """Expand shortened URL. 30 | """ 31 | try: 32 | response = requests.head(url, allow_redirects=True) 33 | return response.url 34 | except requests.exceptions.RequestException: 35 | return url # Return original URL if expansion fails 36 | 37 | def is_url_safe(self, target_url: str) -> Tuple[str, List[Dict[str, str]]]: 38 | """Check if URL is safe using Google Safe Browsing API. 39 | """ 40 | safe_endpoint = f"{self.base_url}/threatMatches:find?key={self.safebrowsing_key}" 41 | expanded_url = self.expand_url(target_url) 42 | 43 | request_body = { 44 | "client": { 45 | "clientId": self.client_id, 46 | "clientVersion": self.client_version 47 | }, 48 | "threatInfo": { 49 | "threatTypes": self.THREAT_TYPES, 50 | "platformTypes": ["ANY_PLATFORM"], 51 | "threatEntryTypes": ["URL"], 52 | "threatEntries": [{"url": target_url}] 53 | } 54 | } 55 | 56 | if expanded_url != target_url: 57 | request_body["threatInfo"]["threatEntries"].append({"url": expanded_url}) 58 | 59 | try: 60 | response = requests.post(safe_endpoint, json=request_body) 61 | response.raise_for_status() 62 | 63 | result = response.json() 64 | 65 | if not result: 66 | return "Not Flagged", [] 67 | 68 | threats = [] 69 | if "matches" in result: 70 | for match in result["matches"]: 71 | threats.append({ 72 | "threat_type": match.get("threatType"), 73 | "threat_url": match.get("threat", {}).get("url"), 74 | }) 75 | 76 | return "Flagged", threats 77 | 78 | except requests.exceptions.RequestException as e: 79 | raise Exception(f"Error checking URL safety: {str(e)}") -------------------------------------------------------------------------------- /config/agents.yaml: -------------------------------------------------------------------------------- 1 | ocr_agent: 2 | assignment: > 3 | You are an OCR specialist. Your role is to: 4 | 1. Extract text from an image using Optical Character Recognition (OCR) 5 | 2. Clean and format the extracted text 6 | 3. Do not perform any analysis on the extracted text 7 | 4. Reply with the extracted text 8 | 5. If there is no text in the image, reply with "NO_TEXT_FOUND" 9 | 10 | link_checker_agent: 11 | assignment: > 12 | You are a Link checker. Your role is to: 13 | 1. Check the extracted text for any URLs 14 | 2. Verify the legitimacy of the URLs using your registered function 15 | 16 | content_agent: 17 | assignment: > 18 | You are a content analysis specialist. Your role is to: 19 | 1. Analyze text for common scam patterns 20 | 2. If available, analyze the results of the URL check: look for any flag related to Malware, Phishing, and Social Engineering. 21 | 3. Identify urgency indicators, threats, or pressure tactics 22 | 5. Check for inconsistencies in messaging 23 | 6. Evaluate legitimacy of any claims or offers 24 | 25 | decision_agent: 26 | assignment: > 27 | You are the final decision maker. Your role is to: 28 | 1. Make a final determination on scam probability 29 | 2. Provide detailed explanation of the decision 30 | 3. Provide a confidence level about your final determination in a scale from 1-5, where 5 is highly confident 31 | 32 | summary_agent: 33 | assignment: > 34 | You are a communication specialist who creates clear, concise summaries of technical analyses. Your role is to: 35 | 1. Synthesize the findings of a scam assessment into user-friendly language 36 | 2. Highlight the most important points that users need to know 37 | 3. Provide actionable recommendations 38 | 4. Shorten your message into one paragraph 39 | 40 | language_translation_agent: 41 | assignment: > 42 | You are a language translation specialist. Your role is to: 43 | 1. Infer the languge of the text extracted from the image, this is the user language 44 | 2. If the user language is english, just continue 45 | 3. If the user language is different than english, translate your summary into the user language 46 | 47 | data_storage_agent: 48 | assignment: > 49 | You are a data storage specialist. Your role is to: 50 | 1. Store using the registered function the following fields: 51 | - extracted text 52 | - your summary 53 | - final determination ('1' for scam, '0' for not_scam) 54 | - confidence level on the final determination in a scale from 1-5, where 5 is highly confident 55 | All the fields are mandatory 56 | 57 | # deprecated after switching to AutoGen v0.4.0 58 | # 59 | # user_proxy: 60 | # assignment: > 61 | # Your role is to coordinate the available agents to carry out a scam assessment process. These are the steps to follow: 62 | # 1. Extract text from an image 63 | # 2. Analyze the text content for scam patterns 64 | # 3. Synthesize the analyses and make final determination 65 | # 4. Generate a summary of the final determination -------------------------------------------------------------------------------- /app.py: -------------------------------------------------------------------------------- 1 | from PIL import Image 2 | import gradio as gr 3 | 4 | from agents import MinervaTeam 5 | from tools.formatter import AutoGenFormatter 6 | 7 | 8 | title = "Minerva: LLM Agents for Scam Protection" 9 | description = """ 10 | 🦉 Minerva uses LLM Agents to analyze screenshots for potential scams.
11 | 📢 It provides the analysis in the language of the extracted text.

12 | 13 | 📄 Try out one of the examples to perform a scam analysis.
14 | ⚙️ The Agentic Workflow is streamed for demonstration purposes.

15 | 16 | 🕵 LLM Agents coordinated as an AutoGen Team in a RoundRobin fashion:
17 | - *OCR Specialist*
18 | - *Link Checker*
19 | - *Content Analyst*
20 | - *Decision Maker*
21 | - *Summary Specialist*
22 | - *Language Translation Specialist*
23 | - *Data Storage Agent*

24 | 25 | 🧑‍💻️ https://github.com/dcarpintero/minerva
26 | 🎓 Submission for RBI Berkeley, CS294/194-196, LLM Agents (Diego Carpintero)

27 | ♥️ Built with AutoGen 0.4.0 and OpenAI. 28 | """ 29 | inputs = gr.components.Image() 30 | outputs = [ 31 | gr.components.Textbox(label="Analysis Result"), 32 | gr.HTML(label="Agentic Workflow (Streaming)") 33 | ] 34 | examples = "examples" 35 | example_labels = ["EN:Gift:Social", "ES:Banking:Social", "EN:Billing:SMS", "EN:Multifactor:Email", "EN:CustomerService:Twitter", "NO_TEXT:Landscape.HAM", "FR:OperaTicket:HAM"] 36 | 37 | agents = MinervaTeam() 38 | formatter = AutoGenFormatter() 39 | 40 | def to_html(texts): 41 | formatted_html = '' 42 | for text in texts: 43 | formatted_html += text.replace('\n', '
') + '
' 44 | return f'
{formatted_html}
' 45 | 46 | async def predict(img): 47 | try: 48 | img = Image.fromarray(img) 49 | stream = await agents.analyze(img) 50 | 51 | streams = [] 52 | messages = [] 53 | async for s in stream: 54 | streams.append(s) 55 | messages.append(await formatter.to_output(s)) 56 | yield ["Pondering, stand by...", to_html(messages)] 57 | 58 | if streams[-1]: 59 | prediction = streams[-1].messages[-4].content 60 | else: 61 | prediction = "No analysis available. Try again later." 62 | 63 | await agents.reset() 64 | yield [prediction, to_html(messages)] 65 | 66 | except Exception as e: 67 | print(e) 68 | yield ["Error during analysis. Try again later.", ""] 69 | 70 | 71 | with gr.Blocks() as demo: 72 | with gr.Tab("Minerva: AI Guardian for Scam Protection"): 73 | with gr.Row(): 74 | gr.Interface( 75 | fn=predict, 76 | inputs=inputs, 77 | outputs=outputs, 78 | examples=examples, 79 | example_labels=example_labels, 80 | description=description, 81 | ).queue(default_concurrency_limit=5) 82 | 83 | demo.launch() -------------------------------------------------------------------------------- /tools/db_connector.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | import pandas as pd 3 | import sqlite3 4 | 5 | from typing import Optional 6 | from datetime import datetime 7 | 8 | class DatabaseConnector: 9 | """Manages database operations for storing analysis results.""" 10 | 11 | def __init__(self, db_path: str = "sqlite3/results.financial.db"): 12 | """Initialize database connection and create table if it doesn't exist. 13 | """ 14 | self.db_path = db_path 15 | self._create_table() 16 | 17 | def _create_table(self) -> None: 18 | query = """ 19 | CREATE TABLE IF NOT EXISTS results ( 20 | id INTEGER PRIMARY KEY AUTOINCREMENT, 21 | text TEXT NOT NULL, 22 | summary TEXT, 23 | is_scam BOOLEAN NOT NULL, 24 | confidence_level INTEGER, 25 | created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, 26 | message_hash TEXT UNIQUE NOT NULL 27 | ) 28 | """ 29 | with sqlite3.connect(self.db_path) as conn: 30 | conn.execute(query) 31 | 32 | def _compute_hash(self, text: str) -> str: 33 | """Compute SHA-256 hash of the input text.""" 34 | return hashlib.sha256(text.encode('utf-8')).hexdigest() 35 | 36 | def store_result( 37 | self, 38 | text: str, 39 | summary: Optional[str], 40 | is_scam: bool, 41 | confidence_level: int 42 | ) -> Optional[int]: # Updated return type hint 43 | """Store analysis result in the database. 44 | """ 45 | message_hash = self._compute_hash(text) 46 | hash_query = "SELECT id FROM results WHERE message_hash = ?" 47 | 48 | query = """ 49 | INSERT INTO results (text, summary, is_scam, confidence_level, message_hash) 50 | VALUES (?, ?, ?, ?, ?) 51 | """ 52 | 53 | with sqlite3.connect(self.db_path) as conn: 54 | cursor = conn.execute(hash_query, (message_hash,)) 55 | existing_id = cursor.fetchone() 56 | 57 | if existing_id: 58 | return None 59 | 60 | cursor = conn.execute( 61 | query, 62 | (text, summary, is_scam, confidence_level, message_hash) 63 | ) 64 | return cursor.lastrowid 65 | 66 | def get_result(self, result_id: int) -> Optional[dict]: 67 | """Retrieve a specific analysis result by ID. 68 | """ 69 | query = "SELECT * FROM results WHERE id = ?" 70 | with sqlite3.connect(self.db_path) as conn: 71 | conn.row_factory = sqlite3.Row 72 | cursor = conn.execute(query, (result_id,)) 73 | result = cursor.fetchone() 74 | 75 | if result: 76 | return dict(result) 77 | return None 78 | 79 | def get_top_k(self, k: int = 10) -> pd.DataFrame: 80 | """Retrieve most recent analysis results as a DataFrame. 81 | """ 82 | query = "SELECT * FROM results ORDER BY created_at DESC LIMIT ?" 83 | with sqlite3.connect(self.db_path) as conn: 84 | df = pd.read_sql_query(query, conn, params=(k,)) 85 | return df 86 | 87 | def get_all(self) -> pd.DataFrame: 88 | """Retrieve all analysis results as a DataFrame. 89 | """ 90 | query = "SELECT * FROM results" 91 | with sqlite3.connect(self.db_path) as conn: 92 | df = pd.read_sql_query(query, conn) 93 | return df 94 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # pdm 105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 106 | #pdm.lock 107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 108 | # in version control. 109 | # https://pdm.fming.dev/latest/usage/project/#working-with-version-control 110 | .pdm.toml 111 | .pdm-python 112 | .pdm-build/ 113 | 114 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 115 | __pypackages__/ 116 | 117 | # Celery stuff 118 | celerybeat-schedule 119 | celerybeat.pid 120 | 121 | # SageMath parsed files 122 | *.sage.py 123 | 124 | # Environments 125 | .env 126 | .venv 127 | env/ 128 | venv/ 129 | ENV/ 130 | env.bak/ 131 | venv.bak/ 132 | 133 | # Spyder project settings 134 | .spyderproject 135 | .spyproject 136 | 137 | # Rope project settings 138 | .ropeproject 139 | 140 | # mkdocs documentation 141 | /site 142 | 143 | # mypy 144 | .mypy_cache/ 145 | .dmypy.json 146 | dmypy.json 147 | 148 | # Pyre type checker 149 | .pyre/ 150 | 151 | # pytype static type analyzer 152 | .pytype/ 153 | 154 | # Cython debug symbols 155 | cython_debug/ 156 | 157 | # PyCharm 158 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 159 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 160 | # and can be added to the global gitignore or merged into this file. For a more nuclear 161 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 162 | #.idea/ 163 | -------------------------------------------------------------------------------- /evals/experiments/scam.financial.csv: -------------------------------------------------------------------------------- 1 | message,is_scam,category,subcategory 2 | "ALERT: Unauthorized access detected on your account. Click hxxp://secur1ty-bank-verify.com to verify your identity immediately. Your account will be suspended in 2 hours.",1,financial,banking_and_payment 3 | "Dear customer, we detected unusual activity on your account. Please verify by texting back your online banking username and password.",1,financial,banking_and_payment 4 | "Revolut: We noticed a charge of $458.23 on your card ending in 7580 that appears unusual for your spending pattern. For your security, you might review this transaction in your Revolut app or at https://revolut.com. Never share your PIN or password with anyone.",0,financial,banking_and_payment 5 | "WARNING: $2,499 transaction pending at Amazon. If not you, click hxxp://chase-secure02.net/verify to cancel transaction immediately!",1,financial,banking_and_payment 6 | "Suspicious withdrawal attempt ($850) detected on your card ending 4355. Reply YES to confirm or NO to block.",1,financial,banking_and_payment 7 | "Revolut: You just made an online payment of EUR 64.47 with your card ending in 7587.",0,financial,banking_and_payment 8 | "URGENT: Your payment of $1,249 has been processed. To cancel this wire transfer, call our security team at +1-855-SCAM immediately.",1,financial,banking_and_payment 9 | "Your wire transfer of $5,000 to J. Smith was declined. To complete, verify at hxxp://wire-secure-verify.net with your account details.",1,financial,banking_and_payment 10 | "Hey Ana! Revolut is offering €100 for new customers right now. Just let me know if you're interested!",0,financial,banking_and_payment 11 | "CRITICAL: Invest in our new crypto fund! Guaranteed 500% returns in 30 days. Send Bitcoin to secure your spot: bc1qxy2kgdygjrsqtzq2n0yrf2493p83kkfjhx0wlh",1,financial,investment 12 | "🚀 LAST CHANCE! Join our exclusive forex trading group. Turn $100 into $5000 weekly. Contact WhatsApp: +44-SCAM-NUMBER",1,financial,investment 13 | "Fidelity Investments: Your mutual fund purchase of $5,000 into FXAIX is confirmed. Settlement date: Dec 20. View your updated portfolio at fidelity.com/investments. For support with your account, visit fidelity.com/customer-service.",0,financial,investment 14 | "BINARY OPTIONS OPPORTUNITY: Start earning $1000/day with our automated trading system. Pay $299 to join: hxxp://binary-profits.xyz",1,financial,investment 15 | "Get rich with our AI trading bot! 100% success rate. Send $500 worth of ETH to activate: 0x742d35Cc6634C0532925a3b844Bc454e4438f44e",1,financial,investment 16 | "Hi Sarah! Just saw Vanguard is offering a great deal on their index funds right now - no minimum investment required!",0,financial,investment 17 | "🔥 NEW CRYPTO ALERT! BitMaxCoin launching tomorrow. Buy now at $0.01, guaranteed to hit $1! Register: hxxp://bitmaxcoin.net",1,financial,investment 18 | "Exclusive Forex Signal Group! Copy our trades for guaranteed profits. Join now: hxxp://forex-signals.xyz. Only $199/month.",1,financial,investment 19 | "Hi Thomas, just saw that Schwab dropped their commission fees to zero! Might be worth checking out if you're still paying fees at E*TRADE.",0,financial,investment 20 | "LOAN APPROVED! Get $50,000 instantly. Send $500 processing fee via Western Union to release funds. Reply YES to proceed.",1,financial,lending 21 | "Pre-approved for $25,000! No credit check needed. Send SSN and bank details to hxxp://quickloans-secure.xyz",1,financial,lending 22 | "Capital One: Your personal loan is confirmed. Check more personalized offers at capitalize.com.",0,financial,lending 23 | "DEBT CONSOLIDATION ALERT: Pay off all debts today! Send $199 via Bitcoin to start: hxxp://debt-freedom.xyz",1,financial,lending 24 | "Your credit card debts can be cleared for pennies on the dollar! Contact debthelp@scam.net for secret method.",1,financial,lending 25 | "Hey David! Just FYI - my credit union is offering really low rates on home loans right now. 4.5% for 30-year fixed. Worth checking out if you're still house hunting. Let me know if you want the details!",0,financial,lending 26 | "STUDENT LOAN FORGIVENESS PROGRAM! Pay $599 to eliminate all student debt. Must act now!",1,financial,lending 27 | "ATTENTION: Qualify for instant loan! Send photos of your credit cards to loans@fastcash.xyz",1,financial,lending 28 | "Bank of America has a special rate on personal loans this month. My colleague just got approved for 6.5% APR.",0,financial,lending -------------------------------------------------------------------------------- /evals/experiments/scam.authority.csv: -------------------------------------------------------------------------------- 1 | message,is_scam,category,subcategory 2 | "URGENT: IRS FINAL NOTICE. You owe $4,875 in back taxes. Pay now at hxxp://irs-payment.xyz or face immediate arrest.",1,authority,government 3 | "Your Social Security number has been suspended due to suspicious activity. Call +1-SCAM-NUMBER immediately to prevent arrest.",1,authority,government 4 | "IRS: Your tax return has been processed. Track your refund at irs.gov/refunds using your filing info. Questions? Call 800-829-1040. The IRS never requests sensitive information by email.",0,authority,government 5 | "IMMIGRATION ALERT: Your visa status is irregular. Pay $999 penalty fee at hxxp://uscis-verify.net to avoid deportation.",1,authority,government 6 | "URGENT: Your Green Card application needs verification. Send copies of passport and SSN to visa@immigration-services.xyz",1,authority,government 7 | "USCIS: Your appointment is confirmed for Dec 21, 10AM at 26 Federal Plaza, NYC. Bring confirmation notice #USC789012 and required documents listed on uscis.gov/appointments.",0,authority,government 8 | "POLICE NOTICE: Warrant issued for your arrest. Pay fine of $750 via Bitcoin to cancel warrant. Contact officer@police-dept.net",1,authority,government 9 | "FINAL WARNING: Court appearance mandatory! Pay $500 processing fee at hxxp://court-services.xyz to dismiss charges.",1,authority,government 10 | "County Court Notice: You are summoned for jury duty starting Jan 15, 2024. Register at courts.state.gov/jury or call 555-0123. No fees required for jury service.",0,authority,government 11 | "IT SECURITY ALERT: Your company email will be suspended. Verify at hxxp://corporate-email-verify.net within 30 minutes.",1,authority,corporate 12 | "URGENT: CEO requests immediate wire transfer of $9,850. Reply with confirmation. Confidential matter - do not discuss.",1,authority,corporate 13 | "Microsoft IT: Your password expires today. Update it at microsoft.com/account or through Microsoft Authenticator app. We never ask for passwords via email.",0,authority,corporate 14 | "HR NOTICE: Employee benefits suspended. Update details at hxxp://hr-benefits.xyz to restore coverage immediately.",1,authority,corporate 15 | "Payroll Processing Error: Send bank details to payroll@company-verify.net for immediate correction of salary.",1,authority,corporate 16 | "Workday: Your W-2 form for 2023 is now available. Access securely at workday.com/benefits. For technical support, contact your HR department at extension 5545.",0,authority,corporate 17 | "VENDOR PAYMENT ALERT: New banking details for invoice processing. Update at hxxp://vendor-verify.net immediately.",1,authority,corporate 18 | "Urgent: CFO needs gift cards for client meeting. Buy $1,000 in Apple cards and send codes to cfo@company-gifts.xyz",1,authority,corporate 19 | "Adobe Security: Sign in attempt from new device detected. Review activity at adobe.com/security. Don't recognize this? Change your password at adobe.com/account.",0,authority,corporate 20 | "BANK SECURITY ALERT: Account frozen due to suspicious activity. Call +1-SCAM-NUMBER immediately to verify identity.",1,authority,financial_institutions 21 | "Your credit card has been charged $899. If unauthorized, click hxxp://bank-secure-verify.net to dispute.",1,authority,financial_institutions 22 | "Chase Fraud Prevention: Did you attempt a purchase of $523.87 at Amazon? Reply YES or NO. View card activity at chase.com. Never share your PIN or password.",0,authority,financial_institutions 23 | "IMMEDIATE ACTION: Credit score dropped 100 points. Pay $299 at hxxp://credit-restore.xyz to protect your score.",1,authority,financial_institutions 24 | "Insurance Claim Warning: Policy canceled unless verified. Send policy details to claims@insurance-verify.net now.",1,authority,financial_institutions 25 | "Geico: Your auto insurance payment of $143.50 is scheduled for Dec 20. Review or update payment info at geico.com/payment. Questions? Call 1-800-207-7847.",0,authority,financial_institutions 26 | "INVESTMENT ALERT: Portfolio requires immediate verification. Login at hxxp://investment-secure.net to prevent freeze.",1,authority,financial_institutions 27 | "Account Processing Error: Send bank statement to verify@payment-processor.xyz to avoid service interruption.",1,authority,financial_institutions 28 | "Fidelity: Unusual login attempt detected from new device. Verify it's you at fidelity.com/security. To report unauthorized access, call 800-544-6666.",0,authority,financial_institutions 29 | -------------------------------------------------------------------------------- /evaluation.py: -------------------------------------------------------------------------------- 1 | import time 2 | from typing import Tuple 3 | 4 | import pandas as pd 5 | 6 | from agents import MinervaTeam 7 | from typing import List, Dict 8 | import logging 9 | 10 | from tools import DatabaseConnector 11 | 12 | class MinervaEvaluator: 13 | def __init__(self): 14 | self.agents = MinervaTeam() 15 | self.db_connector = DatabaseConnector() 16 | 17 | self.logger = logging.getLogger(__name__ + '.MinervaEvaluator') 18 | handler = logging.StreamHandler() 19 | formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') 20 | 21 | handler.setFormatter(formatter) 22 | self.logger.addHandler(handler) 23 | 24 | self.logger.setLevel(logging.INFO) 25 | self.logger.propagate = False 26 | 27 | async def predict(self, msg: str) -> Tuple[str, float]: 28 | """ 29 | Analyze a text message and return the prediction. 30 | """ 31 | start_time = time.perf_counter() 32 | try: 33 | stream = await self.agents.analyze_msg(msg) 34 | 35 | streams = [] 36 | async for s in stream: 37 | streams.append(s) 38 | 39 | if streams and streams[-1] and streams[-1].messages: 40 | prediction = streams[-1].messages[-1].content 41 | else: 42 | prediction = "unknown,unknown,0.0" 43 | 44 | await self.agents.reset() 45 | 46 | latency = time.perf_counter() - start_time 47 | return prediction, latency 48 | 49 | except Exception as e: 50 | self.logger.error(f"Error during prediction: {e}") 51 | latency = time.perf_counter() - start_time 52 | 53 | return "error,unknown,0.0", latency 54 | 55 | async def evaluate_dataset(self, df_evals: pd.DataFrame) -> Dict: 56 | """ 57 | Evaluate the model on a dataset and calculate metrics. 58 | """ 59 | # run evals predictions 60 | latencies = [] 61 | for i, row in df_evals.iterrows(): 62 | self.logger.info(f"{row['category']}.{row['subcategory']}#{i} - {row['message'][:50]}...") 63 | result, latency = await self.predict(row['message']) 64 | latencies.append(latency) 65 | 66 | # reverse preds order, and reset index 67 | df_preds = self.db_connector.get_top_k(i+1) 68 | df_preds = df_preds[::-1].reset_index(drop=True) 69 | 70 | # calculate metrics 71 | metrics = self.calculate_metrics(df_evals, df_preds) 72 | 73 | return { 74 | "metrics": metrics, 75 | "predictions": df_preds, 76 | "latency": sum(latencies) / len(latencies) 77 | } 78 | 79 | def calculate_metrics(self, df_evals: pd.DataFrame, df_preds: pd.DataFrame) -> Dict: 80 | """ 81 | Calculate evaluation metrics. 82 | """ 83 | k = len(df_preds) 84 | df_evals = df_evals.head(k) 85 | 86 | correct = sum(df_evals['is_scam'] == df_preds['is_scam']) 87 | accuracy = correct / k if k > 0 else 0 88 | 89 | df_preds['confidence_level'] = df_preds['confidence_level'].astype(int) 90 | avg_confidence = df_preds['confidence_level'].mean() 91 | 92 | return { 93 | "accuracy": accuracy, 94 | "correct_predictions": correct, 95 | "total_samples": k, 96 | "average_confidence": avg_confidence 97 | } 98 | 99 | async def main(): 100 | evaluator = MinervaEvaluator() 101 | 102 | try: 103 | df_evals = pd.read_csv("./evals/experiments/scam.all.categories.csv") 104 | if not all(col in df_evals.columns for col in ['message', 'category', 'subcategory', 'is_scam']): 105 | raise ValueError("Required columns missing in evaluation dataset") 106 | 107 | except Exception as e: 108 | logging.error(f"Error loading dataset: {e}") 109 | 110 | results = await evaluator.evaluate_dataset(df_evals) 111 | 112 | print("\nEvaluation Results:") 113 | print(f"Accuracy: {results['metrics']['accuracy']:.2%}") 114 | print(f"Correct Predictions: {results['metrics']['correct_predictions']}/{results['metrics']['total_samples']}") 115 | print(f"Average Confidence: {results['metrics']['average_confidence']:.2}") 116 | print(f"Average Latency: {results['latency']:.1f} seconds") 117 | 118 | 119 | if __name__ == "__main__": 120 | import asyncio 121 | asyncio.run(main()) -------------------------------------------------------------------------------- /evals/experiments/scam.customer_service.csv: -------------------------------------------------------------------------------- 1 | message,is_scam,category,subcategory 2 | "WARNING: Virus detected on your Windows PC! Call our Microsoft support team immediately at 1-888-SCAM to prevent data loss. $299 for immediate fix.",1,customer_service,technical_support 3 | "Your computer has been locked! Multiple viruses found. Call Apple Security Team +1-SCAM-NUMBER now or risk permanent damage.",1,customer_service,technical_support 4 | "Microsoft Security Alert: To check if your device is affected by recent malware attacks, run Windows Security scan: Start > Settings > Windows Security. Never share remote access unless you initiated contact. Questions? Visit microsoft.com/security.",0,customer_service,technical_support 5 | "ALERT: Your antivirus subscription expires in 2 hours! Renew now at hxxp://norton-security-renewal.net to prevent infection.",1,customer_service,technical_support 6 | "Your system is infected with 13 viruses! Download our security tool from hxxp://antivirus-protect.xyz. Credit card required.",1,customer_service,technical_support 7 | "Norton: Your security subscription will renew automatically on Jan 15, 2024 ($84.99). Manage your subscription at norton.com/mynorton. Questions? Call 1-855-815-2726.",0,customer_service,technical_support 8 | "Your cloud storage is 99% full! Urgent action required: Click hxxp://cloud-storage-verify.net to prevent data loss.",1,customer_service,technical_support 9 | "CRITICAL: Google Drive storage breach detected. Verify account at hxxp://gdrive-secure.com to protect files.",1,customer_service,technical_support 10 | "Google Drive: You're using 14.8GB of 15GB storage (98%). Upgrade your storage at drive.google.com/storage or manage your files at drive.google.com. Learn more about storage options at support.google.com.",0,customer_service,technical_support 11 | "LEGAL NOTICE: Your case requires immediate attention. Pay $500 retainer fee via Bitcoin to hxxp://legal-services.net for representation.",1,customer_service,professional_services 12 | "Urgent Legal Matter: Settlement pending in your favor. Send $299 processing fee to claim. Contact legal@settlement-claim.net",1,customer_service,professional_services 13 | "Smith & Associates Law Firm: Your consultation is confirmed for Dec 20, 2:00 PM at our downtown office. Free 30-minute initial consultation. Directions and parking info at smithlaw.com/contact. Questions? Call 555-0123.",0,customer_service,professional_services 14 | "INSURANCE ALERT: Your policy will be canceled! Update payment info at hxxp://insurance-verify.net within 6 hours.",1,customer_service,professional_services 15 | "Final Notice: Your health insurance claim needs verification. Send medical history to claims@health-verify.net immediately.",1,customer_service,professional_services 16 | "State Farm: Your auto insurance payment of $143.50 is due Dec 25. Pay securely at statefarm.com or through our mobile app. Questions about your policy? Call 1-800-STATE-FARM.",0,customer_service,professional_services 17 | "URGENT: Your medical records show eligibility for $150,000 settlement. Pay $199 processing fee: hxxp://medical-claim.net",1,customer_service,professional_services 18 | "Immediate Action Required: Send $299 to register for premium healthcare services. WhatsApp +1-SCAM-NUMBER for details.",1,customer_service,professional_services 19 | "Kaiser Permanente: Your prescription is ready for pickup at Main Street Pharmacy. Pay copay of $15 at pickup. Questions? Call your pharmacy at 555-0177 or visit kp.org/prescriptions.",0,customer_service,professional_services 20 | "POWER SHUTOFF NOTICE: Pay $499 immediately or service disconnects today. Click hxxp://utility-pay.net to restore.",1,customer_service,utility_services 21 | "URGENT: Your electricity will be disconnected in 2 hours. Call +1-SCAM-NUMBER now to prevent shutoff.",1,customer_service,utility_services 22 | "PG&E: Your bill of $127.86 is due Dec 22. Pay online at pge.com or by phone at 1-877-704-8470. Payment assistance programs available at pge.com/assistance. We never demand immediate payment or threaten immediate disconnection.",0,customer_service,utility_services 23 | "INTERNET SERVICE ALERT: Network breach detected! Verify account at hxxp://internet-secure.net or face disconnection.",1,customer_service,utility_services 24 | "Your WiFi has been compromised! Call Internet Security Team +1-SCAM-NUMBER immediately or service will be terminated.",1,customer_service,utility_services 25 | "Comcast: Scheduled maintenance in your area on Dec 21, 2-5 AM. No action needed. Track status at xfinity.com/status. Mobile hotspots available for Xfinity Internet customers during maintenance.",0,customer_service,utility_services 26 | "FINAL WARNING: Water service will be disconnected! Pay $399 through hxxp://water-utility-pay.net to prevent shutoff.",1,customer_service,utility_services 27 | "Your water meter shows unusual usage. Verify account at hxxp://water-verify.net or face penalties.",1,customer_service,utility_services 28 | "City Water Department: Your monthly bill of $78.92 is ready. Due date: Jan 5. Pay online at citywater.gov or at any authorized payment location. Questions? Call 555-0148 during business hours.",0,customer_service,utility_services 29 | -------------------------------------------------------------------------------- /evals/experiments/scam.commerce.csv: -------------------------------------------------------------------------------- 1 | message,is_scam,category,subcategory 2 | "Your Amazon order #8829 has been shipped! Track your package: hxxp://amaz0n-tracking.net. Click now to avoid delivery issues.",1,commerce,shopping 3 | "URGENT: Problem with your recent Amazon order. Verify your payment details at hxxp://amazon-secure-verify.com to avoid cancellation.",1,commerce,shopping 4 | "Amazon: Your order #112-7366944-3891234 has shipped and will arrive by 8pm today. Track your delivery at amazon.com/orders or via the Amazon app. Questions? Contact Amazon Customer Service.",0,commerce,shopping 5 | "Your Walmart order has been canceled due to payment issues. To restore order, update payment info: hxxp://walmart-secure.shop/restore",1,commerce,shopping 6 | "SHIPPING ALERT: Package delivery attempted. Final notice - click hxxp://delivery-status-verify.net to reschedule or package returns tomorrow.",1,commerce,shopping 7 | "FedEx: Your package from Best Buy will arrive on Dec 20 between 2-6 PM. Track at fedex.com with tracking number: 7891234567. Need to reschedule? Sign in to fedex.com/delivery-manager.",0,commerce,shopping 8 | "REFUND NOTICE: Your recent purchase refund of $89.99 is pending. Click hxxp://refund-secure.xyz and enter your card details to claim.",1,commerce,shopping 9 | "Your refund request was approved! To receive $143.50, send your bank details to refunds@shopping-returns.net",1,commerce,shopping 10 | "Target: Your refund of $67.89 for order #TGT-456789 has been processed to your original payment method. Refund will appear in 3-5 business days. View details at target.com/orders.",0,commerce,shopping 11 | "iPhone 14 Pro: 90% OFF! Only $199 today. Limited stock. Buy now: hxxp://cheap-phones-sale.net. Pay with Bitcoin for extra discount!",1,commerce,shopping 12 | "AUTHENTIC Ray-Ban sunglasses 85% OFF! Must sell today. Contact WhatsApp: +1-234-SCAM for exclusive deals.",1,commerce,shopping 13 | "Nike.com: Holiday Sale - Up to 50% off select styles. Shop now at nike.com/holiday-deals. Free shipping for Nike Members. Returns always free within 60 days.",0,commerce,shopping 14 | "URGENT: Your eBay account needs verification. Suspicious activity detected. Verify at hxxp://ebay-secure-verify.net to avoid suspension.",1,commerce,marketplace 15 | "Hi, I'm interested in your item. I'll pay extra $100 if you accept. Send your PayPal to buyer.88751@mail.net for immediate payment.",1,commerce,marketplace 16 | "eBay: New message about 'Vintage Camera' from buyer_john. View and respond securely at ebay.com/messages. Remember: Always complete transactions on eBay for purchase protection.",0,commerce,marketplace 17 | "Your Airbnb booking payment failed. Update payment within 2hrs: hxxp://airbnb-payments.site or lose reservation.",1,commerce,marketplace 18 | "SPECIAL OFFER: Luxury apartment, 80% below market rate. Send security deposit via Western Union to secure. Contact +44-SCAM-NUMBER.",1,commerce,marketplace 19 | "Airbnb: Booking confirmed for 'Mountain View Cottage'. Check-in Dec 24, 3PM. Access all details at airbnb.com/trips. Questions? Message your host through the Airbnb app.",0,commerce,marketplace 20 | "TICKET ALERT: Concert tickets 50% off! Limited time offer. Pay through hxxp://ticket-secure-pay.net using bank transfer.",1,commerce,marketplace 21 | "2 VIP concert tickets for sale! Transfer money to my personal account for immediate delivery. WhatsApp +1-SCAM for details.",1,commerce,marketplace 22 | "Ticketmaster: Your order for 'Taylor Swift | The Eras Tour' is confirmed. Access tickets at ticketmaster.com/mybehavior or via the Ticketmaster app. Beware of resale scams - only buy from verified sources.",0,commerce,marketplace 23 | "URGENT INVOICE: Your company owes $2,487 for web services. Pay within 24hrs to avoid legal action: hxxp://invoice-pay-secure.net",1,commerce,business_operations 24 | "Please update your vendor payment information immediately. New account details: SWIFT: SCAM123. Reply with confirmation.",1,commerce,business_operations 25 | "Adobe: Your invoice #AD-456789 for Creative Cloud Business ($52.99) is now available. View and pay securely at adobe.com/account. Due date: Jan 1, 2024. Questions? Contact Adobe Business Support.",0,commerce,business_operations 26 | "FINAL NOTICE: Your business listing will be removed. Pay $299 today: hxxp://business-listing-renew.com to maintain visibility.",1,commerce,business_operations 27 | "Your Yellow Pages listing expires in 24hrs! Send $399 via Bitcoin to fast-track renewal. Contact admin@yp-renewal.net",1,commerce,business_operations 28 | "Google Business Profile: Your business verification is complete for 'Main Street Cafe'. Manage your listing at business.google.com. Need help? Visit support.google.com/business.",0,commerce,business_operations 29 | "Office supplies order confirmed: $1,899. Must pay today. New supplier special offer. Bank details: hxxp://supply-order.net",1,commerce,business_operations 30 | "ATTENTION: Your toner cartridge order ($799) will be auto-billed. To cancel, send company details to orders@supply-scam.com",1,commerce,business_operations 31 | "Staples Business Advantage: Your supply order #STP-789012 totaling $234.56 will ship on Dec 19. View order details and track shipment at staplesadvantage.com. Questions? Call 1-800-3STAPLE.",0,commerce,business_operations 32 | -------------------------------------------------------------------------------- /evals/experiments/scam.opportunity_rewards.csv: -------------------------------------------------------------------------------- 1 | message,is_scam,category,subcategory 2 | "URGENT Work From Home Opportunity! Make $5,000/week guaranteed. No experience needed. Contact recruiter@job-offer.xyz for immediate start.",1,opportunity_rewards,employment 3 | "Congratulations! You've been selected for a remote position. Send $199 for background check to start earning $900/day!",1,opportunity_rewards,employment 4 | "Amazon Careers: Thank you for applying to Customer Service Representative (Job ID: CS4527). Salary range: $18-22/hr + benefits. Next step: Complete your assessment at amazon.jobs/assess/CS4527. Questions? Contact recruiting@amazon.com. Amazon never charges fees for employment.",0,opportunity_rewards,employment 5 | "HIRING NOW: Data Entry Position - $45/hr. Send your bank details to jobs@data-entry-work.net for immediate payment setup.",1,opportunity_rewards,employment 6 | "Work From Home Opportunity! Process payments for our company. Use your bank account, earn 10% commission. Contact +1-SCAM-NUMBER.",1,opportunity_rewards,employment 7 | "Microsoft Careers: Your application for Software Engineer (REQ58769) is under review. Salary: $125,000-$165,000/year + benefits. Schedule your technical interview at careers.microsoft.com/interview/REQ58769. Visit careers.microsoft.com/applicant-faq for common questions.",0,opportunity_rewards,employment 8 | "URGENT: Your resume was selected! Pay $299 training fee via Bitcoin to secure high-paying remote position. Start tomorrow!",1,opportunity_rewards,employment 9 | "Make $800 daily from home! Small investment required ($450) for training materials. WhatsApp +44-SCAM-NUMBER for details.",1,opportunity_rewards,employment 10 | "Google Hiring Event: Join our Virtual Career Fair on Dec 20, 2-5PM EST. Roles: Cloud Solutions Engineer ($130-180K). Register at careers.google.com/events/dec2023. No payment required. All interviews conducted through official Google Meet platform.",0,opportunity_rewards,employment 11 | "CONGRATULATIONS! You won $1,000,000 in the Instagram Lottery! Send $500 processing fee to claim: hxxp://prize-claim.net",1,opportunity_rewards,prizes 12 | "YOU'RE A WINNER! Your email was randomly selected. Send passport copy and bank details to claim@lottery-win.xyz",1,opportunity_rewards,prizes 13 | "Target: Congratulations! You've earned a $50 Target Gift Card through our Circle Rewards program. Redeem in the Target app or at target.com/circle/rewards using code TGT-RF456789. Valid through 3/31/24. Help: 1-800-591-3869. Never pay fees to claim rewards.",0,opportunity_rewards,prizes 14 | "XBOX Sweepstakes Winner! You won a new console. Pay $99 shipping fee at hxxp://xbox-prize.net to receive your prize.",1,opportunity_rewards,prizes 15 | "Amazon Raffle Winner! To receive your free iPhone 14, click hxxp://amazon-prizes.xyz and enter your credit card for verification.",1,opportunity_rewards,prizes 16 | "Walmart: You've won our monthly customer survey drawing! Claim your $100 Walmart Gift Card at any store's customer service desk with ID and confirmation code WM23789456. Questions? Visit walmart.com/giftcard-support or call 1-800-925-6278.",0,opportunity_rewards,prizes 17 | "GAME SHOW WINNER! Pay $299 processing fee via Western Union to claim your $50,000 prize. Contact prize@gameshow-win.net",1,opportunity_rewards,prizes 18 | "Survey Reward: Complete survey and pay $10 fee to receive your free iPad! Click hxxp://survey-rewards.xyz now!",1,opportunity_rewards,prizes 19 | "Best Buy Rewards: Thanks for being an elite member! Your $75 reward certificate is available in your Best Buy account. Redeem online at bestbuy.com/rewards or in stores with your member ID. Certificate expires 1/31/24. Visit bestbuy.com/rewards-faq for details.",0,opportunity_rewards,prizes 20 | "INHERITANCE NOTICE: You are beneficiary to $4.5M USD. Send $1,500 legal fee to claim. Contact barrister@inheritance.net",1,opportunity_rewards,inheritance 21 | "URGENT: Your unknown relative left you $2.8M. Transfer $750 via Bitcoin to start claim process. Reply within 24hrs!",1,opportunity_rewards,inheritance 22 | "Charles Schwab Estate Services: Our records indicate you may be a beneficiary in the Robert Johnson estate. Schedule a free consultation at schwab.com/estates or call 800-435-4000. Note: Schwab never requests upfront fees or personal information by email.",0,opportunity_rewards,inheritance 23 | "CONFIDENTIAL: Late oil merchant left you $5.2M. Send passport and bank details to claim@inheritance-lawyer.xyz",1,opportunity_rewards,inheritance 24 | "Your inheritance claim needs processing! Pay $2,999 to attorney via Western Union. Contact +1-SCAM-NUMBER immediately.",1,opportunity_rewards,inheritance 25 | "JP Morgan Estate Planning: Following up on your inquiry about the Thompson Family Trust distribution. Please schedule an appointment at chase.com/private-bank/estates or visit your local Chase branch. For questions, call Private Client Services: 1-866-265-1727.",0,opportunity_rewards,inheritance 26 | "LAST CHANCE: Unclaimed inheritance of $3.7M expires today! Send $500 via MoneyGram to secure funds. Urgent!",1,opportunity_rewards,inheritance 27 | "Your deceased relative's estate! Send $899 administrative fee to hxxp://estate-claims.net within 12 hours.",1,opportunity_rewards,inheritance 28 | "Vanguard Beneficiary Services: Your inherited IRA documentation is ready for review. Log in to vanguard.com/inheritedira to schedule a consultation. Questions? Call 800-662-2739. No fees required for inherited account services.",0,opportunity_rewards,inheritance -------------------------------------------------------------------------------- /agents.py: -------------------------------------------------------------------------------- 1 | from typing import List, AsyncIterator 2 | import os 3 | 4 | from autogen_agentchat.agents import AssistantAgent, ToolUseAssistantAgent 5 | from autogen_agentchat.conditions import TextMentionTermination 6 | from autogen_agentchat.messages import MultiModalMessage 7 | from autogen_agentchat.teams import RoundRobinGroupChat 8 | from autogen_core import Image as AGImage 9 | from autogen_core.tools import FunctionTool 10 | from autogen_ext.models.openai import OpenAIChatCompletionClient 11 | 12 | from dotenv import load_dotenv, find_dotenv 13 | from PIL import Image 14 | import yaml 15 | 16 | from tools import ImageOCR, URLChecker, DatabaseConnector 17 | 18 | class MinervaTeam: 19 | """ 20 | Minerva Team tasked with Scam Protection using an Agentic Workflow 21 | """ 22 | 23 | def __init__(self, config_path: str = "config/agents.yaml"): 24 | """ 25 | Initialize Minerva with agents and tools 26 | """ 27 | self.load_environment() 28 | self.model = self.initialize_model() 29 | self.config = self.load_config(config_path) 30 | self.agents = self.create_agents() 31 | self.team = self.create_team() 32 | 33 | def load_environment(self): 34 | """Load environment variables""" 35 | load_dotenv(find_dotenv()) 36 | 37 | def load_config(self, config_path: str) -> dict: 38 | """Load agent configurations from YAML file""" 39 | with open(config_path, 'r') as file: 40 | return yaml.safe_load(file) 41 | 42 | def initialize_model(self) -> OpenAIChatCompletionClient: 43 | """Initialize OpenAI model""" 44 | return OpenAIChatCompletionClient( 45 | model="gpt-4o-mini", 46 | api_key=os.getenv("OPENAI_API_KEY") 47 | ) 48 | 49 | def create_agents(self) -> List[AssistantAgent]: 50 | """Create all required agents with their specialized roles and tools""" 51 | 52 | ocr = ImageOCR() 53 | ocr_tool = FunctionTool( 54 | ocr.extract_text, 55 | description="Extracts text from an image path" 56 | ) 57 | 58 | url_checker = URLChecker() 59 | url_checker_tool = FunctionTool( 60 | url_checker.is_url_safe, 61 | description="Checks if a URL is safe" 62 | ) 63 | 64 | db_connector = DatabaseConnector() 65 | db_connector_tool = FunctionTool( 66 | db_connector.store_result, 67 | description="Stores analysis results in a database" 68 | ) 69 | 70 | agents = [] 71 | 72 | agents.append(AssistantAgent( 73 | name="OCR_Specialist", 74 | description="Extracts text from an image", 75 | system_message=self.config['ocr_agent']['assignment'], 76 | model_client=self.model, 77 | #tools=[ocr_tool] # Default OCR to GPT-4o vision capabilities. Uncomment to OCR with tool calling (requires pytesseract) 78 | )) 79 | 80 | agents.append(ToolUseAssistantAgent( 81 | name="Link_Checker", 82 | description="Checks if a Link is safe", 83 | system_message=self.config['link_checker_agent']['assignment'], 84 | model_client=self.model, 85 | registered_tools=[url_checker_tool] 86 | )) 87 | 88 | agents.append(AssistantAgent( 89 | name="Content_Analyst", 90 | description="Analyzes the text for scam patterns", 91 | system_message=self.config['content_agent']['assignment'], 92 | model_client=self.model 93 | )) 94 | 95 | agents.append(AssistantAgent( 96 | name="Decision_Maker", 97 | description="Synthesizes the analyses and make final determination", 98 | system_message=self.config['decision_agent']['assignment'], 99 | model_client=self.model 100 | )) 101 | 102 | agents.append(AssistantAgent( 103 | name="Summary_Agent", 104 | description="Generate a summary of the final determination", 105 | system_message=self.config['summary_agent']['assignment'], 106 | model_client=self.model 107 | )) 108 | 109 | agents.append(AssistantAgent( 110 | name="Language_Translation_Agent", 111 | description="Translate the summary to the user language", 112 | system_message=self.config['language_translation_agent']['assignment'], 113 | model_client=self.model 114 | )) 115 | 116 | agents.append(ToolUseAssistantAgent( 117 | name="DataStorage_Agent", 118 | description="Store the extracted text, summary, determination (1, 0) and confidence level (high, medium low) in a database", 119 | system_message=self.config['data_storage_agent']['assignment'], 120 | model_client=self.model, 121 | registered_tools=[db_connector_tool] 122 | )) 123 | 124 | return agents 125 | 126 | def create_team(self) -> RoundRobinGroupChat: 127 | """Create a team of agents that work together in Round Robin fashion""" 128 | termination = TextMentionTermination("NO_TEXT_FOUND") 129 | 130 | return RoundRobinGroupChat( 131 | self.agents, 132 | max_turns=7, 133 | termination_condition=termination 134 | ) 135 | 136 | async def reset(self): 137 | """Reset team state""" 138 | await self.team.reset() 139 | 140 | async def analyze(self, image: Image) -> AsyncIterator: 141 | """ 142 | Analyze an image for potential scams. 143 | """ 144 | img = AGImage(image) 145 | mm_message = MultiModalMessage(content=[img], source="User") 146 | 147 | return self.team.run_stream(task=mm_message) -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 13 | 14 | ## MINERVA: A Multi-Agent LLM System for Digital Scam Protection 15 | 16 | Digital scams inflict devastating impacts in our society. According to the FBI IC3, $37.4 billion was lost in the United States alone over the past five years due to Internet scams [1]. Beyond these direct financial losses, the hidden costs of processing nearly 4 million associated complaints has overwhelmed institutional and enterprise resources [1]. Victims also face psychological disruptions and diminished trust in emerging technologies. 17 | 18 | To address this challenge, we present MINERVA, a multi-agent system powered by a Large Language Model (gpt-4o-mini) for protecting users from digital scams. Our system implements a cooperative team of seven specialized agents built on the AutoGen framework (v0.4.0) [2]. Each agent combines advanced model capabilities with specific tools to handle distinct aspects of scam detection: optical character recognition, link verification, content analysis, decision making, summarization, linguistic translation, and archiving. 19 | 20 | To evaluate our system's performance, we developed a hierarchical scam taxonomy as the foundation for creating a diverse synthetic dataset. Our multi-agent system achieves higher accuracy compared to a prompt baseline (88.3% vs. 69.5%) while providing more comprehensive functionality. Beyond improving scam detection accuracy and reducing administrative overhead, MINERVA aims to deliver broader societal benefits in digital literacy, trust in AI-powered safety tools, and insights for law enforcement and cybersecurity professionals. 21 | 22 | *[This work was developed for RDI Berkeley, LLM-Agents Course, CS294/194-196. By Diego Carpintero]* 23 | 24 | ### Introduction 25 | 26 | We define an *Agent* as a system that leverages a language model to interpret natural language instructions, perform tasks, interact with its environment, and coopearate with other agents (and a human) towards reaching out a given goal. 27 | 28 | Recent advancements in AI have enabled the development of sophisticated agents capable of reasoning and tool usage, with multi-agent collaboration emerging as a promising paradigm for complex tasks. Our intuition is that scam detection represents an ideal application domain for such multi-agent systems due to its inherent complexity and need for diverse analytical perspectives. Digital scams typically combine multiple deceptive elements — from social engineering tactics to technical manipulation — requiring different types of analysis. A multi-agent approach allows specialized agents to focus on distinct aspects while cooperating through structured dialogue to form comprehensive assessments. 29 | 30 | ### Architecture 31 | 32 |

33 | 34 |

35 |

Minerva: Multi-Agent LLM System Architecture

36 | 37 | Our architecture consists of seven specialized agents coordinated by a group manager that orchestrates interactions in a round-robin fashion. Operating on a publish-subscribe pattern, agents share a common message thread where they publish their findings - from OCR analysis and link verification to content examination, decision making, summarization, linguistic translation, and archiving. The system processes each multimodal message using Python methods, APIs, and database tools, with users submitting the input as a screenshot of a digital communication. 38 | 39 | ### Agents 40 | 41 | Our specialized agents are built on AutoGen v0.4.0: 42 | 43 | - `OCR Agent`: *Extracts text from an image using pytesseract or the LLM-Vision capabilities.* 44 | - `Link Checker`: *Verifies the legitimacy of URLs using Google SafeBrowsing API.* 45 | - `Content Analyst`: *Analyzes the extracted text for scam patterns.* 46 | - `Decision Maker`: *Synthesizes the analyses and makes final determination.* 47 | - `Summarizer`: *Generates a summary of the final determination.* 48 | - `Language Translation Specialist`: *Translates the summary to the user language.* 49 | - `Data Storage Agent`: *Store the extracted text, assessment summary, determination (1, 0) and confidence level [1-5] in a sqlite database* 50 | 51 | with system prompts defined in [./config/agents.yaml](./config/agents.yaml). 52 | 53 | In this example, we show how to create the `Link Checker`, `Language Translation` and `Data Storage` Agents: 54 | 55 | ``` 56 | link_checker_agent: 57 | assignment: > 58 | You are an Internet link checker. Your role is to: 59 | 1. Check the extracted text for any URLs 60 | 2. Verify the legitimacy of the URLs using your registered function 61 | 62 | language_translation_agent: 63 | assignment: > 64 | You are a language translation specialist. Your role is to: 65 | 1. Infer the languge of the text extracted from the image, this is the user language 66 | 2. If the user language is different than english, translate your summary into the user language 67 | 68 | data_storage_agent: 69 | assignment: > 70 | You are a data storage specialist. 71 | Your role is to store using the registered function the following fields: 72 | - extracted text 73 | - summary 74 | - final determination ('1' for scam, '0' for not_scam) 75 | - confidence level on your final determination in a scale from 1-5 [...] 76 | All the fields are mandatory 77 | ``` 78 | 79 | ```python 80 | agents = [] 81 | 82 | agents.append(ToolUseAssistantAgent( 83 | name="Link_Checker", 84 | description="Checks if a Link is safe", 85 | system_message=self.config['link_checker_agent']['assignment'], 86 | model_client=self.model, 87 | registered_tools=[url_checker_tool] 88 | )) 89 | 90 | agents.append(ToolUseAssistantAgent( 91 | name="DataStorage_Agent", 92 | description="Store the extracted text, summary, determination (1, 0) and confidence level (high, medium low) in a database", 93 | system_message=self.config['data_storage_agent']['assignment'], 94 | model_client=self.model, 95 | registered_tools=[db_connector_tool] 96 | )) 97 | 98 | agents.append(AssistantAgent( 99 | name="Language_Translation_Agent", 100 | description="Translate the summary to the user language", 101 | system_message=self.config['language_translation_agent']['assignment'], 102 | model_client=self.model 103 | )) 104 | ``` 105 | Note that `Link Checker` and `Data Storage` are defined as `ToolUseAssistantAgent` as they have access to the tools set in the `registed_tools` parameter, whereas `Language Translation` is an `AssitantAgent` relying on the LLM capabilities to carry out its task. 106 | 107 | ### Tools 108 | 109 | We have decoupled the implementation of our tools into a specific `./tools` module: 110 | 111 | - `db_connector.py`: Manages storing and retrieval of analysis results into a `SQLite` database. 112 | - `image_ocr.py`: Handles OCR (Optical Character Recognition). 113 | - `url_checker.py`: Implements "MALWARE", "SOCIAL_ENGINEERING" "UNWANTED_SOFTWARE", "POTENTIALLY_HARMFUL_APPLICATION" checks for extracted URLs using [Google's SafeBrowsing API](https://developers.google.com/safe-browsing/v4/). 114 | - `formatter.py`: This is a non-agentic tool that formats `TaskResult` and `Response` messages into a string format for streaming the workflow steps into the user interface. 115 | 116 | ### Orchestration 117 | 118 | Our agentic team is created as a `RoundRobinGroupChat`: 119 | 120 | ```python 121 | def create_team(self) -> RoundRobinGroupChat: 122 | """Create a team of agents that work together in Round Robin fashion""" 123 | termination = TextMentionTermination("NO_TEXT_FOUND") 124 | 125 | return RoundRobinGroupChat( 126 | self.agents, 127 | max_turns=7, 128 | termination_condition=termination 129 | ) 130 | 131 | async def reset(self): 132 | """Reset team state""" 133 | await self.team.reset() 134 | ``` 135 | 136 | This defines the following workflow: 137 | - agents cooperate in a round-robin sequence, with each agent performing its specialized task 138 | - the workflow terminates either when all agents complete their tasks or when the OCR specialist cannot extract text from the provided image 139 | - the team state resets after each cycle to ensure independence between detection tasks 140 | 141 | ### Application 142 | 143 | MINERVA scam detection system can be tested through a web interface at https://huggingface.co/spaces/dcarpintero/minerva. This demo supports messages in multiple languages and includes prepared examples for evaluation: 144 | 145 |

146 | 147 |

148 |

Minerva: Analysis with Multi-Lingual Messages

149 | 150 | ### Evaluation 151 | 152 | #### Dataset Creation 153 | 154 | Our evaluation methodology is based on a synthetic dataset developed in two phases. First, we prompted Claude 3.5 Sonnet to generate a hierarchical taxonomy of scam categories, informed by academic research provided in context. The taxonomy comprises five main categories: financial, commerce, authority impersonation, rewards and opportunities, and customer service deception. Using this framework, we generated a balanced dataset with two deceptive messages and one legitimate (control) message at topic level. 155 | 156 | #### Results 157 | 158 | Our initial evaluation compared MINERVA against a prompt baseline (temperature=0.0, seed=42) using the same model version (gpt-4o-mini). Results show that our multi-agent approach achieves higher accuracy (88.3% vs. 69.5%) while requiring additional processing time (20.7 vs. 0.5 seconds). Agentic performance varies across categories, with particularly strong accuracy in service-related scam (~100%) and commerce (~93%). A category-level analysis further revealed that while both approaches reflect similar confidence levels (4.62 vs. 4.68 on a 1-5 scale) on all categories, our system's missed predictions primarily occur in the control group of specific categories. This might suggest that the model tends to express a more defensive assessment in certain contexts such as opportunity/rewards and authority impersonation. 159 | 160 |

161 | 162 |

163 | 164 |

165 | 166 |

167 | 168 | ### Societal Impact 169 | 170 | Beyond improving scam detection accuracy and reducing administrative overhead, this project aims to deliver broader societal benefits. By providing users with detailed, well-reasoned explanations of detected scams, MINERVA enhances digital literacy and builds trust in AI-powered safety tools. Additionally, the archiving task of this agentic workflow enables to create an open, anonymized dataset of categorized scam patterns. This dataset would serve, at least, two purposes: (i) enabling fine-tuning of models with current scam patterns, and (ii) providing insights for law enforcement and cybersecurity professionals to understand emerging threats and attack vectors. 171 | 172 | ### References 173 | 174 | - [1] [FBI's Internet Crime Complaint Center (IC3), 2023 Internet Crime Report](https://www.ic3.gov/AnnualReport/Reports/2023_IC3Report.pdf) 175 | - [2] [AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation](https://arxiv.org/abs/2308.08155) 176 | 177 | ### Author 178 | 179 | Diego Carpintero (https://github.com/dcarpintero) 180 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /evals/experiments/scam.all.categories.csv: -------------------------------------------------------------------------------- 1 | message,is_scam,category,subcategory 2 | "URGENT: IRS FINAL NOTICE. You owe $4,875 in back taxes. Pay now at hxxp://irs-payment.xyz or face immediate arrest.",1,authority,government 3 | "Your Social Security number has been suspended due to suspicious activity. Call +1-SCAM-NUMBER immediately to prevent arrest.",1,authority,government 4 | "IRS: Your tax return has been processed. Track your refund at irs.gov/refunds using your filing info. Questions? Call 800-829-1040. The IRS never requests sensitive information by email.",0,authority,government 5 | "IMMIGRATION ALERT: Your visa status is irregular. Pay $999 penalty fee at hxxp://uscis-verify.net to avoid deportation.",1,authority,government 6 | "URGENT: Your Green Card application needs verification. Send copies of passport and SSN to visa@immigration-services.xyz",1,authority,government 7 | "USCIS: Your appointment is confirmed for Dec 21, 10AM at 26 Federal Plaza, NYC. Bring confirmation notice #USC789012 and required documents listed on uscis.gov/appointments.",0,authority,government 8 | "POLICE NOTICE: Warrant issued for your arrest. Pay fine of $750 via Bitcoin to cancel warrant. Contact officer@police-dept.net",1,authority,government 9 | "FINAL WARNING: Court appearance mandatory! Pay $500 processing fee at hxxp://court-services.xyz to dismiss charges.",1,authority,government 10 | "County Court Notice: You are summoned for jury duty starting Jan 15, 2024. Register at courts.state.gov/jury or call 555-0123. No fees required for jury service.",0,authority,government 11 | "IT SECURITY ALERT: Your company email will be suspended. Verify at hxxp://corporate-email-verify.net within 30 minutes.",1,authority,corporate 12 | "URGENT: CEO requests immediate wire transfer of $9,850. Reply with confirmation. Confidential matter - do not discuss.",1,authority,corporate 13 | "Microsoft IT: Your password expires today. Update it at microsoft.com/account or through Microsoft Authenticator app. We never ask for passwords via email.",0,authority,corporate 14 | "HR NOTICE: Employee benefits suspended. Update details at hxxp://hr-benefits.xyz to restore coverage immediately.",1,authority,corporate 15 | "Payroll Processing Error: Send bank details to payroll@company-verify.net for immediate correction of salary.",1,authority,corporate 16 | "Workday: Your W-2 form for 2023 is now available. Access securely at workday.com/benefits. For technical support, contact your HR department at extension 5545.",0,authority,corporate 17 | "VENDOR PAYMENT ALERT: New banking details for invoice processing. Update at hxxp://vendor-verify.net immediately.",1,authority,corporate 18 | "Urgent: CFO needs gift cards for client meeting. Buy $1,000 in Apple cards and send codes to cfo@company-gifts.xyz",1,authority,corporate 19 | "Adobe Security: Sign in attempt from new device detected. Review activity at adobe.com/security. Don't recognize this? Change your password at adobe.com/account.",0,authority,corporate 20 | "BANK SECURITY ALERT: Account frozen due to suspicious activity. Call +1-SCAM-NUMBER immediately to verify identity.",1,authority,financial_institutions 21 | "Your credit card has been charged $899. If unauthorized, click hxxp://bank-secure-verify.net to dispute.",1,authority,financial_institutions 22 | "Chase Fraud Prevention: Did you attempt a purchase of $523.87 at Amazon? Reply YES or NO. View card activity at chase.com. Never share your PIN or password.",0,authority,financial_institutions 23 | "IMMEDIATE ACTION: Credit score dropped 100 points. Pay $299 at hxxp://credit-restore.xyz to protect your score.",1,authority,financial_institutions 24 | "Insurance Claim Warning: Policy canceled unless verified. Send policy details to claims@insurance-verify.net now.",1,authority,financial_institutions 25 | "Geico: Your auto insurance payment of $143.50 is scheduled for Dec 20. Review or update payment info at geico.com/payment. Questions? Call 1-800-207-7847.",0,authority,financial_institutions 26 | "INVESTMENT ALERT: Portfolio requires immediate verification. Login at hxxp://investment-secure.net to prevent freeze.",1,authority,financial_institutions 27 | "Account Processing Error: Send bank statement to verify@payment-processor.xyz to avoid service interruption.",1,authority,financial_institutions 28 | "Fidelity: Unusual login attempt detected from new device. Verify it's you at fidelity.com/security. To report unauthorized access, call 800-544-6666.",0,authority,financial_institutions 29 | "Your Amazon order #8829 has been shipped! Track your package: hxxp://amaz0n-tracking.net. Click now to avoid delivery issues.",1,commerce,shopping 30 | "URGENT: Problem with your recent Amazon order. Verify your payment details at hxxp://amazon-secure-verify.com to avoid cancellation.",1,commerce,shopping 31 | "Amazon: Your order #112-7366944-3891234 has shipped and will arrive by 8pm today. Track your delivery at amazon.com/orders or via the Amazon app. Questions? Contact Amazon Customer Service.",0,commerce,shopping 32 | "Your Walmart order has been canceled due to payment issues. To restore order, update payment info: hxxp://walmart-secure.shop/restore",1,commerce,shopping 33 | "SHIPPING ALERT: Package delivery attempted. Final notice - click hxxp://delivery-status-verify.net to reschedule or package returns tomorrow.",1,commerce,shopping 34 | "FedEx: Your package from Best Buy will arrive on Dec 20 between 2-6 PM. Track at fedex.com with tracking number: 7891234567. Need to reschedule? Sign in to fedex.com/delivery-manager.",0,commerce,shopping 35 | "REFUND NOTICE: Your recent purchase refund of $89.99 is pending. Click hxxp://refund-secure.xyz and enter your card details to claim.",1,commerce,shopping 36 | "Your refund request was approved! To receive $143.50, send your bank details to refunds@shopping-returns.net",1,commerce,shopping 37 | "Target: Your refund of $67.89 for order #TGT-456789 has been processed to your original payment method. Refund will appear in 3-5 business days. View details at target.com/orders.",0,commerce,shopping 38 | "iPhone 14 Pro: 90% OFF! Only $199 today. Limited stock. Buy now: hxxp://cheap-phones-sale.net. Pay with Bitcoin for extra discount!",1,commerce,shopping 39 | "AUTHENTIC Ray-Ban sunglasses 85% OFF! Must sell today. Contact WhatsApp: +1-234-SCAM for exclusive deals.",1,commerce,shopping 40 | "Nike.com: Holiday Sale - Up to 50% off select styles. Shop now at nike.com/holiday-deals. Free shipping for Nike Members. Returns always free within 60 days.",0,commerce,shopping 41 | "URGENT: Your eBay account needs verification. Suspicious activity detected. Verify at hxxp://ebay-secure-verify.net to avoid suspension.",1,commerce,marketplace 42 | "Hi, I'm interested in your item. I'll pay extra $100 if you accept. Send your PayPal to buyer.88751@mail.net for immediate payment.",1,commerce,marketplace 43 | "eBay: New message about 'Vintage Camera' from buyer_john. View and respond securely at ebay.com/messages. Remember: Always complete transactions on eBay for purchase protection.",0,commerce,marketplace 44 | "Your Airbnb booking payment failed. Update payment within 2hrs: hxxp://airbnb-payments.site or lose reservation.",1,commerce,marketplace 45 | "SPECIAL OFFER: Luxury apartment, 80% below market rate. Send security deposit via Western Union to secure. Contact +44-SCAM-NUMBER.",1,commerce,marketplace 46 | "Airbnb: Booking confirmed for 'Mountain View Cottage'. Check-in Dec 24, 3PM. Access all details at airbnb.com/trips. Questions? Message your host through the Airbnb app.",0,commerce,marketplace 47 | "TICKET ALERT: Concert tickets 50% off! Limited time offer. Pay through hxxp://ticket-secure-pay.net using bank transfer.",1,commerce,marketplace 48 | "2 VIP concert tickets for sale! Transfer money to my personal account for immediate delivery. WhatsApp +1-SCAM for details.",1,commerce,marketplace 49 | "Ticketmaster: Your order for 'Taylor Swift | The Eras Tour' is confirmed. Access tickets at ticketmaster.com/mybehavior or via the Ticketmaster app. Beware of resale scams - only buy from verified sources.",0,commerce,marketplace 50 | "URGENT INVOICE: Your company owes $2,487 for web services. Pay within 24hrs to avoid legal action: hxxp://invoice-pay-secure.net",1,commerce,business_operations 51 | "Please update your vendor payment information immediately. New account details: SWIFT: SCAM123. Reply with confirmation.",1,commerce,business_operations 52 | "Adobe: Your invoice #AD-456789 for Creative Cloud Business ($52.99) is now available. View and pay securely at adobe.com/account. Due date: Jan 1, 2024. Questions? Contact Adobe Business Support.",0,commerce,business_operations 53 | "FINAL NOTICE: Your business listing will be removed. Pay $299 today: hxxp://business-listing-renew.com to maintain visibility.",1,commerce,business_operations 54 | "Your Yellow Pages listing expires in 24hrs! Send $399 via Bitcoin to fast-track renewal. Contact admin@yp-renewal.net",1,commerce,business_operations 55 | "Google Business Profile: Your business verification is complete for 'Main Street Cafe'. Manage your listing at business.google.com. Need help? Visit support.google.com/business.",0,commerce,business_operations 56 | "Office supplies order confirmed: $1,899. Must pay today. New supplier special offer. Bank details: hxxp://supply-order.net",1,commerce,business_operations 57 | "ATTENTION: Your toner cartridge order ($799) will be auto-billed. To cancel, send company details to orders@supply-scam.com",1,commerce,business_operations 58 | "Staples Business Advantage: Your supply order #STP-789012 totaling $234.56 will ship on Dec 19. View order details and track shipment at staplesadvantage.com. Questions? Call 1-800-3STAPLE.",0,commerce,business_operations 59 | "WARNING: Virus detected on your Windows PC! Call our Microsoft support team immediately at 1-888-SCAM to prevent data loss. $299 for immediate fix.",1,customer_service,technical_support 60 | "Your computer has been locked! Multiple viruses found. Call Apple Security Team +1-SCAM-NUMBER now or risk permanent damage.",1,customer_service,technical_support 61 | "Microsoft Security Alert: To check if your device is affected by recent malware attacks, run Windows Security scan: Start > Settings > Windows Security. Never share remote access unless you initiated contact. Questions? Visit microsoft.com/security.",0,customer_service,technical_support 62 | "ALERT: Your antivirus subscription expires in 2 hours! Renew now at hxxp://norton-security-renewal.net to prevent infection.",1,customer_service,technical_support 63 | "Your system is infected with 13 viruses! Download our security tool from hxxp://antivirus-protect.xyz. Credit card required.",1,customer_service,technical_support 64 | "Norton: Your security subscription will renew automatically on Jan 15, 2024 ($84.99). Manage your subscription at norton.com/mynorton. Questions? Call 1-855-815-2726.",0,customer_service,technical_support 65 | "Your cloud storage is 99% full! Urgent action required: Click hxxp://cloud-storage-verify.net to prevent data loss.",1,customer_service,technical_support 66 | "CRITICAL: Google Drive storage breach detected. Verify account at hxxp://gdrive-secure.com to protect files.",1,customer_service,technical_support 67 | "Google Drive: You're using 14.8GB of 15GB storage (98%). Upgrade your storage at drive.google.com/storage or manage your files at drive.google.com. Learn more about storage options at support.google.com.",0,customer_service,technical_support 68 | "LEGAL NOTICE: Your case requires immediate attention. Pay $500 retainer fee via Bitcoin to hxxp://legal-services.net for representation.",1,customer_service,professional_services 69 | "Urgent Legal Matter: Settlement pending in your favor. Send $299 processing fee to claim. Contact legal@settlement-claim.net",1,customer_service,professional_services 70 | "Smith & Associates Law Firm: Your consultation is confirmed for Dec 20, 2:00 PM at our downtown office. Free 30-minute initial consultation. Directions and parking info at smithlaw.com/contact. Questions? Call 555-0123.",0,customer_service,professional_services 71 | "INSURANCE ALERT: Your policy will be canceled! Update payment info at hxxp://insurance-verify.net within 6 hours.",1,customer_service,professional_services 72 | "Final Notice: Your health insurance claim needs verification. Send medical history to claims@health-verify.net immediately.",1,customer_service,professional_services 73 | "State Farm: Your auto insurance payment of $143.50 is due Dec 25. Pay securely at statefarm.com or through our mobile app. Questions about your policy? Call 1-800-STATE-FARM.",0,customer_service,professional_services 74 | "URGENT: Your medical records show eligibility for $150,000 settlement. Pay $199 processing fee: hxxp://medical-claim.net",1,customer_service,professional_services 75 | "Immediate Action Required: Send $299 to register for premium healthcare services. WhatsApp +1-SCAM-NUMBER for details.",1,customer_service,professional_services 76 | "Kaiser Permanente: Your prescription is ready for pickup at Main Street Pharmacy. Pay copay of $15 at pickup. Questions? Call your pharmacy at 555-0177 or visit kp.org/prescriptions.",0,customer_service,professional_services 77 | "POWER SHUTOFF NOTICE: Pay $499 immediately or service disconnects today. Click hxxp://utility-pay.net to restore.",1,customer_service,utility_services 78 | "URGENT: Your electricity will be disconnected in 2 hours. Call +1-SCAM-NUMBER now to prevent shutoff.",1,customer_service,utility_services 79 | "PG&E: Your bill of $127.86 is due Dec 22. Pay online at pge.com or by phone at 1-877-704-8470. Payment assistance programs available at pge.com/assistance. We never demand immediate payment or threaten immediate disconnection.",0,customer_service,utility_services 80 | "INTERNET SERVICE ALERT: Network breach detected! Verify account at hxxp://internet-secure.net or face disconnection.",1,customer_service,utility_services 81 | "Your WiFi has been compromised! Call Internet Security Team +1-SCAM-NUMBER immediately or service will be terminated.",1,customer_service,utility_services 82 | "Comcast: Scheduled maintenance in your area on Dec 21, 2-5 AM. No action needed. Track status at xfinity.com/status. Mobile hotspots available for Xfinity Internet customers during maintenance.",0,customer_service,utility_services 83 | "FINAL WARNING: Water service will be disconnected! Pay $399 through hxxp://water-utility-pay.net to prevent shutoff.",1,customer_service,utility_services 84 | "Your water meter shows unusual usage. Verify account at hxxp://water-verify.net or face penalties.",1,customer_service,utility_services 85 | "City Water Department: Your monthly bill of $78.92 is ready. Due date: Jan 5. Pay online at citywater.gov or at any authorized payment location. Questions? Call 555-0148 during business hours.",0,customer_service,utility_services 86 | "ALERT: Unauthorized access detected on your account. Click hxxp://secur1ty-bank-verify.com to verify your identity immediately. Your account will be suspended in 2 hours.",1,financial,banking_and_payment 87 | "Dear customer, we detected unusual activity on your account. Please verify by texting back your online banking username and password.",1,financial,banking_and_payment 88 | "Revolut: We noticed a charge of $458.23 on your card ending in 7580 that appears unusual for your spending pattern. For your security, you might review this transaction in your Revolut app or at https://revolut.com. Never share your PIN or password with anyone.",0,financial,banking_and_payment 89 | "WARNING: $2,499 transaction pending at Amazon. If not you, click hxxp://chase-secure02.net/verify to cancel transaction immediately!",1,financial,banking_and_payment 90 | "Suspicious withdrawal attempt ($850) detected on your card ending 4355. Reply YES to confirm or NO to block.",1,financial,banking_and_payment 91 | "Revolut: You just made an online payment of EUR 64.47 with your card ending in 7587.",0,financial,banking_and_payment 92 | "URGENT: Your payment of $1,249 has been processed. To cancel this wire transfer, call our security team at +1-855-SCAM immediately.",1,financial,banking_and_payment 93 | "Your wire transfer of $5,000 to J. Smith was declined. To complete, verify at hxxp://wire-secure-verify.net with your account details.",1,financial,banking_and_payment 94 | "Hey Ana! Revolut is offering €100 for new customers right now. Just let me know if you're interested!",0,financial,banking_and_payment 95 | "CRITICAL: Invest in our new crypto fund! Guaranteed 500% returns in 30 days. Send Bitcoin to secure your spot: bc1qxy2kgdygjrsqtzq2n0yrf2493p83kkfjhx0wlh",1,financial,investment 96 | "🚀 LAST CHANCE! Join our exclusive forex trading group. Turn $100 into $5000 weekly. Contact WhatsApp: +44-SCAM-NUMBER",1,financial,investment 97 | "Fidelity Investments: Your mutual fund purchase of $5,000 into FXAIX is confirmed. Settlement date: Dec 20. View your updated portfolio at fidelity.com/investments. For support with your account, visit fidelity.com/customer-service.",0,financial,investment 98 | "BINARY OPTIONS OPPORTUNITY: Start earning $1000/day with our automated trading system. Pay $299 to join: hxxp://binary-profits.xyz",1,financial,investment 99 | "Get rich with our AI trading bot! 100% success rate. Send $500 worth of ETH to activate: 0x742d35Cc6634C0532925a3b844Bc454e4438f44e",1,financial,investment 100 | "Hi Sarah! Just saw Vanguard is offering a great deal on their index funds right now - no minimum investment required!",0,financial,investment 101 | "🔥 NEW CRYPTO ALERT! BitMaxCoin launching tomorrow. Buy now at $0.01, guaranteed to hit $1! Register: hxxp://bitmaxcoin.net",1,financial,investment 102 | "Exclusive Forex Signal Group! Copy our trades for guaranteed profits. Join now: hxxp://forex-signals.xyz. Only $199/month.",1,financial,investment 103 | "Hi Thomas, just saw that Schwab dropped their commission fees to zero! Might be worth checking out if you're still paying fees at E*TRADE.",0,financial,investment 104 | "LOAN APPROVED! Get $50,000 instantly. Send $500 processing fee via Western Union to release funds. Reply YES to proceed.",1,financial,lending 105 | "Pre-approved for $25,000! No credit check needed. Send SSN and bank details to hxxp://quickloans-secure.xyz",1,financial,lending 106 | "Capital One: Your personal loan is confirmed. Check more personalized offers at capitalize.com.",0,financial,lending 107 | "DEBT CONSOLIDATION ALERT: Pay off all debts today! Send $199 via Bitcoin to start: hxxp://debt-freedom.xyz",1,financial,lending 108 | "Your credit card debts can be cleared for pennies on the dollar! Contact debthelp@scam.net for secret method.",1,financial,lending 109 | "Hey David! Just FYI - my credit union is offering really low rates on home loans right now. 4.5% for 30-year fixed. Worth checking out if you're still house hunting. Let me know if you want the details!",0,financial,lending 110 | "STUDENT LOAN FORGIVENESS PROGRAM! Pay $599 to eliminate all student debt. Must act now!",1,financial,lending 111 | "ATTENTION: Qualify for instant loan! Send photos of your credit cards to loans@fastcash.xyz",1,financial,lending 112 | "Bank of America has a special rate on personal loans this month. My colleague just got approved for 6.5% APR.",0,financial,lending 113 | "URGENT Work From Home Opportunity! Make $5,000/week guaranteed. No experience needed. Contact recruiter@job-offer.xyz for immediate start.",1,opportunity_rewards,employment 114 | "Congratulations! You've been selected for a remote position. Send $199 for background check to start earning $900/day!",1,opportunity_rewards,employment 115 | "Amazon Careers: Thank you for applying to Customer Service Representative (Job ID: CS4527). Salary range: $18-22/hr + benefits. Next step: Complete your assessment at amazon.jobs/assess/CS4527. Questions? Contact recruiting@amazon.com. Amazon never charges fees for employment.",0,opportunity_rewards,employment 116 | "HIRING NOW: Data Entry Position - $45/hr. Send your bank details to jobs@data-entry-work.net for immediate payment setup.",1,opportunity_rewards,employment 117 | "Work From Home Opportunity! Process payments for our company. Use your bank account, earn 10% commission. Contact +1-SCAM-NUMBER.",1,opportunity_rewards,employment 118 | "Microsoft Careers: Your application for Software Engineer (REQ58769) is under review. Salary: $125,000-$165,000/year + benefits. Schedule your technical interview at careers.microsoft.com/interview/REQ58769. Visit careers.microsoft.com/applicant-faq for common questions.",0,opportunity_rewards,employment 119 | "URGENT: Your resume was selected! Pay $299 training fee via Bitcoin to secure high-paying remote position. Start tomorrow!",1,opportunity_rewards,employment 120 | "Make $800 daily from home! Small investment required ($450) for training materials. WhatsApp +44-SCAM-NUMBER for details.",1,opportunity_rewards,employment 121 | "Google Hiring Event: Join our Virtual Career Fair on Dec 20, 2-5PM EST. Roles: Cloud Solutions Engineer ($130-180K). Register at careers.google.com/events/dec2023. No payment required. All interviews conducted through official Google Meet platform.",0,opportunity_rewards,employment 122 | "CONGRATULATIONS! You won $1,000,000 in the Instagram Lottery! Send $500 processing fee to claim: hxxp://prize-claim.net",1,opportunity_rewards,prizes 123 | "YOU'RE A WINNER! Your email was randomly selected. Send passport copy and bank details to claim@lottery-win.xyz",1,opportunity_rewards,prizes 124 | "Target: Congratulations! You've earned a $50 Target Gift Card through our Circle Rewards program. Redeem in the Target app or at target.com/circle/rewards using code TGT-RF456789. Valid through 3/31/24. Help: 1-800-591-3869. Never pay fees to claim rewards.",0,opportunity_rewards,prizes 125 | "XBOX Sweepstakes Winner! You won a new console. Pay $99 shipping fee at hxxp://xbox-prize.net to receive your prize.",1,opportunity_rewards,prizes 126 | "Amazon Raffle Winner! To receive your free iPhone 14, click hxxp://amazon-prizes.xyz and enter your credit card for verification.",1,opportunity_rewards,prizes 127 | "Walmart: You've won our monthly customer survey drawing! Claim your $100 Walmart Gift Card at any store's customer service desk with ID and confirmation code WM23789456. Questions? Visit walmart.com/giftcard-support or call 1-800-925-6278.",0,opportunity_rewards,prizes 128 | "GAME SHOW WINNER! Pay $299 processing fee via Western Union to claim your $50,000 prize. Contact prize@gameshow-win.net",1,opportunity_rewards,prizes 129 | "Survey Reward: Complete survey and pay $10 fee to receive your free iPad! Click hxxp://survey-rewards.xyz now!",1,opportunity_rewards,prizes 130 | "Best Buy Rewards: Thanks for being an elite member! Your $75 reward certificate is available in your Best Buy account. Redeem online at bestbuy.com/rewards or in stores with your member ID. Certificate expires 1/31/24. Visit bestbuy.com/rewards-faq for details.",0,opportunity_rewards,prizes 131 | "INHERITANCE NOTICE: You are beneficiary to $4.5M USD. Send $1,500 legal fee to claim. Contact barrister@inheritance.net",1,opportunity_rewards,inheritance 132 | "URGENT: Your unknown relative left you $2.8M. Transfer $750 via Bitcoin to start claim process. Reply within 24hrs!",1,opportunity_rewards,inheritance 133 | "Charles Schwab Estate Services: Our records indicate you may be a beneficiary in the Robert Johnson estate. Schedule a free consultation at schwab.com/estates or call 800-435-4000. Note: Schwab never requests upfront fees or personal information by email.",0,opportunity_rewards,inheritance 134 | "CONFIDENTIAL: Late oil merchant left you $5.2M. Send passport and bank details to claim@inheritance-lawyer.xyz",1,opportunity_rewards,inheritance 135 | "Your inheritance claim needs processing! Pay $2,999 to attorney via Western Union. Contact +1-SCAM-NUMBER immediately.",1,opportunity_rewards,inheritance 136 | "JP Morgan Estate Planning: Following up on your inquiry about the Thompson Family Trust distribution. Please schedule an appointment at chase.com/private-bank/estates or visit your local Chase branch. For questions, call Private Client Services: 1-866-265-1727.",0,opportunity_rewards,inheritance 137 | "LAST CHANCE: Unclaimed inheritance of $3.7M expires today! Send $500 via MoneyGram to secure funds. Urgent!",1,opportunity_rewards,inheritance 138 | "Your deceased relative's estate! Send $899 administrative fee to hxxp://estate-claims.net within 12 hours.",1,opportunity_rewards,inheritance 139 | "Vanguard Beneficiary Services: Your inherited IRA documentation is ready for review. Log in to vanguard.com/inheritedira to schedule a consultation. Questions? Call 800-662-2739. No fees required for inherited account services.",0,opportunity_rewards,inheritance --------------------------------------------------------------------------------