├── fedotllm ├── __init__.py ├── agents │ ├── __init__.py │ ├── automl │ │ ├── __init__.py │ │ ├── templates │ │ │ ├── fedot_predict.py │ │ │ ├── fedot_evaluate.py │ │ │ ├── fedot_train.py │ │ │ ├── skeleton-simple.py │ │ │ ├── load_template.py │ │ │ └── skeleton-complex.py │ │ ├── state.py │ │ ├── structured.py │ │ ├── automl.py │ │ └── nodes.py │ ├── base.py │ ├── utils.py │ ├── scraper.py │ └── translator.py ├── prompts │ ├── __init__.py │ ├── utils.py │ └── automl.py ├── constants.py ├── log.py ├── enviroments.py ├── llm.py ├── data.py └── main.py ├── graph.png ├── image ├── NPS.png ├── lads.jpg ├── table1.png ├── table2.png ├── AutoDS-UI.png └── architecture.png ├── .env_example ├── benchmark └── benchmark_results.csv ├── app_components ├── media_utils.py ├── data_handlers.py ├── session_state.py ├── ui_components.py ├── agent_handler.py └── fragments.py ├── utils ├── config │ ├── loader.py │ └── schema.py └── llm_factory.py ├── docker-compose.yml ├── graph ├── prompts.py ├── state.py ├── lightautoml_template.py ├── code_executor_node.py ├── graph.py ├── llm_nodes.py └── prompts_en.py ├── Dockerfile ├── config.yml ├── environment.yml ├── LICENSE ├── app.py ├── README.md ├── .gitignore └── requirements.txt /fedotllm/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sb-ai-lab/LADS/main/graph.png -------------------------------------------------------------------------------- /image/NPS.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sb-ai-lab/LADS/main/image/NPS.png -------------------------------------------------------------------------------- /image/lads.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sb-ai-lab/LADS/main/image/lads.jpg -------------------------------------------------------------------------------- /image/table1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sb-ai-lab/LADS/main/image/table1.png -------------------------------------------------------------------------------- /image/table2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sb-ai-lab/LADS/main/image/table2.png -------------------------------------------------------------------------------- /image/AutoDS-UI.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sb-ai-lab/LADS/main/image/AutoDS-UI.png -------------------------------------------------------------------------------- /fedotllm/agents/__init__.py: -------------------------------------------------------------------------------- 1 | # This file makes the 'agents' directory a Python package. 2 | -------------------------------------------------------------------------------- /image/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sb-ai-lab/LADS/main/image/architecture.png -------------------------------------------------------------------------------- /.env_example: -------------------------------------------------------------------------------- 1 | GIGACHAT_API_TOKEN='' 2 | OPENAI_API_KEY='' 3 | 4 | LANGFUSE_PUBLIC_KEY='' 5 | LANGFUSE_SECRET_KEY='' -------------------------------------------------------------------------------- /benchmark/benchmark_results.csv: -------------------------------------------------------------------------------- 1 | id,LogisticRegression,LGBM,Tabular NN,ds_agent_history,our_data 2 | employee_promotion,0.831,0.849,0.823,0.8653846153846153,0.845 3 | -------------------------------------------------------------------------------- /fedotllm/agents/automl/__init__.py: -------------------------------------------------------------------------------- 1 | from fedotllm.agents.automl.automl import AutoMLAgent, AutoMLAgentState 2 | 3 | __all__ = ["AutoMLAgent", "AutoMLAgentState"] 4 | -------------------------------------------------------------------------------- /fedotllm/prompts/__init__.py: -------------------------------------------------------------------------------- 1 | import fedotllm.prompts.automl as automl 2 | import fedotllm.prompts.utils as utils 3 | 4 | __all__ = ["automl", "researcher", "supervisor", "utils"] 5 | -------------------------------------------------------------------------------- /app_components/media_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import base64 3 | import streamlit as st 4 | 5 | 6 | def get_base64_encoded_image(image_path: str) -> str: 7 | with open(image_path, "rb") as image_file: 8 | return base64.b64encode(image_file.read()).decode("utf-8") 9 | 10 | 11 | -------------------------------------------------------------------------------- /utils/config/loader.py: -------------------------------------------------------------------------------- 1 | import yaml 2 | from pathlib import Path 3 | from .schema import AppConfig, SecretsConfig 4 | 5 | 6 | def load_config() -> AppConfig: 7 | config_path = Path("config.yml") 8 | with config_path.open("r", encoding="utf-8") as f: 9 | data = yaml.safe_load(f) 10 | 11 | secrets = SecretsConfig() 12 | config = AppConfig(**data, secrets=secrets) 13 | return config.inject_all_secrets() 14 | -------------------------------------------------------------------------------- /fedotllm/agents/automl/templates/fedot_predict.py: -------------------------------------------------------------------------------- 1 | def automl_predict(model: Fedot, features: np.ndarray | pd.DataFrame | pd.Series) -> np.ndarray: 2 | if isinstance(features, (pd.DataFrame, pd.Series)): 3 | features = features.to_numpy() 4 | input_data = InputData.from_numpy(features, None, task=Task({%problem%})) 5 | predictions = model.{%predict_method%} 6 | print(f"Predictions shape: {predictions.shape}") 7 | return predictions -------------------------------------------------------------------------------- /fedotllm/agents/base.py: -------------------------------------------------------------------------------- 1 | from typing import Annotated, TypedDict 2 | 3 | from langchain_core.messages import AnyMessage 4 | from langgraph.graph.message import add_messages 5 | from langgraph.graph.state import CompiledStateGraph 6 | 7 | 8 | class Agent: 9 | def create_graph(self) -> CompiledStateGraph: 10 | raise NotImplementedError 11 | 12 | 13 | class FedotLLMAgentState(TypedDict): 14 | messages: Annotated[list[AnyMessage], add_messages] 15 | -------------------------------------------------------------------------------- /fedotllm/agents/automl/state.py: -------------------------------------------------------------------------------- 1 | from fedotllm.agents.automl.structured import FedotConfig 2 | from fedotllm.agents.base import FedotLLMAgentState 3 | from fedotllm.enviroments import Observation 4 | 5 | 6 | class AutoMLAgentState(FedotLLMAgentState): 7 | reflection: str 8 | fedot_config: FedotConfig 9 | skeleton: str 10 | raw_code: str | None 11 | code: str | None 12 | observation: Observation | None 13 | fix_attempts: int 14 | metrics: str 15 | pipeline: str 16 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | ds-agent: 3 | build: 4 | context: . 5 | dockerfile: Dockerfile 6 | ports: 7 | - "8501:8501" 8 | volumes: 9 | - .:/app 10 | - ./datasets:/app/datasets 11 | environment: 12 | - LANGFUSE_PUBLIC_KEY=${LANGFUSE_PUBLIC_KEY} 13 | - LANGFUSE_SECRET_KEY=${LANGFUSE_SECRET_KEY} 14 | - LANGFUSE_HOST=${LANGFUSE_HOST} 15 | - LANGFUSE_USER=${LANGFUSE_USER} 16 | - LANGFUSE_SESSION=${LANGFUSE_SESSION} 17 | - GIGACHAT_API_TOKEN=${GIGACHAT_API_TOKEN} -------------------------------------------------------------------------------- /fedotllm/constants.py: -------------------------------------------------------------------------------- 1 | # File formats 2 | CSV_SUFFIXES = [".csv"] 3 | PARQUET_SUFFIXES = [".parquet", ".pq"] 4 | EXCEL_SUFFIXES = [".xls", ".xlsx", ".xlsm", ".xlsb", ".odf", ".ods", ".odt"] 5 | ARFF_SUFFIXES = [".arff"] 6 | DATASET_EXTENSIONS = [*CSV_SUFFIXES, *PARQUET_SUFFIXES, *EXCEL_SUFFIXES, *ARFF_SUFFIXES] 7 | 8 | # Initial Session state 9 | DEFAULT_SESSION_VALUES = { 10 | "llm": {}, 11 | "uploaded_files": {}, 12 | "lang": "en", 13 | "messages": [ 14 | { 15 | "role": "assistant", 16 | "content": "Hello! Pick a model, upload the dataset files and send me the task description.", 17 | } 18 | ], 19 | "prev_graph": None, 20 | "output_filename": None, 21 | "output_file": None, 22 | "task_description": None, 23 | "task_running": False, 24 | } 25 | -------------------------------------------------------------------------------- /graph/prompts.py: -------------------------------------------------------------------------------- 1 | from utils.config.loader import load_config 2 | from graph.prompts_ru import GIGACHAT_PROMPTS_RU 3 | from graph.prompts_en import GIGACHAT_PROMPTS_EN 4 | from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder 5 | 6 | 7 | def load_prompt(prompt_name: str, model: str = 'gigachat') -> ChatPromptTemplate: 8 | messages = [] 9 | config = load_config() 10 | prompts = GIGACHAT_PROMPTS_RU if config.general.prompt_language == "ru" else GIGACHAT_PROMPTS_EN 11 | 12 | prompt_data = prompts[prompt_name] 13 | 14 | if 'system' in prompt_data: 15 | messages.append(("system", prompt_data['system'])) 16 | 17 | messages.append(MessagesPlaceholder("history", optional=True)) 18 | 19 | if 'user' in prompt_data: 20 | messages.append(("user", prompt_data['user'])) 21 | 22 | return ChatPromptTemplate(messages) 23 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM continuumio/miniconda3:latest 2 | 3 | WORKDIR /app 4 | 5 | # Copy environment file 6 | COPY environment.yml . 7 | 8 | # Create conda environment 9 | RUN conda env create -f environment.yml 10 | 11 | # Make RUN commands use the new environment 12 | SHELL ["conda", "run", "-n", "ds_agent", "/bin/bash", "-c"] 13 | 14 | # Copy project files (including protobuf directory) 15 | COPY . . 16 | 17 | # Generate protobuf classes 18 | RUN conda run -n ds_agent python3 -m grpc_tools.protoc -I protobuf \ 19 | --python_out=utils/salute_speech \ 20 | --grpc_python_out=utils/salute_speech \ 21 | protobuf/recognition.proto \ 22 | protobuf/storage.proto \ 23 | protobuf/task.proto \ 24 | protobuf/synthesis.proto 25 | 26 | # Set up the container's entry command 27 | EXPOSE 8501 28 | ENTRYPOINT ["conda", "run", "-n", "ds_agent", "streamlit", "run", "app.py", "--server.address=0.0.0.0"] -------------------------------------------------------------------------------- /config.yml: -------------------------------------------------------------------------------- 1 | # llm: 2 | # provider: gigachat 3 | # model_name: GigaChat-2-Max 4 | # verify_ssl: false 5 | # profanity_check: false 6 | # scope: GIGACHAT_API_CORP 7 | # timeout: 600 8 | 9 | llm: 10 | provider: openai 11 | model_name: gpt-4o 12 | base_url: # Add url 13 | 14 | langfuse: 15 | host: # Add host 16 | user: # Add user 17 | 18 | general: 19 | max_improvements: 5 20 | code_generation_config: 'local' 21 | recursion_limit: 50 22 | max_code_execution_time: 3000 23 | prompt_language: "en" 24 | 25 | fedot: 26 | provider: openai 27 | model_name: gpt-4o 28 | base_url: # Add url 29 | fix_tries: 2 30 | templates: 31 | code: "skeleton-simple.py" 32 | train: "fedot_train.py" 33 | evaluate: "fedot_evaluate.py" 34 | predict: "fedot_predict.py" 35 | predictor_init_kwargs: 36 | timeout: 1.0 37 | 38 | model_overrides: 39 | llm_code_generator_agent: 40 | provider: gigachat 41 | model_name: GigaChat-2-Max -------------------------------------------------------------------------------- /fedotllm/log.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | logger = logging.getLogger("FEDOTLLM") 4 | logger.setLevel(logging.DEBUG) # Changed from INFO to DEBUG to allow debug messages 5 | 6 | file_handler = logging.FileHandler("fedotllm.log", mode="w") 7 | file_handler.setLevel(logging.DEBUG) 8 | 9 | console_handler = logging.StreamHandler() 10 | console_handler.setLevel(logging.INFO) 11 | 12 | formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") 13 | file_handler.setFormatter(formatter) 14 | console_handler.setFormatter(formatter) 15 | 16 | logger.addHandler(file_handler) 17 | logger.addHandler(console_handler) 18 | 19 | if __name__ == "__main__": 20 | # Example usage: 21 | logger.info("This is an informational message.") 22 | logger.warning("This is a warning message.") 23 | logger.error("This is an error message.") 24 | logger.critical("This is a critical message.") 25 | 26 | print("Log messages written to fedotllm.log") 27 | -------------------------------------------------------------------------------- /fedotllm/agents/automl/templates/fedot_evaluate.py: -------------------------------------------------------------------------------- 1 | def evaluate_model(model: Fedot, test_features: np.ndarray | pd.DataFrame | pd.Series, test_target: np.ndarray | pd.DataFrame | pd.Series): 2 | if isinstance(test_features, pd.DataFrame) and isinstance(test_target, (pd.DataFrame, pd.Series)): 3 | input_data = InputData.from_dataframe(test_features, test_target, task=Task({%problem%})) 4 | elif isinstance(test_features, np.ndarray) and isinstance(test_target, np.ndarray): 5 | input_data = InputData.from_numpy(test_features, test_target, task=Task({%problem%})) 6 | else: 7 | raise ValueError("Unsupported data types for test_features and test_target. " 8 | "Expected pandas DataFrame and (DataFrame or Series), or numpy ndarray and numpy ndarray." 9 | f"Got: {type(test_features)} and {type(test_target)}") 10 | y_pred = model.{%predict_method%} 11 | print("Model metrics: ", model.get_metrics()) 12 | return model.get_metrics() -------------------------------------------------------------------------------- /graph/state.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | from typing import Annotated, Sequence, List, Optional 3 | from typing_extensions import TypedDict 4 | from langchain_core.messages import AnyMessage 5 | from langgraph.graph.message import add_messages 6 | from e2b_code_interpreter import Sandbox 7 | 8 | 9 | class AgentState(TypedDict): 10 | messages: Annotated[Sequence[AnyMessage], add_messages] 11 | task: str 12 | df: Optional[pd.DataFrame] 13 | df_train: Optional[pd.DataFrame] 14 | df_test: Optional[pd.DataFrame] 15 | df_name: str 16 | sandbox: Sandbox 17 | code_improvement_count: int 18 | current_node: str 19 | feedback: List[str] 20 | generated_code: str 21 | rephrased_plan: str 22 | code_results: str 23 | lama: bool 24 | test_split: bool 25 | improvements_code: List[str] 26 | human_understanding: List[str] 27 | code_generation_config: str 28 | train_code: str 29 | test_code: str 30 | test_df: Optional[pd.DataFrame] 31 | test_df_name: str 32 | -------------------------------------------------------------------------------- /utils/llm_factory.py: -------------------------------------------------------------------------------- 1 | from langchain_gigachat.chat_models import GigaChat 2 | from langchain_openai import ChatOpenAI 3 | 4 | 5 | def create_llm(node_name, config): 6 | 7 | llm_cfg = config.model_overrides.get(node_name) if config.model_overrides and node_name in config.model_overrides else config.llm 8 | 9 | if llm_cfg.provider == "gigachat": 10 | return GigaChat( 11 | credentials=llm_cfg.token.get_secret_value(), 12 | model=llm_cfg.model_name, 13 | scope=llm_cfg.scope, 14 | verify_ssl_certs=llm_cfg.verify_ssl, 15 | profanity_check=llm_cfg.profanity_check, 16 | timeout=llm_cfg.timeout 17 | ) 18 | if llm_cfg.provider == "openai": 19 | return ChatOpenAI( 20 | model_name=llm_cfg.model_name, 21 | openai_api_key=llm_cfg.token.get_secret_value(), 22 | base_url=llm_cfg.base_url, 23 | ) 24 | else: 25 | raise ValueError(f"Unknown LLM provider: {llm_cfg.provider}") 26 | -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- 1 | name: LightAutoDS 2 | channels: 3 | - defaults 4 | - https://repo.anaconda.com/pkgs/main 5 | - https://repo.anaconda.com/pkgs/r 6 | dependencies: 7 | - _libgcc_mutex=0.1=main 8 | - _openmp_mutex=5.1=1_gnu 9 | - bzip2=1.0.8=h5eee18b_6 10 | - ca-certificates=2025.2.25=h06a4308_0 11 | - expat=2.7.1=h6a678d5_0 12 | - ld_impl_linux-64=2.40=h12ee557_0 13 | - libffi=3.4.4=h6a678d5_1 14 | - libgcc-ng=11.2.0=h1234567_1 15 | - libgomp=11.2.0=h1234567_1 16 | - libstdcxx-ng=11.2.0=h1234567_1 17 | - libuuid=1.41.5=h5eee18b_0 18 | - libxcb=1.17.0=h9b100fa_0 19 | - ncurses=6.4=h6a678d5_0 20 | - openssl=3.0.16=h5eee18b_0 21 | - pip=25.1=pyhc872135_2 22 | - pthread-stubs=0.3=h0ce48e5_1 23 | - python=3.10.18=h1a3bd86_0 24 | - readline=8.2=h5eee18b_0 25 | - setuptools=78.1.1=py310h06a4308_0 26 | - sqlite=3.45.3=h5eee18b_0 27 | - tk=8.6.14=h993c535_1 28 | - tzdata=2025b=h04d1e81_0 29 | - wheel=0.45.1=py310h06a4308_0 30 | - xorg-libx11=1.8.12=h9b100fa_1 31 | - xorg-libxau=1.0.12=h9b100fa_0 32 | - xorg-libxdmcp=1.1.5=h9b100fa_0 33 | - xorg-xorgproto=2024.1=h5eee18b_1 34 | - xz=5.6.4=h5eee18b_1 35 | - zlib=1.2.13=h5eee18b_1 36 | -------------------------------------------------------------------------------- /fedotllm/enviroments.py: -------------------------------------------------------------------------------- 1 | import os 2 | import subprocess 3 | from pathlib import Path 4 | 5 | from pydantic import BaseModel, Field 6 | 7 | from fedotllm.log import logger 8 | 9 | 10 | class Observation(BaseModel): 11 | error: bool = Field(default=False) 12 | msg: str = Field(default="") 13 | stdout: str = Field(default="") 14 | stderr: str = Field(default="") 15 | 16 | 17 | def execute_code(path_to_run_code: Path) -> Observation: 18 | try: 19 | result = subprocess.run( 20 | ["python3", "-W", "ignore", path_to_run_code], 21 | capture_output=True, 22 | text=True, 23 | preexec_fn=os.setsid, 24 | ) 25 | logger.debug(f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}") 26 | return Observation( 27 | error=result.returncode != 0, stdout=result.stdout, stderr=result.stderr 28 | ) 29 | except Exception as e: 30 | stderr = f"An unexpected error occurred in the execution harness: {type(e).__name__}: {e}" 31 | logger.error( 32 | f"Unexpected error executing {path_to_run_code}: {e}", exc_info=True 33 | ) 34 | return Observation(error=True, stdout="", stderr=stderr) 35 | -------------------------------------------------------------------------------- /fedotllm/prompts/utils.py: -------------------------------------------------------------------------------- 1 | from typing import Type, TypeVar 2 | 3 | from pydantic import BaseModel 4 | 5 | T = TypeVar("T", bound=BaseModel) 6 | 7 | 8 | def structured_response(response_model: Type[T]): 9 | return f""" 10 | The output must be a JSON object equivalent to type ${response_model.__name__}, according to the following Pydantic definitions: 11 | ===== 12 | ``` 13 | {response_model.model_json_schema()} 14 | ``` 15 | ===== 16 | Important: 17 | 1. Return only valid JSON. No extra explanations, text, or comments. 18 | 2. Ensure that the output can be parsed by a JSON parser directly. 19 | 3. Do not include any non-JSON text or formatting outside the JSON object. 20 | 4. An example is \{{"": ""\}} 21 | """ 22 | 23 | 24 | def ai_assert_prompt(var1, var2, condition: str): 25 | return f""" 26 | You are an intelligent assertion function to evaluate conditions between two variables. 27 | The variables are: 28 | 1. var1: {var1} 29 | 2. var2: {var2} 30 | condition: {condition} 31 | ===== 32 | Important: 33 | 1. Return only true or false. No extra explanations, text, or comments 34 | 2. Ensure that the output can be parsed by a regex pattern: ^(true|false)$ 35 | 3. Do not include any text or formatting outside the true/false value 36 | 4. An example is true or false 37 | """ 38 | -------------------------------------------------------------------------------- /app_components/data_handlers.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import streamlit as st 3 | from io import StringIO, BytesIO 4 | 5 | SUPPORTED_FILE_TYPES = { 6 | 'csv': (pd.read_csv, lambda df, path: df.to_csv(path)), 7 | 'xlsx': (pd.read_excel, lambda df, path: df.to_excel(path)), 8 | 'xls': (pd.read_excel, lambda df, path: df.to_excel(path)), 9 | 'parquet': (pd.read_parquet, lambda df, path: df.to_parquet(path)) 10 | } 11 | 12 | 13 | def load_data(file_content: bytes, file_extension: str) -> pd.DataFrame: 14 | 15 | if file_extension.lower() not in SUPPORTED_FILE_TYPES: 16 | raise ValueError( 17 | f"Unsupported file type: {file_extension.lower()}. " 18 | f"Supported types are {', '.join(SUPPORTED_FILE_TYPES.keys())}." 19 | ) 20 | 21 | try: 22 | reader_func, _ = SUPPORTED_FILE_TYPES[file_extension.lower()] 23 | if file_extension.lower() == 'csv': 24 | df = reader_func(StringIO(file_content.decode("utf-8")), index_col=0) 25 | else: 26 | df = reader_func(BytesIO(file_content)) 27 | return df 28 | except Exception as e: 29 | st.error(f"Error loading data: {str(e)}") 30 | raise 31 | 32 | 33 | def save_file_to_disk(df: pd.DataFrame, file_name: str, file_extension: str) -> None: 34 | try: 35 | _, writer_func = SUPPORTED_FILE_TYPES[file_extension.lower()] 36 | writer_func(df, './datasets/'+file_name) 37 | except Exception as e: 38 | st.error(f"Error saving file to disk: {str(e)}") 39 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2025, ___ 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | 8 | 1. Redistributions of source code must retain the above copyright notice, this 9 | list of conditions and the following disclaimer. 10 | 11 | 2. Redistributions in binary form must reproduce the above copyright notice, 12 | this list of conditions and the following disclaimer in the documentation 13 | and/or other materials provided with the distribution. 14 | 15 | 3. Neither the name of the copyright holder nor the names of its 16 | contributors may be used to endorse or promote products derived from 17 | this software without specific prior written permission. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 22 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 23 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 25 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 26 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 27 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /app_components/session_state.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import streamlit as st 3 | import uuid 4 | 5 | 6 | def initialize_session_state() -> None: 7 | 8 | default_session_state = { 9 | "conversations": {}, 10 | "current_conversation": None, 11 | "user_input_key": 0, 12 | "conversation_progress": {}, 13 | "chat_names": {}, 14 | "uploaded_files": {}, 15 | "uploaded_test_files": {}, 16 | "figures": {}, 17 | "current_human_text": [], 18 | "df_name": None, 19 | "test_df_name": None, 20 | "transcribed_text": "", 21 | "loading_message": "", 22 | "uuid": str(uuid.uuid4()), 23 | "accumulated_status_messages": [], 24 | "extract_metric": [], 25 | "benchmark_history": [], 26 | "last_benchmark_index": -1, 27 | "current_node": None 28 | } 29 | 30 | for key, default_value in default_session_state.items(): 31 | st.session_state.setdefault(key, default_value) 32 | 33 | 34 | def create_new_conversation() -> str: 35 | 36 | conversation_id = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") 37 | st.session_state.conversations[conversation_id] = [] 38 | st.session_state.current_conversation = conversation_id 39 | 40 | st.session_state.conversation_progress[conversation_id] = {} 41 | st.session_state.chat_names[conversation_id] = f"Chat {conversation_id}" 42 | st.session_state.user_input_key += 1 43 | 44 | st.session_state.accumulated_status_messages = [] 45 | 46 | if "shown_human_messages" in st.session_state: 47 | st.session_state.shown_human_messages = set() 48 | 49 | return conversation_id 50 | -------------------------------------------------------------------------------- /fedotllm/agents/automl/templates/fedot_train.py: -------------------------------------------------------------------------------- 1 | def train_model(train_features: np.ndarray | pd.DataFrame, train_target: np.ndarray | pd.DataFrame | pd.Series): 2 | if isinstance(train_features, pd.DataFrame) and isinstance(train_target, (pd.DataFrame, pd.Series)): 3 | input_data = InputData.from_dataframe(train_features, train_target, task=Task({%problem%})) 4 | elif isinstance(train_features, np.ndarray) and isinstance(train_target, np.ndarray): 5 | input_data = InputData.from_numpy(train_features, train_target, task=Task({%problem%})) 6 | else: 7 | raise ValueError("Unsupported data types for train_features and train_target. " 8 | "Expected pandas DataFrame and (DataFrame or Series), or numpy ndarray and numpy ndarray." 9 | f"Got: {type(train_features)} and {type(train_target)}") 10 | 11 | model = Fedot(problem={%problem%}.value, 12 | timeout={%timeout%}, 13 | seed=42, 14 | cv_folds={%cv_folds%}, 15 | preset={%preset%}, 16 | metric={%metric%}, 17 | n_jobs=1, 18 | with_tuning=True, 19 | show_progress=True) 20 | 21 | try: 22 | model.fit(features=input_data) # this is the training step, after this step variable 'model' will be a trained model 23 | except Exception as e: 24 | raise RuntimeError( 25 | f"Model training failed. Please check your data preprocessing carefully. " 26 | f"Common issues include: missing values, incorrect data types, feature scaling problems, " 27 | f"or incompatible target variable format. Original error: {str(e)}" 28 | ) from e 29 | 30 | # Save the pipeline 31 | pipeline = model.current_pipeline 32 | pipeline.save(path=PIPELINE_PATH, create_subdir=False, is_datetime_in_path=False) 33 | 34 | return model -------------------------------------------------------------------------------- /app_components/ui_components.py: -------------------------------------------------------------------------------- 1 | import os 2 | import streamlit as st 3 | 4 | from .media_utils import get_base64_encoded_image 5 | from .fragments import ( 6 | file_upload_fragment, 7 | conversation_management_fragment, 8 | chat_input_fragment, 9 | render_conversation, 10 | ) 11 | 12 | 13 | def render_header(): 14 | logo_path = os.path.join('image', 'lads.jpg') 15 | 16 | st.markdown( 17 | f""" 18 |
19 | 20 |

LightAutoDS-Tab

21 |



22 |
23 | """, 24 | unsafe_allow_html=True 25 | ) 26 | 27 | 28 | def render_sidebar(): 29 | with st.sidebar: 30 | 31 | file_upload_fragment() 32 | 33 | st.divider() 34 | 35 | conversation_management_fragment() 36 | 37 | 38 | def render_conversation_messages(): 39 | if not st.session_state.current_conversation: 40 | return 41 | 42 | messages = st.session_state.conversations[st.session_state.current_conversation] 43 | 44 | exchanges = [] 45 | current_user_message = None 46 | 47 | for message in messages: 48 | if message.get("role") == "user": 49 | current_user_message = message.get("content", "") 50 | elif message.get("role") == "assistant" and current_user_message is not None: 51 | exchanges.append((current_user_message, message)) 52 | current_user_message = None 53 | 54 | tables_results = st.session_state.benchmark_history 55 | for (user_msg, assistant_msg), table_raw in zip(exchanges, tables_results): 56 | render_conversation(user_msg, assistant_msg, table_raw) 57 | 58 | def render_input_section(): 59 | chat_input_fragment() 60 | -------------------------------------------------------------------------------- /app.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import logging 4 | import streamlit as st 5 | 6 | sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'utils')) 7 | 8 | from app_components.session_state import initialize_session_state 9 | from app_components.agent_handler import initialize_services 10 | from app_components.ui_components import ( 11 | render_header, 12 | render_sidebar, 13 | render_conversation_messages, 14 | render_input_section 15 | ) 16 | from utils.config.loader import load_config 17 | from colorlog import ColoredFormatter 18 | 19 | 20 | LOGGING = { 21 | "level": "INFO", 22 | "format": "%(asctime)s %(levelname)s %(name)s %(message)s" 23 | } 24 | 25 | logging.basicConfig(level=LOGGING["level"], format=LOGGING['format']) 26 | logger = logging.getLogger(__name__) 27 | 28 | formatter = ColoredFormatter( 29 | "%(log_color)s%(levelname)s: %(message)s", 30 | log_colors={ 31 | 'DEBUG': 'cyan', 32 | 'INFO': 'green', 33 | 'WARNING': 'yellow', 34 | 'ERROR': 'red', 35 | 'CRITICAL': 'bold_red', 36 | } 37 | ) 38 | 39 | console_handler = logging.StreamHandler() 40 | console_handler.setFormatter(formatter) 41 | 42 | logger.addHandler(console_handler) 43 | 44 | 45 | def main(): 46 | 47 | st.set_page_config(layout="wide", page_title="DS Agent", page_icon="image/lads.jpg") 48 | 49 | config = load_config() 50 | st.session_state.setdefault("config", config) 51 | 52 | initialize_session_state() 53 | render_header() 54 | 55 | init_status = st.empty() 56 | 57 | if "app_initialized" not in st.session_state: 58 | init_status.info("Starting application, please wait...") 59 | initialize_services() 60 | st.session_state.app_initialized = True 61 | init_status.empty() 62 | 63 | render_sidebar() 64 | render_conversation_messages() 65 | 66 | render_input_section() 67 | 68 | 69 | if __name__ == "__main__": 70 | main() 71 | -------------------------------------------------------------------------------- /fedotllm/agents/utils.py: -------------------------------------------------------------------------------- 1 | import json 2 | import re 3 | from typing import Any, Dict 4 | 5 | import json_repair 6 | 7 | from fedotllm.log import logger 8 | 9 | 10 | # if ```python on response, or ``` on response, or whole response is code, return code 11 | def extract_code(response: str) -> str: 12 | """Extract code content from text that may contain code blocks. 13 | 14 | Args: 15 | response: Input text that might contain code blocks 16 | Returns: 17 | Extracted code content or original text if no code blocks found 18 | """ 19 | response = response.strip() 20 | code_match = re.search( 21 | r"```(?:\w+)?\s*(.*?)```", 22 | response, 23 | re.DOTALL, 24 | ) 25 | return code_match.group(1).strip() if code_match else response 26 | 27 | 28 | def parse_json(raw_reply: str | None) -> Dict[str, Any] | None: 29 | """Parse a JSON string from the raw reply.""" 30 | if not raw_reply: 31 | logger.warning("Received empty or None raw reply for JSON parsing.") 32 | return None 33 | 34 | def try_json_loads(data: str) -> Dict[str, Any] | None: 35 | try: 36 | repaired_json = json_repair.repair_json( 37 | data, ensure_ascii=False, return_objects=True 38 | ) 39 | return repaired_json if repaired_json != "" else None 40 | except json.JSONDecodeError as e: 41 | logger.error(f"JSON decoding error: {e}") 42 | return None 43 | 44 | raw_reply = raw_reply.strip() 45 | # Case 1: Check if the JSON is enclosed in triple backticks 46 | json_match = re.search(r"\{.*\}|```(?:json)?\s*(.*?)```", raw_reply, re.DOTALL) 47 | if json_match: 48 | if json_match.group(1): 49 | reply_str = json_match.group(1).strip() 50 | else: 51 | reply_str = json_match.group(0).strip() 52 | reply = try_json_loads(reply_str) 53 | if reply is not None: 54 | return reply 55 | 56 | # Case 2: Assume the entire string is a JSON object 57 | return try_json_loads(raw_reply) 58 | -------------------------------------------------------------------------------- /graph/lightautoml_template.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from lightautoml.automl.presets.tabular_presets import TabularAutoML 3 | from lightautoml.tasks import Task 4 | from sklearn.metrics import roc_auc_score 5 | from sklearn.metrics import r2_score 6 | 7 | import pandas as pd 8 | from sklearn.model_selection import train_test_split 9 | import warnings 10 | 11 | warnings.filterwarnings("ignore") 12 | 13 | 14 | def main(): 15 | parser = argparse.ArgumentParser(description='Run LightAutoML model training') 16 | parser.add_argument('--df_name', type=str, required=True, help='Path to the input CSV file') 17 | parser.add_argument('--task_type', type=str, required=True, help='Type of task (e.g., "binary", "multiclass", "regression")') 18 | parser.add_argument('--task_metric', type=str, required=True, help='metric for the task (e.g., "auc", "f1", "rmse")') 19 | parser.add_argument('--target', type=str, required=True, help='Name of the target column in the dataset') 20 | args = parser.parse_args() 21 | 22 | df_name = args.df_name 23 | task_type = args.task_type 24 | task_metric = args.task_metric 25 | target = args.target 26 | 27 | df = pd.read_csv('datasets/'+df_name) 28 | 29 | train_df, test_df = train_test_split(df, test_size=0.2, random_state=42) 30 | 31 | automl = TabularAutoML( 32 | task=Task( 33 | name=task_type, 34 | metric=task_metric 35 | ), 36 | timeout=30 37 | ) 38 | 39 | oof_preds = automl.fit_predict(train_df, roles={'target': target}).data 40 | test_preds = automl.predict(test_df).data 41 | if task_type == "reg": 42 | print("R2 score on oof data:", r2_score(train_df[target].values, oof_preds[:, 0])) 43 | print("R2 score on test data:", r2_score(test_df[target].values, test_preds[:, 0])) 44 | else: 45 | print("ROC-AUC score on oof data:", roc_auc_score(train_df[target].values, oof_preds[:, 0])) 46 | print("ROC-AUC score on test data:", roc_auc_score(test_df[target].values, test_preds[:, 0])) 47 | 48 | 49 | if __name__ == '__main__': 50 | main() 51 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 |
3 | Logo 4 |

LightAutoDS-Tab

5 |
6 | 7 |
8 | 9 | Python3.10 10 | 11 | 12 |
13 | 14 | ![architecture](./image/architecture.png) 15 | 16 | **LightAutoDS-Tab**, a multi-AutoML agentic system for tasks with tabular data, which combines an LLM-based code generation with several AutoML tools. 17 | 18 | ## ✨ Demo 19 | [Watch the Video](https://www.youtube.com/watch?v=5e8eADd_HWE) 20 | 21 | ## 🧑‍💻 User interface 22 | 23 | 24 | The interface includes two main panels: 25 | 26 | 1. The **right panel** provides detailed *technical insights* into each step of the ML pipeline construction, offering transparency for expert users. 27 | 1. The **left panel** presents a simplified, *non-technical summary* of the process, making it easy for non-experts to follow along and understand the results 28 | 29 | ## 🚀 Quick Start 30 | 31 | **Step 1: Clone the repository** 32 | 33 | ```shell 34 | git clone https://github.com/sb-ai-lab/LADS.git 35 | cd LADS 36 | ``` 37 | 38 | **Step 2: Create conda environment** 39 | 40 | ```shell 41 | conda env create -f environment.yml 42 | conda activate LightAutoDS 43 | ``` 44 | 45 | **Step 3. Set up environment variables** 46 | 47 | You need to create a `.env` file in the root directory of the project. 48 | 49 | ```shell 50 | cp .env_example .env 51 | ``` 52 | 53 | You will need to fill in the required API keys and other environment variables in the `.env` file. 54 | 55 | You can also change some parameters in [`config.yml`](./config.yml). 56 | 57 | **Step 4: Run the application** 58 | 59 | ```shell 60 | streamlit run app.py 61 | ``` 62 | 63 | Your application will be hosted on [http://localhost:8501](http://localhost:8501) by default. 64 | 65 | ## 📊 Result 66 | We evaluated our framework on eight Kaggle ML datasets and compared it with two state-of-the-art open-source solutions: AutoKaggle and AIDE. 67 | 68 | To ensure consistency across competitions, we use the Normalized Performance Score (NPS). This score standardizes the results, with a higher value indicating better performance. 69 | 70 |
71 | 72 |
73 | 74 | 75 | 76 | ## 📜 License 77 | 78 | Distributed under the BSD 3-Clause License. See [`LICENSE`](./LICENSE) for more information. -------------------------------------------------------------------------------- /fedotllm/llm.py: -------------------------------------------------------------------------------- 1 | import os 2 | from typing import Any, Dict, List, Optional, Type, TypeVar 3 | 4 | import litellm 5 | from pydantic import BaseModel 6 | from tenacity import retry, stop_after_attempt, wait_exponential 7 | 8 | from fedotllm import prompts 9 | from fedotllm.agents.utils import parse_json 10 | from fedotllm.log import logger 11 | from utils.config.loader import load_config 12 | 13 | from dotenv import load_dotenv 14 | load_dotenv() 15 | 16 | T = TypeVar("T", bound=BaseModel) 17 | 18 | litellm._logging._disable_debugging() 19 | 20 | 21 | LANGFUSE_PUBLIC_KEY = os.getenv("LANGFUSE_PUBLIC_KEY", "") 22 | LANGFUSE_SECRET_KEY = os.getenv("LANGFUSE_SECRET_KEY", "") 23 | 24 | 25 | if LANGFUSE_PUBLIC_KEY and LANGFUSE_SECRET_KEY: 26 | litellm.success_callback = ["langfuse"] 27 | litellm.failure_callback = ["langfuse"] 28 | 29 | 30 | class AIInference: 31 | def __init__( 32 | self, 33 | api_key: str | None = None, 34 | base_url: str | None = None, 35 | provider: str | None = None, 36 | model: str | None = None, 37 | ): 38 | settings = load_config() 39 | self.base_url = base_url or settings.fedot.base_url 40 | self.model = model or settings.fedot.model_name 41 | self.api_key = api_key or os.getenv("OPENAI_API_KEY") 42 | self.provider = provider or settings.fedot.provider 43 | if self.provider: 44 | self.model = f"{self.provider}/{self.model}" 45 | if not self.api_key: 46 | raise Exception( 47 | "API key not provided and OPENAI_API_KEY environment variable not set" 48 | ) 49 | 50 | self.completion_params = { 51 | "model": self.model, 52 | "api_key": self.api_key, 53 | "base_url": self.base_url, 54 | # "max_completion_tokens": 8000, 55 | "extra_headers": {"X-Title": "FEDOT.LLM"}, 56 | } 57 | 58 | @retry( 59 | stop=stop_after_attempt(5), 60 | wait=wait_exponential(multiplier=1, min=4, max=10), 61 | reraise=True, 62 | ) 63 | def create(self, messages: str, response_model: Type[T]) -> T: 64 | messages = f"{messages}\n{prompts.utils.structured_response(response_model)}" 65 | response = self.query(messages) 66 | json_obj = parse_json(response) if response else None 67 | return response_model.model_validate(json_obj) 68 | 69 | @retry( 70 | stop=stop_after_attempt(5), 71 | wait=wait_exponential(multiplier=1, min=4, max=10), 72 | reraise=True, 73 | ) 74 | def query(self, messages: str | List[Dict[str, Any]]) -> str | None: 75 | messages = ( 76 | [{"role": "user", "content": messages}] 77 | if isinstance(messages, str) 78 | else messages 79 | ) 80 | logger.debug("Sending messages to LLM: %s", messages) 81 | response = litellm.completion( 82 | messages=messages, 83 | **self.completion_params, 84 | ) 85 | logger.debug( 86 | "Received response from LLM: %s", response.choices[0].message.content 87 | ) 88 | return response.choices[0].message.content 89 | 90 | 91 | if __name__ == "__main__": 92 | inference = AIInference() 93 | print(inference.query("Say hello world!")) 94 | -------------------------------------------------------------------------------- /fedotllm/agents/automl/structured.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | from typing import Literal, Optional, Union 3 | 4 | from fedot.core.repository.tasks import TaskTypesEnum 5 | from pydantic import BaseModel, ConfigDict, Field 6 | 7 | 8 | class ProblemType(str, Enum): 9 | CLASSIFICATION = "classification" 10 | REGRESSION = "regression" 11 | TS_FORECASTING = "ts_forecasting" 12 | 13 | 14 | class PresetType(str, Enum): 15 | BEST_QUALITY = "best_quality" 16 | FAST_TRAIN = "fast_train" 17 | STABLE = "stable" 18 | AUTO = "auto" 19 | GPU = "gpu" 20 | TS = "ts" 21 | AUTOML = "automl" 22 | 23 | 24 | class ClassificationMetricsEnum(str, Enum): 25 | ROCAUC = "roc_auc" 26 | precision = "precision" 27 | # f1 = 'f1' 28 | # logloss = 'neg_log_loss' 29 | # ROCAUC_penalty = 'roc_auc_pen' 30 | accuracy = "accuracy" 31 | 32 | 33 | class RegressionMetricsEnum(str, Enum): 34 | RMSE = "rmse" 35 | MSE = "mse" 36 | MSLE = "neg_mean_squared_log_error" 37 | MAPE = "mape" 38 | SMAPE = "smape" 39 | MAE = "mae" 40 | R2 = "r2" 41 | RMSE_penalty = "rmse_pen" 42 | 43 | 44 | class TimeSeriesForecastingMetricsEnum(str, Enum): 45 | MASE = "mase" 46 | RMSE = "rmse" 47 | MSE = "mse" 48 | MSLE = "neg_mean_squared_log_error" 49 | MAPE = "mape" 50 | SMAPE = "smape" 51 | MAE = "mae" 52 | R2 = "r2" 53 | RMSE_penalty = "rmse_pen" 54 | 55 | 56 | class FedotConfig(BaseModel): 57 | model_config = ConfigDict(arbitrary_types_allowed=True) 58 | 59 | problem: TaskTypesEnum = Field( 60 | ..., description="Name of the modelling problem to solve" 61 | ) 62 | timeout: float = Field( 63 | ..., description="Time for model design (in minutes): Default: 1.0" 64 | ) 65 | cv_folds: Optional[int] = Field( 66 | ..., description="Number of folds for cross-validation: Default: None" 67 | ) 68 | preset: PresetType = Field( 69 | ..., 70 | description=( 71 | "Name of the preset for model building. Possible options:\n" 72 | "best_quality -> All models that are available for this data type and task are used\n" 73 | "fast_train -> Models that learn quickly. This includes preprocessing operations (data operations) that only reduce the dimensionality of the data, but cannot increase it. For example, there are no polynomial features and one-hot encoding operations\n" 74 | "stable -> The most reliable preset in which the most stable operations are included\n" 75 | "auto -> Automatically determine which preset should be used\n" 76 | "gpu -> Models that use GPU resources for computation\n" 77 | "ts -> A special preset with models for time series forecasting task\n" 78 | "automl -> A special preset with only AutoML libraries such as TPOT and H2O as operations" 79 | "Default: auto" 80 | ), 81 | ) 82 | metric: Union[ 83 | ClassificationMetricsEnum, 84 | RegressionMetricsEnum, 85 | TimeSeriesForecastingMetricsEnum, 86 | ] = Field( 87 | ..., 88 | description="Choose relevant to problem metric of model quality assessment.", 89 | ) 90 | predict_method: Literal["predict", "predict_proba", "forecast"] = Field( 91 | ..., 92 | description="Method for prediction: predict - for classification and regression, predict_proba - for classification, forecast - for time series forecasting", 93 | ) 94 | -------------------------------------------------------------------------------- /fedotllm/agents/scraper.py: -------------------------------------------------------------------------------- 1 | import warnings 2 | from urllib.parse import urljoin, urlparse 3 | 4 | import requests 5 | from bs4 import BeautifulSoup, MarkupResemblesLocatorWarning 6 | from pydantic import HttpUrl 7 | from tqdm import tqdm 8 | 9 | warnings.filterwarnings("ignore", category=MarkupResemblesLocatorWarning) 10 | 11 | 12 | def extract_sub_links(raw_html: str, url: HttpUrl, base_url: HttpUrl): 13 | parsed_url = urlparse(url) 14 | parsed_base_url = urlparse(base_url) 15 | absolute_paths = set() 16 | 17 | soup = BeautifulSoup(raw_html, "html.parser") 18 | for link_tag in soup.find_all("a", href=True): 19 | link = link_tag["href"] 20 | try: 21 | parsed_link = urlparse(link) 22 | if parsed_link.scheme == "http" or parsed_link.scheme == "https": 23 | absolute_path = link 24 | elif link.startswith("//"): 25 | absolute_path = f"{parsed_url.scheme}:{link}" 26 | else: 27 | absolute_path = urljoin(url, parsed_link.path) 28 | if parsed_link.query: 29 | absolute_path += f"?{parsed_link.query}" 30 | 31 | if urlparse(absolute_path).netloc == parsed_base_url.netloc: 32 | if absolute_path.startswith(base_url): 33 | absolute_paths.add(absolute_path) 34 | except Exception as _: 35 | continue 36 | 37 | return absolute_paths 38 | 39 | 40 | def extract_metadata(raw_html: str, url: str, response: requests.Response) -> dict: 41 | content_type = getattr(response, "headers").get("Content-Type", "") 42 | metadata = {"source": url, "content_type": content_type} 43 | soup = BeautifulSoup(raw_html, "html.parser") 44 | if title := soup.find("title"): 45 | metadata["title"] = title.get_text() 46 | return metadata 47 | 48 | 49 | def recursive_url_loader(url: str, max_depth: int = 10, timeout: int = 10): 50 | base_url = url 51 | visited = set() 52 | documents = [] 53 | 54 | def recursive_scrape(url: str, depth: int): 55 | if depth < 0 or url in visited: 56 | return 57 | 58 | visited.add(url) 59 | try: 60 | response = requests.get(url, timeout=timeout) 61 | 62 | if 400 <= response.status_code <= 599: 63 | raise ValueError(f"Received HTTP status {response.status_code}") 64 | except Exception as _: 65 | return 66 | 67 | document = { 68 | "content": response.text, 69 | "metadata": extract_metadata( 70 | raw_html=response.text, url=url, response=response 71 | ), 72 | } 73 | sub_links = extract_sub_links( 74 | raw_html=response.text, url=url, base_url=base_url 75 | ) 76 | return document, sub_links 77 | 78 | depth_bar = tqdm(total=max_depth + 1, desc="Scraping") 79 | 80 | depth = max_depth 81 | waiting = {base_url} 82 | while True: 83 | links = waiting.copy() 84 | for link in links: 85 | depth_bar.set_postfix_str(f"{link}") 86 | if scraped := recursive_scrape(link, depth): 87 | document, sub_links = scraped 88 | documents.append(document) 89 | waiting.update(sub_links) 90 | waiting.difference_update(visited) 91 | depth = depth - 1 92 | depth_bar.update(1) 93 | if depth < 0 or len(waiting) == 0: 94 | depth_bar.n = max_depth + 1 95 | return documents 96 | -------------------------------------------------------------------------------- /fedotllm/agents/automl/automl.py: -------------------------------------------------------------------------------- 1 | from functools import partial 2 | from pathlib import Path 3 | 4 | from langgraph.graph import END, START, StateGraph 5 | from langgraph.types import Command 6 | 7 | from fedotllm.agents.automl.nodes import ( 8 | evaluate, 9 | extract_metrics, 10 | fix_solution, 11 | generate_automl_config, 12 | generate_code, 13 | generate_report, 14 | if_bug, 15 | insert_templates, 16 | problem_reflection, 17 | run_tests, 18 | select_skeleton, 19 | ) 20 | from fedotllm.data import Dataset 21 | from fedotllm.llm import AIInference 22 | 23 | from .state import AutoMLAgentState 24 | 25 | 26 | class AutoMLAgent: 27 | def __init__(self, inference: AIInference, dataset: Dataset, workspace: Path): 28 | self.inference = inference 29 | self.dataset = dataset 30 | self.workspace = workspace 31 | 32 | def init_state(self, state: AutoMLAgentState): 33 | return Command( 34 | update={ 35 | "reflection": None, 36 | "fedot_config": None, 37 | "skeleton": None, 38 | "raw_code": None, 39 | "code": None, 40 | "observation": None, 41 | "fix_attempts": 0, 42 | "metrics": "", 43 | "pipeline": "", 44 | "report": "", 45 | } 46 | ) 47 | 48 | def create_graph(self): 49 | workflow = StateGraph(AutoMLAgentState) 50 | workflow.add_node("init_state", self.init_state) 51 | workflow.add_node( 52 | "problem_reflection", 53 | partial(problem_reflection, inference=self.inference, dataset=self.dataset), 54 | ) 55 | workflow.add_node( 56 | "generate_automl_config", 57 | partial( 58 | generate_automl_config, 59 | inference=self.inference, 60 | dataset=self.dataset, 61 | ), 62 | ) 63 | workflow.add_node( 64 | "select_skeleton", 65 | partial(select_skeleton, dataset=self.dataset, workspace=self.workspace), 66 | ) 67 | workflow.add_node("insert_templates", insert_templates) 68 | workflow.add_node( 69 | "generate_code", 70 | partial(generate_code, inference=self.inference, dataset=self.dataset), 71 | ) 72 | workflow.add_node("evaluate_main", partial(evaluate, workspace=self.workspace)) 73 | workflow.add_node( 74 | "fix_solution_main", 75 | partial(fix_solution, inference=self.inference, dataset=self.dataset), 76 | ) 77 | workflow.add_node( 78 | "run_tests", 79 | partial(run_tests, workspace=self.workspace, inference=self.inference), 80 | ) 81 | workflow.add_node( 82 | "extract_metrics", partial(extract_metrics, workspace=self.workspace) 83 | ) 84 | workflow.add_node( 85 | "generate_report", partial(generate_report, inference=self.inference) 86 | ) 87 | 88 | workflow.add_edge(START, "init_state") 89 | workflow.add_edge("init_state", "problem_reflection") 90 | workflow.add_edge("problem_reflection", "generate_automl_config") 91 | workflow.add_edge("generate_automl_config", "select_skeleton") 92 | workflow.add_edge("select_skeleton", "generate_code") 93 | workflow.add_edge("generate_code", "insert_templates") 94 | workflow.add_conditional_edges( 95 | "insert_templates", 96 | lambda state: state["code"] is None, 97 | {True: "generate_code", False: "evaluate_main"}, 98 | ) 99 | workflow.add_conditional_edges( 100 | "evaluate_main", 101 | if_bug, 102 | {True: "fix_solution_main", False: "run_tests"}, 103 | ) 104 | workflow.add_edge("fix_solution_main", "insert_templates") 105 | workflow.add_conditional_edges( 106 | "run_tests", 107 | if_bug, 108 | {True: "fix_solution_main", False: "extract_metrics"}, 109 | ) 110 | workflow.add_edge("extract_metrics", "generate_report") 111 | workflow.add_edge("generate_report", END) 112 | return workflow.compile().with_config(config={"run_name": "AutoMLAgent"}) 113 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | #Project specific 2 | /datasets 3 | /models 4 | new_graph.png 5 | .env 6 | 7 | # Byte-compiled / optimized / DLL files 8 | __pycache__/ 9 | *.py[cod] 10 | *$py.class 11 | 12 | # C extensions 13 | *.so 14 | 15 | # Distribution / packaging 16 | .Python 17 | build/ 18 | develop-eggs/ 19 | dist/ 20 | downloads/ 21 | eggs/ 22 | .eggs/ 23 | lib/ 24 | lib64/ 25 | parts/ 26 | sdist/ 27 | var/ 28 | wheels/ 29 | share/python-wheels/ 30 | *.egg-info/ 31 | .installed.cfg 32 | *.egg 33 | MANIFEST 34 | 35 | # PyInstaller 36 | # Usually these files are written by a python script from a template 37 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 38 | *.manifest 39 | *.spec 40 | 41 | # Installer logs 42 | pip-log.txt 43 | pip-delete-this-directory.txt 44 | 45 | # Unit test / coverage reports 46 | htmlcov/ 47 | .tox/ 48 | .nox/ 49 | .coverage 50 | .coverage.* 51 | .cache 52 | nosetests.xml 53 | coverage.xml 54 | *.cover 55 | *.py,cover 56 | .hypothesis/ 57 | .pytest_cache/ 58 | cover/ 59 | 60 | # Translations 61 | *.mo 62 | *.pot 63 | 64 | # Django stuff: 65 | *.log 66 | local_settings.py 67 | db.sqlite3 68 | db.sqlite3-journal 69 | 70 | # Flask stuff: 71 | instance/ 72 | .webassets-cache 73 | 74 | # Scrapy stuff: 75 | .scrapy 76 | 77 | # Sphinx documentation 78 | docs/_build/ 79 | 80 | # PyBuilder 81 | .pybuilder/ 82 | target/ 83 | 84 | # Jupyter Notebook 85 | .ipynb_checkpoints 86 | 87 | # IPython 88 | profile_default/ 89 | ipython_config.py 90 | 91 | # pyenv 92 | # For a library or package, you might want to ignore these files since the code is 93 | # intended to run in multiple environments; otherwise, check them in: 94 | # .python-version 95 | 96 | # pipenv 97 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 98 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 99 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 100 | # install all needed dependencies. 101 | #Pipfile.lock 102 | 103 | # UV 104 | # Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. 105 | # This is especially recommended for binary packages to ensure reproducibility, and is more 106 | # commonly ignored for libraries. 107 | #uv.lock 108 | 109 | # poetry 110 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 111 | # This is especially recommended for binary packages to ensure reproducibility, and is more 112 | # commonly ignored for libraries. 113 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 114 | #poetry.lock 115 | 116 | # pdm 117 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 118 | #pdm.lock 119 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 120 | # in version control. 121 | # https://pdm.fming.dev/latest/usage/project/#working-with-version-control 122 | .pdm.toml 123 | .pdm-python 124 | .pdm-build/ 125 | 126 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 127 | __pypackages__/ 128 | 129 | # Celery stuff 130 | celerybeat-schedule 131 | celerybeat.pid 132 | 133 | # SageMath parsed files 134 | *.sage.py 135 | 136 | # Environments 137 | .env 138 | .venv 139 | env/ 140 | venv/ 141 | ENV/ 142 | env.bak/ 143 | venv.bak/ 144 | 145 | # Spyder project settings 146 | .spyderproject 147 | .spyproject 148 | 149 | # Rope project settings 150 | .ropeproject 151 | 152 | # mkdocs documentation 153 | /site 154 | 155 | # mypy 156 | .mypy_cache/ 157 | .dmypy.json 158 | dmypy.json 159 | 160 | # Pyre type checker 161 | .pyre/ 162 | 163 | # pytype static type analyzer 164 | .pytype/ 165 | 166 | # Cython debug symbols 167 | cython_debug/ 168 | 169 | # PyCharm 170 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 171 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 172 | # and can be added to the global gitignore or merged into this file. For a more nuclear 173 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 174 | #.idea/ 175 | 176 | # Ruff stuff: 177 | .ruff_cache/ 178 | 179 | # PyPI configuration file 180 | .pypirc 181 | 182 | # Output files 183 | output/ 184 | utils/salute_speech/ 185 | benchmark/benchmark_results.csv -------------------------------------------------------------------------------- /utils/config/schema.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Dict, Optional, Literal 2 | from pydantic import BaseModel as PydanticBaseModel, SecretStr, Field 3 | from pydantic_settings import BaseSettings, SettingsConfigDict 4 | 5 | 6 | class SecretInjectableModel(PydanticBaseModel): 7 | def inject_secrets(self, secrets: Any, context: Optional[Dict[str, Any]] = None): 8 | context = context or {} 9 | data = self.model_dump() 10 | 11 | for name, field in self.model_fields.items(): 12 | if field.json_schema_extra is None: 13 | continue 14 | 15 | metadata = field.json_schema_extra.get("metadata") 16 | if not metadata: 17 | continue 18 | 19 | source = metadata.get("secret_source") 20 | if not source: 21 | continue 22 | 23 | if isinstance(source, dict): 24 | key = context.get("provider") 25 | if not key: 26 | continue 27 | secret_name = source.get(key) 28 | if not secret_name: 29 | continue 30 | else: 31 | secret_name = source 32 | 33 | secret_value = getattr(secrets, secret_name, None) 34 | if secret_value is not None: 35 | 36 | if isinstance(secret_value, SecretStr): 37 | data[name] = secret_value.get_secret_value() 38 | else: 39 | data[name] = secret_value 40 | 41 | return self.__class__(**data) 42 | 43 | 44 | class LLMConfig(SecretInjectableModel): 45 | provider: Literal["gigachat", "openai"] = "gigachat" 46 | model_name: str = "GigaChat-2-Max" 47 | verify_ssl: bool = False 48 | profanity_check: bool = True 49 | scope: str = "GIGACHAT_API_CORP" 50 | timeout: Optional[int] = None 51 | base_url: Optional[str] = None 52 | token: Optional[SecretStr] = Field( 53 | None, 54 | json_schema_extra={"metadata": {"secret_source": { 55 | "gigachat": "GIGACHAT_API_TOKEN", 56 | "openai": "OPENAI_API_KEY" 57 | }}} 58 | ) 59 | 60 | 61 | class LangfuseConfig(SecretInjectableModel): 62 | host: Optional[str] 63 | user: Optional[str] = '' 64 | public_key: Optional[SecretStr] = Field(None, json_schema_extra={"metadata": {"secret_source": "LANGFUSE_PUBLIC_KEY"}}) 65 | secret_key: Optional[SecretStr] = Field(None, json_schema_extra={"metadata": {"secret_source": "LANGFUSE_SECRET_KEY"}}) 66 | 67 | 68 | class AgentConfig(SecretInjectableModel): 69 | max_improvements: int = 5 70 | recursion_limit: int = 1000 71 | max_code_execution_time: int = 3000 72 | code_generation_config: Optional[str] = 'local' 73 | e2b_token: Optional[SecretStr] = Field(None, json_schema_extra={"metadata": {"secret_source": "E2B_API_KEY"}}) 74 | prompt_language: Literal["ru", "en"] = "ru" 75 | 76 | 77 | class FedotTemplates(SecretInjectableModel): 78 | code: str 79 | train: str 80 | evaluate: str 81 | predict: str 82 | 83 | 84 | class FedotConfig(SecretInjectableModel): 85 | provider: str = "openai" 86 | model_name: str = "gpt-4o" 87 | base_url: Optional[str] = None 88 | fix_tries: int = 2 89 | templates: FedotTemplates 90 | predictor_init_kwargs: Dict[str, Any] = Field(default_factory=dict) 91 | 92 | 93 | class SecretsConfig(BaseSettings): 94 | model_config = SettingsConfigDict(env_file=".env", extra="ignore") 95 | 96 | GIGACHAT_API_TOKEN: Optional[SecretStr] = None 97 | OPENAI_API_KEY: Optional[SecretStr] = None 98 | E2B_API_KEY: Optional[SecretStr] = None 99 | SALUTE_API_KEY: Optional[SecretStr] = None 100 | LANGFUSE_SECRET_KEY: Optional[SecretStr] = None 101 | LANGFUSE_PUBLIC_KEY: Optional[SecretStr] = None 102 | 103 | 104 | class AppConfig(SecretInjectableModel): 105 | llm: LLMConfig 106 | fedot: FedotConfig 107 | langfuse: Optional[LangfuseConfig] = None 108 | general: AgentConfig 109 | 110 | secrets: SecretsConfig 111 | 112 | model_overrides: Optional[Dict[str, LLMConfig]] = None 113 | 114 | def inject_all_secrets(self): 115 | self.llm = self.llm.inject_secrets(self.secrets, context=self.llm.model_dump()) 116 | if self.langfuse: 117 | self.langfuse = self.langfuse.inject_secrets(self.secrets) 118 | if self.model_overrides: 119 | for key, val in self.model_overrides.items(): 120 | self.model_overrides[key] = val.inject_secrets(self.secrets, context=val.model_dump()) 121 | return self 122 | -------------------------------------------------------------------------------- /fedotllm/data.py: -------------------------------------------------------------------------------- 1 | import io 2 | from pathlib import Path 3 | from typing import List 4 | 5 | import pandas as pd 6 | from scipy.io.arff import loadarff 7 | 8 | from fedotllm.constants import ( 9 | ARFF_SUFFIXES, 10 | CSV_SUFFIXES, 11 | DATASET_EXTENSIONS, 12 | EXCEL_SUFFIXES, 13 | PARQUET_SUFFIXES, 14 | ) 15 | 16 | 17 | def load_pd(data): 18 | if isinstance(data, (Path, str)): 19 | path = data 20 | if isinstance(path, str): 21 | path = Path(path) 22 | format = None 23 | if path.suffix in EXCEL_SUFFIXES: 24 | format = "excel" 25 | elif path.suffix in PARQUET_SUFFIXES: 26 | format = "parquet" 27 | elif path.suffix in CSV_SUFFIXES: 28 | format = "csv" 29 | elif path.suffix in ARFF_SUFFIXES: 30 | format = "arff" 31 | else: 32 | if format is None: 33 | raise Exception(f"file format for {path.suffix} not supported!") 34 | else: 35 | raise Exception("file format " + format + " not supported!") 36 | 37 | match format: 38 | case "excel": 39 | return pd.read_excel(path, engine="calamine") 40 | case "parquet": 41 | try: 42 | return pd.read_parquet(path, engine="fastparquet") 43 | except Exception: 44 | return pd.read_parquet(path, engine="pyarrow") 45 | case "arff": 46 | return pd.DataFrame(loadarff(path)[0]) 47 | case "csv": 48 | return pd.read_csv(path) 49 | else: 50 | return pd.DataFrame(data) 51 | 52 | 53 | def missing_values(df: pd.DataFrame) -> pd.DataFrame: 54 | missing = df.isna().sum() 55 | missing = missing[missing > 0].sort_values(ascending=False) 56 | missing_pct = (missing / len(df)).round(3) * 100 57 | missing_df = pd.DataFrame({"Missing": missing, "Percent": missing_pct}) 58 | return missing_df 59 | 60 | 61 | class Split: 62 | """ 63 | Split within dataset object 64 | """ 65 | 66 | def __init__(self, name: str, data: pd.DataFrame): 67 | self.name = name 68 | self.data = data 69 | 70 | 71 | class Dataset: 72 | def __init__(self, splits: List[Split], path: Path): 73 | self.splits = splits 74 | self.path = path 75 | 76 | @classmethod 77 | def from_path(cls, path: Path): 78 | """ 79 | Load Dataset a folder with dataset objects 80 | 81 | Args: 82 | path: Path to folder with Dataset data 83 | """ 84 | 85 | # Loading all splits in folder 86 | splits = [] 87 | if path.is_dir(): 88 | files = [x for x in path.glob("**/*") if x.is_file()] 89 | else: 90 | files = [path] 91 | for file in files: 92 | file_path = file.absolute() 93 | if file_path.suffix in DATASET_EXTENSIONS: 94 | file_dataframe = load_pd(file_path) 95 | split = Split(data=file_dataframe, name=file.name) 96 | splits.append(split) 97 | 98 | return Dataset(splits=splits, path=path) 99 | 100 | def get_train_split(self): 101 | # heuristics to find train split 102 | for split in self.splits: 103 | if "train" in split.name.lower(): 104 | train_split = split 105 | break 106 | else: 107 | # Find splits with max column count 108 | max_cols = max(split.data.shape[1] for split in self.splits) 109 | max_col_splits = [ 110 | split for split in self.splits if split.data.shape[1] == max_cols 111 | ] 112 | 113 | # If multiple splits have the same column count, take the one with more rows 114 | if len(max_col_splits) > 1: 115 | train_split = max(max_col_splits, key=lambda split: split.data.shape[0]) 116 | else: 117 | train_split = max_col_splits[0] 118 | return train_split 119 | 120 | def dataset_eda(self): 121 | """Generate exploratory data analysis summary only for the split with maximum columns.""" 122 | if not self.splits: 123 | return "No data splits available" 124 | # heuristics to find train split 125 | train_split = self.get_train_split() 126 | df = train_split.data 127 | eda = "" 128 | if df.shape[1] <= 10: 129 | eda += "\n===== 1. BASIC INFO =====\n" 130 | 131 | buf = io.StringIO() 132 | df.info(buf=buf) 133 | 134 | info_str = buf.getvalue() 135 | eda += info_str 136 | 137 | eda += "\n===== 2. MISSING VALUES =====\n" 138 | eda += missing_values(df).to_markdown() 139 | return eda 140 | 141 | def dataset_preview(self, sample_size: int = 11): 142 | preview = "" 143 | train_split = self.get_train_split() 144 | if train_split.data.shape[1] > 10: 145 | preview += f"File: {train_split.name}\n" 146 | preview += train_split.data.sample(sample_size).to_markdown() 147 | preview += "\n\n" 148 | for split in self.splits: 149 | preview += f"File: {split.name}\n" 150 | preview += f"Columns: {split.data.columns.tolist()}\n" 151 | preview += "\n\n" 152 | else: 153 | for split in self.splits: 154 | preview += f"File: {split.name}\n" 155 | preview += split.data.sample(sample_size).to_markdown() 156 | preview += "\n\n" 157 | return preview 158 | 159 | def __str__(self): 160 | return self.dataset_preview() 161 | -------------------------------------------------------------------------------- /fedotllm/main.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from typing import Any, AsyncIterator, Callable, List, Optional 3 | 4 | import pandas as pd 5 | from langchain_core.messages import AIMessage, HumanMessage 6 | from langchain_core.runnables.schema import StreamEvent 7 | 8 | from fedotllm.agents.automl import AutoMLAgent 9 | from fedotllm.agents.translator import TranslatorAgent 10 | from fedotllm.data import Dataset 11 | from fedotllm.llm import AIInference 12 | from fedotllm.log import logger 13 | 14 | 15 | 16 | class FedotAI: 17 | def __init__( 18 | self, 19 | task_path: Path | str, 20 | inference: Optional[AIInference] = None, 21 | handlers: Optional[List[Callable[[StreamEvent], None]]] = None, 22 | workspace: Path | str | None = None, 23 | ): 24 | if isinstance(task_path, str): 25 | task_path = Path(task_path) 26 | self.task_path = task_path.resolve() 27 | assert self.task_path.is_dir(), ( 28 | "Task path does not exist or is not a directory." 29 | ) 30 | 31 | self.inference = inference if inference is not None else AIInference() 32 | self.handlers = handlers if handlers is not None else [] 33 | 34 | if isinstance(workspace, str): 35 | workspace = Path(workspace) 36 | self.workspace = workspace 37 | 38 | def ainvoke(self, message: str): 39 | logger.info( 40 | f"FedotAI ainvoke called. Input message (first 100 chars): '{message[:100]}...'" 41 | ) 42 | if not self.workspace: 43 | self.workspace = Path( 44 | f"fedotllm-output-{pd.Timestamp.now().strftime('%Y%m%d_%H%M%S')}" 45 | ) 46 | logger.info(f"Workspace for ainvoke created at: {self.workspace}") 47 | 48 | dataset = Dataset.from_path(self.task_path) 49 | translator_agent = TranslatorAgent(inference=self.inference) 50 | 51 | logger.info("FedotAI ainvoke: Translating input message to English.") 52 | translated_message = translator_agent.translate_input_to_english(message) 53 | logger.info( 54 | f"FedotAI ainvoke: Input message translated to (first 100 chars): '{translated_message[:100]}...'" 55 | ) 56 | 57 | automl_agent = AutoMLAgent( 58 | inference=self.inference, dataset=dataset, workspace=self.workspace 59 | ).create_graph() 60 | 61 | raw_response = automl_agent.invoke( 62 | {"messages": [HumanMessage(content=translated_message)]} 63 | ) 64 | 65 | logger.debug( 66 | f"FedotAI ainvoke: Raw response from SupervisorAgent: {raw_response}" 67 | ) 68 | 69 | if ( 70 | raw_response 71 | and "messages" in raw_response 72 | and isinstance(raw_response["messages"], list) 73 | and len(raw_response["messages"]) > 0 74 | ): 75 | last_message_original = raw_response["messages"][-1] 76 | logger.debug( 77 | f"FedotAI ainvoke: Original last_message from Supervisor: {last_message_original}" 78 | ) 79 | 80 | if hasattr(last_message_original, "content"): 81 | ai_message_content = last_message_original.content 82 | logger.info( 83 | f"FedotAI ainvoke: Before output translation. Source lang: {translator_agent.source_language}. Content (first 100): '{ai_message_content[:100]}...'" 84 | ) 85 | 86 | translated_output = ( 87 | translator_agent.translate_output_to_source_language( 88 | ai_message_content 89 | ) 90 | ) 91 | logger.info( 92 | f"FedotAI ainvoke: After output translation. Translated content (first 100): '{translated_output[:100]}...'" 93 | ) 94 | 95 | if isinstance(last_message_original, AIMessage): 96 | # Create new AIMessage, preserving other attributes 97 | # Ensure all attributes are correctly handled, using defaults if necessary 98 | new_ai_message = AIMessage( 99 | content=translated_output, 100 | id=getattr(last_message_original, "id", None), 101 | response_metadata=getattr( 102 | last_message_original, "response_metadata", {} 103 | ), 104 | tool_calls=getattr(last_message_original, "tool_calls", []), 105 | tool_call_chunks=getattr( 106 | last_message_original, "tool_call_chunks", [] 107 | ), 108 | usage_metadata=getattr( 109 | last_message_original, "usage_metadata", None 110 | ), 111 | ) 112 | raw_response["messages"][-1] = new_ai_message 113 | logger.debug( 114 | f"FedotAI ainvoke: Updated AIMessage with translated content: {new_ai_message}" 115 | ) 116 | else: 117 | logger.warning( 118 | f"FedotAI ainvoke: Last message is not AIMessage (type: {type(last_message_original)}), direct content update might be insufficient or ineffective if immutable." 119 | ) 120 | # Attempting to update content directly if mutable, though AIMessage is preferred. 121 | if hasattr(last_message_original, "content"): 122 | last_message_original.content = translated_output 123 | logger.debug( 124 | f"FedotAI ainvoke: Attempted to update content of non-AIMessage. New last_message: {last_message_original}" 125 | ) 126 | 127 | else: 128 | logger.warning( 129 | "FedotAI ainvoke: Last message in response has no 'content' attribute." 130 | ) 131 | else: 132 | logger.warning( 133 | "FedotAI ainvoke: No messages found in raw_response or response structure is unexpected." 134 | ) 135 | 136 | return raw_response 137 | -------------------------------------------------------------------------------- /graph/code_executor_node.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | import sys 4 | import json 5 | import tempfile 6 | import subprocess 7 | 8 | from graph.state import AgentState 9 | from langchain_core.messages import AIMessage 10 | 11 | lightautoml_template = 'graph/lightautoml_template.py' 12 | 13 | PYTHON_REGEX = r"```python-execute(.+?)```" 14 | JSON_REGEX = r"```json(.+?)```" 15 | 16 | lightautoml_error = """В результате выполнения кода {lightautoml_template} возникла ошибка: 17 | ``` 18 | {process_err} 19 | ``` 20 | Исправь ошибку 21 | """ 22 | 23 | lightautoml_result = """Результат выполнения кода {lightautoml_template}: 24 | ``` 25 | {process_stdout} 26 | ``` 27 | """ 28 | 29 | matplotlib_setup = """ 30 | import matplotlib 31 | matplotlib.use('Agg') 32 | import matplotlib.pyplot as plt 33 | plt.ioff() # Turn off interactive mode 34 | """ 35 | 36 | local_exec_result = """Результат выполнения кода: 37 | ``` 38 | {process_stdout} 39 | ```""" 40 | 41 | local_exec_error = """В результате выполнения кода возникла ошибка: 42 | ``` 43 | {process_stderr} 44 | ``` 45 | Исправь ошибку""" 46 | 47 | timeout = 3000 48 | 49 | e2b_exec_error = """В результате выполнения кода возникла ошибка: 50 | ``` 51 | {execution_error_traceback} 52 | ``` 53 | Исправь ошибку""" 54 | 55 | e2b_exec_result = """Результат выполнения блока кода: 56 | ``` 57 | {logs} 58 | {text_results} 59 | ```""" 60 | 61 | 62 | def execute_e2b_code(sandbox, code: str) -> str: 63 | result = '' 64 | 65 | execution = sandbox.run_code(code) 66 | 67 | if execution.error: 68 | result = e2b_exec_error.format( 69 | execution_error_traceback=execution.error.traceback 70 | ) 71 | else: 72 | logs = '\n'.join(execution.logs.stdout) 73 | text_results = "\n".join([result.text for result in execution.results if result.text]) 74 | result_text = e2b_exec_result.format( 75 | logs=logs, 76 | text_results=text_results 77 | ) 78 | result = result_text.strip() 79 | 80 | return result 81 | 82 | 83 | def execute_code_locally(code: str) -> str: 84 | 85 | result = '' 86 | 87 | with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False) as temp_file: 88 | temp_file.write(code) 89 | temp_file.flush() 90 | 91 | try: 92 | process = subprocess.run( 93 | [sys.executable, temp_file.name], 94 | capture_output=True, 95 | text=True, 96 | timeout=timeout 97 | ) 98 | 99 | if process.returncode == 0: 100 | result = local_exec_result.format(process_stdout=process.stdout) 101 | else: 102 | result = local_exec_error.format(process_stderr=process.stderr) 103 | 104 | except subprocess.TimeoutExpired: 105 | result = f"Код превысил время выполнения ({timeout} секунд)" 106 | finally: 107 | os.unlink(temp_file.name) 108 | 109 | return result 110 | 111 | 112 | def execute_lightautoml_locally(state: AgentState): 113 | messages = state['messages'] 114 | json_block = re.findall(JSON_REGEX, messages[-1].content, re.DOTALL | re.MULTILINE)[0] 115 | config = json.loads(json_block) 116 | result = '' 117 | try: 118 | process = subprocess.run( 119 | [ 120 | sys.executable, 121 | lightautoml_template, 122 | "--df_name", state['df_name'], 123 | "--task_type", config['task_type'], 124 | "--target", config['target'], 125 | "--task_metric", config['task_metric'] 126 | ], 127 | capture_output=True, 128 | text=True, 129 | timeout=timeout 130 | ) 131 | 132 | if process.returncode == 0: 133 | result_text = lightautoml_result.format( 134 | lightautoml_template=lightautoml_template, 135 | process_stdout=process.stdout 136 | ) 137 | result = result_text.strip() 138 | else: 139 | result = lightautoml_error.format( 140 | lightautoml_template=lightautoml_template, 141 | process_err=process.stderr 142 | ) 143 | 144 | except subprocess.TimeoutExpired: 145 | result = f"Блок {lightautoml_template} превысил время выполнения ({timeout} секунд)" 146 | 147 | return result 148 | 149 | 150 | def execute_train_test(state: AgentState): 151 | messages = state['messages'] 152 | last_content = messages[-1].content 153 | 154 | code_blocks = re.findall(PYTHON_REGEX, last_content, re.DOTALL | re.MULTILINE) 155 | train_code = code_blocks[0].strip() if len(code_blocks) > 0 else "" 156 | test_code = code_blocks[1].strip() if len(code_blocks) > 1 else "" 157 | 158 | result_train = execute_code_locally(train_code) 159 | result_test = execute_code_locally(test_code) 160 | 161 | result = AIMessage(content=f"Результаты выполнения кода для обучения:\n{result_train}\n\nРезультаты выполнения кода для тестирования:\n{result_test}") 162 | return {"messages": result, "train_code": train_code, "test_code": test_code} 163 | 164 | 165 | def execute_code(state: AgentState): 166 | messages = state['messages'] 167 | code_blocks = re.findall(PYTHON_REGEX, messages[-1].content, re.DOTALL | re.MULTILINE) 168 | code_to_execute = "\n".join(code_blocks) 169 | full_code = matplotlib_setup + code_to_execute + "\nplt.close('all')" 170 | execution_location = state['code_generation_config'] 171 | 172 | if state['lama']: 173 | result = execute_lightautoml_locally(state) 174 | # if state['test_split']: 175 | # train = code_blocks[0] 176 | # test = code_blocks[1] 177 | 178 | # result_train = execute_code_locally(train) 179 | # result_test = execute_code_locally(test) 180 | # result = f"Результаты выполнения кода для обучения:\n{result_train}\n\nРезультаты выполнения кода для тестирования:\n{result_test}" 181 | # code_to_execute = f"train:\n{train}\ntest:\n{test}" 182 | 183 | else: 184 | if execution_location == 'e2b': 185 | sandbox = state['sandbox'] 186 | result = execute_e2b_code(sandbox, full_code) 187 | 188 | if execution_location == 'local': 189 | result = execute_code_locally(full_code) 190 | 191 | return {"messages": AIMessage(content=result), 'generated_code': code_to_execute, 'code_results': result, 'lama': False, 'test_split': False} 192 | 193 | -------------------------------------------------------------------------------- /fedotllm/agents/automl/templates/skeleton-simple.py: -------------------------------------------------------------------------------- 1 | ### UNMODIFIABLE IMPORT BEGIN ### 2 | import random 3 | from pathlib import Path 4 | import pandas as pd 5 | import numpy as np 6 | from typing import Tuple 7 | from fedot.api.main import Fedot 8 | from fedot.core.data.data import InputData 9 | from fedot.core.repository.tasks import ( 10 | Task, 11 | TaskTypesEnum, 12 | ) # classification, regression, ts_forecasting. 13 | from automl import train_model, evaluate_model, automl_predict 14 | 15 | ### UNMODIFIABLE IMPORT END ### 16 | # USER CODE BEGIN IMPORTS # 17 | from sklearn.model_selection import train_test_split 18 | from sklearn.impute import SimpleImputer 19 | # USER CODE END IMPORTS # 20 | 21 | SEED = 42 22 | random.seed(SEED) 23 | np.random.seed(SEED) 24 | 25 | ### UNMODIFIABLE CODE BEGIN ### 26 | DATASET_PATH = Path("{%dataset_path%}") # path for saving and loading dataset(s) 27 | WORKSPACE_PATH = Path("{%work_dir_path%}") 28 | PIPELINE_PATH = WORKSPACE_PATH / "pipeline" # path for saving and loading pipelines 29 | SUBMISSION_PATH = WORKSPACE_PATH / "submission.csv" # path for saving submission file 30 | EVAL_SET_SIZE = 0.2 # 20% of the data for evaluation 31 | ### UNMODIFIABLE CODE END ### 32 | # --- TODO: Update these paths for your specific competition --- 33 | TRAIN_FILE = DATASET_PATH / "train.csv" # Replace with your actual filename 34 | TEST_FILE = DATASET_PATH / "test.csv" # Replace with your actual filename 35 | SAMPLE_SUBMISSION_FILE = ( 36 | DATASET_PATH / "sample_submission.csv" 37 | ) # Replace with your actual filename or None 38 | 39 | 40 | # USER CODE BEGIN LOAD_DATA # 41 | def load_data(): 42 | # TODO: this function is for loading a dataset from user’s local storage 43 | return train, X_test 44 | 45 | 46 | # USER CODE END LOAD_DATA # 47 | 48 | 49 | def transform_data(dataset: pd.DataFrame) -> Tuple[np.ndarray, np.ndarray]: 50 | """ 51 | Function to transform data into a format that can be used for training the model. 52 | Used on both Train and Test data. Test data may initially not contain target columns. 53 | """ 54 | 55 | # TODO: Specify target columns 56 | target_columns = [] 57 | 58 | # Separating features and target if present 59 | data = dataset.copy(deep=True) 60 | has_target = any(col in data.columns for col in target_columns) 61 | if has_target: 62 | features = data.drop(columns=target_columns) 63 | target = data[target_columns].values 64 | else: 65 | features = data 66 | target = None 67 | 68 | # Imputing missing values - 'mean' strategy for numeric columns, 'most_frequent' otherwise 69 | numeric_cols = features.select_dtypes(include=[np.number]).columns 70 | categorical_cols = features.select_dtypes(exclude=[np.number]).columns 71 | if len(numeric_cols) > 0: 72 | numeric_imputer = SimpleImputer(strategy="mean") 73 | features[numeric_cols] = numeric_imputer.fit_transform(features[numeric_cols]) 74 | if len(categorical_cols) > 0: 75 | categorical_imputer = SimpleImputer(strategy="most_frequent") 76 | features[categorical_cols] = categorical_imputer.fit_transform( 77 | features[categorical_cols] 78 | ) 79 | 80 | # TODO: Drop all columns from features that are not important for prdictions. All other dataset transformations are STRICTLY FORBIDDEN. 81 | # TODO: Before any operations, make sure to check whether columns you operate on are present in data. Do not raise exceptions. 82 | 83 | return features.values, target 84 | 85 | 86 | # The main function to orchestrate the data loading, feature engineering, model training and model evaluation 87 | def create_model(): 88 | """ 89 | Function to execute the ML pipeline. 90 | """ 91 | # USER CODE BEGIN CREATE MODEL # 92 | # TODO: Step 1. Retrieve or load a dataset from hub (if available) or user’s local storage, start path from the DATASET_PATH 93 | train, X_test = load_data() 94 | 95 | # TODO: Step 2. Create a train-test split of the data by splitting the ‘dataset‘ into train_data and test_data. 96 | # Create a train-validation split 97 | # Note: EVAL_SET_SIZE is a constant defined above, corresponding to 20% of the data for evaluation 98 | # Note: You may need to use stratified sampling if the target is categorical 99 | train_data, eval_test_data = train_test_split( 100 | train, test_size=EVAL_SET_SIZE, random_state=SEED 101 | ) # corresponding to 80%, 20% of ‘dataset‘ 102 | 103 | train_features, train_target = transform_data(train_data) 104 | eval_test_features, eval_test_target = transform_data(eval_test_data) 105 | test_features, _ = transform_data(X_test) 106 | 107 | # TODO: Step 3. Train AutoML model. AutoML performs feature engineering and model training. 108 | model = train_model(train_features, train_target) 109 | 110 | # TODO: Step 4. evaluate the trained model using the defined "evaluate_model" function model_performance, model_complexity = evaluate_model() 111 | model_performance = evaluate_model(model, eval_test_features, eval_test_target) 112 | 113 | # TODO: Step 5. Evaluate predictions for the test datase using AutoML Framework 114 | # **YOU MUST USE automl_predict()** 115 | # Prediction result will not have an ID column, only a column for target (or columns for multiple targets) 116 | # If output submission should have an ID column, add it to the prediction. 117 | # If ID column has numeric type, convert it to integer 118 | predictions: np.ndarray = automl_predict(model, test_features) # returns 2D array 119 | output = pd.DataFrame(predictions, columns=[...]) 120 | 121 | # USER CODE END CREATE MODEL # 122 | # If target submission format is not numeric, convert predictions to expected format 123 | # For example: convert probabilities to class labels, apply inverse transformations, 124 | # or map numeric predictions back to categorical labels if needed 125 | output.to_csv(SUBMISSION_PATH, index=False) 126 | return model_performance 127 | 128 | 129 | ### UNMODIFIABLE CODE BEGIN ### 130 | def main(): 131 | """ 132 | Main function to execute the ML pipeline. 133 | """ 134 | print("Files and directories:") 135 | paths = { 136 | "Dataset Path": DATASET_PATH, 137 | "Workspace Path": WORKSPACE_PATH, 138 | "Pipeline Path": PIPELINE_PATH, 139 | "Submission Path": SUBMISSION_PATH, 140 | "Train File": TRAIN_FILE, 141 | "Test File": TEST_FILE, 142 | "Sample Submission File": SAMPLE_SUBMISSION_FILE, 143 | } 144 | for name, path in paths.items(): 145 | print(f"{name}: {path}") 146 | 147 | model_performance = create_model() 148 | print("Model Performance on Test Set:", model_performance) 149 | 150 | 151 | if __name__ == "__main__": 152 | main() 153 | ### UNMODIFIABLE CODE END ### 154 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | aiohappyeyeballs==2.6.1 2 | aiohttp==3.11.16 3 | aioice==0.10.1 4 | aiortc==1.12.0 5 | aiosignal==1.3.2 6 | alabaster==0.7.16 7 | alembic==1.15.2 8 | altair==5.5.0 9 | annotated-types==0.7.0 10 | anyio==4.9.0 11 | anytree==2.13.0 12 | argon2-cffi==23.1.0 13 | argon2-cffi-bindings==21.2.0 14 | asttokens==3.0.0 15 | async-timeout==4.0.3 16 | attrs==25.3.0 17 | autodocsumm==0.2.14 18 | AutoWoE==1.3.3 19 | av==14.3.0 20 | babel==2.17.0 21 | backoff==2.2.1 22 | backports.tarfile==1.2.0 23 | beautifulsoup4==4.13.3 24 | bitsandbytes==0.46.0 25 | blinker==1.9.0 26 | bokeh==3.7.3 27 | bs4==0.0.2 28 | build==1.2.2.post1 29 | CacheControl==0.14.3 30 | cachetools==5.5.2 31 | catboost==1.2.8 32 | certifi==2025.1.31 33 | cffi==1.17.1 34 | charset-normalizer==3.4.1 35 | cleo==2.1.0 36 | click==8.1.8 37 | cloudpickle==3.1.1 38 | cmaes==0.11.1 39 | coloredlogs==15.0.1 40 | colorlog==6.9.0 41 | contourpy==1.3.1 42 | crashtest==0.4.1 43 | cryptography==44.0.3 44 | cycler==0.12.1 45 | dask==2025.3.0 46 | dataclasses-json==0.6.7 47 | datasets==3.6.0 48 | decorator==5.2.1 49 | depq==1.5.5 50 | dill==0.3.8 51 | distlib==0.3.9 52 | distributed==2025.3.0 53 | distro==1.9.0 54 | dnspython==2.7.0 55 | docutils==0.20.1 56 | dotenv==0.9.9 57 | dulwich==0.22.8 58 | dynaconf==3.2.11 59 | e2b==1.2.0 60 | e2b-code-interpreter==1.1.1 61 | et_xmlfile==2.0.0 62 | ete3==3.1.3 63 | exceptiongroup==1.3.0 64 | executing==2.2.0 65 | fastembed==0.6.0 66 | fastjsonschema==2.21.1 67 | fedot==0.7.5 68 | filelock==3.18.0 69 | findpython==0.6.3 70 | flatbuffers==25.2.10 71 | fonttools==4.56.0 72 | frozenlist==1.5.0 73 | fsspec==2025.3.0 74 | func_timeout==4.3.5 75 | future==1.0.0 76 | gigachat==0.1.39 77 | gitdb==4.0.12 78 | GitPython==3.1.44 79 | google-crc32c==1.7.1 80 | grandalf==0.8 81 | graphviz==0.20.3 82 | greenlet==3.1.1 83 | grpcio==1.71.0 84 | grpcio-tools==1.71.0 85 | h11==0.14.0 86 | h2==4.2.0 87 | holidays==0.69 88 | hpack==4.1.0 89 | httpcore==1.0.7 90 | httpx==0.28.1 91 | httpx-sse==0.4.0 92 | huggingface-hub==0.30.1 93 | humanfriendly==10.0 94 | hyperframe==6.1.0 95 | hyperopt==0.2.7 96 | idna==3.10 97 | ifaddr==0.2.0 98 | imageio==2.37.0 99 | imagesize==1.4.1 100 | imbalanced-learn==0.13.0 101 | imblearn==0.0 102 | importlib_metadata==8.7.0 103 | iniconfig==2.1.0 104 | installer==0.7.0 105 | iOpt==0.2.22 106 | ipython==8.37.0 107 | jaraco.classes==3.4.0 108 | jaraco.context==6.0.1 109 | jaraco.functools==4.1.0 110 | jedi==0.19.2 111 | jeepney==0.9.0 112 | Jinja2==3.1.6 113 | jiter==0.9.0 114 | joblib==1.4.2 115 | json2html==1.3.0 116 | json_repair==0.47.1 117 | jsonpatch==1.33 118 | jsonpickle==4.1.1 119 | jsonpointer==3.0.0 120 | jsonschema==4.23.0 121 | jsonschema-specifications==2024.10.1 122 | keyring==25.6.0 123 | kiwisolver==1.4.8 124 | langchain==0.3.23 125 | langchain-community==0.3.21 126 | langchain-core==0.3.51 127 | langchain-gigachat==0.3.8 128 | langchain-openai==0.3.9 129 | langchain-qdrant==0.2.0 130 | langchain-text-splitters==0.3.8 131 | langdetect==1.0.9 132 | langfuse==2.60.0 133 | langgraph==0.3.18 134 | langgraph-checkpoint==2.0.21 135 | langgraph-prebuilt==0.1.3 136 | langgraph-sdk==0.1.58 137 | langsmith==0.3.18 138 | liac-arff==2.5.0 139 | LightAutoML==0.4.1 140 | lightgbm==4.6.0 141 | litellm==1.73.0 142 | loader==2017.9.11 143 | locket==1.0.0 144 | loguru==0.7.3 145 | mabwiser==2.7.4 146 | Mako==1.3.9 147 | Markdown==3.8 148 | MarkupSafe==2.1.1 149 | marshmallow==3.26.1 150 | matplotlib==3.10.1 151 | matplotlib-inline==0.1.7 152 | minio==7.2.15 153 | mlxtend==0.23.4 154 | mmh3==5.1.0 155 | more-itertools==10.7.0 156 | mpmath==1.3.0 157 | msgpack==1.1.0 158 | multidict==6.3.2 159 | multiprocess==0.70.16 160 | mypy-extensions==1.0.0 161 | narwhals==1.32.0 162 | networkx==3.2.1 163 | nltk==3.9.1 164 | numpy==1.26.2 165 | nvidia-cublas-cu12==12.4.5.8 166 | nvidia-cuda-cupti-cu12==12.4.127 167 | nvidia-cuda-nvrtc-cu12==12.4.127 168 | nvidia-cuda-runtime-cu12==12.4.127 169 | nvidia-cudnn-cu12==9.1.0.70 170 | nvidia-cufft-cu12==11.2.1.3 171 | nvidia-curand-cu12==10.3.5.147 172 | nvidia-cusolver-cu12==11.6.1.9 173 | nvidia-cusparse-cu12==12.3.1.170 174 | nvidia-cusparselt-cu12==0.6.2 175 | nvidia-nccl-cu12==2.21.5 176 | nvidia-nvjitlink-cu12==12.4.127 177 | nvidia-nvtx-cu12==12.4.127 178 | onnxruntime==1.21.0 179 | openai==1.88.0 180 | openml==0.15.1 181 | openpyxl==3.1.5 182 | optuna==4.2.1 183 | orjson==3.10.15 184 | packaging==24.2 185 | pandas==2.2.3 186 | parso==0.8.4 187 | partd==1.4.2 188 | pathos==0.3.2 189 | patsy==1.0.1 190 | pbs-installer==2025.6.12 191 | pexpect==4.9.0 192 | pillow==11.1.0 193 | pkginfo==1.12.1.2 194 | platformdirs==4.3.8 195 | plotly==6.0.1 196 | pluggy==1.5.0 197 | poetry-core==1.9.1 198 | portalocker==2.10.1 199 | pox==0.3.6 200 | ppft==1.7.7 201 | prompt_toolkit==3.0.51 202 | propcache==0.3.1 203 | protobuf==5.29.4 204 | psutil==7.0.0 205 | ptyprocess==0.7.0 206 | pure_eval==0.2.3 207 | py4j==0.10.9.9 208 | py_rust_stemmers==0.1.5 209 | pyaml==25.5.0 210 | pyarrow==19.0.1 211 | pycparser==2.22 212 | pycryptodome==3.22.0 213 | pydantic==2.10.6 214 | pydantic-settings==2.8.1 215 | pydantic_core==2.27.2 216 | pydeck==0.9.1 217 | pydub==0.25.1 218 | pyee==13.0.0 219 | Pygments==2.19.1 220 | pylibsrtp==0.12.0 221 | pyngrok==7.2.8 222 | pyOpenSSL==25.0.0 223 | pyparsing==3.2.3 224 | pyproject_hooks==1.2.0 225 | pytest==8.3.5 226 | python-dateutil==2.9.0.post0 227 | python-dotenv==1.0.1 228 | pytz==2025.1 229 | pyvis==0.2.1 230 | PyYAML==6.0.2 231 | qdrant-client==1.13.3 232 | RapidFuzz==3.13.0 233 | readthedocs-sphinx-search==0.3.2 234 | referencing==0.36.2 235 | regex==2024.11.6 236 | requests==2.32.3 237 | requests-toolbelt==1.0.0 238 | roman-numerals-py==3.1.0 239 | rpds-py==0.24.0 240 | safetensors==0.5.3 241 | SALib==1.5.1 242 | scikit-base==0.12.3 243 | scikit-learn==1.5.2 244 | scikit-optimize==0.10.2 245 | scipy==1.12.0 246 | seaborn==0.13.2 247 | SecretStorage==3.3.3 248 | sentence-transformers==4.0.2 249 | shellingham==1.5.4 250 | six==1.17.0 251 | sklearn-compat==0.1.3 252 | sktime==0.37.0 253 | smart-open==7.1.0 254 | smmap==5.0.2 255 | sniffio==1.3.1 256 | snowballstemmer==2.2.0 257 | sortedcontainers==2.4.0 258 | sounddevice==0.5.1 259 | soupsieve==2.6 260 | Sphinx==7.1.2 261 | sphinx-rtd-theme==3.0.2 262 | sphinxcontrib-applehelp==2.0.0 263 | sphinxcontrib-details-directive==0.1.0 264 | sphinxcontrib-devhelp==2.0.0 265 | sphinxcontrib-htmlhelp==2.1.0 266 | sphinxcontrib-jquery==4.1 267 | sphinxcontrib-jsmath==1.0.1 268 | sphinxcontrib-qthelp==2.0.0 269 | sphinxcontrib-serializinghtml==2.0.0 270 | SQLAlchemy==2.0.39 271 | stack-data==0.6.3 272 | statsmodels==0.14.0 273 | streamlit==1.44.1 274 | streamlit-bokeh-events==0.1.2 275 | streamlit-scrollable-textbox==0.0.3 276 | streamlit-webrtc==0.62.4 277 | StrEnum==0.4.15 278 | sympy==1.13.1 279 | tabulate==0.9.0 280 | tblib==3.1.0 281 | tenacity==9.0.0 282 | testfixtures==8.3.0 283 | thegolem==0.4.1 284 | threadpoolctl==3.6.0 285 | tiktoken==0.9.0 286 | tokenizers==0.21.1 287 | toml==0.10.2 288 | tomli==2.2.1 289 | tomlkit==0.13.3 290 | toolz==1.0.0 291 | torch==2.6.0 292 | tornado==6.5.1 293 | tqdm==4.66.6 294 | traitlets==5.14.3 295 | transformers==4.51.0 296 | triton==3.2.0 297 | trove-classifiers==2025.5.9.12 298 | types-requests==2.32.0.20250306 299 | typing==3.7.4.3 300 | typing-inspect==0.9.0 301 | typing_extensions==4.14.0 302 | tzdata==2025.1 303 | urllib3==2.3.0 304 | virtualenv==20.31.2 305 | watchdog==6.0.0 306 | wcwidth==0.2.13 307 | wrapt==1.17.2 308 | xgboost==2.1.4 309 | xmltodict==0.14.2 310 | xxhash==3.5.0 311 | xyzservices==2025.4.0 312 | yarl==1.19.0 313 | zict==3.0.0 314 | zipp==3.23.0 315 | zss==1.2.0 316 | zstandard==0.23.0 317 | -------------------------------------------------------------------------------- /fedotllm/prompts/automl.py: -------------------------------------------------------------------------------- 1 | def code_generation_prompt(reflection: str, dataset_path: str, skeleton: str) -> str: 2 | return f""" 3 | You are a helpful and intelligent assistant specializing in solving machine learning tasks. Your role is to complete and optimize Python scripts based on user instructions while adhering strictly to the specified constraints. 4 | # About the Dataset: 5 | [Task] 6 | {reflection} 7 | 8 | [Path to Dataset] 9 | {dataset_path} 10 | 11 | Below is the Python script you need to complete. Your implementation must begin with a Python code block (` ```python `) and strictly produce executable code without requiring further modifications. 12 | 13 | [solution.py] 14 | ```python 15 | {skeleton} 16 | ``` 17 | 18 | **Key Rules and Constraints**: 19 | 1. Do **not delete any comments** in the provided code. 20 | 2. Do **not modify code enclosed between** the designated markers (`### comment ### code ### comment ###`). This code is autogenerated and will be regenerated upon project restarts. 21 | 3. You are **prohibited from using any methods or attributes** from the Fedot framework classes (e.g., `Fedot`, `Pipeline`), except those that are **already used in the provided code** or **explicitly mentioned in the comments**. 22 | 4. You are allowed to write and modify code **only within the 'USER CODE' sections**. All other sections will be regenerated upon project restarts. 23 | 24 | Write the whole code below. 25 | ```python 26 | """ 27 | 28 | 29 | def fix_solution_prompt( 30 | reflection: str, 31 | dataset_path: str, 32 | code_recent_solution: str, 33 | stdout: str, 34 | stderr: str, 35 | msg: str = "", 36 | ) -> str: 37 | return f""" 38 | You are a senior machine learning engineer. Analyze the following information: the task description with reflections, the path to the dataset, the Python code from a previous solution, and the resulting stdout and stderr. Identify and correct the specific error that caused the failure without altering any other logic. Provide the complete corrected Python script in a code block. 39 | 40 | # Task with Reflections 41 | {reflection} 42 | 43 | # Dataset Path 44 | {dataset_path} 45 | 46 | # Previous Python Solution 47 | ```python 48 | {code_recent_solution} 49 | ``` 50 | {"# Execution Message: " + msg if msg else ""} 51 | 52 | # Execution Output 53 | ```text 54 | {stdout} 55 | ``` 56 | 57 | # Error Trace 58 | ```text 59 | {stderr} 60 | ``` 61 | 62 | Write the full fixed code below. 63 | ```python 64 | """ 65 | 66 | 67 | def generate_configuration_prompt(reflection: str) -> str: 68 | return f""" 69 | You are a machine learning expert tasked with solving a given machine learning problem. 70 | Review the problem description provided within the `` section, including any reflections or additional context. 71 | Your objective is to define the optimal parameters for an automated machine learning model fitting framework, ensuring alignment with the stated goals, rules, and constraints of the task. 72 | If specific parameter values or constraints are not provided, use default values that are commonly accepted as best practices. 73 | 74 | 75 | {reflection} 76 | 77 | """ 78 | 79 | 80 | def problem_reflection_prompt(data_files_and_content: str, dataset_eda: str) -> str: 81 | return f""" 82 | Please conduct a comprehensive analysis of the competition, focusing on the following aspects: 83 | 1. Competition Overview: Understand the background and context of the topic. 84 | 2. Files: Analyze each provided file, detailing its purpose and how it should be used in the competition. 85 | 3. Problem Definition: Clarify the problem's definition and requirements. 86 | 4. Data Information: Gather detailed information about the data, including its structure and contents. 87 | 4.1 Data type: 88 | 4.1.1. ID type: features that are unique identifiers for each data point, which will NOT be used in the model training. 89 | 4.1.2. Numerical type: features that are numerical values. 90 | 4.1.3. Categorical type: features that are categorical values. 91 | 4.1.4 Datetime type: features that are datetime values. 92 | 4.2 Detailed data description 93 | 5. Target Variable: Identify the target variable that needs to be predicted or optimized, which is provided in the training set but not in the test set. 94 | 6. Evaluation Metrics: Determine the evaluation metrics that will be used to assess the submissions. 95 | 7. Submission Format: Understand the required format for the final submission. 96 | 8. Other Key Aspects: Highlight any other important aspects that could influence the approach to the competition. 97 | Ensure that the analysis is thorough, with a strong emphasis on : 98 | 1. Understanding the purpose and usage of each file provided. 99 | 2. Figuring out the target variable and evaluation metrics. 100 | 3. Classification of the features. 101 | 102 | # Available Data File And Content in The File 103 | {data_files_and_content} 104 | 105 | # EDA 106 | {dataset_eda} 107 | """ 108 | 109 | 110 | # # Available Data File And Content in The File 111 | # {data_files_and_content} 112 | 113 | # # EDA 114 | # {dataset_eda} 115 | # """ 116 | 117 | 118 | def reporter_prompt(metrics: str, pipeline: str, code: str) -> str: 119 | return f""" 120 | You are an expert in machine learning tasked with evaluating and reporting on an ML model designed to address the problem. 121 | 122 | Your report should adhere to the following instructions: 123 | - Be concise and styled like a Substack blog summary. 124 | - Structure the content using bullet points for readability. 125 | - Use Markdown formatting (e.g., headers, **bold text**, `code snippets`, tables) for clarity. 126 | - Explain key points in popular science language, accessible to readers from diverse fields. 127 | - Include code snippets and interpretation of results in layman-friendly terms. 128 | - Provide essential context while avoiding discussions of empty or missing values. 129 | - Do not include suggestions for next steps or future work. 130 | 131 | **Report Outline:** 132 | 1. **Overview** 133 | - Problem description 134 | - Goal: Summarize the purpose of the model in plain terms for a general audience. 135 | 136 | 2. **Data Preprocessing** 137 | - Describe the data preprocessing steps used before modeling in plain, accessible language. 138 | - Provide illustrative examples to clarify specific preprocessing steps. 139 | - If data normalization was applied, describe it as: 140 | "Normalization ensures all features are on the same scale, improving model performance. For example, a scaling process converts values like 'age' (5-90) to 0-1." 141 | - If missing values were imputed, describe it as: 142 | "Missing values in the dataset were replaced using mean imputation to ensure uniformity. For example, in the column 'Income', the mean value of $50,000 was substituted for missing entries." 143 | 144 | 3. **Pipeline Summary** 145 | - Summarize the steps in `{pipeline}` using accessible language and optionally include illustrative examples. 146 | - Key Parameters: 147 | | Model | Parameters | Explanation | 148 | |---------|--------------|---------------| 149 | | CatBoost| num_trees: 3000, learning_rate: 0.03, max_depth: 5, l2_leaf_reg: 0.01 | CatBoost was choosen because | 150 | | Model 2 | Parameter 2 | Explanation 2 | 151 | 152 | 4. **Code Highlights:** 153 | - Include relevant code snippets wrapped in Markdown Python blocks: 154 | ```python 155 | {code} 156 | ``` 157 | - Add a brief explanation of what the code does and why it's a key. 158 | 1. Data Preprocessing (Short Key Snippets) 159 | 2. Model Training, Evaluation, Prediction 160 | 3. Submission File Creation 161 | 4. Other Key Snippets 162 | 163 | 5. **Metrics** 164 | - Share performance metric `{metrics}` and briefly describe what each metric signifies (e.g., "Accuracy tells us how often the model gets it right"): 165 | 166 | 6. **Takeaways** 167 | - Wrap up with a concise summary of results, emphasizing their significance in a real-world context. For example: "This model predicts X with an accuracy of Y%, demonstrating its potential in Z applications." 168 | 169 | Engage your audience with a relatable, professional tone that simplifies complex ideas without oversimplifying the context. Ensure the report can resonate with both experts and non-experts alike. 170 | """ 171 | -------------------------------------------------------------------------------- /app_components/agent_handler.py: -------------------------------------------------------------------------------- 1 | import time 2 | import uuid 3 | import re 4 | import logging 5 | import streamlit as st 6 | from typing import List, Tuple 7 | 8 | from utils.config.loader import load_config 9 | from e2b_code_interpreter import Sandbox 10 | from langfuse.callback import CallbackHandler 11 | from graph.graph import graph_builder 12 | from sklearn.model_selection import train_test_split 13 | from .data_handlers import save_file_to_disk 14 | 15 | 16 | logger = logging.getLogger(__name__) 17 | 18 | METRIC = "ROC-AUC" 19 | 20 | def initialize_services(): 21 | if "services_initialized" not in st.session_state: 22 | now = time.time() 23 | with st.spinner("Initializing services..."): 24 | config = load_config() 25 | if config.general.e2b_token: 26 | sandbox = Sandbox(api_key=config.general.e2b_token.get_secret_value()) 27 | sandbox.set_timeout(1000) 28 | st.session_state.sandbox = sandbox 29 | 30 | agent = graph_builder() 31 | logger.info(f"Time until graph is built: {time.time() - now}") 32 | if config.langfuse: 33 | session_id = st.session_state.get("uuid", str(uuid.uuid4())) 34 | langfuse_handler = CallbackHandler( 35 | public_key=config.langfuse.public_key.get_secret_value(), 36 | secret_key=config.langfuse.secret_key.get_secret_value(), 37 | host=config.langfuse.host, 38 | user_id=config.langfuse.user, 39 | session_id=session_id 40 | ) 41 | st.session_state.langfuse_handler = langfuse_handler 42 | 43 | st.session_state.config = config 44 | st.session_state.agent = agent 45 | st.session_state.services_initialized = True 46 | logger.info(f"Time until whole session state is set: {str(time.time() - now)}") 47 | 48 | 49 | def build_conversation_history() -> List[Tuple[str, str]]: 50 | conversation_history = [] 51 | current_conv_messages = st.session_state.conversations.get(st.session_state.current_conversation, []) 52 | 53 | for message in current_conv_messages: 54 | role = message.get("role") 55 | content = message.get("content", "") 56 | 57 | if role == "user": 58 | conversation_history.append(("user", content)) 59 | elif role == "assistant": 60 | conversation_history.append(("assistant", content)) 61 | return conversation_history 62 | 63 | 64 | def stream_agent_response_for_frontend(): 65 | 66 | config = st.session_state.get("config", {}) 67 | sandbox = st.session_state.get("sandbox", None) 68 | agent = st.session_state.get("agent", None) 69 | langfuse_handler = st.session_state.get("langfuse_handler", None) 70 | rec_lim = config.general.recursion_limit 71 | 72 | if "shown_human_messages" not in st.session_state: 73 | st.session_state.shown_human_messages = set() 74 | 75 | if (st.session_state.current_conversation not in st.session_state.conversations): 76 | st.error("Error: Current conversation not found.") 77 | return 78 | 79 | conversation_messages = (st.session_state.conversations[st.session_state.current_conversation]) 80 | if not conversation_messages: 81 | st.error("Error: No messages in current conversation.") 82 | return 83 | 84 | conversation_history = build_conversation_history() 85 | 86 | 87 | df_name = st.session_state.get("df_name") 88 | test_df_name = st.session_state.get("test_df_name") 89 | if df_name and df_name in st.session_state.uploaded_files and not test_df_name: 90 | full_df = st.session_state.uploaded_files[df_name]["df"] 91 | X_train, X_test = train_test_split(full_df, test_size=0.2, random_state=42) 92 | 93 | if "." in df_name: 94 | base, ext = df_name.rsplit('.', 1) 95 | train_name = f"train.{ext}" 96 | test_name = f"test.{ext}" 97 | else: 98 | train_name = f"train" 99 | test_name = f"test" 100 | 101 | st.session_state.uploaded_files[train_name] = { 102 | 'df': X_train, 103 | 'type': st.session_state.uploaded_files[df_name]['type'], 104 | 'df_name': train_name 105 | } 106 | st.session_state.uploaded_test_files[test_name] = { 107 | 'df': X_test, 108 | 'type': st.session_state.uploaded_files[df_name]['type'], 109 | 'df_name': test_name 110 | } 111 | 112 | # Save split datasets to disk 113 | file_ext = st.session_state.uploaded_files[df_name]['type'] 114 | save_file_to_disk(X_train, train_name, file_ext) 115 | save_file_to_disk(X_test, test_name, file_ext) 116 | 117 | st.session_state.df_name = train_name 118 | st.session_state.test_df_name = test_name 119 | 120 | df = None 121 | df_name = st.session_state.df_name 122 | test_df = None 123 | test_df_name = st.session_state.get("test_df_name") 124 | if test_df_name and test_df_name in st.session_state.uploaded_test_files: 125 | test_df = st.session_state.uploaded_test_files[test_df_name]["df"] 126 | if df_name and df_name in st.session_state.uploaded_files: 127 | df = st.session_state.uploaded_files[df_name]["df"] 128 | 129 | try: 130 | agent_config = {"recursion_limit": rec_lim} 131 | 132 | if langfuse_handler: 133 | agent_config["callbacks"] = [langfuse_handler] 134 | 135 | agent_message = {"messages": conversation_history} 136 | agent_message["code_generation_config"] = config.general.code_generation_config 137 | 138 | if sandbox: 139 | agent_message["sandbox"] = sandbox 140 | if df is not None: 141 | agent_message["df"] = df 142 | agent_message["df_name"] = df_name 143 | if test_df is not None: 144 | agent_message["test_df"] = test_df 145 | agent_message["test_df_name"] = test_df_name 146 | 147 | for values in agent.stream(agent_message, stream_mode="values", config=agent_config): 148 | human_content = None 149 | current_node = values.get("current_node") 150 | matches = None 151 | 152 | 153 | hu_list = values.get("human_understanding", []) 154 | current_node = values.get("current_node") 155 | st.session_state.current_node = current_node 156 | 157 | if hu_list: 158 | for hu_content in hu_list: 159 | if isinstance(hu_content, list): 160 | hu_content_str = "\n".join(str(item) for item in hu_content) 161 | else: 162 | hu_content_str = str(hu_content) 163 | 164 | if hu_content_str not in st.session_state.shown_human_messages: 165 | st.session_state.shown_human_messages.add(hu_content_str) 166 | human_content = hu_content_str 167 | break 168 | 169 | if current_node == "result_summarization_agent" or current_node == "fedot_config_generator": 170 | matches = re.findall(fr'{METRIC}: ([0-9]*\.[0-9]+)', values["messages"][-1].content) 171 | elif current_node == "lightautoml_local_executor": 172 | matches = re.findall(r'test data: ([0-9]*\.[0-9]+)', values["messages"][-1].content) 173 | if matches is not None: 174 | for match in matches: 175 | metric = float(match) 176 | st.session_state.extract_metric.append(metric) 177 | 178 | message = values["messages"][-1] 179 | 180 | if current_node is None: 181 | continue 182 | 183 | node_message_content = f"**{current_node}:** {message.content}" 184 | 185 | yield { 186 | "type": "assistant_message_chunk", 187 | "node_name": current_node, 188 | "content": node_message_content, 189 | "human_content": human_content, 190 | } 191 | 192 | except RecursionError: 193 | logger.error( 194 | "Maximum recursion depth reached during agent processing." 195 | ) 196 | yield { 197 | "type": "assistant_message_chunk", 198 | "node_name": "Error", 199 | "content": ( 200 | "Processing stopped due to reaching maximum recursion depth." 201 | ), 202 | "human_content": None, 203 | } 204 | except Exception as e: 205 | st.error(f"Error during agent processing: {str(e)}") 206 | logger.error(f"Error during agent processing: {str(e)}") 207 | yield { 208 | "type": "assistant_message_chunk", 209 | "node_name": "Error", 210 | "content": f"An error occurred: {str(e)}", 211 | "human_content": None, 212 | } -------------------------------------------------------------------------------- /graph/graph.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | from langgraph.graph import END, StateGraph, START 4 | 5 | from graph.state import AgentState 6 | from graph.code_executor_node import execute_code, execute_train_test 7 | from graph.llm_nodes import ( 8 | input_node, 9 | rephraser_agent, 10 | code_generation_agent, 11 | validate_solution, 12 | code_improvement_agent, 13 | automl_router, 14 | lightautoml_generator, 15 | feedback_for_code_improvement_agent, 16 | human_explanation_agent, 17 | train_inference_split, 18 | check_train_test_inference, 19 | code_router, 20 | no_code_agent, 21 | result_summarization_agent, 22 | fedot_generator, 23 | final, 24 | ) 25 | from utils.llm_factory import create_llm 26 | from utils.config.loader import load_config 27 | 28 | INPUT_NODE = "input_node" 29 | INPUT_AGENT = "rephraser_agent" 30 | CODE_GENERATOR_AGENT = "code_generator_agent" 31 | CODE_EXECUTOR = "code_executor" 32 | TASK_VALIDATOR = "task_validator" 33 | CODE_IMPROVEMENT_AGENT = "code_improvement_agent" 34 | ANSWER_GENERATOR = "answer_generator" 35 | HUMAN_EXPLANATION = "human_explanation_planning" 36 | TASK_VALIDATOR_EXPLANATION = "human_explanation_validator" 37 | CODE_IMPROVEMENT_EXPLANATION = "human_explanation_improvement" 38 | RESULT_EXPLANATION = "human_explanation_results" 39 | FEEDBACK_FOR_CODE_IMPROVEMENT = "feedback_for_code_improvement_agent" 40 | FEEDBACK_FOR_CODE_RESULTS = "feedback_for_code_results_agent" 41 | TRAIN_INFERENCE_SPLITTER = "train_inference_splitter" 42 | CHECK_TRAIN_TEST_INFERENCE = "check_train_test_inference" 43 | EXECUTE_TRAIN_TEST = "execute_train_test" 44 | 45 | AUTOML_ROUTER_AGENT = "automl_router" 46 | LIGHTAUTOML_CONFIG_GENERATOR_AGENT = "lightautoml_config_generator" 47 | LIGHTAUTOML_LOCAL_EXECUTOR = "lightautoml_local_executor" 48 | FEDOT_CONFIG_GENERATOR_AGENT = "fedot_config_generator" 49 | 50 | CODE_ROUTER = "code_router" 51 | NO_CODE_AGENT = "no_code_agent" 52 | RESULT_SUMMARIZATION_AGENT = "result_summarization_agent" 53 | 54 | 55 | ERROR_REGEX = r"(?:" + "|".join([ 56 | r"Traceback $$most recent call last$$:", 57 | r"Error:", 58 | r"Exception:", 59 | r"ValueError:", 60 | r"NameError:", 61 | r"SyntaxError:", 62 | ]) + r")" 63 | 64 | 65 | def code_generation_retry(state: AgentState) -> str: 66 | last_message = state['messages'][-1] 67 | if re.findall(ERROR_REGEX, last_message.content, re.DOTALL | re.MULTILINE): 68 | return CODE_GENERATOR_AGENT 69 | return RESULT_SUMMARIZATION_AGENT 70 | 71 | 72 | def task_validation_retry(state: AgentState) -> str: 73 | last_message = state['messages'][-1].content 74 | if "VALID NO" in last_message: 75 | return TASK_VALIDATOR_EXPLANATION 76 | elif "VALID YES" in last_message: 77 | return FEEDBACK_FOR_CODE_IMPROVEMENT 78 | return CODE_GENERATOR_AGENT 79 | 80 | 81 | def check_number_improvements(state: AgentState) -> str: 82 | if state['code_improvement_count'] >= 3: 83 | return ANSWER_GENERATOR 84 | return CODE_GENERATOR_AGENT 85 | 86 | def code_router_func(state: AgentState) -> str: 87 | last_message = state['messages'][-1].content 88 | if "YES" in last_message: 89 | return AUTOML_ROUTER_AGENT 90 | else: 91 | return NO_CODE_AGENT 92 | 93 | def automl_router_func(state: AgentState) -> str: 94 | last_message = state['messages'][-1].content 95 | if "LAMA" in last_message: 96 | return LIGHTAUTOML_CONFIG_GENERATOR_AGENT 97 | elif "FEDOT" in last_message: 98 | return FEDOT_CONFIG_GENERATOR_AGENT 99 | else: 100 | return INPUT_AGENT 101 | 102 | def train_inference_router(state: AgentState) -> str: 103 | last_message = state['messages'][-1].content 104 | if "VALID" in last_message: 105 | return ANSWER_GENERATOR 106 | else: 107 | return EXECUTE_TRAIN_TEST 108 | 109 | def add_node_name(state: AgentState, node_name: str) -> AgentState: 110 | state['current_node'] = node_name 111 | return state 112 | 113 | 114 | def graph_builder() -> StateGraph: 115 | 116 | config = load_config() 117 | 118 | workflow = StateGraph(AgentState) 119 | 120 | nodes = { 121 | LIGHTAUTOML_LOCAL_EXECUTOR: execute_code, 122 | CODE_EXECUTOR: execute_code, 123 | EXECUTE_TRAIN_TEST: execute_train_test, 124 | INPUT_NODE: input_node 125 | } 126 | 127 | llm_nodes = { 128 | AUTOML_ROUTER_AGENT: automl_router, 129 | LIGHTAUTOML_CONFIG_GENERATOR_AGENT: lightautoml_generator, 130 | FEDOT_CONFIG_GENERATOR_AGENT: fedot_generator, 131 | INPUT_AGENT: rephraser_agent, 132 | CODE_GENERATOR_AGENT: code_generation_agent, 133 | TASK_VALIDATOR: validate_solution, 134 | CODE_IMPROVEMENT_AGENT: code_improvement_agent, 135 | HUMAN_EXPLANATION: human_explanation_agent, 136 | TASK_VALIDATOR_EXPLANATION: human_explanation_agent, 137 | CODE_IMPROVEMENT_EXPLANATION: human_explanation_agent, 138 | FEEDBACK_FOR_CODE_IMPROVEMENT: feedback_for_code_improvement_agent, 139 | FEEDBACK_FOR_CODE_RESULTS: feedback_for_code_improvement_agent, 140 | TRAIN_INFERENCE_SPLITTER: train_inference_split, 141 | CHECK_TRAIN_TEST_INFERENCE: check_train_test_inference, 142 | CODE_ROUTER: code_router, 143 | NO_CODE_AGENT: no_code_agent, 144 | RESULT_SUMMARIZATION_AGENT: result_summarization_agent, 145 | RESULT_EXPLANATION: human_explanation_agent, 146 | ANSWER_GENERATOR: final, 147 | } 148 | 149 | for node_name, node_func in nodes.items(): 150 | workflow.add_node(node_name, lambda x, f=node_func, n=node_name: add_node_name(f(x), n)) 151 | 152 | for node_name, node_func in llm_nodes.items(): 153 | workflow.add_node(node_name, lambda x, f=node_func, n=node_name: add_node_name(f(x, create_llm(n, config)), n)) 154 | 155 | workflow.add_edge(START, INPUT_NODE) 156 | workflow.add_edge(INPUT_NODE, CODE_ROUTER) 157 | workflow.add_conditional_edges( 158 | CODE_ROUTER, 159 | code_router_func, 160 | {AUTOML_ROUTER_AGENT: AUTOML_ROUTER_AGENT, 161 | NO_CODE_AGENT: NO_CODE_AGENT} 162 | ) 163 | workflow.add_edge(NO_CODE_AGENT, END) 164 | 165 | workflow.add_conditional_edges( 166 | AUTOML_ROUTER_AGENT, 167 | automl_router_func, 168 | { 169 | LIGHTAUTOML_CONFIG_GENERATOR_AGENT: LIGHTAUTOML_CONFIG_GENERATOR_AGENT, 170 | FEDOT_CONFIG_GENERATOR_AGENT: FEDOT_CONFIG_GENERATOR_AGENT, 171 | INPUT_AGENT: INPUT_AGENT 172 | } 173 | ) 174 | workflow.add_edge(LIGHTAUTOML_CONFIG_GENERATOR_AGENT, LIGHTAUTOML_LOCAL_EXECUTOR) 175 | workflow.add_edge(LIGHTAUTOML_LOCAL_EXECUTOR, END) 176 | 177 | workflow.add_edge(FEDOT_CONFIG_GENERATOR_AGENT, END) 178 | 179 | workflow.add_edge(INPUT_AGENT, HUMAN_EXPLANATION) 180 | 181 | workflow.add_edge(HUMAN_EXPLANATION, CODE_GENERATOR_AGENT) 182 | workflow.add_edge(CODE_GENERATOR_AGENT, CODE_EXECUTOR) 183 | workflow.add_conditional_edges( 184 | CODE_EXECUTOR, 185 | code_generation_retry, 186 | {RESULT_SUMMARIZATION_AGENT: RESULT_SUMMARIZATION_AGENT, 187 | CODE_GENERATOR_AGENT: CODE_GENERATOR_AGENT} 188 | ) 189 | 190 | workflow.add_edge(RESULT_SUMMARIZATION_AGENT, RESULT_EXPLANATION) 191 | workflow.add_edge(RESULT_EXPLANATION, TASK_VALIDATOR) 192 | 193 | workflow.add_conditional_edges( 194 | TASK_VALIDATOR, 195 | task_validation_retry, 196 | { 197 | TASK_VALIDATOR_EXPLANATION: TASK_VALIDATOR_EXPLANATION, 198 | FEEDBACK_FOR_CODE_IMPROVEMENT: FEEDBACK_FOR_CODE_IMPROVEMENT, 199 | CODE_GENERATOR_AGENT: CODE_GENERATOR_AGENT 200 | } 201 | ) 202 | 203 | workflow.add_edge(TASK_VALIDATOR_EXPLANATION, TRAIN_INFERENCE_SPLITTER) 204 | workflow.add_edge(FEEDBACK_FOR_CODE_IMPROVEMENT, CODE_IMPROVEMENT_AGENT) 205 | workflow.add_edge(CODE_IMPROVEMENT_AGENT, CODE_IMPROVEMENT_EXPLANATION) 206 | 207 | workflow.add_conditional_edges( 208 | CODE_IMPROVEMENT_EXPLANATION, 209 | check_number_improvements, 210 | { 211 | TRAIN_INFERENCE_SPLITTER: TRAIN_INFERENCE_SPLITTER, 212 | CODE_GENERATOR_AGENT: CODE_GENERATOR_AGENT 213 | } 214 | ) 215 | 216 | workflow.add_edge(TRAIN_INFERENCE_SPLITTER, EXECUTE_TRAIN_TEST) 217 | workflow.add_edge(EXECUTE_TRAIN_TEST, CHECK_TRAIN_TEST_INFERENCE) 218 | workflow.add_conditional_edges( 219 | CHECK_TRAIN_TEST_INFERENCE, 220 | train_inference_router, 221 | { 222 | ANSWER_GENERATOR: ANSWER_GENERATOR, 223 | EXECUTE_TRAIN_TEST: EXECUTE_TRAIN_TEST 224 | } 225 | ) 226 | 227 | workflow.add_edge(ANSWER_GENERATOR, END) 228 | try: 229 | workflow.compile().get_graph(xray=False).draw_mermaid_png(output_file_path='new_graph.png') 230 | except Exception: 231 | # skipping graph picture generation 232 | print(workflow.compile().get_graph().print_ascii()) 233 | pass 234 | return workflow.compile() -------------------------------------------------------------------------------- /graph/llm_nodes.py: -------------------------------------------------------------------------------- 1 | import re 2 | import os 3 | import shutil 4 | 5 | from graph.state import AgentState 6 | from graph.prompts import load_prompt 7 | 8 | from fedotllm.llm import AIInference 9 | from fedotllm.main import FedotAI 10 | 11 | 12 | PYTHON_REGEX = r"```python-execute(.+?)```" 13 | 14 | # Additional Functions 15 | 16 | 17 | def construct_user_input(state: AgentState) -> str: 18 | user_input = f"Задача: {state['task']}\n" 19 | if "df" in state: 20 | user_input += f"Превью датасета: {state['df'].head().to_string()}\n" 21 | user_input += f"Колонки, которые есть в датасете: {state['df'].columns}\n" 22 | if "df_name" in state: 23 | user_input += f"Название файла с датасетом: {state['df_name']}\n" 24 | return user_input 25 | 26 | 27 | def extract_python_code(text): 28 | matches = re.findall(PYTHON_REGEX, text, re.DOTALL) 29 | return matches[0].strip() if matches else None 30 | 31 | 32 | def find_message_with_code(state: AgentState): 33 | for i in range(1, len(state['messages'])): 34 | if re.findall(PYTHON_REGEX, state['messages'][-i].content, re.DOTALL | re.MULTILINE): 35 | extracted_code = extract_python_code(state['messages'][-i].content) 36 | break 37 | else: 38 | extracted_code = state['messages'][-i].content 39 | return extracted_code 40 | 41 | 42 | 43 | # Agent 44 | 45 | 46 | def input_node(state: AgentState) -> AgentState: 47 | state['task'] = state['messages'][-1].content 48 | default_state = { 49 | 'code_for_test': [], 50 | 'feedback': [], 51 | 'code_improvement_count': 0, 52 | 'improvements_code': [], 53 | 'human_understanding': [], 54 | 'generated_code': "", 55 | 'code_results': "", 56 | 'rephrased_plan': "", 57 | 'lama': False, 58 | "test_split": False, 59 | "test_df": None, 60 | "test_df_name": "", 61 | } 62 | 63 | for key, value in default_state.items(): 64 | if key not in state: 65 | state[key] = value 66 | 67 | return state 68 | 69 | 70 | def rephraser_agent(state: AgentState, llm): 71 | 72 | user_input = construct_user_input(state) 73 | prompt_template = load_prompt('rephraser') 74 | chain = prompt_template | llm 75 | message = chain.invoke({"user_input": user_input}) 76 | message.content = '\n' + message.content 77 | state['rephrased_plan'] = message.content.strip() 78 | return {"messages": message} 79 | 80 | 81 | def code_router(state: AgentState, llm): 82 | 83 | prompt_template = load_prompt('code_router') 84 | chain = prompt_template | llm 85 | response = chain.invoke({"task": state['task']}) 86 | return {"messages": response} 87 | 88 | 89 | def no_code_agent(state: AgentState, llm): 90 | 91 | prompt_template = load_prompt('no_code') 92 | chain = prompt_template | llm 93 | user_input = construct_user_input(state) 94 | response = chain.invoke({"text": user_input, "history": state['messages']}) 95 | response.content = '\n' + response.content 96 | return {"messages": response} 97 | 98 | 99 | def result_explanation_agent(state: AgentState, llm): 100 | 101 | prompt_template = load_prompt('result_explanation') 102 | chain = prompt_template | llm 103 | 104 | last_two_message = [msg.content for msg in state['messages'][-2:]] 105 | response = chain.invoke({"text": last_two_message}) 106 | 107 | response.content = '\n' + response.content 108 | return {"messages": response} 109 | 110 | 111 | def result_summarization_agent(state: AgentState, llm): 112 | prompt_template = load_prompt('result_summarization') 113 | chain = prompt_template | llm 114 | 115 | last_two_message = [msg.content for msg in state['messages'][-2:]] 116 | response = chain.invoke({"text": last_two_message}) 117 | 118 | response.content = '\n' + response.content 119 | return {"messages": response} 120 | 121 | 122 | def automl_router(state: AgentState, llm): 123 | 124 | prompt_template = load_prompt('automl_router') 125 | chain = prompt_template | llm 126 | response = chain.invoke({"task": state['task']}) 127 | return {"messages": response} 128 | 129 | 130 | def lightautoml_generator(state: AgentState, llm): 131 | 132 | prompt_template = load_prompt('lightautoml_parser') 133 | chain = prompt_template | llm 134 | response = chain.invoke({ 135 | "task": state['task'], 136 | "file_name": state['df_name'], 137 | "df_columns": state['df'].columns.tolist(), 138 | "df_head": state['df'].head().to_string() 139 | }) 140 | response.content = '\n' + response.content.strip() 141 | return {"messages": response, 'lama': True} 142 | 143 | 144 | def fedot_generator(state: AgentState, llm) -> str: 145 | 146 | output_path = os.path.join(os.getcwd(), 'output') 147 | if os.path.exists(output_path): 148 | shutil.rmtree(output_path) 149 | os.makedirs(output_path, exist_ok=True) 150 | 151 | task_path = 'datasets/' #+ state['df_name'] 152 | 153 | fedot_ai = FedotAI( 154 | task_path=task_path, 155 | inference=AIInference(), 156 | workspace=output_path, 157 | ) 158 | output = fedot_ai.ainvoke(message=state['task']) 159 | 160 | # Extract the fedotllm agent message for Code interpretation 161 | fedotllm_message = output['messages'][-1].content if len(output['messages']) > 1 else "No fedotllm message available" 162 | 163 | current_understanding = state.get('human_understanding', []) 164 | updated_understanding = current_understanding + [f"**FedotLLM Agent Report:**\n{fedotllm_message}"] 165 | 166 | prompt_template = load_prompt('fedot_parser') 167 | chain = prompt_template | llm 168 | response = chain.invoke({"results": output['messages'][1].content}) 169 | 170 | return {"messages": response, "human_understanding": updated_understanding} 171 | 172 | 173 | def human_explanation_agent(state: AgentState, llm): 174 | 175 | human_prompts = { 176 | 'rephraser_agent': 'human_explanation_planning', 177 | 'task_validator': 'human_explanation_validator', 178 | 'code_improvement_agent': 'human_explanation_improvement', 179 | 'result_summarization_agent': 'human_explanation_results' 180 | } 181 | 182 | prompt_template = load_prompt(human_prompts.get(state['current_node'], 'human_explanation')) 183 | chain = prompt_template | llm 184 | 185 | last_message = state['messages'][-1].content 186 | response = chain.invoke({"text": last_message, "history": state['messages']}) 187 | 188 | explanation_text = response.content.strip() 189 | current_understanding = state.get('human_understanding', []) 190 | updated_understanding = current_understanding + [explanation_text] 191 | 192 | return { 193 | "messages": response, 194 | "human_understanding": updated_understanding 195 | } 196 | 197 | 198 | def code_generation_agent(state: AgentState, llm): 199 | 200 | prompt_template = load_prompt('code_generator') 201 | chain = prompt_template | llm 202 | user_input = construct_user_input(state) 203 | response = chain.invoke({"user_input": user_input, "history": state['messages']}) 204 | response.content = '\n' + response.content 205 | return {"messages": response} 206 | 207 | 208 | def validate_solution(state: AgentState, llm): 209 | 210 | user_input = construct_user_input(state) 211 | 212 | prompt_template = load_prompt('validate_solution') 213 | chain = prompt_template | llm 214 | solution = "Код:\n```python-execute" + state["generated_code"] + '\n```' 215 | solution += "Резульат выполнения кода: " + ''.join(state['code_results']) 216 | 217 | message = chain.invoke({"user_input": user_input, "solution": solution, "rephrased_plan": state['rephrased_plan']}) 218 | 219 | return {"messages": message} 220 | 221 | 222 | def feedback_for_code_improvement_agent(state: AgentState, llm_base): 223 | 224 | generated_code = state['generated_code'][-1] 225 | code_result = state['code_results'][-1] if state['code_results'] else "Нет результатов выполнения кода." 226 | 227 | combined_message = f"Сгенерированный код:\n{generated_code}\n\nРезультат выполнения кода:\n{code_result}" 228 | 229 | user_prompt = load_prompt('output_result_filter') 230 | chain = user_prompt | llm_base 231 | response = chain.invoke({"result": combined_message}) 232 | 233 | past_feedback = state.get('feedback', []) 234 | if state.get('improvements_code'): 235 | latest_improvement = state['improvements_code'][-1] 236 | past_feedback.append({f"Улучшение {state['code_improvement_count']}": latest_improvement["improve"].content}) 237 | res = {f"Результат {state['code_improvement_count']}": response.content} 238 | past_feedback.append(res) 239 | 240 | return {"feedback": past_feedback, "messages": response} 241 | 242 | 243 | def code_improvement_agent(state: AgentState, llm): 244 | 245 | prompt_template = load_prompt('code_improvement') 246 | user_input = construct_user_input(state) 247 | feedback = state['feedback'][-1] if state['feedback'] else "Нет предыдущих улучшений." 248 | code = state['generated_code'][-1] 249 | 250 | chain = prompt_template | llm 251 | message = chain.invoke({"user_input": user_input, "code": code, "solution": state['generated_code'][-1], "feedback": feedback}) 252 | 253 | improvements = state['improvements_code'] 254 | improvements.append({"improve": message}) 255 | 256 | return {"messages": message, "code_improvement_count": state['code_improvement_count']+1, "improvements_code": improvements} 257 | 258 | 259 | def train_inference_split(state: AgentState, llm): 260 | prompt_template = load_prompt('train_inference_split') 261 | chain = prompt_template | llm 262 | response = chain.invoke({"code": state['generated_code'], "train_dataset_name": state['df_name'], "test_dataset_name": state['test_df_name']}) 263 | 264 | return {"messages": response, "test_split": True} 265 | 266 | 267 | def check_train_test_inference(state: AgentState, llm): 268 | last_message = state['messages'][-1].content 269 | promt_template = load_prompt('train_test_checker') 270 | chain = promt_template | llm 271 | response = chain.invoke({"code_result": last_message, "train_code": state['train_code'], "test_code": state['test_code']}) 272 | 273 | return {"messages": response} 274 | 275 | 276 | def final(state: AgentState, llm): 277 | prompt_message = load_prompt('output_summarization') 278 | chain = prompt_message | llm 279 | message = chain.invoke({"task": state['task'], "base": state['human_understanding'][1], "feedback": state['feedback']}) 280 | message.content = message.content 281 | 282 | os.makedirs('./code', exist_ok=True) 283 | with open('./code/train.py', 'w', encoding='utf-8') as f: 284 | f.write(state.get('train_code', '')) 285 | with open('./code/test.py', 'w', encoding='utf-8') as f: 286 | f.write(state.get('test_code', '')) 287 | return {"messages": message} -------------------------------------------------------------------------------- /fedotllm/agents/automl/templates/load_template.py: -------------------------------------------------------------------------------- 1 | import re 2 | from pathlib import Path 3 | from typing import Union, Set, Optional 4 | 5 | Path.templates_root = Path(__file__).parent 6 | 7 | def render_sub_templates(text: str, loaded_templates: Optional[Set[Path]] = None) -> str: 8 | """ 9 | Processes the input text by replacing template placeholders with their corresponding content. 10 | It also aggregates import statements from all sub-templates and inserts them appropriately. 11 | 12 | Args: 13 | text (str): The input text containing template placeholders. 14 | loaded_templates (Optional[Set[Path]]): Set of already loaded templates to prevent circular references. 15 | 16 | Returns: 17 | str: The processed text with all templates rendered and imports aggregated. 18 | """ 19 | if loaded_templates is None: 20 | loaded_templates = set() 21 | 22 | sub_templates = re.findall(r"<%%\s*(.*?)\s*%%>", text) 23 | sub_imports = set() 24 | 25 | for sub_template in sub_templates: 26 | sub_template_path = sub_template.strip().lower() 27 | sub_template_content = load_template(sub_template_path, loaded_templates) 28 | 29 | # Extract and remove import statements 30 | imports = _extract_imports(sub_template_content) 31 | sub_imports.update(imports) 32 | sub_template_content = _remove_imports(sub_template_content, imports).strip() 33 | 34 | # Replace the placeholder with the template content, preserving indentation 35 | text = _replace_placeholder_with_content(text, sub_template, sub_template_content) 36 | 37 | # Aggregate and insert imports 38 | if sub_imports: 39 | imports_str = "\n".join(sorted(sub_imports)) 40 | text = _insert_imports(text, imports_str) 41 | 42 | return text.strip() 43 | 44 | def _extract_imports(content: str) -> Set[str]: 45 | """ 46 | Extracts import statements from the given content. 47 | 48 | Args: 49 | content (str): The content to extract imports from. 50 | 51 | Returns: 52 | Set[str]: A set of import statements. 53 | """ 54 | import_pattern = re.compile( 55 | r"^(import\s+\S+|from\s+\S+\s+import\s+\S+)", 56 | re.MULTILINE 57 | ) 58 | imports = set(match.group(1) for match in import_pattern.finditer(content)) 59 | return imports 60 | 61 | def _remove_imports(content: str, imports: Set[str]) -> str: 62 | """ 63 | Removes the specified import statements from the content. 64 | 65 | Args: 66 | content (str): The original content. 67 | imports (Set[str]): The import statements to remove. 68 | 69 | Returns: 70 | str: The content without the specified imports. 71 | """ 72 | for imp in imports: 73 | content = content.replace(imp, "") 74 | return content 75 | 76 | def _replace_placeholder_with_content(text: str, placeholder: str, content: str) -> str: 77 | """ 78 | Replaces the template placeholder in the text with the actual content, 79 | preserving the original indentation. 80 | 81 | Args: 82 | text (str): The original text containing the placeholder. 83 | placeholder (str): The name of the placeholder to replace. 84 | content (str): The content to insert in place of the placeholder. 85 | 86 | Returns: 87 | str: The text with the placeholder replaced by the content. 88 | """ 89 | pattern = re.compile(rf"(?P^\s*)<%%\s*{re.escape(placeholder)}\s*%%>", re.MULTILINE) 90 | match = pattern.search(text) 91 | if match: 92 | indent = match.group("indent") 93 | indented_content = "\n".join( 94 | f"{indent}{line}" if line.strip() else line 95 | for line in content.splitlines() 96 | ) 97 | text = pattern.sub(indented_content, text, count=1) 98 | else: 99 | # Fallback if pattern does not match 100 | text = text.replace(f"<%%{placeholder}%%>", content) 101 | return text 102 | 103 | def _insert_imports(text: str, imports_str: str) -> str: 104 | """ 105 | Inserts aggregated import statements into the text after initial comments. 106 | 107 | Args: 108 | text (str): The original text. 109 | imports_str (str): The aggregated import statements as a single string. 110 | 111 | Returns: 112 | str: The text with imports inserted. 113 | """ 114 | comment_pattern = re.compile(r'^(\s*#.*\n)*') 115 | match = comment_pattern.match(text) 116 | if match: 117 | comments = match.group(0) 118 | rest_of_text = text[len(comments):] 119 | text = f"{comments}{imports_str}\n{rest_of_text}" 120 | else: 121 | text = f"{imports_str}\n{text}" 122 | return text 123 | 124 | def load_template(template_path: Union[str, Path], loaded_templates: Optional[Set[Path]] = None) -> str: 125 | """ 126 | Loads and processes a template file by resolving its sub-templates, 127 | aggregating import statements, and preserving indentation. 128 | 129 | Args: 130 | template_path (Union[str, Path]): Path to the template file. 131 | loaded_templates (Optional[Set[Path]]): Set of already loaded templates to prevent circular dependencies. 132 | 133 | Returns: 134 | str: The processed template content with aggregated imports and properly indented inserted templates. 135 | 136 | Raises: 137 | ValueError: If a circular template reference is detected. 138 | FileNotFoundError: If the template file does not exist. 139 | """ 140 | if loaded_templates is None: 141 | loaded_templates = set() 142 | 143 | template_path = Path(template_path) 144 | if not template_path.is_absolute(): 145 | template_path = Path.templates_root / template_path 146 | 147 | if template_path in loaded_templates: 148 | raise ValueError(f"Circular template reference detected: {template_path}") 149 | 150 | if not template_path.exists(): 151 | raise FileNotFoundError(f"Template file not found: {template_path}") 152 | 153 | loaded_templates.add(template_path) 154 | try: 155 | file_content = template_path.read_text() 156 | processed_content = render_sub_templates(file_content, loaded_templates) 157 | finally: 158 | loaded_templates.remove(template_path) 159 | 160 | return processed_content.strip() 161 | 162 | def render_template(template: str, **kwargs) -> str: 163 | """ 164 | Replaces placeholders in the template with corresponding keyword arguments. 165 | 166 | Args: 167 | template (str): The template string containing placeholders like {% var %}. 168 | **kwargs: Variable names and their values to replace in the template. 169 | 170 | Returns: 171 | str: The rendered template with all placeholders replaced. 172 | 173 | Raises: 174 | ValueError: If any placeholder does not have a corresponding keyword argument. 175 | """ 176 | import re 177 | 178 | pattern = re.compile(r"{%\s*(\w+)\s*%}") 179 | not_found = [] 180 | 181 | def replace_match(match): 182 | var_name = match.group(1) 183 | if var_name in kwargs: 184 | value = kwargs[var_name] 185 | return str(value) 186 | else: 187 | not_found.append(var_name) 188 | return match.group(0) # Keep the placeholder unchanged 189 | 190 | rendered = pattern.sub(replace_match, template) 191 | 192 | if not_found: 193 | raise ValueError(f"Variables not found for placeholders: {not_found}") 194 | 195 | return rendered 196 | 197 | def insert_template(code: str, template: str) -> str: 198 | """ 199 | Inserts a sub-template into the main code by replacing the placeholder. 200 | 201 | The placeholder format in the code should be: 202 | ### template ### 203 | ...sub-template content... 204 | ### template ### 205 | 206 | After insertion, the entire code is processed through `render_templates` to handle 207 | any nested templates. 208 | 209 | Args: 210 | code (str): The main code where the template will be inserted. 211 | template (str): The name of the template to insert. 212 | 213 | Returns: 214 | str: The code with the template inserted and all templates rendered. 215 | 216 | Raises: 217 | ValueError: If the template placeholder is not found in the code. 218 | """ 219 | # Normalize the template name 220 | template = template.lower().strip() 221 | sub_imports: Set[str] = set() 222 | 223 | # Define regex pattern to match the placeholder with optional whitespace and multiline content 224 | # Capturing groups: 225 | # 1. Leading whitespace before the first ### 226 | # 2. The entire placeholder including the content to replace 227 | pattern = re.compile( 228 | rf"(?P^[ \t]*)###\s*{re.escape(template)}\s*###.*?###\s*{re.escape(template)}\s*###", 229 | re.IGNORECASE | re.DOTALL | re.MULTILINE 230 | ) 231 | 232 | def replacement(match: re.Match) -> str: 233 | """ 234 | Replacement function to load and insert the sub-template content with preserved indentation. 235 | 236 | Args: 237 | match (re.Match): The regex match object. 238 | 239 | Returns: 240 | str: The loaded and correctly indented sub-template content. 241 | 242 | Raises: 243 | ValueError: If loading the template fails. 244 | """ 245 | try: 246 | leading_whitespace = match.group("indent") or "" 247 | 248 | # Load the template content using `load_template`, which internally calls `render_templates` 249 | loaded_content = load_template(template, sub_imports) 250 | 251 | # Extract and remove import statements from the loaded content 252 | extracted_imports = _extract_imports(loaded_content) 253 | sub_imports.update(extracted_imports) 254 | loaded_content = _remove_imports(loaded_content, extracted_imports).strip() 255 | 256 | # Indent each line of the loaded content to match the placeholder's indentation 257 | indented_content = "\n".join( 258 | f"{leading_whitespace}{line}" if line.strip() else line 259 | for line in loaded_content.splitlines() 260 | ) 261 | 262 | return indented_content 263 | except Exception as e: 264 | raise ValueError(f"Failed to load and render template '{template}': {e}") from e 265 | 266 | # Perform the substitution using the replacement function 267 | new_code, count = pattern.subn(replacement, code) 268 | 269 | if count == 0: 270 | raise ValueError(f"Template placeholder '### {template} ###' not found in the code.") 271 | 272 | # After insertion, ensure that any nested templates within the code are rendered 273 | rendered_code = render_sub_templates(new_code, sub_imports) 274 | 275 | if sub_imports: 276 | imports_str = "\n".join(sorted(sub_imports)) 277 | rendered_code = _insert_imports(rendered_code, imports_str) 278 | 279 | return rendered_code.strip() -------------------------------------------------------------------------------- /fedotllm/agents/translator.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | from langdetect import LangDetectException, detect 4 | 5 | from fedotllm.llm import AIInference 6 | from fedotllm.log import logger 7 | 8 | 9 | class TranslatorAgent: 10 | def __init__(self, inference: AIInference): 11 | self.source_language = None 12 | self.inference = inference 13 | self.code_block_placeholder_prefix = "__CODE_BLOCK_PLACEHOLDER__" 14 | logger.info("TranslatorAgent initialized with provided AIInference instance.") 15 | 16 | def _extract_code_blocks(self, text: str) -> tuple[str, dict[str, str]]: 17 | code_blocks_map = {} 18 | pattern = re.compile(r"(```(?:[a-zA-Z0-9_.-]+)?\n)(.*?)(\n```)", re.DOTALL) 19 | idx = 0 20 | 21 | def replace_match(match): 22 | nonlocal idx 23 | placeholder = f"{self.code_block_placeholder_prefix}_{idx}__" 24 | code_blocks_map[placeholder] = match.group(0) 25 | idx += 1 26 | return placeholder 27 | 28 | processed_text = pattern.sub(replace_match, text) 29 | if idx > 0: 30 | logger.debug( 31 | f"_extract_code_blocks: Found and replaced {idx} code blocks. Placeholders: {list(code_blocks_map.keys())}" 32 | ) 33 | else: 34 | logger.debug("_extract_code_blocks: No code blocks found in text.") 35 | return processed_text, code_blocks_map 36 | 37 | def _reinsert_code_blocks(self, text: str, code_blocks_map: dict[str, str]) -> str: 38 | if not code_blocks_map: 39 | logger.debug("_reinsert_code_blocks: No code blocks in map to reinsert.") 40 | return text 41 | 42 | logger.debug( 43 | f"_reinsert_code_blocks: Attempting to reinsert {len(code_blocks_map)} code block(s) from map: {list(code_blocks_map.keys())}" 44 | ) 45 | reinserted_count = 0 46 | not_found_placeholders = [] 47 | for placeholder, original_code_block in code_blocks_map.items(): 48 | escaped_placeholder = re.escape(placeholder) 49 | text_after_sub, num_replacements = re.subn( 50 | escaped_placeholder, lambda m: original_code_block, text, count=1 51 | ) 52 | logger.debug( 53 | f"_reinsert_code_blocks: Searching for placeholder: '{placeholder}'. Found and replaced: {num_replacements > 0} (Count: {num_replacements})" 54 | ) 55 | if num_replacements > 0: 56 | reinserted_count += 1 57 | text = text_after_sub 58 | else: 59 | logger.warning( 60 | f"_reinsert_code_blocks: Placeholder '{placeholder}' not found in the translated text for re-insertion." 61 | ) 62 | not_found_placeholders.append(placeholder) 63 | if reinserted_count == len(code_blocks_map) and not not_found_placeholders: 64 | logger.debug( 65 | f"_reinsert_code_blocks: Successfully reinserted all {reinserted_count} code blocks." 66 | ) 67 | else: 68 | logger.warning( 69 | f"_reinsert_code_blocks: Reinserted {reinserted_count} out of {len(code_blocks_map)} code blocks from map. " 70 | f"Placeholders not found in text: {not_found_placeholders if not_found_placeholders else 'None (but counts mismatch, check logic)'}" 71 | ) 72 | return text 73 | 74 | def _translate_text( 75 | self, text: str, target_language: str, source_language: str = "auto" 76 | ) -> str: 77 | logger.info( 78 | f"TranslatorAgent._translate_text: Attempting translation from '{source_language}' to '{target_language}' using self.inference.query." 79 | ) 80 | logger.debug( 81 | f"Original text for _translate_text (first 200 chars): '{text[:200]}...'" 82 | ) 83 | 84 | # If the input text is empty, return it directly 85 | if not text: 86 | logger.debug( 87 | "Input text to _translate_text is empty. Returning empty string." 88 | ) 89 | return "" 90 | 91 | processed_text, code_blocks_map = self._extract_code_blocks(text) 92 | if text != processed_text: 93 | logger.debug( 94 | f"Text after code block extraction (first 200 chars): '{processed_text[:200]}...'" 95 | ) 96 | 97 | prompt_source_lang_description = source_language # Default 98 | if str(source_language).lower() == "auto" or source_language is None: 99 | prompt_source_lang_description = "the auto-detected source language" 100 | elif str(source_language).lower() == "en": 101 | prompt_source_lang_description = "English" 102 | else: # Use the language code directly for others 103 | prompt_source_lang_description = str(source_language) 104 | 105 | prompt = ( 106 | f"Translate the following text from {prompt_source_lang_description} to {target_language}. " 107 | f"It is absolutely crucial to preserve the original formatting exactly. " 108 | f"This includes all markdown syntax: headers (e.g., #, ##), lists (e.g., -, *, 1.), " 109 | f"bold (e.g., **text** or __text__), italics (e.g., *text* or _text_), " 110 | f"strikethrough (e.g., ~~text~~), links (e.g., [text](url)), images (e.g., ![alt](url)), " 111 | f"tables (using pipe and hyphen syntax), and blockquotes (e.g., > text). " 112 | f"The text provided may contain placeholders like '{self.code_block_placeholder_prefix}_NUMBER__' " 113 | f"(e.g., {self.code_block_placeholder_prefix}_0__, {self.code_block_placeholder_prefix}_1__). " 114 | f"These placeholders represent original code blocks and MUST NOT be translated or altered in any way. " 115 | f"They must be preserved exactly as they appear in the input text. " 116 | f"Only translate the surrounding text. " 117 | f"If the text (excluding placeholders) is already in {target_language} and requires no translation, " 118 | f"return it as is, ensuring placeholders are also returned as is.\n\n" 119 | f"Text to translate (placeholders like {self.code_block_placeholder_prefix}_0__ must be kept as is):\n{processed_text}" 120 | ) 121 | 122 | try: 123 | logger.debug( 124 | f"TranslatorAgent: Sending prompt to self.inference.query (from '{source_language}' to '{target_language}'):\n{prompt}" 125 | ) 126 | response_text = self.inference.query(prompt) 127 | logger.debug( 128 | f"TranslatorAgent: Received response from self.inference.query. Type: {type(response_text)}. Content (first 200 chars): '{str(response_text)[:200]}...'" 129 | ) 130 | 131 | if isinstance(response_text, str): 132 | if ( 133 | response_text == "" and processed_text != "" 134 | ): # LLM returned empty string for non-empty input 135 | logger.info( 136 | "LLM returned an empty string. Assuming it's an intentional empty translation." 137 | ) 138 | translated_text_with_placeholders = ( 139 | "" # Use empty string for re-insertion 140 | ) 141 | elif response_text == processed_text: 142 | logger.info( 143 | f"Text from {source_language} to {target_language} may not have been translated by LLM (output is same as input to LLM)." 144 | ) 145 | translated_text_with_placeholders = response_text 146 | else: # Non-empty, different from processed_text 147 | translated_text_with_placeholders = response_text 148 | logger.info( 149 | f"Successfully translated text from {source_language} to {target_language} using self.inference.query." 150 | ) 151 | else: # Not a string (None, int, etc.) 152 | logger.warning( 153 | f"LLM query did not return a valid string (type: {type(response_text)}, value: '{str(response_text)[:200]}...'). Returning original text." 154 | ) 155 | return text # Return original, unprocessed text 156 | 157 | except Exception as e: 158 | logger.error( 159 | f"Error during translation using self.inference.query from '{source_language}' to '{target_language}': {e}", 160 | exc_info=True, 161 | ) 162 | return text # Return original, unprocessed text 163 | 164 | logger.debug( 165 | f"TranslatorAgent._translate_text: Text before re-inserting code blocks (placeholders should be visible):\n{translated_text_with_placeholders}" 166 | ) 167 | final_translated_text = self._reinsert_code_blocks( 168 | translated_text_with_placeholders, code_blocks_map 169 | ) 170 | logger.debug( 171 | f"Final translated text after reinserting code blocks (target: {target_language}, first 200 chars): '{final_translated_text[:200]}...'" 172 | ) 173 | return final_translated_text 174 | 175 | def translate_input_to_english(self, message: str) -> str: 176 | logger.info( 177 | f"TranslatorAgent: Received input message for translation to English (first 200 chars): '{message[:200]}...'" 178 | ) 179 | if not message: 180 | logger.info("Input message is empty. Skipping detection and translation.") 181 | self.source_language = None # Explicitly set, as no detection happens 182 | return "" 183 | try: 184 | self.source_language = detect(message) 185 | except LangDetectException as e: 186 | self.source_language = "en" # Default if detection fails 187 | logger.warning( 188 | f"Language detection failed for input message (defaulting to 'en'): {e}", 189 | exc_info=True, 190 | ) 191 | 192 | logger.info( 193 | f"TranslatorAgent: Source language for input set to: {self.source_language}" 194 | ) 195 | 196 | if self.source_language != "en": 197 | logger.info(f"Translating input from {self.source_language} to English.") 198 | return self._translate_text( 199 | message, target_language="en", source_language=self.source_language 200 | ) 201 | else: 202 | logger.info("Input is already English. No translation needed.") 203 | return message 204 | 205 | def translate_output_to_source_language(self, message: str) -> str: 206 | logger.info( 207 | f"TranslatorAgent: Attempting output translation. Current source_language: {self.source_language}" 208 | ) 209 | 210 | if not message: # Handles empty string, None, etc. 211 | logger.info("Output message is empty. Skipping translation.") 212 | return "" # Return empty string consistently 213 | 214 | if self.source_language and self.source_language != "en": 215 | logger.info(f"Translating output from English to {self.source_language}.") 216 | logger.info( 217 | f"English message for output translation (first 200 chars): '{message[:200]}...'" 218 | ) 219 | return self._translate_text( 220 | message, target_language=self.source_language, source_language="en" 221 | ) 222 | elif not self.source_language: 223 | logger.warning( 224 | "Cannot translate output: source_language not set (input might not have been processed or detection failed)." 225 | ) 226 | return message 227 | else: # source_language is 'en' 228 | logger.info( 229 | f"Output translation not needed (source language was '{self.source_language}'). Returning original English message." 230 | ) 231 | logger.debug( 232 | f"Original English message for output (first 200 chars): '{message[:200]}...'" 233 | ) 234 | return message 235 | -------------------------------------------------------------------------------- /app_components/fragments.py: -------------------------------------------------------------------------------- 1 | import streamlit as st 2 | import pandas as pd 3 | from typing import Dict, Any, List, Optional 4 | 5 | from .agent_handler import stream_agent_response_for_frontend 6 | from .data_handlers import load_data, save_file_to_disk 7 | from .session_state import create_new_conversation 8 | from .data_handlers import SUPPORTED_FILE_TYPES 9 | 10 | COLUMN_SHAPES = [1, 1] 11 | BENCHMARK_CSV_PATH = "benchmark/benchmark_results.csv" 12 | ID = "employee_promotion" 13 | 14 | def get_benchmarks_from_csv(benchmark_csv_path, id): 15 | df = pd.read_csv(benchmark_csv_path) 16 | row = df[df['id'] == id].iloc[0] 17 | return row 18 | 19 | def update_ds_agent_history(benchmark_csv_path, id, ds_agent_result): 20 | df = pd.read_csv(benchmark_csv_path) 21 | df.loc[df['id'] == id, 'ds_agent_history'] = ds_agent_result 22 | df.to_csv(benchmark_csv_path, index=False) 23 | 24 | 25 | def extract_final_response(assistant_message: Dict[str, Any]) -> str: 26 | if 'progress' in assistant_message and assistant_message['progress']: 27 | valid_progress_entries = [str(p) for p in assistant_message['progress'] if p is not None] 28 | if valid_progress_entries: 29 | final_response = valid_progress_entries[-1] 30 | return final_response 31 | else: 32 | return assistant_message.get('content', '') 33 | else: 34 | return assistant_message.get('content', '') 35 | 36 | 37 | def render_status_boxes( 38 | progress_messages: List[str], 39 | interpretation_messages: List[str], 40 | progress_title: str = "Processed pipeline", 41 | interpretation_title: str = "Code interpretation", 42 | state: str = "complete", 43 | expanded: bool = True, 44 | status_placeholder: Optional[st.empty] = None, 45 | pipeline_placeholder: Optional[st.empty] = None, 46 | ): 47 | 48 | subcol1, subcol2 = st.columns(COLUMN_SHAPES) 49 | 50 | with subcol2: 51 | if status_placeholder and state == "running": 52 | with status_placeholder.container(height=700): 53 | with st.status(progress_title, state=state, expanded=expanded): 54 | for msg in progress_messages: 55 | with st.chat_message("assistant"): 56 | st.markdown(msg) 57 | else: 58 | with st.status(progress_title, state=state, expanded=expanded): 59 | valid_progress_entries = [str(p) for p in progress_messages if p is not None] 60 | if len(valid_progress_entries) > 1: 61 | progress_for_box = valid_progress_entries[:-1] 62 | else: 63 | progress_for_box = [] 64 | for msg in progress_for_box: 65 | with st.chat_message("assistant"): 66 | st.markdown(msg) 67 | 68 | with subcol1: 69 | if pipeline_placeholder and state == "running": 70 | with pipeline_placeholder.container(height=700): 71 | with st.status(interpretation_title, state=state, expanded=expanded): 72 | valid_human_entries = [str(h) for h in interpretation_messages if h is not None] 73 | for msg in valid_human_entries: 74 | with st.chat_message("assistant"): 75 | st.markdown(msg) 76 | 77 | elif not (status_placeholder and state == "running"): 78 | with st.status(interpretation_title, state=state, expanded=expanded): 79 | valid_human_entries = [str(h) for h in interpretation_messages if h is not None] 80 | for msg in valid_human_entries: 81 | with st.chat_message("assistant"): 82 | st.markdown(msg) 83 | 84 | 85 | @st.fragment 86 | def file_upload_fragment(): 87 | 88 | train_file = st.file_uploader("📥 Upload training dataset", type=list(SUPPORTED_FILE_TYPES.keys()), key="train_file") 89 | test_file = st.file_uploader("📥 Upload test dataset for predictions", type=list(SUPPORTED_FILE_TYPES.keys()), key="test_file") 90 | 91 | if train_file is not None: 92 | try: 93 | sandbox = st.session_state.get("sandbox", None) 94 | 95 | file_name = train_file.name 96 | file_type = file_name.split('.')[-1].lower() 97 | file_content = train_file.getvalue() 98 | 99 | st.session_state.loading_message = f"Loading {file_name}..." 100 | status_placeholder = st.empty() 101 | status_placeholder.info(st.session_state.loading_message) 102 | 103 | df = load_data(file_content, file_type) 104 | st.write(df.head()) 105 | 106 | if sandbox is not None: 107 | sandbox.files.write(file_name, file_content) 108 | save_file_to_disk(df, file_name, file_type) 109 | else: 110 | save_file_to_disk(df, file_name, file_type) 111 | 112 | st.session_state.uploaded_files[file_name] = { 113 | 'df': df, 114 | 'type': file_type, 115 | 'df_name': file_name 116 | } 117 | 118 | st.session_state.df_name = file_name 119 | 120 | st.session_state.loading_message = "" 121 | status_placeholder.empty() 122 | st.success(f"File '{file_name}' successfully uploaded!") 123 | 124 | except Exception as e: 125 | st.error(f"Error uploading file: {str(e)}") 126 | 127 | if st.session_state.uploaded_files: 128 | st.markdown("### Uploaded Files") 129 | for file_name in st.session_state.uploaded_files: 130 | st.text(f"• {file_name}") 131 | 132 | if test_file is not None: 133 | try: 134 | sandbox = st.session_state.get("sandbox", None) 135 | 136 | file_name = test_file.name 137 | file_type = file_name.split('.')[-1].lower() 138 | file_content = test_file.getvalue() 139 | 140 | st.session_state.loading_message = f"Loading test dataset {file_name}..." 141 | status_placeholder = st.empty() 142 | status_placeholder.info(st.session_state.loading_message) 143 | 144 | df = load_data(file_content, file_type) 145 | st.write(df.head()) 146 | 147 | if sandbox is not None: 148 | sandbox.files.write(file_name, file_content) 149 | save_file_to_disk(df, file_name, file_type) 150 | else: 151 | save_file_to_disk(df, file_name, file_type) 152 | 153 | st.session_state.uploaded_test_files[file_name] = { 154 | 'df': df, 155 | 'type': file_type, 156 | 'df_name': file_name 157 | } 158 | 159 | st.session_state.test_df_name = file_name 160 | 161 | st.session_state.loading_message = "" 162 | status_placeholder.empty() 163 | st.success(f"Test file '{file_name}' successfully uploaded!") 164 | 165 | except Exception as e: 166 | st.error(f"Error uploading test file: {str(e)}") 167 | 168 | if st.session_state.uploaded_test_files: 169 | st.markdown("### Uploaded Test Files") 170 | for file_name in st.session_state.uploaded_test_files: 171 | st.text(f"• {file_name}") 172 | 173 | 174 | def switch_conversation(conv_id): 175 | st.session_state.current_conversation = conv_id 176 | st.session_state.user_input_key += 1 177 | st.session_state.accumulated_status_messages = [] 178 | 179 | if "shown_human_messages" in st.session_state: 180 | st.session_state.shown_human_messages = set() 181 | 182 | st.rerun() 183 | 184 | 185 | @st.fragment 186 | def conversation_management_fragment(): 187 | if st.button("New Chat"): 188 | create_new_conversation() 189 | st.rerun() 190 | 191 | st.markdown("### Conversations") 192 | 193 | if not st.session_state.conversations: 194 | create_new_conversation() 195 | 196 | for conv_id in st.session_state.conversations.keys(): 197 | 198 | if st.button(st.session_state.chat_names.get(conv_id, f"Chat {conv_id}"), key=f"btn_{conv_id}"): 199 | switch_conversation(conv_id) 200 | 201 | 202 | def setup_chat_placeholders(): 203 | user_message_placeholder = st.empty() 204 | subcol1, subcol2 = st.columns(COLUMN_SHAPES) 205 | 206 | with subcol2: 207 | status_box_placeholder = st.empty() 208 | with subcol1: 209 | human_pipeline_content_placeholder = st.empty() 210 | 211 | return user_message_placeholder, status_box_placeholder, human_pipeline_content_placeholder 212 | 213 | 214 | 215 | 216 | def process_agent_events(status_placeholder, pipeline_placeholder): 217 | st.session_state.accumulated_status_messages = [] 218 | accumulated_interpretation_messages = [] 219 | temp_assistant_messages = [] 220 | 221 | agent_event_iterator = stream_agent_response_for_frontend() 222 | 223 | for event in agent_event_iterator: 224 | if event["type"] == "assistant_message_chunk": 225 | content = event["content"] 226 | human_content = event.get("human_content", None) 227 | node_name = event.get("node_name", "") 228 | 229 | if node_name not in ["human_explanation_planning", "human_explanation_validator", "human_explanation_improvement", "human_explanation_results"]: 230 | st.session_state.accumulated_status_messages.append(content) 231 | 232 | if human_content: 233 | accumulated_interpretation_messages.append(human_content) 234 | 235 | render_status_boxes( 236 | st.session_state.accumulated_status_messages, 237 | accumulated_interpretation_messages, 238 | "Processing request...", 239 | "Code interpretation...", 240 | "running", 241 | True, 242 | status_placeholder, 243 | pipeline_placeholder 244 | ) 245 | 246 | temp_assistant_messages.append({ 247 | "role": "assistant", 248 | "content": content, 249 | "progress": [], 250 | "human": accumulated_interpretation_messages.copy(), 251 | "images": [] 252 | }) 253 | 254 | return temp_assistant_messages, accumulated_interpretation_messages 255 | 256 | 257 | def finalize_conversation(temp_assistant_messages, accumulated_interpretation_messages, current_conv_id): 258 | if temp_assistant_messages: 259 | combined_content = "".join([msg["content"] for msg in temp_assistant_messages]) 260 | consolidated_message = { 261 | "role": "assistant", 262 | "content": combined_content, 263 | "progress": st.session_state.accumulated_status_messages.copy(), 264 | "human": accumulated_interpretation_messages.copy(), 265 | "images": [] 266 | } 267 | st.session_state.conversations[current_conv_id].append(consolidated_message) 268 | 269 | def get_table_results(): 270 | row = get_benchmarks_from_csv(BENCHMARK_CSV_PATH, ID) 271 | 272 | if st.session_state.get("extract_metric", []): 273 | ds_agent_result = max(st.session_state.extract_metric) 274 | else: 275 | ds_agent_result = row['our_data'] 276 | 277 | data = { 278 | 'Logistic Regression': row['LogisticRegression'], 279 | 'LGBM': row['LGBM'], 280 | 'Tabular NN': row['Tabular NN'], 281 | 'LADS': ds_agent_result, 282 | } 283 | 284 | if st.session_state.current_node == "no_code_agent": 285 | data = None 286 | 287 | st.session_state.benchmark_history.append(data) 288 | 289 | def cleanup_and_rerun(user_message_placeholder, status_placeholder, pipeline_placeholder): 290 | status_placeholder.empty() 291 | pipeline_placeholder.empty() 292 | user_message_placeholder.empty() 293 | st.session_state.user_input_key += 1 294 | st.rerun() 295 | 296 | 297 | @st.fragment 298 | def chat_input_fragment(): 299 | user_message_placeholder, status_box_placeholder, human_pipeline_content_placeholder = setup_chat_placeholders() 300 | 301 | 302 | with st.form("chat_form", clear_on_submit=True): 303 | default_input = st.session_state.get("transcribed_text", "") 304 | user_input = st.text_input( 305 | "Your message:", 306 | value=default_input, 307 | key=f"user_input_{st.session_state.user_input_key}" 308 | ) 309 | submit_button = st.form_submit_button("Send") 310 | 311 | if submit_button and user_input and st.session_state.current_conversation: 312 | current_conv_id = st.session_state.current_conversation 313 | 314 | if "shown_human_messages" in st.session_state: 315 | st.session_state.shown_human_messages = set() 316 | 317 | user_message_data = {"role": "user", "content": user_input} 318 | st.session_state.conversations[current_conv_id].append(user_message_data) 319 | 320 | with user_message_placeholder.container(): 321 | with st.chat_message("user"): 322 | st.markdown(user_input) 323 | 324 | if "transcribed_text" in st.session_state: 325 | st.session_state.transcribed_text = "" 326 | 327 | status_placeholder = status_box_placeholder.empty() 328 | pipeline_placeholder = human_pipeline_content_placeholder.empty() 329 | 330 | temp_assistant_messages, accumulated_interpretation_messages = process_agent_events(status_placeholder, pipeline_placeholder) 331 | get_table_results() 332 | finalize_conversation(temp_assistant_messages, accumulated_interpretation_messages, current_conv_id) 333 | cleanup_and_rerun(user_message_placeholder, status_placeholder, pipeline_placeholder) 334 | 335 | 336 | @st.fragment 337 | def render_conversation(user_message: str, assistant_message: Dict[str, Any], table_raw=None): 338 | 339 | with st.chat_message("user"): 340 | st.markdown(user_message) 341 | 342 | render_status_boxes( 343 | assistant_message['progress'], 344 | assistant_message['human'], 345 | "Processed pipeline", 346 | "Code interpretation!", 347 | "complete", 348 | False 349 | ) 350 | 351 | with st.chat_message("assistant"): 352 | final_response_text = extract_final_response(assistant_message) 353 | st.markdown(final_response_text) 354 | 355 | with st.container(): 356 | if st.session_state.benchmark_history and table_raw is not None: 357 | st.markdown("#### Benchmark") 358 | df = pd.DataFrame([table_raw]) 359 | if st.session_state.benchmark_history[-1] is not None: 360 | update_ds_agent_history(BENCHMARK_CSV_PATH, ID, st.session_state.benchmark_history[-1]['LADS']) 361 | st.dataframe(df.style.highlight_max(axis=1, color="#39FF14"), use_container_width=True) 362 | st.markdown("---") 363 | 364 | -------------------------------------------------------------------------------- /fedotllm/agents/automl/nodes.py: -------------------------------------------------------------------------------- 1 | import re 2 | from pathlib import Path 3 | 4 | import pandas as pd 5 | from fedot.api.main import Fedot 6 | from golem.core.dag.graph_utils import graph_structure 7 | from langchain_core.messages import HumanMessage, convert_to_openai_messages 8 | from langgraph.types import Command 9 | 10 | from fedotllm import prompts 11 | from fedotllm.agents.automl.state import AutoMLAgentState 12 | from fedotllm.agents.automl.structured import FedotConfig 13 | from fedotllm.agents.automl.templates.load_template import ( 14 | load_template, 15 | render_template, 16 | ) 17 | from fedotllm.agents.utils import extract_code 18 | from fedotllm.data import Dataset 19 | from fedotllm.enviroments import ( 20 | Observation, 21 | execute_code, 22 | ) 23 | from fedotllm.llm import AIInference 24 | from fedotllm.log import logger 25 | from utils.config.loader import load_config 26 | 27 | PREDICT_METHOD_MAP = { 28 | "predict": "predict(features=input_data)", 29 | "forecast": "forecast(pre_history=input_data)", 30 | "predict_proba": "predict_proba(features=input_data)", 31 | } 32 | 33 | 34 | def problem_reflection( 35 | state: AutoMLAgentState, inference: AIInference, dataset: Dataset 36 | ): 37 | logger.info("Running problem reflection") 38 | 39 | messages = convert_to_openai_messages(state["messages"]) 40 | messages.append( 41 | { 42 | "role": "user", 43 | "content": prompts.automl.problem_reflection_prompt( 44 | data_files_and_content=dataset.dataset_preview(), 45 | dataset_eda=dataset.dataset_eda(), 46 | ), 47 | } 48 | ) 49 | reflection = inference.query(messages) 50 | return Command(update={"reflection": reflection}) 51 | 52 | 53 | def generate_automl_config( 54 | state: AutoMLAgentState, inference: AIInference, dataset: Dataset 55 | ): 56 | logger.info("Running generate automl config") 57 | 58 | config = inference.create( 59 | prompts.automl.generate_configuration_prompt( 60 | reflection=state["reflection"], 61 | ), 62 | response_model=FedotConfig, 63 | ) 64 | 65 | return Command(update={"fedot_config": config}) 66 | 67 | 68 | def select_skeleton(state: AutoMLAgentState, dataset: Dataset, workspace: Path): 69 | logger.info("Running select skeleton") 70 | fedot_config = state["fedot_config"] 71 | 72 | # Get prediction method 73 | predict_method = PREDICT_METHOD_MAP.get(fedot_config.predict_method) 74 | 75 | if predict_method is None: 76 | raise ValueError(f"Unknown predict method: {fedot_config.predict_method}") 77 | 78 | code_template = load_template(load_config().fedot.templates.code) 79 | code_template = render_template( 80 | template=code_template, 81 | dataset_path=dataset.path, 82 | work_dir_path=workspace.resolve(), 83 | ) 84 | 85 | return Command(update={"skeleton": code_template}) 86 | 87 | 88 | def generate_code(state: AutoMLAgentState, inference: AIInference, dataset: Dataset): 89 | logger.info("Generating code") 90 | codegen_prompt = prompts.automl.code_generation_prompt( 91 | reflection=state["reflection"], 92 | skeleton=state["skeleton"], 93 | dataset_path=str(dataset.path.absolute()), 94 | ) 95 | code = inference.query(codegen_prompt) 96 | extracted_code = extract_code(code) 97 | return Command(update={"raw_code": extracted_code}) 98 | 99 | 100 | def insert_templates(state: AutoMLAgentState): 101 | logger.info("Running insert templates") 102 | code = state["raw_code"] 103 | fedot_config = state["fedot_config"] 104 | config = load_config() 105 | predict_method = PREDICT_METHOD_MAP.get(fedot_config.predict_method) 106 | 107 | try: 108 | templates = { 109 | config.fedot.templates.train: { 110 | "params": { 111 | "problem": str(fedot_config.problem), 112 | "timeout": fedot_config.timeout, 113 | "cv_folds": fedot_config.cv_folds, 114 | "preset": f"'{fedot_config.preset.value}'", 115 | "metric": f"'{fedot_config.metric.value}'", 116 | **load_config().fedot.predictor_init_kwargs, 117 | } 118 | }, 119 | config.fedot.templates.evaluate: { 120 | "params": { 121 | "problem": str(fedot_config.problem), 122 | "predict_method": predict_method, 123 | } 124 | }, 125 | config.fedot.templates.predict: { 126 | "params": { 127 | "problem": str(fedot_config.problem), 128 | "predict_method": predict_method, 129 | } 130 | }, 131 | } 132 | 133 | rendered_templates = [] 134 | for template_name, config in templates.items(): 135 | template = load_template(template_name) 136 | rendered = render_template(template=template, **config["params"]) 137 | rendered_templates.append(rendered) 138 | 139 | code = code.replace( 140 | "from automl import train_model, evaluate_model, automl_predict", 141 | "\n".join(rendered_templates), 142 | ) 143 | 144 | logger.debug(f"Updated code: \n{code}") 145 | return Command(update={"code": code}) 146 | 147 | except Exception as e: 148 | logger.error(f"Failed to insert templates: {str(e)}") 149 | return Command(update={"code": None}) 150 | 151 | 152 | def _generate_code_file(code: str, output_dir: Path): 153 | output_dir.mkdir(parents=True, exist_ok=True) 154 | with open(output_dir / "solution.py", "w") as f: 155 | f.write(code) 156 | return output_dir / "solution.py" 157 | 158 | 159 | def evaluate(state: AutoMLAgentState, workspace: Path): 160 | logger.info("Running evaluate") 161 | code_path = _generate_code_file(state["code"], workspace) 162 | observation = execute_code(path_to_run_code=code_path) 163 | if observation.error: 164 | logger.error(observation.stderr) 165 | logger.debug( 166 | f"Evaluate result\nIs Error: {observation.error}\nStdout: {observation.stdout}\nStderr: {observation.stderr}" 167 | ) 168 | return Command(update={"observation": observation}) 169 | 170 | 171 | def if_bug(state: AutoMLAgentState): 172 | if ( 173 | state["observation"].error 174 | and state["fix_attempts"] < load_config().fedot.fix_tries 175 | ): 176 | return True 177 | if state["fix_attempts"] >= load_config().fedot.fix_tries: 178 | logger.error("Too many fix tries") 179 | return False 180 | 181 | 182 | def fix_solution(state: AutoMLAgentState, inference: AIInference, dataset: Dataset): 183 | logger.info("Running fix solution") 184 | 185 | fix_prompt = prompts.automl.fix_solution_prompt( 186 | reflection=state["reflection"], 187 | dataset_path=str(dataset.path.absolute()), 188 | code_recent_solution=state["raw_code"], 189 | msg=state["observation"].msg, 190 | stderr=state["observation"].stderr, 191 | stdout=state["observation"].stdout, 192 | ) 193 | 194 | fixed_solution = inference.query(fix_prompt) 195 | extracted_code = extract_code(fixed_solution) 196 | return Command( 197 | update={"raw_code": extracted_code, "fix_attempts": state["fix_attempts"] + 1} 198 | ) 199 | 200 | 201 | def extract_metrics(state: AutoMLAgentState, workspace: Path): 202 | logger.info("Running extract_metrics") 203 | 204 | def _parse_metrics(raw_output: str) -> str | None: 205 | pattern = r"Model metrics:\s*(\{.*?\})" 206 | if match := re.search(pattern, raw_output): 207 | return match.group(1).strip() 208 | return "Metrics not found" 209 | 210 | try: 211 | state["metrics"] = _parse_metrics(state["observation"].stdout) 212 | logger.info(f"Metrics: {state['metrics']}") 213 | 214 | pipeline_path = workspace / "pipeline" 215 | if pipeline_path.exists(): 216 | model = Fedot(problem="classification") 217 | model.load(pipeline_path) 218 | state["pipeline"] = graph_structure(model.current_pipeline) 219 | logger.info(f"Pipeline: {state['pipeline']}") 220 | else: 221 | logger.warning("Pipeline not found at expected path") 222 | state["pipeline"] = "Pipeline not found" 223 | except Exception as e: 224 | logger.error(f"Failed to extract metrics: {str(e)}") 225 | state["metrics"] = "Metrics not found" 226 | state["pipeline"] = "Pipeline not found" 227 | 228 | return state 229 | 230 | 231 | def run_tests(state: AutoMLAgentState, workspace: Path, inference: AIInference): 232 | logger.info("Running tests") 233 | 234 | def extract_metrics(raw_output: str) -> Observation: 235 | if match := re.search(r"Model metrics:\s*(\{.*?\})", raw_output): 236 | return Observation(error=False, msg=match.group(1).strip()) 237 | return Observation( 238 | error=True, 239 | msg="Metrics not found. Check if you use `evaluate` function and it was executed successfully.", 240 | ) 241 | 242 | def extract_pipeline(workspace: Path) -> Observation: 243 | pipeline_path = workspace / "pipeline" 244 | if not pipeline_path.exists(): 245 | return Observation( 246 | error=True, 247 | msg="Pipeline not found. Check if you use `train_model` function and it was executed successfully.", 248 | ) 249 | try: 250 | model = Fedot(problem="classification") 251 | model.load(pipeline_path) 252 | return Observation(error=False, msg=graph_structure(model.current_pipeline)) 253 | except Exception as e: 254 | return Observation(error=True, msg=f"Pipeline loading failed: {str(e)}") 255 | 256 | def check_submission_file(workspace: Path) -> Observation: 257 | submission_file = workspace / "submission.csv" 258 | return Observation( 259 | error=not submission_file.exists(), 260 | msg="Submission file exists." 261 | if submission_file.exists() 262 | else f"Submission file not found. Check if you save submission file successfully to {submission_file}.", 263 | ) 264 | 265 | def test_submission_format(args: tuple) -> Observation: 266 | raw_output, inference = args 267 | submission_file = workspace / "submission.csv" 268 | print("DEBUG: RAW OUTPUT\n", raw_output) 269 | 270 | if not ( 271 | match := re.search( 272 | r"Sample Submission File:\s*(.*?)$", raw_output, re.MULTILINE 273 | ) 274 | ): 275 | return Observation( 276 | error=True, 277 | msg="Sample submission file format not found. Print `Sample Submission File: {sample_submission}` in your code so I can check it.", 278 | ) 279 | 280 | sample_path = match.group(1).strip() 281 | print(f"Sample submission file path: {sample_path}") 282 | if not sample_path.endswith(".csv"): 283 | return Observation( 284 | error=True, 285 | msg="Sample Submission file format is incorrect. It should be a CSV file (.csv).", 286 | ) 287 | 288 | if not submission_file.exists() or submission_file.suffix != ".csv": 289 | return Observation( 290 | error=True, 291 | msg="Submission file format is incorrect. It should be a CSV file (.csv).", 292 | ) 293 | 294 | try: 295 | sample_df = pd.read_csv(sample_path) 296 | submission_df = pd.read_csv(submission_file) 297 | 298 | if submission_df.empty: 299 | return Observation(error=True, msg="Submission file is empty.") 300 | 301 | if not submission_df.columns.equals(sample_df.columns): 302 | return Observation( 303 | error=True, 304 | msg=f"Submission file columns don't match. Expected: {list(sample_df.columns)}, Got: {list(submission_df.columns)}", 305 | ) 306 | 307 | if submission_df.shape[1] != sample_df.shape[1]: 308 | return Observation( 309 | error=True, 310 | msg=f"Submission file has wrong number of columns. Expected: {sample_df.shape[1]}, Got: {submission_df.shape[1]}", 311 | ) 312 | 313 | # LLM validation for deeper format checking 314 | try: 315 | submission_sample = submission_df.head(3).to_string( 316 | max_rows=3, max_cols=10 317 | ) 318 | sample_submission_sample = sample_df.head(3).to_string( 319 | max_rows=3, max_cols=10 320 | ) 321 | 322 | result = inference.query( 323 | prompts.utils.ai_assert_prompt( 324 | var1=submission_sample, 325 | var2=sample_submission_sample, 326 | condition=( 327 | "Compare the submission file format with the sample submission file format to determine if they have the same structure by verifying the following:" 328 | "1. Column names match exactly. 2. Data types in corresponding columns are compatible. 3. The overall structure, including column order and presence, is consistent.\n" 329 | "IMPORTANT: \n" 330 | "1. Ignore differences in the values within the columns." 331 | "2. Focus solely on structure, column names, and data types." 332 | ), 333 | ) 334 | ) 335 | 336 | if result.strip().lower() != "true": 337 | return Observation( 338 | error=True, 339 | msg=f"Submission file format does not match expected format. Expected: {sample_submission_sample}, Got: {submission_sample}", 340 | ) 341 | except Exception: 342 | pass # LLM validation is optional, pandas validation is sufficient 343 | 344 | return Observation(error=False, msg="Submission file format is correct.") 345 | 346 | except Exception as e: 347 | return Observation( 348 | error=True, msg=f"Error validating submission format: {str(e)}" 349 | ) 350 | 351 | # Run all tests 352 | tests = [ 353 | (extract_metrics, state["observation"].stdout), 354 | (extract_pipeline, workspace), 355 | (check_submission_file, workspace), 356 | (test_submission_format, (state["observation"].stdout, inference)), 357 | ] 358 | 359 | for test_func, param in tests: 360 | result = test_func(param) 361 | if result.error: 362 | logger.error(f"Test failed: {result.msg}") 363 | state["observation"].error = True 364 | state["observation"].msg += f"\nTest failed: {result.msg}" 365 | else: 366 | logger.info(f"Test passed: {result.msg}") 367 | state["observation"].msg += f"\nTest passed: {result.msg}" 368 | 369 | return state 370 | 371 | 372 | def generate_report(state: AutoMLAgentState, inference: AIInference): 373 | if state["code"] and state["pipeline"]: 374 | messages = state["messages"] 375 | messages.append( 376 | { 377 | "role": "user", 378 | "content": prompts.automl.reporter_prompt( 379 | metrics=state["metrics"], 380 | pipeline=state["pipeline"], 381 | code=state["code"], 382 | ), 383 | } 384 | ) 385 | response = inference.query(convert_to_openai_messages(messages)) 386 | else: 387 | response = "Solution not found. Please try again." 388 | return Command( 389 | update={"messages": HumanMessage(content=response, role="AutoMLAgent")} 390 | ) 391 | -------------------------------------------------------------------------------- /graph/prompts_en.py: -------------------------------------------------------------------------------- 1 | from typing import Dict 2 | 3 | code_generator_system_prompt: str = """You are a Senior Python Developer with deep understanding of the Python tech stack and libraries. Your task is to solve the user's problem by providing clean, optimized, and professionally written code. 4 | 5 | Code requirements: 6 | - Write the entire code in one place without splitting it into parts. If an explanation is needed, write it at the end or as comments within the code. 7 | - The code must not contain any input requirements (like input or other forms)! Use only the information already available to you. 8 | - Code execution: Wrap the code you send in a markdown tag ```python-execute\n``` for execution. 9 | - Code quality: Your code must follow PEP8 standards. 10 | - Error handling: If an error occurs during execution, fix it and provide the corrected version. 11 | - Imports: Start your code with a clear and correct import block. 12 | - Data types: Ensure proper type conversion in operations. 13 | - Response format: Use markdown to format your responses. 14 | - Execution result: Responses must be based only on the outputs from running the code. 15 | - If the task requires, print the metric using print(). 16 | - Datasets are located in the datasets/ folder, and save models in the models/ folder. 17 | - If you need to analyze a dataset, first print its column names using print(df.columns), and optionally display the first 5 rows with print(df.head()). 18 | - Print dataset information (like df.describe()) to better understand how to work with it. 19 | - If there is already a solution or code in the messages, rewrite it entirely yourself according to these rules! 20 | 21 | These requirements ensure your code's quality and compliance with modern development standards. 22 | """ 23 | 24 | code_generator_user_prompt: str = """Based on the previous messages, help me solve my task: 25 | {user_input}""" 26 | 27 | rephraser_system_prompt: str = """You are an experienced data analyst and machine learning engineer who understands the task and creates a plan for solving it. 28 | Help the user formulate a step-by-step plan. 29 | Write down each step needed to solve the problem clearly. 30 | Do not write code or solve the task, just describe the overall plan. 31 | Use short bullet points, no need to elaborate in detail. 32 | """ 33 | 34 | rephraser_user_prompt: str = """Help me formulate a clear plan to solve the task. 35 | {user_input} 36 | """ 37 | 38 | validate_solution_system_prompt: str = """You are an experienced data analyst and machine learning engineer evaluating whether a solution to the task is correct and, if so, whether it needs improvement. 39 | Your job is to verify the correctness of the solution and give feedback. 40 | Reply 'VALID NO' if the answer is correct and the result is good and the user didn't ask for improvement. 41 | Reply 'VALID YES' if the answer is correct but the result is not good enough and needs improvement. Do not answer 'VALID YES' if the user didn't request improvement or the result is sufficiently good. 42 | If the answer is incorrect, reply 'WRONG' and give detailed feedback. 43 | Do not write any additional text, respond only with: "VALID YES", "VALID NO", or "WRONG". 44 | """ 45 | 46 | validate_solution_user_prompt: str = """{user_input} 47 | Solution plan: 48 | {rephrased_plan} 49 | Solution: 50 | {solution} 51 | """ 52 | 53 | code_improvement_system_prompt: str = """You are an experienced data analyst and machine learning engineer who understands how ML models work. 54 | You must: 55 | 1. Explain why the result was obtained, based on the code. 56 | 2. Suggest how to improve this result. Focus on feature engineering, model configuration, and other aspects to improve performance. 57 | Do not suggest multiple options; always provide ONLY ONE method for improving the code. Give a textual instruction without any code. 58 | Make sure you are not repeating steps from previous iterations, but you may consider refining some approaches. 59 | """ 60 | 61 | code_improvement_user_prompt: str = """{user_input} 62 | Code: 63 | ```python-execute 64 | {code} 65 | ``` 66 | Solution: 67 | {solution} 68 | 69 | These are all previous iterations’ improvements and results: 70 | {feedback} 71 | """ 72 | 73 | output_summarization_system_prompt: str = """You are an experienced data analyst and machine learning engineer who understands the task and can summarize solution results. 74 | Your task is to summarize the results obtained while working on the problem. For each improvement, provide the result and a description of the improvement approach. 75 | Answer in this format: 76 | 77 | User's task 78 | 79 | Baseline result 80 | 81 | Improvement 1 82 | 83 | Result 1 84 | 85 | Improvement 2 86 | 87 | Result 2 88 | ... 89 | 90 | Last improvement 91 | 92 | Last result 93 | 94 | If there's only one result, write just "Result" without mentioning a baseline. 95 | Describe the final result in detail: how it works, what models were used, what metrics were obtained, etc. 96 | """ 97 | 98 | output_summarization_user_prompt: str = """My task is: 99 | {task} 100 | This is the basic result and description of the approach: 101 | {base} 102 | These are all the results and descriptions of approaches with code improvement: 103 | {feedback} 104 | """ 105 | 106 | output_result_filter: str = """Extract from the following text which models were used and which metrics were obtained, if mentioned: 107 | {result} 108 | """ 109 | 110 | automl_router_system_prompt: str = """You are an experienced machine learning developer who understands the specific methods needed to solve a task. 111 | Determine whether the user wants to solve the given task using the LightAutoML library, Fedot, or another automated machine learning method. 112 | Carefully analyze the user's request to understand if automl, LightAutoML or Fedot is explicitly mentioned. If the request is general and does not explicitly mention automl, LightAutoML or Fedot, assume that the user does not want to use them and respond with the single word "NO". 113 | If LightAutoML is specified, respond with the single word "LAMA". 114 | If Fedot is specified, respond with the single word "FEDOT". 115 | If automl is specified, respond with the single word "LAMA" or "FEDOT", you can choose. 116 | """ 117 | 118 | automl_router_user_prompt: str = """Based on the task: 119 | ```{task}``` 120 | determine whether to use LightAutoML or FEDOT to solve it, othervise return NO! 121 | """ 122 | 123 | lightautoml_parser_system_prompt: str = """You are an experienced machine learning engineer who formulates tasks in machine learning terms. 124 | Your job is to generate a training config for an ML model based on input data. 125 | For regression, use "r2-score" as the task_metric and "reg" as the task_type. 126 | For classification, use "auc" as the task_metric and "binary" as the task_type. 127 | Always respond in the format: 128 | 129 | ```json 130 | {{ 131 | "task_type": "", 132 | "target": "", 133 | "task_metric": "" 134 | }} 135 | ``` 136 | """ 137 | 138 | lightautoml_parser_user_prompt: str = """Based on the user's task, column names, a few rows from the dataset, and the file name, generate a training config. 139 | User task: {task} 140 | File name: {file_name} 141 | Column names: {df_columns} 142 | Sample rows: 143 | {df_head} 144 | """ 145 | 146 | human_explanation_system_prompt: str = """ You are an experienced data scientist who understands how ML models work. 147 | Your task is to explain things in a way that regular people can understand, even those with minimal knowledge of machine learning. They might know what a model is but not understand terms like "target" or what a metric means. 148 | Explain briefly, clearly, and in plain language that anyone — even grandmas or top executives — can understand! 149 | """ 150 | 151 | human_explanation_user_prompt: str = """This is the text you need to explain: 152 | {text} 153 | """ 154 | 155 | human_explanation_planning_user_prompt: str = """This is the text you need to explain: 156 | {text} 157 | Explain it in such a way that you first say: 158 | This is the task solution plan, and then list the steps without explanations! 159 | Do not explain the steps, just write in a maximum of 5 words! 160 | Bold all steps and important words! 161 | """ 162 | human_explanation_results_user_prompt: str = """This is the text you need to explain: 163 | {text} 164 | Explain it like this: 165 | First, state which models were used to solve this task (bold all models), and then state the metrics obtained (bold all metrics). 166 | Do not explain the models and metrics! 167 | """ 168 | human_explanation_valid_user_prompt: str = """ 169 | Simply say that the agent successfully built the models and the agent believes the results are good enough! 170 | Bold important words! 171 | """ 172 | 173 | human_explanation_improvement_user_prompt: str = """This is the text you need to explain: 174 | {text} 175 | Explain it like this: 176 | - First, simply say which previous model was used, 177 | - Then briefly in one sentence explain why the results of this model are unsatisfactory, 178 | - Finally, briefly in two sentences explain how this model can be improved. 179 | Bold important words! 180 | Write explanations in bullet points! 181 | """ 182 | 183 | train_inference_split_system_prompt: str = """You are an experienced machine learning engineer who understands how ML works. 184 | Your task is to split the code into two parts: training and inference. 185 | The first part should contain only the model training code, and the second part only the inference code. 186 | If the model isn't saved during training, add code to save it. 187 | If the model isn't loaded during inference, add code to load it. 188 | In the training code, use the training dataset name. In the inference code, use the test dataset name. 189 | Always save predictions during inference to a file named dataset_name + "_predictions.csv", including the ID for which the prediction was made. 190 | Save any files, models, etc., in the code/ folder. 191 | Datasets are located in the datasets/ folder. 192 | Always answer in the format: 193 | train_code: 194 | ```python-execute 195 | ... 196 | ``` 197 | test_code: 198 | ```python-execute 199 | ... 200 | ``` 201 | Don't write any additional text, only markdown code blocks with the python-execute tag. 202 | """ 203 | 204 | train_inference_split_user_prompt: str = """Help me split this code: 205 | ```python-execute 206 | {code} 207 | ``` 208 | into model training code and inference code. 209 | Training dataset name: {train_dataset_name} 210 | Test dataset name: {test_dataset_name} 211 | """ 212 | 213 | train_test_checker_system_prompt = """ 214 | You are an experienced machine learning engineer who understands how code works and where errors may occur. 215 | Your task is to help the user verify if the generated code is correct. 216 | If it’s incorrect – fix the code and return the corrected version. 217 | If it’s correct – just respond VALID. 218 | 219 | If the code is INCORRECT, follow these rules: 220 | The first part should only contain model training code, the second part only inference code. 221 | If the model isn't saved during training, add saving. 222 | If the model isn't loaded during inference, add loading. 223 | 224 | Always respond in this format: 225 | train_code: 226 | ```python-execute 227 | ... 228 | ``` 229 | test_code: 230 | ```python-execute 231 | ... 232 | ``` 233 | Do not write any additional text, only code in markdown blocks with the python-execute tag. 234 | """ 235 | 236 | train_test_checker_user_prompt: str = """Help me verify the correctness of this code. 237 | Here is the result of its execution: 238 | {code_result} 239 | Here is the training code: 240 | {train_code} 241 | Here is the inference code: 242 | {test_code} 243 | """ 244 | 245 | code_router_system_prompt: str = """ 246 | You must understand whether code is needed to solve the task or not. 247 | Carefully analyze the user's request and determine if writing code is necessary to solve their problem. 248 | If code is needed, answer with only one word "YES", otherwise answer with only one word "NO". 249 | """ 250 | 251 | code_router_user_prompt: str = """Based on the task: 252 | ```{task}``` 253 | determine whether code needs to be used or not. 254 | """ 255 | 256 | 257 | no_code_system_prompt: str = """ You are an experienced data scientist and analyst who understands data and business. 258 | Answer questions clearly and concisely, in language understandable to non-specialists! 259 | """ 260 | 261 | no_code_user_prompt: str = """You need to explain this task: 262 | {text} 263 | """ 264 | 265 | result_summarization_system_prompt: str = """ 266 | You need to briefly explain which model was used and which metric was obtained! 267 | 268 | Always return the results in the following format: 269 | Models: 270 | - model_1: model_name 271 | - model_2: model_name 272 | ... 273 | - model_n: model_name 274 | 275 | model_name can be: LogisticRegression, RandomForest, XGBoost, CatBoost, SVM, ... 276 | 277 | Metrics: 278 | - metric_1: metric_result 279 | - metric_2: metric_result 280 | ... 281 | - metric_n: metric_result 282 | 283 | metric_i can be: ROC-AUC, F1, RMSE, ACCURACY, PRECISION, RECALL, ... 284 | Always write metrics as: ROC-AUC, F1, RMSE, ACCURACY, PRECISION, RECALL, ...! 285 | """ 286 | 287 | result_summarization_user_prompt: str = """Based on the code and result: 288 | ```{text}``` 289 | describe which model was used and which metrics were obtained. 290 | """ 291 | 292 | fedot_parser_system_prompt: str = """You are an experienced data scientist who understands how machine learning models work. 293 | Your task is to summarize the description and state which model was used and what metric was obtained. 294 | Always return results in the following format: 295 | Models: 296 | - model_1: model_name 297 | - model_2: model_name 298 | ... 299 | - model_n: model_name 300 | model_name - can be: LogisticRegression, RandomForest, XGBoost, CatBoost, SVM, ... 301 | Metrics: 302 | - metric_1: metric_result 303 | - metric_2: metric_result 304 | ... 305 | - metric_n: metric_result 306 | Metrics can be: ROC-AUC, F1, RMSE, ACCURACY, PRECISION, RECALL, ... 307 | Always write the metrics like this: ROC-AUC, F1, RMSE, ACCURACY, PRECISION, RECALL, ...! 308 | and return - (model_name) metric: result 309 | """ 310 | 311 | fedot_parser_user_prompt: str = """Based on the results, summarize the description: 312 | Results: {results} 313 | """ 314 | 315 | GIGACHAT_PROMPTS_EN: Dict[str, Dict[str, str]] = { 316 | "code_generator": { 317 | "system": code_generator_system_prompt, 318 | "user": code_generator_user_prompt 319 | }, 320 | "rephraser": { 321 | "system": rephraser_system_prompt, 322 | "user": rephraser_user_prompt 323 | }, 324 | "validate_solution": { 325 | "system": validate_solution_system_prompt, 326 | "user": validate_solution_user_prompt 327 | }, 328 | "code_improvement": { 329 | "system": code_improvement_system_prompt, 330 | "user": code_improvement_user_prompt 331 | }, 332 | "output_summarization": { 333 | "system": output_summarization_system_prompt, 334 | "user": output_summarization_user_prompt 335 | }, 336 | "output_result_filter": { 337 | "system": "", 338 | "user": output_result_filter, 339 | }, 340 | "automl_router": { 341 | "system": automl_router_system_prompt, 342 | "user": automl_router_user_prompt 343 | }, 344 | "lightautoml_parser": { 345 | "system": lightautoml_parser_system_prompt, 346 | "user": lightautoml_parser_user_prompt 347 | }, 348 | "human_explanation": { 349 | "system": human_explanation_system_prompt, 350 | "user": human_explanation_user_prompt 351 | }, 352 | "train_inference_split": { 353 | "system": train_inference_split_system_prompt, 354 | "user": train_inference_split_user_prompt 355 | }, 356 | "train_test_checker": { 357 | "system": train_test_checker_system_prompt, 358 | "user": train_test_checker_user_prompt 359 | }, 360 | "code_router": { 361 | "system": code_router_system_prompt, 362 | "user": code_router_user_prompt 363 | }, 364 | "no_code": { 365 | "system": no_code_system_prompt, 366 | "user": no_code_user_prompt 367 | }, 368 | "result_summarization": { 369 | "system": result_summarization_system_prompt, 370 | "user": result_summarization_user_prompt 371 | }, 372 | "human_explanation_planning": { 373 | "system": human_explanation_system_prompt, 374 | "user": human_explanation_planning_user_prompt 375 | }, 376 | "human_explanation_results": { 377 | "system": human_explanation_system_prompt, 378 | "user": human_explanation_results_user_prompt 379 | }, 380 | "human_explanation_validator": { 381 | "system": human_explanation_system_prompt, 382 | "user": human_explanation_valid_user_prompt 383 | }, 384 | "human_explanation_improvement": { 385 | "system": human_explanation_system_prompt, 386 | "user": human_explanation_improvement_user_prompt 387 | }, 388 | "fedot_parser": { 389 | "system": fedot_parser_system_prompt, 390 | "user": fedot_parser_user_prompt 391 | }, 392 | } 393 | -------------------------------------------------------------------------------- /fedotllm/agents/automl/templates/skeleton-complex.py: -------------------------------------------------------------------------------- 1 | ### UNMODIFIABLE IMPORT BEGIN ### 2 | import random 3 | from pathlib import Path 4 | import numpy as np 5 | import pandas as pd 6 | from fedot.api.main import Fedot 7 | from fedot.core.pipelines.pipeline import Pipeline 8 | from fedot.core.data.data import InputData 9 | from fedot.core.repository.tasks import Task, TaskTypesEnum # classification, regression, ts_forecasting. 10 | from automl import train_model, evaluate_model, automl_predict 11 | ### UNMODIFIABLE IMPORT END ### 12 | # USER CODE BEGIN IMPORTS # 13 | from sklearn.model_selection import train_test_split 14 | from sklearn.impute import SimpleImputer 15 | from sklearn.preprocessing import StandardScaler, LabelEncoder, OneHotEncoder 16 | # USER CODE END IMPORTS # 17 | 18 | SEED = 42 19 | random.seed(SEED) 20 | np.random.seed(SEED) 21 | 22 | ### UNMODIFIABLE CODE BEGIN ### 23 | DATASET_PATH = Path("{%dataset_path%}") # path for saving and loading dataset(s) 24 | WORKSPACE_PATH = Path("{%work_dir_path%}") 25 | PIPELINE_PATH = WORKSPACE_PATH / "pipeline" # path for saving and loading pipelines 26 | SUBMISSION_PATH = WORKSPACE_PATH / "submission.csv" 27 | EVAL_SET_SIZE = 0.2 # 20% of the data for evaluation 28 | ### UNMODIFIABLE CODE END ### 29 | 30 | # --- TODO: Update these paths for your specific competition --- 31 | TRAIN_FILE = DATASET_PATH / "train.csv" # Replace with your actual filename 32 | TEST_FILE = DATASET_PATH / "test.csv" # Replace with your actual filename 33 | SAMPLE_SUBMISSION_FILE = DATASET_PATH / "sample_submission.csv" # Replace with your actual filename or None 34 | ID_COLUMN = "id" # Replace with your actual ID column name, if any 35 | TARGET_COLUMNS = ["target"] # Replace with your actual target column name(s) 36 | 37 | 38 | def load_data(): 39 | """Loads train, test, and optionally sample submission files.""" 40 | try: 41 | train_df = pd.read_csv(TRAIN_FILE) # TODO: Adjust pandas loader if needed 42 | test_df = pd.read_csv(TEST_FILE) # TODO: Adjust pandas loader if needed 43 | sample_sub_df = None 44 | if SAMPLE_SUBMISSION_FILE and (DATASET_PATH / SAMPLE_SUBMISSION_FILE).exists(): 45 | sample_sub_df = pd.read_csv(DATASET_PATH / SAMPLE_SUBMISSION_FILE) # TODO: Adjust pandas loader if needed 46 | else: 47 | print("Sample submission file not found or not specified.") 48 | print("Data loaded successfully.") 49 | print(f"Train shape: {train_df.shape}, Test shape: {test_df.shape}") 50 | return train_df, test_df, sample_sub_df 51 | except FileNotFoundError as e: 52 | print(f"Error loading data: {e}. Please check filenames.") 53 | return None, None, None 54 | 55 | # --------------------------------------------------------------------------- # 56 | # Section 2: Data Cleaning 57 | # --------------------------------------------------------------------------- # 58 | 59 | def cleaning_data(df_train: pd.DataFrame, df_test: pd.DataFrame, target_cols: list): 60 | print("\nCleaning data...") 61 | # Make copies to avoid modifying original dataframes 62 | X_train = df_train.copy() 63 | X_test = df_test.copy() 64 | 65 | # --- Step 0: Handle specific data cleaning tasks (e.g., inconsistent values, typos) --- 66 | # This section remains for custom, early-stage cleaning if necessary. 67 | 68 | y_train = None 69 | # Check if all target columns are in X_train 70 | if all(col in X_train.columns for col in target_cols): 71 | y_train = X_train[target_cols].copy() 72 | X_train = X_train.drop(columns=target_cols) 73 | else: 74 | missing_cols = [col for col in target_cols if col not in X_train.columns] 75 | print(f"Warning: Target column(s) {missing_cols} not in training data.") 76 | # Optionally, remove existing target columns if some are missing 77 | present_target_cols = [col for col in target_cols if col in X_train.columns] 78 | if present_target_cols: 79 | y_train = X_train[present_target_cols].copy() # or handle as error 80 | X_train = X_train.drop(columns=present_target_cols) 81 | print(f"Using available target columns: {present_target_cols}") 82 | 83 | 84 | # Remove target columns from test if present 85 | test_cols_to_drop = [col for col in target_cols if col in X_test.columns] 86 | if test_cols_to_drop: 87 | print(f"Warning: Target column(s) {test_cols_to_drop} found in test data and removed.") 88 | X_test.drop(columns=test_cols_to_drop, inplace=True) 89 | 90 | # Remove ID column from feature lists if it's present and not a feature 91 | if ID_COLUMN in X_train: X_train.drop(columns=[ID_COLUMN], inplace=True) 92 | if ID_COLUMN in X_test: X_test.drop(columns=[ID_COLUMN], inplace=True) 93 | 94 | return X_train, X_test, y_train 95 | 96 | # --------------------------------------------------------------------------- # 97 | # Section 3: Data Preprocessing 98 | # --------------------------------------------------------------------------- # 99 | def preprocess_data(train_features: pd.DataFrame, test_features: pd.DataFrame): 100 | """Cleans and transforms data into a suitable format for modeling 101 | 1. Feature Engineering & Preprocessing: 102 | - Column Dropping: Drop irrelevant columns (e.g., IDs, high cardinality text not used, columns with too many NaNs if not imputed). 103 | - Missing Value Imputation: 104 | - Use `SimpleImputer(strategy='mean')` for numerical or `SimpleImputer(strategy='most_frequent')` for categorical. 105 | - IMPORTANT: Fit imputers ONLY on training data. Then, use the FITTED imputer to `.transform()` train, validation, AND test sets. 106 | This template's structure implies `transform_data` is called multiple times. You'll need to manage fitted transformers 107 | (e.g., define them globally or pass them if they are stateful and fitted on the first call with training data). 108 | A simpler approach for one-shot filling might be to perform stateless transformations or ensure any fitting 109 | logic is conditional (e.g. `if not hasattr(self, 'imputer'): self.imputer.fit(X_train)`). 110 | - Categorical Encoding: 111 | - For nominal: `OneHotEncoder(handle_unknown='ignore')`. 112 | - For ordinal: `OrdinalEncoder()`. 113 | - Fit encoders ONLY on training data and transform all sets. 114 | - Numerical Scaling (Optional but often beneficial for some models): 115 | - `StandardScaler()`. Fit ONLY on training data. 116 | - Feature Creation (Domain-specific): Create new features from existing ones if it makes sense for the problem. 117 | - Data Type Conversion: Ensure all features are numeric (int/float) before returning. 118 | 119 | 2. Consistency: 120 | - Transformations applied to training data MUST be applied identically to test data. 121 | - Ensure the order of columns is consistent if not inherently handled by transformers. 122 | - DO NOT drop rows from the test set (for submission) unless absolutely necessary and accounted for, as it will affect submission alignment. 123 | """ 124 | 125 | print("\nPreprocessing data...") 126 | # Make copies to avoid modifying original dataframes 127 | X_train = train_features.copy() 128 | X_test = test_features.copy() 129 | 130 | # --- Step 0: Column Dropping --- 131 | # Leave only important features 132 | important_features = [ ] 133 | X_train = X_train[important_features] 134 | X_test = X_test[important_features] 135 | 136 | # --- Step 1: Identify feature types (based on training data) --- 137 | numerical_cols = X_train.select_dtypes(include=np.number).columns.tolist() 138 | categorical_cols = X_train.select_dtypes(include=['object', 'category']).columns.tolist() 139 | # --- Step 2: Handle Missing Values (Fit on Train, Transform Train & Test) --- 140 | # Numerical Imputation 141 | # if numerical_cols: 142 | # num_imputer = SimpleImputer(strategy='median') 143 | # X_train[numerical_cols] = num_imputer.fit_transform(X_train[numerical_cols]) 144 | # X_test[numerical_cols] = num_imputer.transform(X_test[numerical_cols]) 145 | 146 | # Categorical Imputation 147 | # if categorical_cols: 148 | # cat_imputer = SimpleImputer(strategy='most_frequent') # Or 'constant' with fill_value='Missing' 149 | # X_train[categorical_cols] = cat_imputer.fit_transform(X_train[categorical_cols]) 150 | # X_test[categorical_cols] = cat_imputer.transform(X_test[categorical_cols]) 151 | 152 | 153 | # --- Step 3: Encode Categorical Features (Fit on Train, Transform Train & Test) --- 154 | # Using OneHotEncoder for columns with few unique values and LabelEncoder for others. 155 | # Store encoders if inverse_transform or consistent application to new data is needed. 156 | # For robust OHE, ensure all categories seen during fit are handled during transform. 157 | # Important Note: 158 | # --------- 159 | # OneHotEncoder sparse attribute is DEPRECATED! **Must use sparse_output instead.** 160 | # ---------- 161 | # Example: 162 | # --------- 163 | # ohe = OneHotEncoder(handle_unknown='ignore', sparse_output=False) # sparse_output=False for DataFrame compatibility 164 | # ohe.fit(X_train[[col]]) 165 | # train_encoded_names = ohe.get_feature_names_out([col]) 166 | # # Transform training data 167 | # train_encoded_df = pd.DataFrame(ohe.transform(X_train[[col]]), columns=train_encoded_names, index=X_train.index) 168 | # X_train = pd.concat([X_train.drop(columns=[col]), train_encoded_df], axis=1) 169 | # # Transform test data 170 | # test_encoded_df = pd.DataFrame(ohe.transform(X_test[[col]]), columns=train_encoded_names, index=X_test.index) 171 | # X_test = pd.concat([X_test.drop(columns=[col]), test_encoded_df], axis=1) 172 | # --------- 173 | # le = LabelEncoder() 174 | # X_train[col] = le.fit_transform(X_train[col].astype(str)) 175 | # X_test[col] = X_test[col].astype(str).map(lambda s: le.transform([s])[0] if s in le.classes_ else -1) # -1 for unknown 176 | 177 | # --- Step 4: Feature Scaling (Fit on Train, Transform Train & Test) --- 178 | # Scale numerical features (original numerical + label encoded if they are treated as numerical) 179 | # Exclude OHE columns if you prefer not to scale them, though scaling them is often fine. 180 | # Example: 181 | # --------- 182 | # scaler = StandardScaler() 183 | # X_train[cols_to_scale] = scaler.fit_transform(X_train[cols_to_scale]) 184 | # X_test[cols_to_scale] = scaler.transform(X_test[cols_to_scale]) 185 | 186 | # --- Step 5: Other transformations (e.g., log transform skewed features) --- 187 | # Apply these transformations consistently after fitting on training data if applicable. 188 | # Example: Log transform (fit on train, transform train & test) 189 | # for col in numerical_cols: # Use original numerical_cols or an updated list 190 | # if col in X_train.columns and X_train[col].skew() > 1.0: 191 | # print(f"Log transforming skewed column: {col}") 192 | # # Add 1 to prevent log(0) issues, ensure positive values 193 | # X_train[col] = np.log1p(X_train[col] - X_train[col].min()) 194 | # X_test[col] = np.log1p(X_test[col] - X_test[col].min()) # Apply same shift as train 195 | 196 | # --- Step 6: Feature Engineering --- 197 | # Create new features from existing ones if it makes sense for the problem. 198 | # Use domain knowledge to create new features if it makes sense. 199 | # Example: 200 | # --------- 201 | # X_train['feature_name'] = X_train['feature_name'].map({}) 202 | # X_train['feature_name'] = X_train['feature_name'] + X_train['feature_name_2'] 203 | 204 | print("Preprocessing complete.") 205 | print(f"Train processed shape: {X_train.shape}, Test processed shape: {X_test.shape}") 206 | return X_train, X_test 207 | 208 | # --------------------------------------------------------------------------- # 209 | # Section 6: Final Model Training & Submission 210 | # --------------------------------------------------------------------------- # 211 | def create_submission_file(test_ids, test_predictions, sample_sub_df, submission_filename="submission.csv"): 212 | """Creates the submission file in the required format.""" 213 | print(f"\nCreating submission file: {submission_filename}") 214 | # Flatten the predictions if they're 2D 215 | if isinstance(test_predictions, np.ndarray) and test_predictions.ndim > 1: 216 | test_predictions = test_predictions.flatten() 217 | if not isinstance(test_ids, (pd.Series, np.ndarray)): 218 | try: 219 | # Attempt to get ID from sample submission if test_ids is not directly usable 220 | if sample_sub_df is not None and ID_COLUMN in sample_sub_df.columns: 221 | test_ids = sample_sub_df[ID_COLUMN] 222 | else: # Fallback if ID_COLUMN not in sample submission 223 | test_ids = np.arange(len(test_predictions)) # Simple range if no IDs 224 | print(f"Warning: Using default range for test_ids as {ID_COLUMN} not found in sample submission.") 225 | except Exception as e: 226 | print(f"Warning: Could not load sample submission to get test_ids: {e}. Using default range.") 227 | test_ids = np.arange(len(test_predictions)) 228 | 229 | 230 | submission_df = pd.DataFrame(test_predictions, columns=[TARGET_COLUMNS]) 231 | submission_df.insert(0, ID_COLUMN, test_ids) 232 | 233 | 234 | submission_file_path = WORKSPACE_PATH / submission_filename 235 | submission_df.to_csv(submission_file_path, index=False) 236 | print(f"Submission file created at: {submission_file_path}") 237 | print("Submission file head:") 238 | print(submission_df.head()) 239 | return submission_df 240 | 241 | # --------------------------------------------------------------------------- # 242 | # Main Orchestration Logic 243 | # --------------------------------------------------------------------------- # 244 | def main(): 245 | """Main function to orchestrate the ML pipeline.""" 246 | print("Starting Kaggle ML Workflow...") 247 | # --- TODO: Define problem_type ('classification', 'regression', 'multi-label classification', 'ts_forecasting') --- 248 | current_problem_type = 'classification' # Example 249 | 250 | # --- Step 0: Load Data --- 251 | train_df, test_df, sample_sub_df = load_data() 252 | if train_df is None or test_df is None: 253 | print("Exiting due to data loading failure.") 254 | return 255 | 256 | # Store original test IDs for submission 257 | original_test_ids = test_df[ID_COLUMN] if ID_COLUMN in test_df.columns else test_df.index 258 | 259 | # --- Step 1: Cleaning --- 260 | X_train_cleaned, X_test_cleaned, y_train_full = cleaning_data(train_df, test_df, TARGET_COLUMNS) 261 | 262 | # --- Step 2: Preprocessing --- 263 | # The target column name (TARGET_COLUMN) is passed to preprocess_data 264 | X_train_processed, X_test_processed = preprocess_data(X_train_cleaned, X_test_cleaned) 265 | 266 | # --- Create a fixed validation split from the processed full training data --- 267 | print(f"\nCreating a fixed train-evaluation split (Eval size: {EVAL_SET_SIZE*100}%) ...") 268 | # For multi-target, set stratify=None 269 | stratify_target = None 270 | if current_problem_type.startswith('classification') and len(TARGET_COLUMNS) == 1: 271 | stratify_target = y_train_full # Only stratify for single-target classification 272 | X_train_main, X_eval_holdout, y_train_main, y_eval_holdout = train_test_split( 273 | X_train_processed, y_train_full, 274 | test_size=EVAL_SET_SIZE, 275 | random_state=SEED, 276 | stratify=stratify_target 277 | ) 278 | 279 | # --- Step 3: Train AutoML model --- 280 | # You must use pre-defined train_model function to train the model. 281 | model = train_model(X_train_main, y_train_main) 282 | 283 | # --- Step 4: Evaluate the trained model --- 284 | # You must use pre-defined evaluate_model function to evaluate the model. 285 | model_performance = evaluate_model(model, X_eval_holdout, y_eval_holdout) 286 | 287 | # --- Step 5: Evaluate predictions for the test dataset using AutoML Framework --- 288 | # You must use pre-defined automl_predict function to predict the model. 289 | predictions:np.ndarray = automl_predict(model, X_test_processed) # returns 2D array 290 | 291 | # --- Step 6: Create submission file --- 292 | if predictions is not None: 293 | create_submission_file(original_test_ids, predictions, sample_sub_df, "submission.csv") 294 | else: 295 | print("No test predictions generated, skipping submission file creation.") 296 | print("\nKaggle ML Workflow finished.") 297 | 298 | if __name__ == "__main__": 299 | print("Files and directories:") 300 | paths = { 301 | "Dataset Path": DATASET_PATH, 302 | "Workspace Path": WORKSPACE_PATH, 303 | "Pipeline Path": PIPELINE_PATH, 304 | "Submission Path": SUBMISSION_PATH, 305 | "Train File": TRAIN_FILE, 306 | "Test File": TEST_FILE, 307 | "Sample Submission File": SAMPLE_SUBMISSION_FILE 308 | } 309 | for name, path in paths.items(): 310 | print(f"{name}: {path}") 311 | main() --------------------------------------------------------------------------------