├── .gitignore ├── CODE_OF_CONDUCT.md ├── LICENSE ├── README.md ├── SECURITY.md ├── SUPPORT.md ├── all_feedback.jpg ├── llfbench ├── __init__.py ├── agents │ ├── __init__.py │ ├── abstract_agent.py │ ├── ag_agent.py │ ├── basic_ai_agent.py │ ├── llm.py │ ├── user_agent.py │ └── utils.py ├── envs │ ├── __init__.py │ ├── alfworld │ │ ├── __init__.py │ │ ├── alfworld.py │ │ ├── alfworld_download.py │ │ ├── base_config.yaml │ │ ├── eval_config.yaml │ │ ├── prompts.py │ │ └── wrapper.py │ ├── bandits │ │ ├── __init__.py │ │ ├── prompts.py │ │ └── wrapper.py │ ├── env_wrappers.py │ ├── gridworld │ │ ├── __init__.py │ │ ├── gridworld.py │ │ ├── prompts.py │ │ ├── room.py │ │ ├── scene.py │ │ └── wrapper.py │ ├── highway │ │ ├── __init__.py │ │ ├── prompts.py │ │ └── wrapper.py │ ├── llf_env.py │ ├── metaworld │ │ ├── __init__.py │ │ ├── gains.py │ │ ├── prompts.py │ │ └── wrapper.py │ ├── optimization │ │ ├── README.md │ │ ├── __init__.py │ │ ├── loss_descent.py │ │ ├── prompts.py │ │ └── wrapper.py │ ├── poem │ │ ├── README.md │ │ ├── __init__.py │ │ ├── formal_poems.py │ │ ├── prompts.py │ │ └── wrapper.py │ ├── reco │ │ ├── README.md │ │ ├── __init__.py │ │ ├── factual_movie_data_2023_12_16.pkl │ │ ├── movie_rec.py │ │ ├── prompts.py │ │ └── wrapper.py │ └── utils.py └── utils │ ├── __init__.py │ ├── parser_utils.py │ └── utils.py ├── partial_feedback.jpg ├── setup.py └── tests ├── test_agents.py └── test_envs.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/.gitignore -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/CODE_OF_CONDUCT.md -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/README.md -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/SECURITY.md -------------------------------------------------------------------------------- /SUPPORT.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/SUPPORT.md -------------------------------------------------------------------------------- /all_feedback.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/all_feedback.jpg -------------------------------------------------------------------------------- /llfbench/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/__init__.py -------------------------------------------------------------------------------- /llfbench/agents/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /llfbench/agents/abstract_agent.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/agents/abstract_agent.py -------------------------------------------------------------------------------- /llfbench/agents/ag_agent.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/agents/ag_agent.py -------------------------------------------------------------------------------- /llfbench/agents/basic_ai_agent.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/agents/basic_ai_agent.py -------------------------------------------------------------------------------- /llfbench/agents/llm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/agents/llm.py -------------------------------------------------------------------------------- /llfbench/agents/user_agent.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/agents/user_agent.py -------------------------------------------------------------------------------- /llfbench/agents/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/agents/utils.py -------------------------------------------------------------------------------- /llfbench/envs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/envs/__init__.py -------------------------------------------------------------------------------- /llfbench/envs/alfworld/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/envs/alfworld/__init__.py -------------------------------------------------------------------------------- /llfbench/envs/alfworld/alfworld.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/envs/alfworld/alfworld.py -------------------------------------------------------------------------------- /llfbench/envs/alfworld/alfworld_download.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/envs/alfworld/alfworld_download.py -------------------------------------------------------------------------------- /llfbench/envs/alfworld/base_config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/envs/alfworld/base_config.yaml -------------------------------------------------------------------------------- /llfbench/envs/alfworld/eval_config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/envs/alfworld/eval_config.yaml -------------------------------------------------------------------------------- /llfbench/envs/alfworld/prompts.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/envs/alfworld/prompts.py -------------------------------------------------------------------------------- /llfbench/envs/alfworld/wrapper.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/envs/alfworld/wrapper.py -------------------------------------------------------------------------------- /llfbench/envs/bandits/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/envs/bandits/__init__.py -------------------------------------------------------------------------------- /llfbench/envs/bandits/prompts.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/envs/bandits/prompts.py -------------------------------------------------------------------------------- /llfbench/envs/bandits/wrapper.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/envs/bandits/wrapper.py -------------------------------------------------------------------------------- /llfbench/envs/env_wrappers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/envs/env_wrappers.py -------------------------------------------------------------------------------- /llfbench/envs/gridworld/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/envs/gridworld/__init__.py -------------------------------------------------------------------------------- /llfbench/envs/gridworld/gridworld.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/envs/gridworld/gridworld.py -------------------------------------------------------------------------------- /llfbench/envs/gridworld/prompts.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/envs/gridworld/prompts.py -------------------------------------------------------------------------------- /llfbench/envs/gridworld/room.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/envs/gridworld/room.py -------------------------------------------------------------------------------- /llfbench/envs/gridworld/scene.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/envs/gridworld/scene.py -------------------------------------------------------------------------------- /llfbench/envs/gridworld/wrapper.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/envs/gridworld/wrapper.py -------------------------------------------------------------------------------- /llfbench/envs/highway/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/envs/highway/__init__.py -------------------------------------------------------------------------------- /llfbench/envs/highway/prompts.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/envs/highway/prompts.py -------------------------------------------------------------------------------- /llfbench/envs/highway/wrapper.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/envs/highway/wrapper.py -------------------------------------------------------------------------------- /llfbench/envs/llf_env.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/envs/llf_env.py -------------------------------------------------------------------------------- /llfbench/envs/metaworld/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/envs/metaworld/__init__.py -------------------------------------------------------------------------------- /llfbench/envs/metaworld/gains.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/envs/metaworld/gains.py -------------------------------------------------------------------------------- /llfbench/envs/metaworld/prompts.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/envs/metaworld/prompts.py -------------------------------------------------------------------------------- /llfbench/envs/metaworld/wrapper.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/envs/metaworld/wrapper.py -------------------------------------------------------------------------------- /llfbench/envs/optimization/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/envs/optimization/README.md -------------------------------------------------------------------------------- /llfbench/envs/optimization/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/envs/optimization/__init__.py -------------------------------------------------------------------------------- /llfbench/envs/optimization/loss_descent.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/envs/optimization/loss_descent.py -------------------------------------------------------------------------------- /llfbench/envs/optimization/prompts.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/envs/optimization/prompts.py -------------------------------------------------------------------------------- /llfbench/envs/optimization/wrapper.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/envs/optimization/wrapper.py -------------------------------------------------------------------------------- /llfbench/envs/poem/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/envs/poem/README.md -------------------------------------------------------------------------------- /llfbench/envs/poem/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/envs/poem/__init__.py -------------------------------------------------------------------------------- /llfbench/envs/poem/formal_poems.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/envs/poem/formal_poems.py -------------------------------------------------------------------------------- /llfbench/envs/poem/prompts.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/envs/poem/prompts.py -------------------------------------------------------------------------------- /llfbench/envs/poem/wrapper.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/envs/poem/wrapper.py -------------------------------------------------------------------------------- /llfbench/envs/reco/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/envs/reco/README.md -------------------------------------------------------------------------------- /llfbench/envs/reco/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/envs/reco/__init__.py -------------------------------------------------------------------------------- /llfbench/envs/reco/factual_movie_data_2023_12_16.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/envs/reco/factual_movie_data_2023_12_16.pkl -------------------------------------------------------------------------------- /llfbench/envs/reco/movie_rec.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/envs/reco/movie_rec.py -------------------------------------------------------------------------------- /llfbench/envs/reco/prompts.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/envs/reco/prompts.py -------------------------------------------------------------------------------- /llfbench/envs/reco/wrapper.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/envs/reco/wrapper.py -------------------------------------------------------------------------------- /llfbench/envs/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/envs/utils.py -------------------------------------------------------------------------------- /llfbench/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/utils/__init__.py -------------------------------------------------------------------------------- /llfbench/utils/parser_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/utils/parser_utils.py -------------------------------------------------------------------------------- /llfbench/utils/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/llfbench/utils/utils.py -------------------------------------------------------------------------------- /partial_feedback.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/partial_feedback.jpg -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/setup.py -------------------------------------------------------------------------------- /tests/test_agents.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/tests/test_agents.py -------------------------------------------------------------------------------- /tests/test_envs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/LLF-Bench/HEAD/tests/test_envs.py --------------------------------------------------------------------------------