├── .env.example ├── .github └── CONTRIBUTING.md ├── .gitignore ├── .pre-commit-config.yaml ├── .secrets.example ├── dev │ ├── github_client_secret │ ├── google_client_secret │ ├── mongo_express_admin_password │ ├── mongodb_root_password │ ├── redis_password │ └── secret_key ├── openai_api_key ├── prod │ ├── github_client_secret │ ├── google_client_secret │ ├── mongo_express_admin_password │ ├── mongodb_root_password │ ├── redis_password │ └── secret_key └── together_api_key ├── Dockerfile ├── Dockerfile.prod ├── Dockerfile.traefik ├── LICENSE ├── README.md ├── admin_scripts ├── __init__.py ├── admin_client.py ├── create_teams.py └── increase_budget.py ├── app ├── __init__.py ├── api │ ├── __init__.py │ ├── api_v1 │ │ ├── __init__.py │ │ ├── api.py │ │ └── endpoints │ │ │ ├── __init__.py │ │ │ ├── budget.py │ │ │ ├── chat.py │ │ │ ├── defense.py │ │ │ ├── key.py │ │ │ ├── models.py │ │ │ ├── oauth2.py │ │ │ ├── scores.py │ │ │ ├── secret.py │ │ │ ├── submission.py │ │ │ ├── teams.py │ │ │ ├── users.py │ │ │ └── utils.py │ ├── deps.py │ └── documentation_text.py ├── config.py ├── crud │ ├── __init__.py │ ├── base.py │ ├── crud_api_key.py │ ├── crud_chat.py │ ├── crud_defense.py │ ├── crud_secret.py │ ├── crud_secret_guess.py │ ├── crud_team.py │ ├── crud_team_budget.py │ └── crud_user.py ├── enums.py ├── frontend.py ├── gradio │ ├── __init__.py │ ├── attack.py │ └── defense.py ├── internals │ ├── __init__.py │ ├── base_acc_20.json │ ├── code_exec.py │ ├── defenses_11.json │ ├── llm.py │ ├── openai_accs_20.json │ ├── output_filter.py │ └── utility_eval.py ├── limits.py ├── main.py ├── models │ ├── __init__.py │ ├── api_key.py │ ├── chat.py │ ├── defense.py │ ├── secret.py │ ├── team.py │ ├── team_budget.py │ └── user.py ├── schemas │ ├── __init__.py │ ├── api_key.py │ ├── chat.py │ ├── defense.py │ ├── generate.py │ ├── message.py │ ├── scores.py │ ├── secret.py │ ├── team_budget.py │ ├── token.py │ └── user.py └── security.py ├── compose.prod.yml ├── compose.yml ├── data ├── download_arb.py ├── final_scores.json ├── pick_mmlu.py └── small_data │ ├── abstract_algebra_test.20.csv │ ├── anatomy_test.20.csv │ ├── arb_law_0.20.csv │ ├── arb_law_1.20.csv │ ├── arb_law_2.20.csv │ ├── arb_law_3.20.csv │ ├── arb_law_4.20.csv │ ├── arb_law_5.20.csv │ ├── arb_law_6.20.csv │ ├── arb_law_7.20.csv │ ├── arb_law_8.20.csv │ ├── arb_law_9.20.csv │ ├── astronomy_test.20.csv │ ├── business_ethics_test.20.csv │ ├── clinical_knowledge_test.20.csv │ ├── college_computer_science_test.20.csv │ ├── college_mathematics_test.20.csv │ ├── college_medicine_test.20.csv │ ├── college_physics_test.20.csv │ ├── computer_security_test.20.csv │ ├── conceptual_physics_test.20.csv │ ├── conceptual_physics_test.csv │ ├── electrical_engineering_test.20.csv │ ├── elementary_mathematics_test.20.csv │ ├── global_facts_test.20.csv │ ├── high_school_biology_test.20.csv │ ├── high_school_chemistry_test.20.csv │ ├── high_school_computer_science_test.20.csv │ ├── high_school_european_history_test.20.csv │ ├── high_school_geography_test.20.csv │ ├── high_school_government_and_politics_test.20.csv │ ├── high_school_macroeconomics_test.20.csv │ ├── high_school_mathematics_test.20.csv │ ├── high_school_microeconomics_test.20.csv │ ├── high_school_physics_test.20.csv │ ├── high_school_psychology_test.20.csv │ ├── high_school_statistics_test.20.csv │ ├── high_school_us_history_test.20.csv │ ├── high_school_world_history_test.20.csv │ ├── human_aging_test.20.csv │ ├── international_law_test.20.csv │ ├── jurisprudence_test.20.csv │ ├── logical_fallacies_test.20.csv │ ├── machine_learning_test.20.csv │ ├── management_test.20.csv │ ├── marketing_test.20.csv │ ├── medical_genetics_test.20.csv │ ├── miscellaneous_test.20.csv │ ├── moral_disputes_test.20.csv │ ├── nutrition_test.20.csv │ ├── philosophy_test.20.csv │ ├── prehistory_test.20.csv │ ├── professional_medicine_test.20.csv │ ├── professional_psychology_test.20.csv │ ├── public_relations_test.20.csv │ ├── security_studies_test.20.csv │ ├── sociology_test.20.csv │ ├── us_foreign_policy_test.20.csv │ ├── virology_test.20.csv │ └── world_religions_test.20.csv ├── frontpage ├── README.md ├── build.sh ├── content │ └── index.md ├── copy_to_app.sh ├── serve.sh ├── static │ ├── img │ │ ├── cispa.png │ │ ├── cmu.png │ │ ├── elsa.png │ │ ├── ethz.png │ │ └── google-deepmind.png │ └── style.css └── template.html ├── prestart.sh ├── pyproject.toml ├── requirements.txt ├── static ├── example_attack.py ├── example_defense.py ├── img │ ├── cispa.png │ ├── cmu.png │ ├── elsa.png │ ├── ethz.png │ └── google-deepmind.png ├── rules.pdf ├── style.css └── table_style.css ├── templates ├── index.html └── leaderboard.html ├── tests ├── __init__.py ├── basic_api_test.py ├── test_validate_code.py └── utility_eval_test.py ├── traefik.dev.toml └── traefik.prod.toml /.env.example: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/.env.example -------------------------------------------------------------------------------- /.github/CONTRIBUTING.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/.github/CONTRIBUTING.md -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/.gitignore -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/.pre-commit-config.yaml -------------------------------------------------------------------------------- /.secrets.example/dev/github_client_secret: -------------------------------------------------------------------------------- 1 | TODO: get from https://github.com/settings/applications/new -------------------------------------------------------------------------------- /.secrets.example/dev/google_client_secret: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/.secrets.example/dev/google_client_secret -------------------------------------------------------------------------------- /.secrets.example/dev/mongo_express_admin_password: -------------------------------------------------------------------------------- 1 | TODO generate with `openssl rand -hex 32` -------------------------------------------------------------------------------- /.secrets.example/dev/mongodb_root_password: -------------------------------------------------------------------------------- 1 | TODO generate with `openssl rand -hex 32` -------------------------------------------------------------------------------- /.secrets.example/dev/redis_password: -------------------------------------------------------------------------------- 1 | TODO generate with `openssl rand -hex 32` -------------------------------------------------------------------------------- /.secrets.example/dev/secret_key: -------------------------------------------------------------------------------- 1 | TODO generate with `openssl rand -hex 32` -------------------------------------------------------------------------------- /.secrets.example/openai_api_key: -------------------------------------------------------------------------------- 1 | sk-TODO get from OpenAI website -------------------------------------------------------------------------------- /.secrets.example/prod/github_client_secret: -------------------------------------------------------------------------------- 1 | TODO: get from https://github.com/settings/applications/new -------------------------------------------------------------------------------- /.secrets.example/prod/google_client_secret: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/.secrets.example/prod/google_client_secret -------------------------------------------------------------------------------- /.secrets.example/prod/mongo_express_admin_password: -------------------------------------------------------------------------------- 1 | TODO generate with `openssl rand -hex 32` -------------------------------------------------------------------------------- /.secrets.example/prod/mongodb_root_password: -------------------------------------------------------------------------------- 1 | TODO generate with `openssl rand -hex 32` -------------------------------------------------------------------------------- /.secrets.example/prod/redis_password: -------------------------------------------------------------------------------- 1 | TODO generate with `openssl rand -hex 32` -------------------------------------------------------------------------------- /.secrets.example/prod/secret_key: -------------------------------------------------------------------------------- 1 | TODO generate with `openssl rand -hex 32` -------------------------------------------------------------------------------- /.secrets.example/together_api_key: -------------------------------------------------------------------------------- 1 | TODO get from Together AI website -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/Dockerfile -------------------------------------------------------------------------------- /Dockerfile.prod: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/Dockerfile.prod -------------------------------------------------------------------------------- /Dockerfile.traefik: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/Dockerfile.traefik -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/README.md -------------------------------------------------------------------------------- /admin_scripts/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /admin_scripts/admin_client.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/admin_scripts/admin_client.py -------------------------------------------------------------------------------- /admin_scripts/create_teams.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/admin_scripts/create_teams.py -------------------------------------------------------------------------------- /admin_scripts/increase_budget.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/admin_scripts/increase_budget.py -------------------------------------------------------------------------------- /app/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/api/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/api/api_v1/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/api/api_v1/__init__.py -------------------------------------------------------------------------------- /app/api/api_v1/api.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/api/api_v1/api.py -------------------------------------------------------------------------------- /app/api/api_v1/endpoints/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/api/api_v1/endpoints/budget.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/api/api_v1/endpoints/budget.py -------------------------------------------------------------------------------- /app/api/api_v1/endpoints/chat.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/api/api_v1/endpoints/chat.py -------------------------------------------------------------------------------- /app/api/api_v1/endpoints/defense.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/api/api_v1/endpoints/defense.py -------------------------------------------------------------------------------- /app/api/api_v1/endpoints/key.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/api/api_v1/endpoints/key.py -------------------------------------------------------------------------------- /app/api/api_v1/endpoints/models.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/api/api_v1/endpoints/models.py -------------------------------------------------------------------------------- /app/api/api_v1/endpoints/oauth2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/api/api_v1/endpoints/oauth2.py -------------------------------------------------------------------------------- /app/api/api_v1/endpoints/scores.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/api/api_v1/endpoints/scores.py -------------------------------------------------------------------------------- /app/api/api_v1/endpoints/secret.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/api/api_v1/endpoints/secret.py -------------------------------------------------------------------------------- /app/api/api_v1/endpoints/submission.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/api/api_v1/endpoints/submission.py -------------------------------------------------------------------------------- /app/api/api_v1/endpoints/teams.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/api/api_v1/endpoints/teams.py -------------------------------------------------------------------------------- /app/api/api_v1/endpoints/users.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/api/api_v1/endpoints/users.py -------------------------------------------------------------------------------- /app/api/api_v1/endpoints/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/api/api_v1/endpoints/utils.py -------------------------------------------------------------------------------- /app/api/deps.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/api/deps.py -------------------------------------------------------------------------------- /app/api/documentation_text.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/api/documentation_text.py -------------------------------------------------------------------------------- /app/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/config.py -------------------------------------------------------------------------------- /app/crud/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/crud/__init__.py -------------------------------------------------------------------------------- /app/crud/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/crud/base.py -------------------------------------------------------------------------------- /app/crud/crud_api_key.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/crud/crud_api_key.py -------------------------------------------------------------------------------- /app/crud/crud_chat.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/crud/crud_chat.py -------------------------------------------------------------------------------- /app/crud/crud_defense.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/crud/crud_defense.py -------------------------------------------------------------------------------- /app/crud/crud_secret.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/crud/crud_secret.py -------------------------------------------------------------------------------- /app/crud/crud_secret_guess.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/crud/crud_secret_guess.py -------------------------------------------------------------------------------- /app/crud/crud_team.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/crud/crud_team.py -------------------------------------------------------------------------------- /app/crud/crud_team_budget.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/crud/crud_team_budget.py -------------------------------------------------------------------------------- /app/crud/crud_user.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/crud/crud_user.py -------------------------------------------------------------------------------- /app/enums.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/enums.py -------------------------------------------------------------------------------- /app/frontend.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/frontend.py -------------------------------------------------------------------------------- /app/gradio/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/gradio/__init__.py -------------------------------------------------------------------------------- /app/gradio/attack.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/gradio/attack.py -------------------------------------------------------------------------------- /app/gradio/defense.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/gradio/defense.py -------------------------------------------------------------------------------- /app/internals/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/internals/base_acc_20.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/internals/base_acc_20.json -------------------------------------------------------------------------------- /app/internals/code_exec.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/internals/code_exec.py -------------------------------------------------------------------------------- /app/internals/defenses_11.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/internals/defenses_11.json -------------------------------------------------------------------------------- /app/internals/llm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/internals/llm.py -------------------------------------------------------------------------------- /app/internals/openai_accs_20.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/internals/openai_accs_20.json -------------------------------------------------------------------------------- /app/internals/output_filter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/internals/output_filter.py -------------------------------------------------------------------------------- /app/internals/utility_eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/internals/utility_eval.py -------------------------------------------------------------------------------- /app/limits.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/limits.py -------------------------------------------------------------------------------- /app/main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/main.py -------------------------------------------------------------------------------- /app/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/models/__init__.py -------------------------------------------------------------------------------- /app/models/api_key.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/models/api_key.py -------------------------------------------------------------------------------- /app/models/chat.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/models/chat.py -------------------------------------------------------------------------------- /app/models/defense.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/models/defense.py -------------------------------------------------------------------------------- /app/models/secret.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/models/secret.py -------------------------------------------------------------------------------- /app/models/team.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/models/team.py -------------------------------------------------------------------------------- /app/models/team_budget.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/models/team_budget.py -------------------------------------------------------------------------------- /app/models/user.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/models/user.py -------------------------------------------------------------------------------- /app/schemas/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/schemas/__init__.py -------------------------------------------------------------------------------- /app/schemas/api_key.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/schemas/api_key.py -------------------------------------------------------------------------------- /app/schemas/chat.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/schemas/chat.py -------------------------------------------------------------------------------- /app/schemas/defense.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/schemas/defense.py -------------------------------------------------------------------------------- /app/schemas/generate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/schemas/generate.py -------------------------------------------------------------------------------- /app/schemas/message.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/schemas/message.py -------------------------------------------------------------------------------- /app/schemas/scores.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/schemas/scores.py -------------------------------------------------------------------------------- /app/schemas/secret.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/schemas/secret.py -------------------------------------------------------------------------------- /app/schemas/team_budget.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/schemas/team_budget.py -------------------------------------------------------------------------------- /app/schemas/token.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/schemas/token.py -------------------------------------------------------------------------------- /app/schemas/user.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/schemas/user.py -------------------------------------------------------------------------------- /app/security.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/app/security.py -------------------------------------------------------------------------------- /compose.prod.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/compose.prod.yml -------------------------------------------------------------------------------- /compose.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/compose.yml -------------------------------------------------------------------------------- /data/download_arb.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/download_arb.py -------------------------------------------------------------------------------- /data/final_scores.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/final_scores.json -------------------------------------------------------------------------------- /data/pick_mmlu.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/pick_mmlu.py -------------------------------------------------------------------------------- /data/small_data/abstract_algebra_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/abstract_algebra_test.20.csv -------------------------------------------------------------------------------- /data/small_data/anatomy_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/anatomy_test.20.csv -------------------------------------------------------------------------------- /data/small_data/arb_law_0.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/arb_law_0.20.csv -------------------------------------------------------------------------------- /data/small_data/arb_law_1.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/arb_law_1.20.csv -------------------------------------------------------------------------------- /data/small_data/arb_law_2.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/arb_law_2.20.csv -------------------------------------------------------------------------------- /data/small_data/arb_law_3.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/arb_law_3.20.csv -------------------------------------------------------------------------------- /data/small_data/arb_law_4.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/arb_law_4.20.csv -------------------------------------------------------------------------------- /data/small_data/arb_law_5.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/arb_law_5.20.csv -------------------------------------------------------------------------------- /data/small_data/arb_law_6.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/arb_law_6.20.csv -------------------------------------------------------------------------------- /data/small_data/arb_law_7.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/arb_law_7.20.csv -------------------------------------------------------------------------------- /data/small_data/arb_law_8.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/arb_law_8.20.csv -------------------------------------------------------------------------------- /data/small_data/arb_law_9.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/arb_law_9.20.csv -------------------------------------------------------------------------------- /data/small_data/astronomy_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/astronomy_test.20.csv -------------------------------------------------------------------------------- /data/small_data/business_ethics_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/business_ethics_test.20.csv -------------------------------------------------------------------------------- /data/small_data/clinical_knowledge_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/clinical_knowledge_test.20.csv -------------------------------------------------------------------------------- /data/small_data/college_computer_science_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/college_computer_science_test.20.csv -------------------------------------------------------------------------------- /data/small_data/college_mathematics_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/college_mathematics_test.20.csv -------------------------------------------------------------------------------- /data/small_data/college_medicine_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/college_medicine_test.20.csv -------------------------------------------------------------------------------- /data/small_data/college_physics_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/college_physics_test.20.csv -------------------------------------------------------------------------------- /data/small_data/computer_security_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/computer_security_test.20.csv -------------------------------------------------------------------------------- /data/small_data/conceptual_physics_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/conceptual_physics_test.20.csv -------------------------------------------------------------------------------- /data/small_data/conceptual_physics_test.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/conceptual_physics_test.csv -------------------------------------------------------------------------------- /data/small_data/electrical_engineering_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/electrical_engineering_test.20.csv -------------------------------------------------------------------------------- /data/small_data/elementary_mathematics_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/elementary_mathematics_test.20.csv -------------------------------------------------------------------------------- /data/small_data/global_facts_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/global_facts_test.20.csv -------------------------------------------------------------------------------- /data/small_data/high_school_biology_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/high_school_biology_test.20.csv -------------------------------------------------------------------------------- /data/small_data/high_school_chemistry_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/high_school_chemistry_test.20.csv -------------------------------------------------------------------------------- /data/small_data/high_school_computer_science_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/high_school_computer_science_test.20.csv -------------------------------------------------------------------------------- /data/small_data/high_school_european_history_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/high_school_european_history_test.20.csv -------------------------------------------------------------------------------- /data/small_data/high_school_geography_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/high_school_geography_test.20.csv -------------------------------------------------------------------------------- /data/small_data/high_school_government_and_politics_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/high_school_government_and_politics_test.20.csv -------------------------------------------------------------------------------- /data/small_data/high_school_macroeconomics_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/high_school_macroeconomics_test.20.csv -------------------------------------------------------------------------------- /data/small_data/high_school_mathematics_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/high_school_mathematics_test.20.csv -------------------------------------------------------------------------------- /data/small_data/high_school_microeconomics_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/high_school_microeconomics_test.20.csv -------------------------------------------------------------------------------- /data/small_data/high_school_physics_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/high_school_physics_test.20.csv -------------------------------------------------------------------------------- /data/small_data/high_school_psychology_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/high_school_psychology_test.20.csv -------------------------------------------------------------------------------- /data/small_data/high_school_statistics_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/high_school_statistics_test.20.csv -------------------------------------------------------------------------------- /data/small_data/high_school_us_history_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/high_school_us_history_test.20.csv -------------------------------------------------------------------------------- /data/small_data/high_school_world_history_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/high_school_world_history_test.20.csv -------------------------------------------------------------------------------- /data/small_data/human_aging_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/human_aging_test.20.csv -------------------------------------------------------------------------------- /data/small_data/international_law_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/international_law_test.20.csv -------------------------------------------------------------------------------- /data/small_data/jurisprudence_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/jurisprudence_test.20.csv -------------------------------------------------------------------------------- /data/small_data/logical_fallacies_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/logical_fallacies_test.20.csv -------------------------------------------------------------------------------- /data/small_data/machine_learning_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/machine_learning_test.20.csv -------------------------------------------------------------------------------- /data/small_data/management_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/management_test.20.csv -------------------------------------------------------------------------------- /data/small_data/marketing_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/marketing_test.20.csv -------------------------------------------------------------------------------- /data/small_data/medical_genetics_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/medical_genetics_test.20.csv -------------------------------------------------------------------------------- /data/small_data/miscellaneous_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/miscellaneous_test.20.csv -------------------------------------------------------------------------------- /data/small_data/moral_disputes_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/moral_disputes_test.20.csv -------------------------------------------------------------------------------- /data/small_data/nutrition_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/nutrition_test.20.csv -------------------------------------------------------------------------------- /data/small_data/philosophy_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/philosophy_test.20.csv -------------------------------------------------------------------------------- /data/small_data/prehistory_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/prehistory_test.20.csv -------------------------------------------------------------------------------- /data/small_data/professional_medicine_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/professional_medicine_test.20.csv -------------------------------------------------------------------------------- /data/small_data/professional_psychology_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/professional_psychology_test.20.csv -------------------------------------------------------------------------------- /data/small_data/public_relations_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/public_relations_test.20.csv -------------------------------------------------------------------------------- /data/small_data/security_studies_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/security_studies_test.20.csv -------------------------------------------------------------------------------- /data/small_data/sociology_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/sociology_test.20.csv -------------------------------------------------------------------------------- /data/small_data/us_foreign_policy_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/us_foreign_policy_test.20.csv -------------------------------------------------------------------------------- /data/small_data/virology_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/virology_test.20.csv -------------------------------------------------------------------------------- /data/small_data/world_religions_test.20.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/data/small_data/world_religions_test.20.csv -------------------------------------------------------------------------------- /frontpage/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/frontpage/README.md -------------------------------------------------------------------------------- /frontpage/build.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/frontpage/build.sh -------------------------------------------------------------------------------- /frontpage/content/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/frontpage/content/index.md -------------------------------------------------------------------------------- /frontpage/copy_to_app.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/frontpage/copy_to_app.sh -------------------------------------------------------------------------------- /frontpage/serve.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | npx http-server public 3 | -------------------------------------------------------------------------------- /frontpage/static/img/cispa.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/frontpage/static/img/cispa.png -------------------------------------------------------------------------------- /frontpage/static/img/cmu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/frontpage/static/img/cmu.png -------------------------------------------------------------------------------- /frontpage/static/img/elsa.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/frontpage/static/img/elsa.png -------------------------------------------------------------------------------- /frontpage/static/img/ethz.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/frontpage/static/img/ethz.png -------------------------------------------------------------------------------- /frontpage/static/img/google-deepmind.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/frontpage/static/img/google-deepmind.png -------------------------------------------------------------------------------- /frontpage/static/style.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/frontpage/static/style.css -------------------------------------------------------------------------------- /frontpage/template.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/frontpage/template.html -------------------------------------------------------------------------------- /prestart.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/prestart.sh -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/pyproject.toml -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/requirements.txt -------------------------------------------------------------------------------- /static/example_attack.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/static/example_attack.py -------------------------------------------------------------------------------- /static/example_defense.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/static/example_defense.py -------------------------------------------------------------------------------- /static/img/cispa.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/static/img/cispa.png -------------------------------------------------------------------------------- /static/img/cmu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/static/img/cmu.png -------------------------------------------------------------------------------- /static/img/elsa.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/static/img/elsa.png -------------------------------------------------------------------------------- /static/img/ethz.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/static/img/ethz.png -------------------------------------------------------------------------------- /static/img/google-deepmind.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/static/img/google-deepmind.png -------------------------------------------------------------------------------- /static/rules.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/static/rules.pdf -------------------------------------------------------------------------------- /static/style.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/static/style.css -------------------------------------------------------------------------------- /static/table_style.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/static/table_style.css -------------------------------------------------------------------------------- /templates/index.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/templates/index.html -------------------------------------------------------------------------------- /templates/leaderboard.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/templates/leaderboard.html -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/basic_api_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/tests/basic_api_test.py -------------------------------------------------------------------------------- /tests/test_validate_code.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/tests/test_validate_code.py -------------------------------------------------------------------------------- /tests/utility_eval_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/tests/utility_eval_test.py -------------------------------------------------------------------------------- /traefik.dev.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/traefik.dev.toml -------------------------------------------------------------------------------- /traefik.prod.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ethz-spylab/satml-llm-ctf/HEAD/traefik.prod.toml --------------------------------------------------------------------------------