├── .dockerignore ├── .env.template ├── .github └── workflows │ ├── admin-deploy.yml │ ├── admin-worker-deploy.yml │ ├── api-deploy.yml │ ├── docker-deploy.yml │ ├── minecraft-builder-deploy.yml │ ├── pr-build.yml │ ├── production-deploy.yml │ ├── render-worker-deploy.yml │ ├── scheduler-deploy.yml │ ├── server-worker-deploy.yml │ └── worker-deploy.yml ├── .gitignore ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.md ├── bin ├── grant-user-role └── manual-postgres-setup ├── deploy ├── admin-worker-deploy.sh ├── docker-cleanup.sh ├── render-worker-deploy.sh ├── scheduler-deploy.sh ├── server-worker-deploy.sh └── worker-deploy.sh ├── deps ├── admin-worker-requirements.in ├── admin-worker-requirements.txt ├── api-requirements.in ├── api-requirements.txt ├── dev-requirements.in ├── dev-requirements.txt ├── known-constraints.in ├── render-worker-requirements.in ├── render-worker-requirements.txt ├── requirements.in ├── requirements.txt ├── server-worker-requirements.in ├── server-worker-requirements.txt ├── worker-requirements.in └── worker-requirements.txt ├── dev └── seed-data.sql ├── docker-compose.yml ├── docs ├── build-local-gameservers.md ├── getting-started │ ├── docker-guide.md │ ├── python-guide.md │ └── role-grant.md └── technical-details │ ├── architecture.md │ └── biome-tinting.md ├── images ├── admin-api.Dockerfile ├── admin-worker.Dockerfile ├── api.Dockerfile ├── builder-runner │ ├── builder-runner.Dockerfile │ ├── command.sh │ ├── package-lock.json │ └── package.json ├── deps-builder.Dockerfile ├── render-worker.Dockerfile ├── server-worker.Dockerfile └── worker.Dockerfile ├── notebooks └── .gitkeep ├── pyproject.toml ├── src └── mc_bench │ ├── __init__.py │ ├── apps │ ├── __init__.py │ ├── admin_api │ │ ├── __init__.py │ │ ├── __main__.py │ │ ├── app.py │ │ ├── celery.py │ │ ├── config.py │ │ ├── lifespan.py │ │ ├── routers │ │ │ ├── __init__.py │ │ │ ├── auth.py │ │ │ ├── generations.py │ │ │ ├── infra.py │ │ │ ├── models.py │ │ │ ├── prompts.py │ │ │ ├── runs.py │ │ │ ├── samples.py │ │ │ ├── tags.py │ │ │ ├── templates.py │ │ │ └── users.py │ │ └── transport_types │ │ │ ├── __init__.py │ │ │ ├── generic.py │ │ │ ├── requests.py │ │ │ └── responses.py │ ├── admin_worker │ │ ├── __init__.py │ │ ├── app.py │ │ ├── config.py │ │ ├── js_scripts │ │ │ ├── __init__.py │ │ │ └── mockScript.js │ │ ├── script-eslintrc.js │ │ └── tasks │ │ │ ├── __init__.py │ │ │ ├── example_task.py │ │ │ ├── generation.py │ │ │ └── run.py │ ├── api │ │ ├── __init__.py │ │ ├── __main__.py │ │ ├── app.py │ │ ├── celery.py │ │ ├── config.py │ │ ├── lifespan.py │ │ ├── prepared_statements.py │ │ ├── routers │ │ │ ├── __init__.py │ │ │ ├── comparison.py │ │ │ └── user.py │ │ └── transport_types │ │ │ ├── generic.py │ │ │ ├── requests.py │ │ │ └── responses.py │ ├── render_worker │ │ ├── __init__.py │ │ ├── app.py │ │ ├── config.py │ │ └── tasks │ │ │ ├── __init__.py │ │ │ └── run.py │ ├── scheduler │ │ ├── __init__.py │ │ ├── __main__.py │ │ ├── config.py │ │ ├── loop.py │ │ └── manager.py │ ├── server_worker │ │ ├── __init__.py │ │ ├── app.py │ │ ├── config.py │ │ ├── js_scripts │ │ │ ├── __init__.py │ │ │ ├── build.js │ │ │ └── export.js │ │ ├── tasks │ │ │ ├── __init__.py │ │ │ └── run.py │ │ └── templates.py │ └── worker │ │ ├── __init__.py │ │ ├── app.py │ │ ├── config.py │ │ └── tasks │ │ ├── __init__.py │ │ └── elo_calculation.py │ ├── auth │ ├── __init__.py │ ├── clients │ │ ├── __init__.py │ │ ├── _base.py │ │ ├── _github.py │ │ └── _google.py │ ├── emails.py │ └── permissions.py │ ├── clients │ ├── __init__.py │ ├── alibaba_cloud.py │ ├── anthropic.py │ ├── deepseek.py │ ├── gemini.py │ ├── grok.py │ ├── mcbench_admin_api.py │ ├── mistral.py │ ├── openai.py │ ├── openrouter.py │ ├── reka.py │ └── zhipuai.py │ ├── config │ └── __init__.py │ ├── constants.py │ ├── events │ ├── __init__.py │ └── types │ │ ├── __init__.py │ │ ├── _base.py │ │ └── event_types.py │ ├── migrations │ ├── README │ ├── __init__.py │ ├── __main__.py │ ├── env.py │ ├── script.py.mako │ └── versions │ │ ├── 0076eb8acb15_add_external_id_to_artifact.py │ │ ├── 00f736acf066_add_prompt_model_and_template_join_.py │ │ ├── 019b9b3a6b6c_add_test_set_id_to_scoring_comparison.py │ │ ├── 0a61b7e99cdc_add_experimental_state_and_permissions_.py │ │ ├── 0d6ed46a9eb3_add_deepseek_model_and_provider.py │ │ ├── 0df207ed67f9_add_artifact_kinds.py │ │ ├── 1035a263c03c_add_role_external_id.py │ │ ├── 163167df595b_support_render_stage.py │ │ ├── 188ed0c14a15_distribute_approved_samples_to_test_sets.py │ │ ├── 1b9505818e11_made_comparison_sample_id_non_nullable.py │ │ ├── 1c4a87ae15fd_add_specification_generation_table_and_.py │ │ ├── 1ddb45880af9_initial_role_creation.py │ │ ├── 1eeb63dd99cc_add_scheduler_control_table.py │ │ ├── 1eef5895e9f2_add_code_validation_stage.py │ │ ├── 1f5438c5d056_add_active_field_to_prompt_table.py │ │ ├── 2066aab9985b_add_sample_reviewer_role.py │ │ ├── 213494c85c66_add_comparison_rank_table.py │ │ ├── 2bd521bcef5d_add_status_columns.py │ │ ├── 2c198d5ab7b5_add_all_tables.py │ │ ├── 2dd0e918c173_add_log_bridge_tables.py │ │ ├── 3491851af8f5_set_model_name_to_not_nullable.py │ │ ├── 34cd8ee7f32c_add_user_identification_token_table.py │ │ ├── 36407e97cfac_add_model_id_to_prompt_leaderboard.py │ │ ├── 38dedff6cd15_fix_permissions.py │ │ ├── 3bc7893bf74c_add_model_generation_and_run_permissions.py │ │ ├── 3cb7f6ca1320_add_sorting_strategy_config.py │ │ ├── 4085c38e19e8_add_provider_class_rows.py │ │ ├── 41cc0a28bcde_add_voting_permission_and_role.py │ │ ├── 473407e9d86e_add_calculate_score_to_tag.py │ │ ├── 47fc21ce40a3_add_leaderboard_tables.py │ │ ├── 4ce0397c4cf7_add_basic_roles_and_permissions_for_.py │ │ ├── 4d220a984a72_add_metrics.py │ │ ├── 4e797f372395_add_role_permission_table.py │ │ ├── 4f05d418156c_add_permissions_for_models.py │ │ ├── 503daea838dc_add_artifact_kinds.py │ │ ├── 514a2c809b29_add_specification_tag_and_specification_.py │ │ ├── 5ca7be2d7150_add_new_artifact_kinds_for_comparison_.py │ │ ├── 5f1a3b5bd548_add_scoped_researcher_roles.py │ │ ├── 5fdfa056d511_remove_sample_columns_from_comparison.py │ │ ├── 64cfe697cc9c_add_indexes_for_comparison_queries.py │ │ ├── 665d4b9d70a6_add_run_stage_modeling.py │ │ ├── 688482827ae4_change_openai_and_gemini_provider_class.py │ │ ├── 6c98791fa0b4_add_task_id_and_heartbeat_to_run_stage.py │ │ ├── 6d48eca377ff_add_proper_roles_for_writing_to_sample_.py │ │ ├── 6e4874c96cbc_added_external_id_to_provider_class.py │ │ ├── 6ef6c3fb5778_added_a_comparison_sample_id_to_sample_.py │ │ ├── 70abece80ae7_fix_api_db_permissions.py │ │ ├── 7242824efbe7_fix_updated_last_modified.py │ │ ├── 7a050a3306c3_add_session_and_identification_to_.py │ │ ├── 7a30bccaa129_add_an_admin_viewer_role.py │ │ ├── 7d62170745c6_fix_user_table.py │ │ ├── 7fc9dbcb65bd_fix_service_permissions_for_leaderboard.py │ │ ├── 848301f47914_add_run_states.py │ │ ├── 8863c7c9ffd4_add_default_test_set_id_to_generation.py │ │ ├── 8877773aa6b6_added_proper_index_to_sample_sample.py │ │ ├── 8b6615c73a4f_add_sample_and_voting_admin_permissions.py │ │ ├── 8f1d29465102_add_test_set_and_update_sample_with_.py │ │ ├── 8f3ea520a73e_add_sample_id_column_to_sample_artifact.py │ │ ├── 8fff682fdaa3_add_new_log_action.py │ │ ├── 9037d32770e9_fix_permissions_for_new_tables.py │ │ ├── 91d6a6a71bf0_add_sample_tracking_logs_and_notes.py │ │ ├── 938175add4e2_add_prompt_leaderboard_table.py │ │ ├── 93bbc1a50a86_add_run_stage_indexes_for_scheduler_.py │ │ ├── 95ab1bc41af5_insert_github_as_auth_provider.py │ │ ├── 9b7eed150ea2_make_user_id_nullable_in_comparison.py │ │ ├── 9d12f9191c09_added_a_active_field_to_sample_sample.py │ │ ├── 9f9e88722da4_add_permissions_for_prompts.py │ │ ├── __init__.py │ │ ├── a0225a967e80_added_is_default_to_provider_table.py │ │ ├── a0bb11b95fff_add_default_experimental_state_to_.py │ │ ├── a4bee97b3630_ensure_auth_user_role_has_proper_unique_.py │ │ ├── a4cbcbdc4a91_add_make_artifact_external_id_not_.py │ │ ├── ab2b7d5586e0_added_a_raw_column_and_updated_result_.py │ │ ├── ab2c29f9cc0c_insert_google_as_auth_provider.py │ │ ├── ac88d1e18c73_add_build_size_to_prompt.py │ │ ├── ae55800708f6_add_grok_model_and_provider.py │ │ ├── b097774ba21f_change_model_and_provider_and_provider_.py │ │ ├── b118428bde0b_add_processed_comparison_table.py │ │ ├── b241f88ec24a_add_external_id_to_experimental_state_.py │ │ ├── b3bfae61b787_add_user_admin_role.py │ │ ├── bcaf6ca2021c_add_artifact_kinds.py │ │ ├── c26cc0577ee4_add_log_actions_not_kinds_active_states.py │ │ ├── c7757bc234ac_make_sample_ids_nullable_in_comparison.py │ │ ├── d0abd80553ec_fix_user_table.py │ │ ├── d618a24f0bed_add_generation_state_and_run_state_.py │ │ ├── d75b6128c146_initial_schema_creation.py │ │ ├── d9dbc81fd67d_add_normalized_username_and_display_.py │ │ ├── db79a9c13551_add_new_providers.py │ │ ├── dbedc0bcab8f_added_name_to_provider_table.py │ │ ├── dfa4bbbc994d_make_template_minecraft_version_not_.py │ │ ├── e1181f674ab9_add_additional_run_state_states.py │ │ ├── e69015b27346_fix_admin_api_permissions.py │ │ ├── eb378aad6fd8_add_enqueued_run_stage_state.py │ │ ├── f0186a991fa2_initial_extension_setup.py │ │ ├── f17051270c7b_add_artifact_kinds.py │ │ ├── f1b7837b000d_add_minecraft_version_to_template.py │ │ ├── f37ddbc4738c_fix_research_note_deleted_and_research_.py │ │ ├── f5fda8b15a78_add_run_stage_stage_slug_for_.py │ │ ├── f67f7ce9da43_implement_cascade_delete_on_provider_.py │ │ ├── fd623475cd7c_added_a_comparison_correlation_id_based_.py │ │ └── ffb71aaa0034_added_active_field_to_model_table.py │ ├── minecraft │ ├── __init__.py │ ├── biome_lookup.py │ ├── rendering │ │ ├── __init__.py │ │ └── cache.py │ ├── resources │ │ └── __init__.py │ ├── schematic.py │ ├── server │ │ ├── __init__.py │ │ └── __main__.py │ └── testing.py │ ├── models │ ├── __init__.py │ ├── _base.py │ ├── comparison.py │ ├── experimental_state.py │ ├── log.py │ ├── model.py │ ├── prompt.py │ ├── provider │ │ ├── README.md │ │ ├── __init__.py │ │ ├── _base.py │ │ ├── alibaba.py │ │ ├── anthropic.py │ │ ├── deepseek.py │ │ ├── gemini.py │ │ ├── grok.py │ │ ├── mistral.py │ │ ├── openai.py │ │ ├── openrouter.py │ │ ├── reka.py │ │ └── zhipuai.py │ ├── run.py │ ├── scheduler_control.py │ ├── template.py │ └── user.py │ ├── schema │ ├── __init__.py │ ├── object_store │ │ ├── __init__.py │ │ └── runs.py │ └── postgres │ │ ├── __init__.py │ │ ├── _metadata.py │ │ ├── auth │ │ ├── __init__.py │ │ ├── _auth_provider.py │ │ ├── _auth_provider_email_hash.py │ │ ├── _permission.py │ │ ├── _role.py │ │ ├── _role_permission.py │ │ ├── _user.py │ │ ├── _user_identification_token.py │ │ └── _user_role.py │ │ ├── research │ │ ├── __init__.py │ │ ├── _experimental_state.py │ │ ├── _log.py │ │ ├── _log_action.py │ │ ├── _model_experimental_state_proposal.py │ │ ├── _model_log.py │ │ ├── _note.py │ │ ├── _note_kind.py │ │ ├── _prompt_experimental_state_proposal.py │ │ ├── _prompt_log.py │ │ ├── _sample_log.py │ │ ├── _template_experimental_state_proposal.py │ │ └── _template_log.py │ │ ├── sample │ │ ├── __init__.py │ │ ├── _artifact.py │ │ ├── _artifact_kind.py │ │ ├── _sample.py │ │ └── _test_set.py │ │ ├── scoring │ │ ├── __init__.py │ │ ├── _comparison.py │ │ ├── _comparison_rank.py │ │ ├── _metric.py │ │ ├── _model_leaderboard.py │ │ ├── _processed_comparison.py │ │ ├── _prompt_leaderboard.py │ │ ├── _sample_approval_state.py │ │ └── _sample_leaderboard.py │ │ └── specification │ │ ├── __init__.py │ │ ├── _generation.py │ │ ├── _generation_state.py │ │ ├── _model.py │ │ ├── _prompt.py │ │ ├── _prompt_tag.py │ │ ├── _provider.py │ │ ├── _provider_class.py │ │ ├── _run.py │ │ ├── _run_stage.py │ │ ├── _run_stage_state.py │ │ ├── _run_state.py │ │ ├── _scheduler_control.py │ │ ├── _stage.py │ │ ├── _tag.py │ │ └── _template.py │ ├── server │ ├── __init__.py │ └── auth.py │ ├── util │ ├── __init__.py │ ├── cache.py │ ├── celery.py │ ├── docker.py │ ├── elo.py │ ├── logging.py │ ├── object_store.py │ ├── postgres.py │ ├── redis.py │ ├── text.py │ └── uuid.py │ └── worker │ ├── __init__.py │ └── run_stage.py └── tests ├── test_elo.py ├── test_export.py └── test_text.py /.env.template: -------------------------------------------------------------------------------- 1 | export POSTGRES_HOST="localhost" 2 | export POSTGRES_DB="mc-bench" 3 | export POSTGRES_USER="mc-bench-admin" 4 | export POSTGRES_PASSWORD="mc-bench" 5 | export POSTGRES_SSLMODE="disable" 6 | export POSTGRES_PORT=5432 7 | 8 | export GITHUB_CLIENT_ID="{ GITHUB OAUTH CLIENT ID }" 9 | export GITHUB_CLIENT_SECRET="{ GITHUB OAUTH CLIENT SECRET }" 10 | 11 | # This is insecure and these secrets should only be used in local development setting 12 | export SECRET_KEY="0wWkP79giyTBM3qclIaFwBFoMfNRo8pmrLmiDz_bYN8" 13 | export GITHUB_EMAIL_SALT="iH7UU7YbypaFYqB2UslAv0LkR2lsEqNevrvwQbJ8bU8" 14 | 15 | # TODO: Update to local tags 16 | export MINECRAFT_SERVER_IMAGE='registry.digitalocean.com/mcbench/minecraft-server:built' 17 | export MINECRAFT_BUILDER_IMAGE='registry.digitalocean.com/mcbench/minecraft-builder:built' 18 | 19 | export ACCESS_TOKEN_EXPIRE_MINUTES=3 20 | export REFRESH_TOKEN_EXPIRE_MINUTES=60 21 | 22 | # API Keys for LLM Services 23 | export ANTHROPIC_API_KEY="your-anthropic-key" 24 | export OPENAI_API_KEY="your-openai-key" 25 | export GOOGLE_API_KEY="your-google-key" 26 | export OPENROUTER_API_KEY="your-openrouter-key" 27 | export XAI_API_KEY="your-xai-key" 28 | export DEEPSEEK_API_KEY="your-deepseek-key" 29 | export FAST_RENDER="true" 30 | export HUMANIZE_LOGS="true" 31 | export BUILD_DELAY_MS="75" 32 | 33 | # Logging configuration 34 | export LOG_LEVEL="INFO" # DEBUG, INFO, WARNING, ERROR, CRITICAL 35 | 36 | # Log interval settings (how frequently to log at INFO level) 37 | export LOG_INTERVAL_BLOCKS="100" # Log every N blocks placed 38 | export LOG_INTERVAL_MATERIALS="10" # Log every N materials baked 39 | export LOG_INTERVAL_COMMANDS="50" # Log every N build commands 40 | export LOG_INTERVAL_EXPORT_PERCENT="10" # Log every N percent of export progress 41 | -------------------------------------------------------------------------------- /.github/workflows/admin-deploy.yml: -------------------------------------------------------------------------------- 1 | name: Admin API Deploy 2 | 3 | on: 4 | workflow_dispatch: 5 | inputs: 6 | commit_hash: 7 | description: 'Commit hash to deploy to production' 8 | required: true 9 | type: string 10 | 11 | env: 12 | REGISTRY: "registry.digitalocean.com" 13 | DOCKER_LOGIN_USERNAME: huntcsg@gmail.com 14 | 15 | jobs: 16 | verify-commit: 17 | runs-on: ubuntu-latest 18 | steps: 19 | - name: Checkout code 20 | uses: actions/checkout@v3 21 | with: 22 | fetch-depth: 0 23 | 24 | - name: Verify commit is on main 25 | run: | 26 | if ! git branch --contains ${{ github.event.inputs.commit_hash }} | grep -q "main"; then 27 | echo "Error: Commit ${{ github.event.inputs.commit_hash }} is not on the main branch" 28 | exit 1 29 | fi 30 | 31 | tag-admin-api-production: 32 | needs: verify-commit 33 | runs-on: ubuntu-latest 34 | environment: production 35 | steps: 36 | - name: Set up Docker Buildx 37 | uses: docker/setup-buildx-action@v3 38 | 39 | - name: Install doctl 40 | uses: digitalocean/action-doctl@v2 41 | with: 42 | token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }} 43 | 44 | - name: Log in to DO Container Registry 45 | run: doctl registry login --expiry-seconds 1200 46 | 47 | - name: Tag Admin API image as production 48 | run: | 49 | echo "Tagging admin-api as production..." 50 | docker buildx imagetools create \ 51 | ${{ env.REGISTRY }}/mcbench/admin-api:${{ github.event.inputs.commit_hash }} \ 52 | --tag ${{ env.REGISTRY }}/mcbench/admin-api:production -------------------------------------------------------------------------------- /.github/workflows/api-deploy.yml: -------------------------------------------------------------------------------- 1 | name: API Deploy 2 | 3 | on: 4 | workflow_dispatch: 5 | inputs: 6 | commit_hash: 7 | description: 'Commit hash to deploy to production' 8 | required: true 9 | type: string 10 | 11 | env: 12 | REGISTRY: "registry.digitalocean.com" 13 | DOCKER_LOGIN_USERNAME: huntcsg@gmail.com 14 | 15 | jobs: 16 | verify-commit: 17 | runs-on: ubuntu-latest 18 | steps: 19 | - name: Checkout code 20 | uses: actions/checkout@v3 21 | with: 22 | fetch-depth: 0 23 | 24 | - name: Verify commit is on main 25 | run: | 26 | if ! git branch --contains ${{ github.event.inputs.commit_hash }} | grep -q "main"; then 27 | echo "Error: Commit ${{ github.event.inputs.commit_hash }} is not on the main branch" 28 | exit 1 29 | fi 30 | 31 | tag-api-production: 32 | needs: verify-commit 33 | runs-on: ubuntu-latest 34 | environment: production 35 | steps: 36 | - name: Set up Docker Buildx 37 | uses: docker/setup-buildx-action@v3 38 | 39 | - name: Install doctl 40 | uses: digitalocean/action-doctl@v2 41 | with: 42 | token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }} 43 | 44 | - name: Log in to DO Container Registry 45 | run: doctl registry login --expiry-seconds 1200 46 | 47 | - name: Tag API image as production 48 | run: | 49 | echo "Tagging api as production..." 50 | docker buildx imagetools create \ 51 | ${{ env.REGISTRY }}/mcbench/api:${{ github.event.inputs.commit_hash }} \ 52 | --tag ${{ env.REGISTRY }}/mcbench/api:production -------------------------------------------------------------------------------- /.github/workflows/minecraft-builder-deploy.yml: -------------------------------------------------------------------------------- 1 | name: Minecraft Builder Deploy 2 | 3 | on: 4 | workflow_dispatch: 5 | inputs: 6 | commit_hash: 7 | description: 'Commit hash to deploy to production' 8 | required: true 9 | type: string 10 | 11 | env: 12 | REGISTRY: "registry.digitalocean.com" 13 | DOCKER_LOGIN_USERNAME: huntcsg@gmail.com 14 | 15 | jobs: 16 | verify-commit: 17 | runs-on: ubuntu-latest 18 | steps: 19 | - name: Checkout code 20 | uses: actions/checkout@v3 21 | with: 22 | fetch-depth: 0 23 | 24 | - name: Verify commit is on main 25 | run: | 26 | if ! git branch --contains ${{ github.event.inputs.commit_hash }} | grep -q "main"; then 27 | echo "Error: Commit ${{ github.event.inputs.commit_hash }} is not on the main branch" 28 | exit 1 29 | fi 30 | 31 | tag-minecraft-builder-production: 32 | needs: verify-commit 33 | runs-on: ubuntu-latest 34 | environment: production 35 | steps: 36 | - name: Set up Docker Buildx 37 | uses: docker/setup-buildx-action@v3 38 | 39 | - name: Install doctl 40 | uses: digitalocean/action-doctl@v2 41 | with: 42 | token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }} 43 | 44 | - name: Log in to DO Container Registry 45 | run: doctl registry login --expiry-seconds 1200 46 | 47 | - name: Tag Minecraft Builder image as production 48 | run: | 49 | echo "Tagging minecraft-builder as production..." 50 | docker buildx imagetools create \ 51 | ${{ env.REGISTRY }}/mcbench/minecraft-builder:${{ github.event.inputs.commit_hash }} \ 52 | --tag ${{ env.REGISTRY }}/mcbench/minecraft-builder:production -------------------------------------------------------------------------------- /.github/workflows/pr-build.yml: -------------------------------------------------------------------------------- 1 | name: PR Build 2 | 3 | on: 4 | pull_request: 5 | branches: [ "main" ] 6 | 7 | env: 8 | REGISTRY: "registry.digitalocean.com" 9 | DOCKER_LOGIN_USERNAME: huntcsg@gmail.com 10 | 11 | jobs: 12 | build-images: 13 | runs-on: ubuntu-latest 14 | strategy: 15 | matrix: 16 | image: 17 | - name: server-worker 18 | path: server-worker.Dockerfile 19 | - name: render-worker 20 | path: render-worker.Dockerfile 21 | - name: minecraft-builder 22 | path: builder-runner.Dockerfile 23 | context: images/builder-runner 24 | - name: admin-api 25 | path: admin-api.Dockerfile 26 | - name: admin-worker 27 | path: admin-worker.Dockerfile 28 | - name: api 29 | path: api.Dockerfile 30 | - name: worker 31 | path: worker.Dockerfile 32 | steps: 33 | - name: Checkout code 34 | uses: actions/checkout@v3 35 | 36 | - name: Set up Docker Buildx 37 | uses: docker/setup-buildx-action@v3 38 | 39 | - name: Install doctl 40 | uses: digitalocean/action-doctl@v2 41 | with: 42 | token: ${{ secrets.DIGITALOCEAN_ACCESS_TOKEN }} 43 | 44 | - name: Log in to DO Container Registry 45 | run: doctl registry login --expiry-seconds 1200 46 | 47 | - name: Build image 48 | uses: docker/build-push-action@v5 49 | with: 50 | context: ${{ matrix.image.context || '.' }} 51 | file: ${{ matrix.image.context || 'images' }}/${{ matrix.image.path }} 52 | push: false 53 | cache-from: type=registry,ref=${{ env.REGISTRY }}/mcbench/${{ matrix.image.name }}:buildcache 54 | cache-to: type=registry,ref=${{ env.REGISTRY }}/mcbench/${{ matrix.image.name }}:buildcache,mode=max 55 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Unless otherwise indicated, all code herein is licensed for use under the following license: 2 | 3 | MIT License 4 | 5 | Copyright (c) 2024 mcbench contributors 6 | 7 | Permission is hereby granted, free of charge, to any person obtaining a copy 8 | of this software and associated documentation files (the "Software"), to deal 9 | in the Software without restriction, including without limitation the rights 10 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | copies of the Software, and to permit persons to whom the Software is 12 | furnished to do so, subject to the following conditions: 13 | 14 | The above copyright notice and this permission notice shall be included in all 15 | copies or substantial portions of the Software. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 | SOFTWARE. 24 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include src/mc_bench/ *.js 2 | recursive-include src/mc_bench/ *.json 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MC Bench Backend 2 | 3 | The Future of AI Evaluation 4 | MC-Bench changes how we evaluate AI models by challenging them to create Minecraft builds. 5 | 6 | # Development Setup 7 | 8 | [Docker Quick Start](docs/getting-started/docker-guide.md) (recommended) 9 | 10 | [Python Quick Start](docs/getting-started/python-guide.md) 11 | 12 | 13 | # Architecture 14 | [Architecture Guide](docs/technical-details/architecture.md) 15 | 16 | # Formatting/Linting 17 | 18 | Run the ruff formatter 19 | ```shell 20 | make fmt 21 | ```` 22 | 23 | Run the ruff checker 24 | ```shell 25 | make check 26 | ``` 27 | 28 | Run the ruff checker with --fix option 29 | ```shell 30 | make check-fix 31 | ``` 32 | -------------------------------------------------------------------------------- /deploy/docker-cleanup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Remove all unused containers, networks, images and volumes 4 | docker system prune -f 5 | 6 | # Remove all images (that aren't currently in use) 7 | docker images -q | xargs -r docker rmi -------------------------------------------------------------------------------- /deps/admin-worker-requirements.in: -------------------------------------------------------------------------------- 1 | celery[redis]>=5.4.0 2 | psycopg2-binary>=2.9.10 3 | sqlalchemy>=2.0.36 4 | alembic>=1.14.0 5 | anthropic 6 | openai 7 | mistralai 8 | reka-api 9 | jinja2 10 | flower 11 | requests 12 | mc-data-files==3.83.1rc0 13 | python-jose[cryptography] 14 | 15 | -------------------------------------------------------------------------------- /deps/api-requirements.in: -------------------------------------------------------------------------------- 1 | Authlib 2 | itsdangerous 3 | python-jose[cryptography] 4 | requests 5 | fastapi[standard]>=0.115.3 6 | pyhumps 7 | jinja2 8 | scalar-fastapi 9 | valx 10 | regex 11 | -------------------------------------------------------------------------------- /deps/dev-requirements.in: -------------------------------------------------------------------------------- 1 | ruff==0.7.1 2 | pip-tools==7.4.1 3 | pytest==8.3.3 4 | docker 5 | jupyter -------------------------------------------------------------------------------- /deps/known-constraints.in: -------------------------------------------------------------------------------- 1 | numpy<2.0 2 | -------------------------------------------------------------------------------- /deps/render-worker-requirements.in: -------------------------------------------------------------------------------- 1 | celery[redis]>=5.4.0 2 | psycopg2-binary>=2.9.10 3 | sqlalchemy>=2.0.36 4 | alembic>=1.14.0 5 | jinja2 6 | flower 7 | requests 8 | bpy 9 | mathutils 10 | Pillow 11 | nbt 12 | mc-data-files==3.83.1rc0 13 | mc-assets==1.13.0rc0 14 | structlog 15 | -------------------------------------------------------------------------------- /deps/requirements.in: -------------------------------------------------------------------------------- 1 | celery[redis]>=5.4.0 2 | psycopg2-binary>=2.9.10 3 | sqlalchemy>=2.0.36 4 | alembic>=1.14.0 5 | jinja2 6 | minio 7 | structlog 8 | pydantic 9 | backoff 10 | -------------------------------------------------------------------------------- /deps/server-worker-requirements.in: -------------------------------------------------------------------------------- 1 | javascript 2 | docker 3 | celery[redis]>=5.4.0 4 | psycopg2-binary>=2.9.10 5 | sqlalchemy>=2.0.36 6 | requests 7 | mc-data-files==3.83.1rc0 8 | -------------------------------------------------------------------------------- /deps/worker-requirements.in: -------------------------------------------------------------------------------- 1 | celery[redis]>=5.4.0 2 | psycopg2-binary>=2.9.10 3 | sqlalchemy>=2.0.36 4 | requests 5 | -------------------------------------------------------------------------------- /docs/build-local-gameservers.md: -------------------------------------------------------------------------------- 1 | # Build local gameservers 2 | 3 | The Minecraft benchmark runs ephemeral minecraft game servers. For licensing reasons we cannot ship or make these publicably available. 4 | 5 | You can however build your own locally. 6 | The easiest way to do this is to checkout [gameservers](https://github.com/mc-bench/gameservers). 7 | 8 | This repository contains Dockerfiles for building various versions of the minecraft server. 9 | 10 | Each version is on a different branch. 11 | 12 | Current versions (branches) in use: 13 | - [Java Minecraft 1.21.1](https://github.com/mc-bench/gameservers/tree/minecraft-1.21.1) 14 | 15 | 16 | # Build the server (Java Minecraft 1.21.1) 17 | 18 | ## 0. Path requirements 19 | The application expects that the game server code lives the a folder next to the `mc-bench-backend`, like the following: 20 | 21 | ``` 22 | ├── gameservers 23 | ├── mc-bench-backend 24 | └── mc-bench-frontend 25 | ``` 26 | ## 1. Navigate up folder structure 27 | 28 | ```bash 29 | cd .. 30 | ``` 31 | 32 | ## 2. Clone Repo 33 | ```bash 34 | git clone git@github.com:mc-bench/gameservers.git 35 | ``` 36 | 37 | ## 3. Checkout correct branch 38 | 39 | ```bash 40 | git checkout minecraft-1.21.1 41 | ``` 42 | 43 | ## 4. Build and tag the docker image 44 | ```bash 45 | docker build -t registry.digitalocean.com/mcbench/gameservers:minecraft-1.21.1-latest . 46 | ``` 47 | 48 | 49 | That's it! You can now run ephemeral game servers locally. 50 | -------------------------------------------------------------------------------- /docs/getting-started/role-grant.md: -------------------------------------------------------------------------------- 1 | # Grant Roles 2 | 3 | Running the frontend and the backend, signup and create a username. 4 | 5 | Then use: 6 | 7 | ```python 8 | ./bin/grant-user-role grant --username {your username} --role admin 9 | ``` 10 | To see the usernames use: 11 | 12 | ```shell 13 | 14 | ./bin/grant-user-role list-users 15 | ``` 16 | 17 | To see the roles use: 18 | 19 | ```shell 20 | 21 | ./bin/grant-user-role list-roles 22 | ``` 23 | -------------------------------------------------------------------------------- /images/admin-api.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.12.7 2 | 3 | COPY deps/requirements.txt requirements.txt 4 | COPY deps/api-requirements.txt api-requirements.txt 5 | RUN pip install -r requirements.txt -r api-requirements.txt 6 | 7 | COPY . /usr/lib/mc-bench-backend 8 | RUN pip install /usr/lib/mc-bench-backend[api] 9 | 10 | CMD ["uvicorn", "mc_bench.apps.admin_api.__main__:app", "--proxy-headers", "--port", "8000", "--host", "0.0.0.0"] 11 | -------------------------------------------------------------------------------- /images/admin-worker.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM mcbenchmark/minecraft-builder-base:2024-12-11 2 | 3 | RUN npm install -g eslint 4 | 5 | COPY deps/requirements.txt requirements.txt 6 | COPY deps/admin-worker-requirements.txt admin-worker-requirements.txt 7 | RUN pip install -r requirements.txt -r admin-worker-requirements.txt 8 | 9 | COPY . /usr/lib/mc-bench-backend 10 | RUN pip install /usr/lib/mc-bench-backend[admin-worker] 11 | 12 | ENV NUM_WORKERS=4 13 | ENTRYPOINT [] 14 | CMD exec celery -A mc_bench.apps.admin_worker worker -Q admin,generation,prompt,parse,validate,post_process,prepare --concurrency $NUM_WORKERS -n $WORKER_NAME 15 | -------------------------------------------------------------------------------- /images/api.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.12.7 2 | 3 | COPY deps/requirements.txt requirements.txt 4 | COPY deps/api-requirements.txt api-requirements.txt 5 | RUN pip install -r requirements.txt -r api-requirements.txt 6 | 7 | COPY . /usr/lib/mc-bench-backend 8 | RUN pip install /usr/lib/mc-bench-backend[api] 9 | 10 | CMD ["uvicorn", "mc_bench.apps.api.__main__:app", "--proxy-headers", "--port", "8000", "--host", "0.0.0.0"] 11 | -------------------------------------------------------------------------------- /images/builder-runner/builder-runner.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM mcbenchmark/minecraft-builder-base:2024-12-11 2 | 3 | RUN apt-get update && apt-get install -y procps && rm -rf /var/lib/apt/lists/* 4 | 5 | WORKDIR /app 6 | 7 | COPY package.json package-lock.json ./ 8 | 9 | RUN npm install 10 | 11 | COPY command.sh ./ 12 | 13 | RUN chmod +x command.sh 14 | 15 | ENTRYPOINT ["./command.sh"] 16 | -------------------------------------------------------------------------------- /images/builder-runner/command.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | set -e 3 | 4 | cp /build-scripts/build-script.js build-script.js 5 | 6 | xvfb-run -a node build-script.js 7 | exit_status=$? 8 | pkill Xvfb || true 9 | exit $exit_status -------------------------------------------------------------------------------- /images/builder-runner/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "builder-runner", 3 | "dependencies": { 4 | "buffer": "^6.0.3", 5 | "canvas": "^2.11.2", 6 | "gl": "^8.1.6", 7 | "minecraft-data": "^3.83.1", 8 | "mineflayer": "^4.25.0", 9 | "node-canvas-webgl": "^0.3.0", 10 | "prettier": "^3.4.1", 11 | "prismarine-chunk": "^1.36.0", 12 | "prismarine-viewer": "^1.33.0", 13 | "prismarine-world": "^3.6.3", 14 | "three": "^0.171.0", 15 | "vec3": "^0.1.10" 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /images/deps-builder.Dockerfile: -------------------------------------------------------------------------------- 1 | ARG PYTHON_VERSION 2 | FROM python:${PYTHON_VERSION}-slim 3 | 4 | RUN pip install pip-tools 5 | -------------------------------------------------------------------------------- /images/render-worker.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.11.7 2 | 3 | RUN apt-get update && apt-get install -y blender 4 | 5 | COPY deps/requirements.txt requirements.txt 6 | COPY deps/render-worker-requirements.txt render-worker-requirements.txt 7 | RUN pip install -r requirements.txt -r render-worker-requirements.txt 8 | 9 | COPY . /usr/lib/mc-bench-backend 10 | RUN pip install /usr/lib/mc-bench-backend[render-worker] 11 | 12 | ENV NUM_WORKERS=1 13 | 14 | ENTRYPOINT [] 15 | CMD exec celery -A mc_bench.apps.render_worker worker -Q render --concurrency $NUM_WORKERS -n $WORKER_NAME 16 | -------------------------------------------------------------------------------- /images/server-worker.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.12.7 2 | 3 | COPY deps/requirements.txt requirements.txt 4 | COPY deps/server-worker-requirements.txt server-worker-requirements.txt 5 | RUN pip install -r requirements.txt -r server-worker-requirements.txt 6 | 7 | COPY . /usr/lib/mc-bench-backend 8 | RUN pip install /usr/lib/mc-bench-backend[server-worker] 9 | 10 | ENV NUM_WORKERS=1 11 | 12 | ENTRYPOINT [] 13 | CMD exec celery -A mc_bench.apps.server_worker worker -Q server --concurrency $NUM_WORKERS -n $WORKER_NAME 14 | -------------------------------------------------------------------------------- /images/worker.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.12.7 2 | 3 | COPY deps/requirements.txt requirements.txt 4 | COPY deps/worker-requirements.txt worker-requirements.txt 5 | RUN pip install -r requirements.txt -r worker-requirements.txt 6 | 7 | COPY . /usr/lib/mc-bench-backend 8 | RUN pip install /usr/lib/mc-bench-backend[worker] 9 | 10 | ENV NUM_WORKERS=1 11 | 12 | ENTRYPOINT [] 13 | CMD exec celery -A mc_bench.apps.worker worker -Q default --concurrency $NUM_WORKERS -n $WORKER_NAME 14 | -------------------------------------------------------------------------------- /notebooks/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mc-bench/mc-bench-backend/360838f5692597b2bbe1c2622e7ac60c13e1871b/notebooks/.gitkeep -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "mc-bench" 3 | version = "0.1.0" 4 | description = "Add your description here" 5 | readme = "README.md" 6 | license = {file = "LICENSE"} 7 | authors = [ 8 | { name = "hunter", email = "huntcsg@gmail.com" } 9 | ] 10 | requires-python = ">=3.10" 11 | dynamic = ["dependencies", "optional-dependencies"] 12 | 13 | [project.scripts] 14 | mc-bench-alembic = "mc_bench.migrations.__main__:main" 15 | 16 | [build-system] 17 | requires = ["setuptools>=61"] 18 | build-backend = "setuptools.build_meta" 19 | 20 | [tool.setuptools.dynamic] 21 | dependencies = { file = "deps/requirements.in" } 22 | 23 | [tool.setuptools.dynamic.optional-dependencies] 24 | dev = {file = "deps/dev-requirements.in"} 25 | api = {file = "deps/api-requirements.in"} 26 | worker = {file = "deps/worker-requirements.in"} 27 | admin-worker = {file = "deps/admin-worker-requirements.in"} 28 | server-worker = {file = "deps/server-worker-requirements.in"} 29 | render-worker = {file = "deps/render-worker-requirements.in"} 30 | 31 | [tool.setuptools.packages.find] 32 | where = ["src"] 33 | 34 | [tool.ruff.lint.pyflakes] 35 | allowed-unused-imports = [ 36 | "mc_bench.schema.postgres.specification" 37 | ] 38 | 39 | [tool.ruff.lint] 40 | ignore = ["E712", "E711"] 41 | -------------------------------------------------------------------------------- /src/mc_bench/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mc-bench/mc-bench-backend/360838f5692597b2bbe1c2622e7ac60c13e1871b/src/mc_bench/__init__.py -------------------------------------------------------------------------------- /src/mc_bench/apps/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mc-bench/mc-bench-backend/360838f5692597b2bbe1c2622e7ac60c13e1871b/src/mc_bench/apps/__init__.py -------------------------------------------------------------------------------- /src/mc_bench/apps/admin_api/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mc-bench/mc-bench-backend/360838f5692597b2bbe1c2622e7ac60c13e1871b/src/mc_bench/apps/admin_api/__init__.py -------------------------------------------------------------------------------- /src/mc_bench/apps/admin_api/__main__.py: -------------------------------------------------------------------------------- 1 | from .app import app 2 | 3 | __all__ = ["app"] 4 | -------------------------------------------------------------------------------- /src/mc_bench/apps/admin_api/celery.py: -------------------------------------------------------------------------------- 1 | from mc_bench.util.celery import make_client_celery_app 2 | 3 | celery = make_client_celery_app() 4 | 5 | 6 | def send_task(name, *args, **kwargs): 7 | kwargs.setdefault("queue", "admin") 8 | return celery.send_task(name, *args, **kwargs) 9 | 10 | 11 | def create_runs( 12 | generation_id, prompt_ids, model_ids, template_ids, num_samples, progress_token 13 | ): 14 | return send_task( 15 | "generation.create_runs", 16 | queue="generation", 17 | kwargs=dict( 18 | generation_id=generation_id, 19 | prompt_ids=prompt_ids, 20 | model_ids=model_ids, 21 | template_ids=template_ids, 22 | num_samples=num_samples, 23 | ), 24 | headers={"token": progress_token}, 25 | ) 26 | -------------------------------------------------------------------------------- /src/mc_bench/apps/admin_api/config.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | 4 | 5 | class Settings: 6 | JWT_SECRET_KEY = os.environ["SECRET_KEY"] 7 | ALGORITHM = "HS256" 8 | ACCESS_TOKEN_EXPIRE_MINUTES = 60 * 24 * 7 9 | HUMANIZE_LOGS = os.environ.get("HUMANIZE_LOGS", "false") == "true" 10 | LOG_LEVEL_STR = os.environ.get("LOG_LEVEL", "INFO") 11 | LOG_LEVEL = getattr(logging, LOG_LEVEL_STR.upper(), logging.INFO) 12 | 13 | 14 | settings = Settings() 15 | -------------------------------------------------------------------------------- /src/mc_bench/apps/admin_api/lifespan.py: -------------------------------------------------------------------------------- 1 | from contextlib import asynccontextmanager 2 | 3 | from mc_bench.events import on_event 4 | from mc_bench.events.types import ( 5 | GenerationStateChanged, 6 | RunStageStateChanged, 7 | RunStateChanged, 8 | ) 9 | from mc_bench.models.run import Generation, Run, RunStage 10 | from mc_bench.util.postgres import get_session 11 | 12 | 13 | @asynccontextmanager 14 | async def lifespan(app): 15 | session = get_session() 16 | engine = session.bind 17 | 18 | on_event(RunStageStateChanged, RunStage.state_change_handler) 19 | on_event(RunStateChanged, Run.state_change_handler) 20 | on_event(GenerationStateChanged, Generation.state_change_handler) 21 | 22 | yield 23 | 24 | # Close all connections and dispose of the engine 25 | engine.dispose() 26 | -------------------------------------------------------------------------------- /src/mc_bench/apps/admin_api/routers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mc-bench/mc-bench-backend/360838f5692597b2bbe1c2622e7ac60c13e1871b/src/mc_bench/apps/admin_api/routers/__init__.py -------------------------------------------------------------------------------- /src/mc_bench/apps/admin_api/routers/auth.py: -------------------------------------------------------------------------------- 1 | from fastapi import Depends 2 | from fastapi.routing import APIRouter 3 | from sqlalchemy import select 4 | from sqlalchemy.orm import Session 5 | 6 | from mc_bench.apps.admin_api.config import settings 7 | from mc_bench.apps.admin_api.transport_types.generic import ListResponse 8 | from mc_bench.apps.admin_api.transport_types.responses import ( 9 | RoleResponse, 10 | ) 11 | from mc_bench.auth.permissions import PERM 12 | from mc_bench.models.user import Role 13 | from mc_bench.server.auth import AuthManager 14 | from mc_bench.util.postgres import get_managed_session 15 | 16 | auth_router = APIRouter() 17 | 18 | am = AuthManager( 19 | jwt_secret=settings.JWT_SECRET_KEY, 20 | jwt_algorithm=settings.ALGORITHM, 21 | ) 22 | 23 | 24 | @auth_router.get( 25 | "/api/auth/role", 26 | dependencies=[ 27 | Depends(am.require_any_scopes([PERM.USER.ADMIN])), 28 | ], 29 | response_model=ListResponse[RoleResponse], 30 | ) 31 | def get_roles( 32 | db: Session = Depends(get_managed_session), 33 | ): 34 | roles = list(db.scalars(select(Role))) 35 | payload = { 36 | "data": [role.to_dict() for role in roles], 37 | "total": len(roles), 38 | } 39 | 40 | return payload 41 | -------------------------------------------------------------------------------- /src/mc_bench/apps/admin_api/routers/tags.py: -------------------------------------------------------------------------------- 1 | from fastapi import Depends 2 | from fastapi.routing import APIRouter 3 | from sqlalchemy import select 4 | from sqlalchemy.orm import Session 5 | 6 | from mc_bench.apps.admin_api.config import settings 7 | from mc_bench.apps.admin_api.transport_types.generic import ListResponse 8 | from mc_bench.apps.admin_api.transport_types.responses import ( 9 | TagResponse, 10 | ) 11 | from mc_bench.auth.permissions import PERM 12 | from mc_bench.models.prompt import Tag 13 | from mc_bench.server.auth import AuthManager 14 | from mc_bench.util.postgres import get_managed_session 15 | 16 | tag_router = APIRouter() 17 | 18 | am = AuthManager( 19 | jwt_secret=settings.JWT_SECRET_KEY, 20 | jwt_algorithm=settings.ALGORITHM, 21 | ) 22 | 23 | 24 | @tag_router.get( 25 | "/api/tag", 26 | dependencies=[ 27 | Depends( 28 | am.require_any_scopes( 29 | [PERM.PROMPT.ADMIN, PERM.PROMPT.READ, PERM.PROMPT.WRITE] 30 | ) 31 | ), 32 | ], 33 | response_model=ListResponse[TagResponse], 34 | ) 35 | def get_tags( 36 | db: Session = Depends(get_managed_session), 37 | ): 38 | tags = list(db.scalars(select(Tag).order_by(Tag.created.desc()))) 39 | payload = { 40 | "data": [tag.to_dict() for tag in tags], 41 | "total": len(tags), 42 | } 43 | 44 | return payload 45 | -------------------------------------------------------------------------------- /src/mc_bench/apps/admin_api/transport_types/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mc-bench/mc-bench-backend/360838f5692597b2bbe1c2622e7ac60c13e1871b/src/mc_bench/apps/admin_api/transport_types/__init__.py -------------------------------------------------------------------------------- /src/mc_bench/apps/admin_api/transport_types/generic.py: -------------------------------------------------------------------------------- 1 | from typing import Generic, List, TypeVar 2 | 3 | import humps 4 | from pydantic import BaseModel, ConfigDict 5 | 6 | T = TypeVar("T") 7 | 8 | 9 | class Base(BaseModel): 10 | model_config = ConfigDict( 11 | from_attributes=True, # Enables ORM model parsing 12 | alias_generator=humps.camelize, # Converts snake_case to camelCase 13 | populate_by_name=True, # Allows accessing fields by snake_case name 14 | arbitrary_types_allowed=True, 15 | protected_namespaces=(), 16 | ) 17 | 18 | 19 | class ListResponse(Base, Generic[T]): 20 | data: List[T] 21 | total: int 22 | -------------------------------------------------------------------------------- /src/mc_bench/apps/admin_worker/__init__.py: -------------------------------------------------------------------------------- 1 | from . import tasks 2 | from .app import app 3 | 4 | __all__ = [ 5 | "app", 6 | "tasks", 7 | ] 8 | -------------------------------------------------------------------------------- /src/mc_bench/apps/admin_worker/app.py: -------------------------------------------------------------------------------- 1 | from mc_bench.events import on_event 2 | from mc_bench.events.types import ( 3 | GenerationStateChanged, 4 | RunStageStateChanged, 5 | RunStateChanged, 6 | ) 7 | from mc_bench.models.run import Generation, Run, RunStage 8 | from mc_bench.util.celery import make_worker_celery_app 9 | from mc_bench.util.logging import configure_logging 10 | 11 | from .config import settings 12 | 13 | configure_logging(humanize=settings.HUMANIZE_LOGS, level=settings.LOG_LEVEL) 14 | 15 | app = make_worker_celery_app( 16 | { 17 | "worker_max_tasks_per_child": 16, 18 | } 19 | ) 20 | 21 | # Event handler registration 22 | on_event(RunStageStateChanged, RunStage.state_change_handler) 23 | on_event(RunStateChanged, Run.state_change_handler) 24 | on_event(GenerationStateChanged, Generation.state_change_handler) 25 | -------------------------------------------------------------------------------- /src/mc_bench/apps/admin_worker/config.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | 4 | 5 | class Settings: 6 | INTERNAL_OBJECT_BUCKET = os.environ["INTERNAL_OBJECT_BUCKET"] 7 | EXTERNAL_OBJECT_BUCKET = os.environ["EXTERNAL_OBJECT_BUCKET"] 8 | HUMANIZE_LOGS = os.environ.get("HUMANIZE_LOGS", "false") == "true" 9 | LOG_LEVEL_STR = os.environ.get("LOG_LEVEL", "INFO") 10 | LOG_LEVEL = getattr(logging, LOG_LEVEL_STR.upper(), logging.INFO) 11 | 12 | 13 | settings = Settings() 14 | -------------------------------------------------------------------------------- /src/mc_bench/apps/admin_worker/js_scripts/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mc-bench/mc-bench-backend/360838f5692597b2bbe1c2622e7ac60c13e1871b/src/mc_bench/apps/admin_worker/js_scripts/__init__.py -------------------------------------------------------------------------------- /src/mc_bench/apps/admin_worker/script-eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | files: ["**/*.js"], 3 | languageOptions: { 4 | ecmaVersion: "latest", 5 | sourceType: "module" 6 | }, 7 | linterOptions: { 8 | reportUnusedDisableDirectives: true, 9 | }, 10 | rules: { 11 | // Catch syntax errors and undefined variables 12 | "no-undef": "error", // Disallow undeclared variables 13 | "no-unreachable": "error", // Disallow unreachable code 14 | "no-constant-condition": "error", // Disallow constant conditions in loops/if 15 | "no-dupe-args": "error", // Disallow duplicate function parameters 16 | "no-dupe-keys": "error", // Disallow duplicate object keys 17 | "no-dupe-else-if": "error", // Disallow duplicate conditions in if-else chains 18 | "no-func-assign": "error", // Disallow reassigning function declarations 19 | "no-import-assign": "error", // Disallow assigning to imported bindings 20 | "no-obj-calls": "error", // Disallow calling global objects as functions 21 | "no-sparse-arrays": "error", // Disallow sparse arrays with empty slots 22 | "no-unexpected-multiline": "error", // Disallow confusing multiline expressions 23 | "use-isnan": "error", // Require using isNaN() to check for NaN 24 | "valid-typeof": "error" // Enforce comparing typeof with valid strings 25 | } 26 | }; 27 | -------------------------------------------------------------------------------- /src/mc_bench/apps/admin_worker/tasks/__init__.py: -------------------------------------------------------------------------------- 1 | from . import example_task, generation, run 2 | 3 | __all__ = [ 4 | "example_task", 5 | "generation", 6 | "run", 7 | ] 8 | -------------------------------------------------------------------------------- /src/mc_bench/apps/admin_worker/tasks/example_task.py: -------------------------------------------------------------------------------- 1 | from ..app import app 2 | 3 | 4 | @app.task(name="admin_example_task") 5 | def example_task(): 6 | return {"ok": True} 7 | -------------------------------------------------------------------------------- /src/mc_bench/apps/api/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mc-bench/mc-bench-backend/360838f5692597b2bbe1c2622e7ac60c13e1871b/src/mc_bench/apps/api/__init__.py -------------------------------------------------------------------------------- /src/mc_bench/apps/api/__main__.py: -------------------------------------------------------------------------------- 1 | from .app import app 2 | 3 | __all__ = ["app"] 4 | -------------------------------------------------------------------------------- /src/mc_bench/apps/api/app.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from fastapi import FastAPI 4 | from fastapi.middleware.cors import CORSMiddleware 5 | from scalar_fastapi import get_scalar_api_reference 6 | 7 | from mc_bench.apps.api.routers.comparison import comparison_router 8 | from mc_bench.apps.api.routers.user import user_router 9 | from mc_bench.util.logging import configure_logging 10 | 11 | from .config import settings 12 | from .lifespan import lifespan 13 | 14 | configure_logging(humanize=settings.HUMANIZE_LOGS, level=settings.LOG_LEVEL) 15 | 16 | allow_origins = os.environ.get("CORS_ALLOWED_ORIGIN", "").split(",") 17 | 18 | 19 | app = FastAPI(lifespan=lifespan) 20 | 21 | app.add_middleware( 22 | CORSMiddleware, 23 | allow_credentials=True, 24 | allow_origins=allow_origins, 25 | allow_methods=["*"], 26 | allow_headers=["*"], 27 | expose_headers=["X-MCBench-Session", "X-MCBench-Identification"], 28 | ) 29 | 30 | app.include_router(user_router) 31 | app.include_router(comparison_router) 32 | 33 | 34 | @app.get("/scalar", include_in_schema=False) 35 | async def scalar_docs(): 36 | return get_scalar_api_reference( 37 | openapi_url="/openapi.json", 38 | title="MC Bench API", 39 | ) 40 | -------------------------------------------------------------------------------- /src/mc_bench/apps/api/celery.py: -------------------------------------------------------------------------------- 1 | from mc_bench.util.celery import make_client_celery_app 2 | 3 | celery = make_client_celery_app() 4 | 5 | 6 | def send_task(name, *args, **kwargs): 7 | kwargs.setdefault("queue", "default") 8 | return celery.send_task(name, *args, **kwargs) 9 | -------------------------------------------------------------------------------- /src/mc_bench/apps/api/config.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | 4 | 5 | class Settings: 6 | GITHUB_CLIENT_ID = os.environ["GITHUB_CLIENT_ID"] 7 | GITHUB_CLIENT_SECRET = os.environ["GITHUB_CLIENT_SECRET"] 8 | JWT_SECRET_KEY = os.environ["SECRET_KEY"] 9 | GITHUB_EMAIL_SALT = os.environ["GITHUB_EMAIL_SALT"] 10 | EMAIL_SALT = os.environ["GITHUB_EMAIL_SALT"] 11 | ALGORITHM = "HS256" 12 | ACCESS_TOKEN_EXPIRE_MINUTES = int(os.environ.get("ACCESS_TOKEN_EXPIRE_MINUTES", 60 * 24 * 7)) 13 | REFRESH_TOKEN_EXPIRE_MINUTES = int( 14 | os.environ.get("REFRESH_TOKEN_EXPIRE_MINUTES", 60 * 24 * 7) 15 | ) 16 | AUTO_GRANT_ADMIN_ROLE = os.environ.get("AUTO_GRANT_ADMIN_ROLE", "false") == "true" 17 | 18 | EXTERNAL_OBJECT_BUCKET = os.environ.get("EXTERNAL_OBJECT_BUCKET") 19 | INTERNAL_OBJECT_BUCKET = os.environ.get("INTERNAL_OBJECT_BUCKET") 20 | 21 | # New settings for Google OAuth 22 | GOOGLE_CLIENT_ID = os.environ["GOOGLE_CLIENT_ID"] 23 | GOOGLE_CLIENT_SECRET = os.environ["GOOGLE_CLIENT_SECRET"] 24 | GOOGLE_REDIRECT_URI = os.environ["GOOGLE_REDIRECT_URI"] 25 | HUMANIZE_LOGS = os.environ.get("HUMANIZE_LOGS", "false") == "true" 26 | LOG_LEVEL_STR = os.environ.get("LOG_LEVEL", "INFO") 27 | LOG_LEVEL = getattr(logging, LOG_LEVEL_STR.upper(), logging.INFO) 28 | 29 | 30 | settings = Settings() 31 | -------------------------------------------------------------------------------- /src/mc_bench/apps/api/lifespan.py: -------------------------------------------------------------------------------- 1 | from contextlib import asynccontextmanager 2 | 3 | import sqlalchemy 4 | 5 | from mc_bench.auth.clients import GithubOauthClient, GoogleOauthClient 6 | from mc_bench.models.user import AuthProvider 7 | from mc_bench.util.postgres import get_session 8 | from mc_bench.util.redis import RedisDatabase, get_redis_pool 9 | 10 | from .config import settings 11 | from .prepared_statements import COMPARISON_BATCH_QUERY 12 | 13 | github_oauth_client = GithubOauthClient( 14 | client_id=settings.GITHUB_CLIENT_ID, 15 | client_secret=settings.GITHUB_CLIENT_SECRET, 16 | salt=settings.GITHUB_EMAIL_SALT, 17 | ) 18 | 19 | google_oauth_client = GoogleOauthClient( 20 | client_id=settings.GOOGLE_CLIENT_ID, 21 | client_secret=settings.GOOGLE_CLIENT_SECRET, 22 | redirect_uri=settings.GOOGLE_REDIRECT_URI, 23 | ) 24 | 25 | 26 | @asynccontextmanager 27 | async def lifespan(app): 28 | session = get_session() 29 | engine = session.bind 30 | 31 | # Fill cache with pool 32 | redis_pool = get_redis_pool(RedisDatabase.COMPARISON) 33 | 34 | AuthProvider.register_client_factory("github", lambda: github_oauth_client) 35 | AuthProvider.register_client_factory("google", lambda: google_oauth_client) 36 | 37 | @sqlalchemy.event.listens_for(engine, "connect") 38 | def prepare_statements(dbapi_connection, connection_record): 39 | """Prepare statements when a new connection is created in the pool""" 40 | cursor = dbapi_connection.cursor() 41 | try: 42 | cursor.execute( 43 | f"PREPARE comparison_batch_query(integer, integer) AS {COMPARISON_BATCH_QUERY}" 44 | ) 45 | dbapi_connection.commit() 46 | finally: 47 | cursor.close() 48 | 49 | yield 50 | 51 | engine.dispose() 52 | redis_pool.close() 53 | -------------------------------------------------------------------------------- /src/mc_bench/apps/api/routers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mc-bench/mc-bench-backend/360838f5692597b2bbe1c2622e7ac60c13e1871b/src/mc_bench/apps/api/routers/__init__.py -------------------------------------------------------------------------------- /src/mc_bench/apps/api/transport_types/generic.py: -------------------------------------------------------------------------------- 1 | from typing import Generic, List, TypeVar 2 | 3 | import humps 4 | from pydantic import BaseModel, ConfigDict 5 | 6 | T = TypeVar("T") 7 | 8 | 9 | class Base(BaseModel): 10 | model_config = ConfigDict( 11 | from_attributes=True, # Enables ORM model parsing 12 | alias_generator=humps.camelize, # Converts snake_case to camelCase 13 | populate_by_name=True, # Allows accessing fields by snake_case name 14 | arbitrary_types_allowed=True, 15 | protected_namespaces=(), 16 | ) 17 | 18 | 19 | class ListResponse(Base, Generic[T]): 20 | data: List[T] 21 | total: int 22 | -------------------------------------------------------------------------------- /src/mc_bench/apps/api/transport_types/requests.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | from typing import Dict, List, Optional 3 | 4 | from .generic import Base 5 | 6 | 7 | class NewComparisonBatchRequest(Base): 8 | batch_size: int = 3 9 | metric_id: uuid.UUID 10 | files: Optional[List[str]] = ("gltf_scene",) 11 | 12 | 13 | class ComparisonDetailRequest(Base): 14 | token: uuid.UUID 15 | samples: List[uuid.UUID] 16 | 17 | 18 | class UserComparisonRequest(Base): 19 | comparison_details: ComparisonDetailRequest 20 | # Best to worst, samples with the same index are considered tied 21 | # Example: [A, B, C] means A is best, B is second, C is worst 22 | # Example: [A, [B, C], D] means A is best, B and C are tied for second, D is worst 23 | # The outer list structure can be flat or nested to represent ties 24 | ordered_sample_ids: List[uuid.UUID | List[uuid.UUID]] 25 | 26 | 27 | class SignupRequest(Base): 28 | username: str 29 | signup_auth_provider: str 30 | signup_auth_provider_data: Dict[str, str] 31 | 32 | 33 | class LoginRequest(Base): 34 | login_auth_provider: str 35 | login_auth_provider_data: Dict[str, str] 36 | 37 | 38 | class CreateUserRequest(Base): 39 | username: str 40 | -------------------------------------------------------------------------------- /src/mc_bench/apps/render_worker/__init__.py: -------------------------------------------------------------------------------- 1 | from . import tasks 2 | from .app import app 3 | 4 | __all__ = [ 5 | "app", 6 | "tasks", 7 | ] 8 | -------------------------------------------------------------------------------- /src/mc_bench/apps/render_worker/app.py: -------------------------------------------------------------------------------- 1 | from mc_bench.events import on_event 2 | from mc_bench.events.types import ( 3 | GenerationStateChanged, 4 | RunStageStateChanged, 5 | RunStateChanged, 6 | ) 7 | from mc_bench.models.run import Generation, Run, RunStage 8 | from mc_bench.util.celery import make_worker_celery_app 9 | from mc_bench.util.logging import configure_logging 10 | 11 | from .config import settings 12 | 13 | configure_logging(humanize=settings.HUMANIZE_LOGS, level=settings.LOG_LEVEL) 14 | 15 | app = make_worker_celery_app( 16 | dict( 17 | worker_prefetch_multiplier=1, 18 | ) 19 | ) 20 | 21 | # Event handler registration 22 | on_event(RunStageStateChanged, RunStage.state_change_handler) 23 | on_event(RunStateChanged, Run.state_change_handler) 24 | on_event(GenerationStateChanged, Generation.state_change_handler) 25 | -------------------------------------------------------------------------------- /src/mc_bench/apps/render_worker/config.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | 4 | 5 | class Settings: 6 | INTERNAL_OBJECT_BUCKET = os.environ["INTERNAL_OBJECT_BUCKET"] 7 | EXTERNAL_OBJECT_BUCKET = os.environ["EXTERNAL_OBJECT_BUCKET"] 8 | FAST_RENDER = os.environ.get("FAST_RENDER") == "true" 9 | HUMANIZE_LOGS = os.environ.get("HUMANIZE_LOGS") == "true" 10 | BLENDER_RENDER_CORES = int(os.environ.get("BLENDER_RENDER_CORES", 1)) 11 | LOG_LEVEL_STR = os.environ.get("LOG_LEVEL", "INFO") 12 | LOG_LEVEL = getattr(logging, LOG_LEVEL_STR.upper(), logging.INFO) 13 | # Configure how frequently to log block placement at INFO level 14 | LOG_INTERVAL_BLOCKS = int(os.environ.get("LOG_INTERVAL_BLOCKS", "100")) 15 | # Configure how frequently to log materials baked at INFO level 16 | LOG_INTERVAL_MATERIALS = int(os.environ.get("LOG_INTERVAL_MATERIALS", "10")) 17 | 18 | 19 | settings = Settings() 20 | -------------------------------------------------------------------------------- /src/mc_bench/apps/render_worker/tasks/__init__.py: -------------------------------------------------------------------------------- 1 | from . import run 2 | 3 | __all__ = [ 4 | "run", 5 | ] 6 | -------------------------------------------------------------------------------- /src/mc_bench/apps/scheduler/__init__.py: -------------------------------------------------------------------------------- 1 | # Import main entry points from the scheduler modules 2 | from mc_bench.apps.scheduler.manager import main 3 | 4 | # Expose the main function for direct imports 5 | __all__ = ["main"] 6 | -------------------------------------------------------------------------------- /src/mc_bench/apps/scheduler/__main__.py: -------------------------------------------------------------------------------- 1 | from mc_bench.apps.scheduler.manager import main 2 | 3 | if __name__ == "__main__": 4 | main() 5 | -------------------------------------------------------------------------------- /src/mc_bench/apps/server_worker/__init__.py: -------------------------------------------------------------------------------- 1 | from . import tasks 2 | from .app import app 3 | 4 | __all__ = [ 5 | "app", 6 | "tasks", 7 | ] 8 | -------------------------------------------------------------------------------- /src/mc_bench/apps/server_worker/app.py: -------------------------------------------------------------------------------- 1 | from mc_bench.events import on_event 2 | from mc_bench.events.types import ( 3 | GenerationStateChanged, 4 | RunStageStateChanged, 5 | RunStateChanged, 6 | ) 7 | from mc_bench.models.run import Generation, Run, RunStage 8 | from mc_bench.util.celery import make_worker_celery_app 9 | from mc_bench.util.logging import configure_logging 10 | 11 | from .config import settings 12 | 13 | configure_logging(humanize=settings.HUMANIZE_LOGS, level=settings.LOG_LEVEL) 14 | 15 | app = make_worker_celery_app() 16 | 17 | # Event handler registration 18 | on_event(RunStageStateChanged, RunStage.state_change_handler) 19 | on_event(RunStateChanged, Run.state_change_handler) 20 | on_event(GenerationStateChanged, Generation.state_change_handler) 21 | -------------------------------------------------------------------------------- /src/mc_bench/apps/server_worker/config.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | 4 | 5 | class Settings: 6 | INTERNAL_OBJECT_BUCKET = os.environ["INTERNAL_OBJECT_BUCKET"] 7 | EXPORT_STRUCTURE_VIEWS = os.environ.get("EXPORT_STRUCTURE_VIEWS", "true") == "true" 8 | EXPOSE_SERVER_PORTS = os.environ.get("EXPOSE_SERVER_PORTS", "false") == "true" 9 | HUMANIZE_LOGS = os.environ.get("HUMANIZE_LOGS", "false") == "true" 10 | BUILD_DELAY = os.environ.get("BUILD_DELAY_MS", "25") 11 | LOG_LEVEL_STR = os.environ.get("LOG_LEVEL", "INFO") 12 | LOG_LEVEL = getattr(logging, LOG_LEVEL_STR.upper(), logging.INFO) 13 | # Configure how frequently to log build commands at INFO level 14 | LOG_INTERVAL_COMMANDS = int(os.environ.get("LOG_INTERVAL_COMMANDS", "50")) 15 | # Configure how frequently to log export progress (as a percentage) 16 | LOG_INTERVAL_EXPORT_PERCENT = int( 17 | os.environ.get("LOG_INTERVAL_EXPORT_PERCENT", "10") 18 | ) 19 | 20 | 21 | settings = Settings() 22 | -------------------------------------------------------------------------------- /src/mc_bench/apps/server_worker/js_scripts/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | HERE = os.path.dirname(os.path.abspath(__file__)) 4 | 5 | 6 | build_script = os.path.join(HERE, "build.js") 7 | export_script = os.path.join(HERE, "export.js") 8 | 9 | 10 | def get_build_script(): 11 | with open(build_script, "r") as f: 12 | return f.read() 13 | 14 | 15 | def get_export_script(): 16 | with open(export_script, "r") as f: 17 | return f.read() 18 | -------------------------------------------------------------------------------- /src/mc_bench/apps/server_worker/tasks/__init__.py: -------------------------------------------------------------------------------- 1 | from . import run 2 | 3 | __all__ = [ 4 | "run", 5 | ] 6 | -------------------------------------------------------------------------------- /src/mc_bench/apps/server_worker/templates.py: -------------------------------------------------------------------------------- 1 | from .js_scripts import get_build_script, get_export_script 2 | 3 | build_template = get_build_script() 4 | 5 | export_template = get_export_script() 6 | -------------------------------------------------------------------------------- /src/mc_bench/apps/worker/__init__.py: -------------------------------------------------------------------------------- 1 | from . import tasks 2 | from .app import app 3 | 4 | __all__ = [ 5 | "app", 6 | "tasks", 7 | ] 8 | -------------------------------------------------------------------------------- /src/mc_bench/apps/worker/app.py: -------------------------------------------------------------------------------- 1 | from mc_bench.util.celery import make_worker_celery_app 2 | from mc_bench.util.logging import configure_logging 3 | 4 | from .config import settings 5 | 6 | configure_logging(humanize=settings.HUMANIZE_LOGS, level=settings.LOG_LEVEL) 7 | 8 | app = make_worker_celery_app() 9 | -------------------------------------------------------------------------------- /src/mc_bench/apps/worker/config.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | 4 | 5 | class Settings: 6 | HUMANIZE_LOGS = os.environ.get("HUMANIZE_LOGS", "false") == "true" 7 | LOG_LEVEL_STR = os.environ.get("LOG_LEVEL", "INFO") 8 | LOG_LEVEL = getattr(logging, LOG_LEVEL_STR.upper(), logging.INFO) 9 | 10 | # ELO calculation settings 11 | ELO_BATCH_SIZE = int(os.environ.get("ELO_BATCH_SIZE", "1000")) 12 | ELO_K_FACTOR = float(os.environ.get("ELO_K_FACTOR", "32.0")) 13 | ELO_DEFAULT_SCORE = float(os.environ.get("ELO_DEFAULT_SCORE", "1000.0")) 14 | ELO_MIN_SCORE = float(os.environ.get("ELO_MIN_SCORE", "100.0")) 15 | 16 | 17 | settings = Settings() 18 | -------------------------------------------------------------------------------- /src/mc_bench/apps/worker/tasks/__init__.py: -------------------------------------------------------------------------------- 1 | from .elo_calculation import elo_calculation 2 | 3 | __all__ = [ 4 | "elo_calculation", 5 | ] 6 | -------------------------------------------------------------------------------- /src/mc_bench/auth/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mc-bench/mc-bench-backend/360838f5692597b2bbe1c2622e7ac60c13e1871b/src/mc_bench/auth/__init__.py -------------------------------------------------------------------------------- /src/mc_bench/auth/clients/__init__.py: -------------------------------------------------------------------------------- 1 | from ._base import AuthenticationClient 2 | from ._github import GithubOauthClient 3 | from ._google import GoogleOauthClient 4 | 5 | __all__ = [ 6 | "AuthenticationClient", 7 | "GithubOauthClient", 8 | "GoogleOauthClient", 9 | ] 10 | -------------------------------------------------------------------------------- /src/mc_bench/auth/clients/_base.py: -------------------------------------------------------------------------------- 1 | import abc 2 | from typing import List 3 | 4 | 5 | class AuthenticationClient(abc.ABC): 6 | @abc.abstractmethod 7 | def get_username(self, **kwargs) -> str: 8 | pass 9 | 10 | @abc.abstractmethod 11 | def get_user_id(self, **kwargs) -> str: 12 | pass 13 | 14 | @abc.abstractmethod 15 | def get_user_emails(self, **kwargs) -> List[str]: 16 | pass 17 | 18 | @abc.abstractmethod 19 | def get_access_token(self, **kwargs) -> str: 20 | pass 21 | -------------------------------------------------------------------------------- /src/mc_bench/auth/emails.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | 3 | 4 | def hash_email(email: str, salt: str) -> str: 5 | """ 6 | Hash an email address using SHA-256 with salt from environment. 7 | """ 8 | if not salt: 9 | raise ValueError("salt must have some contents") 10 | 11 | normalized = email.lower().strip() 12 | 13 | hasher = hashlib.sha256() 14 | hasher.update(salt.encode("utf-8")) 15 | hasher.update(normalized.encode("utf-8")) 16 | return hasher.digest().hex() 17 | -------------------------------------------------------------------------------- /src/mc_bench/auth/permissions.py: -------------------------------------------------------------------------------- 1 | class PERM: 2 | class PROMPT: 3 | READ = "prompt:read" 4 | WRITE = "prompt:write" 5 | ADMIN = "prompt:admin" 6 | REVIEW = "prompt:review" 7 | 8 | class EXPERIMENT: 9 | PROPOSE = "prompt:experiment:propose" 10 | APPROVE = "prompt:experiment:approve" 11 | 12 | class TEMPLATE: 13 | READ = "template:read" 14 | WRITE = "template:write" 15 | ADMIN = "template:admin" 16 | REVIEW = "template:review" 17 | 18 | class EXPERIMENT: 19 | PROPOSE = "template:experiment:propose" 20 | APPROVE = "template:experiment:approve" 21 | 22 | class MODEL: 23 | READ = "model:read" 24 | WRITE = "model:write" 25 | ADMIN = "model:admin" 26 | REVIEW = "model:review" 27 | 28 | class EXPERIMENT: 29 | PROPOSE = "model:experiment:propose" 30 | APPROVE = "model:experiment:approve" 31 | 32 | class GENERATION: 33 | READ = "generation:read" 34 | WRITE = "generation:write" 35 | ADMIN = "generation:admin" 36 | 37 | class RUN: 38 | READ = "run:read" 39 | WRITE = "run:write" 40 | ADMIN = "run:admin" 41 | PROGRESS_WRITE = "run:progress:write" 42 | 43 | class SAMPLE: 44 | READ = "sample:read" 45 | REVIEW = "sample:review" 46 | ADMIN = "sample:admin" 47 | 48 | class VOTING: 49 | ADMIN = "voting:admin" 50 | VOTE = "voting:vote" 51 | 52 | class USER: 53 | ADMIN = "user:admin" 54 | -------------------------------------------------------------------------------- /src/mc_bench/clients/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mc-bench/mc-bench-backend/360838f5692597b2bbe1c2622e7ac60c13e1871b/src/mc_bench/clients/__init__.py -------------------------------------------------------------------------------- /src/mc_bench/clients/alibaba_cloud.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import openai 4 | 5 | 6 | class AlibabaCloudClient: 7 | def __init__(self): 8 | self.client = openai.OpenAI( 9 | api_key=os.environ["ALIBABA_CLOUD_API_KEY"], 10 | base_url="https://dashscope-intl.aliyuncs.com/compatible-mode/v1", 11 | ) 12 | 13 | def send_prompt(self, **kwargs): 14 | prompt_in_kwargs = "prompt" in kwargs 15 | messages_in_kwargs = "messages" in kwargs 16 | assert not (messages_in_kwargs and prompt_in_kwargs) 17 | assert messages_in_kwargs or prompt_in_kwargs 18 | assert "model" in kwargs 19 | 20 | if prompt_in_kwargs: 21 | kwargs["messages"] = [ 22 | { 23 | "role": "user", 24 | "content": kwargs.pop("prompt"), 25 | } 26 | ] 27 | try: 28 | chat_completion = self.client.chat.completions.create(**kwargs) 29 | return chat_completion.choices[0].message.content 30 | except Exception as e: 31 | raise Exception(f"Error calling Alibaba Cloud API: {str(e)}") 32 | -------------------------------------------------------------------------------- /src/mc_bench/clients/anthropic.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import anthropic 4 | 5 | 6 | class AnthropicClient: 7 | def __init__(self): 8 | self.client = anthropic.Anthropic( 9 | api_key=os.environ["ANTHROPIC_API_KEY"], 10 | ) 11 | 12 | def send_prompt(self, **kwargs): 13 | prompt_in_kwargs = "prompt" in kwargs 14 | messages_in_kwargs = "messages" in kwargs 15 | assert not (messages_in_kwargs and prompt_in_kwargs) 16 | assert messages_in_kwargs or prompt_in_kwargs 17 | assert "model" in kwargs 18 | 19 | if prompt_in_kwargs: 20 | kwargs["messages"] = [ 21 | { 22 | "role": "user", 23 | "content": kwargs.pop("prompt"), 24 | } 25 | ] 26 | 27 | message = self.client.messages.create(**kwargs) 28 | 29 | return message.content[0].text 30 | -------------------------------------------------------------------------------- /src/mc_bench/clients/deepseek.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import openai 4 | 5 | 6 | class DeepSeekClient: 7 | def __init__(self): 8 | self.client = openai.OpenAI( 9 | api_key=os.environ["DEEPSEEK_API_KEY"], 10 | base_url="https://api.deepseek.com/v1", 11 | ) 12 | 13 | def send_prompt(self, **kwargs): 14 | prompt_in_kwargs = "prompt" in kwargs 15 | messages_in_kwargs = "messages" in kwargs 16 | assert not (messages_in_kwargs and prompt_in_kwargs) 17 | assert messages_in_kwargs or prompt_in_kwargs 18 | assert "model" in kwargs 19 | 20 | if prompt_in_kwargs: 21 | kwargs["messages"] = [ 22 | { 23 | "role": "user", 24 | "content": kwargs.pop("prompt"), 25 | } 26 | ] 27 | try: 28 | chat_completion = self.client.chat.completions.create(**kwargs) 29 | return chat_completion.choices[0].message.content 30 | except Exception as e: 31 | raise Exception(f"Error calling DeepSeek API: {str(e)}") 32 | -------------------------------------------------------------------------------- /src/mc_bench/clients/gemini.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import openai 4 | 5 | 6 | class GeminiClient: 7 | def __init__(self): 8 | self.client = openai.OpenAI( 9 | api_key=os.environ["GOOGLE_API_KEY"], 10 | base_url="https://generativelanguage.googleapis.com/v1beta/openai/", 11 | ) 12 | 13 | def send_prompt(self, **kwargs): 14 | prompt_in_kwargs = "prompt" in kwargs 15 | messages_in_kwargs = "messages" in kwargs 16 | assert not (messages_in_kwargs and prompt_in_kwargs) 17 | assert messages_in_kwargs or prompt_in_kwargs 18 | assert "model" in kwargs 19 | 20 | if prompt_in_kwargs: 21 | kwargs["messages"] = [ 22 | { 23 | "role": "user", 24 | "content": kwargs.pop("prompt"), 25 | } 26 | ] 27 | try: 28 | chat_completion = self.client.chat.completions.create(**kwargs) 29 | return chat_completion.choices[0].message.content 30 | except Exception as e: 31 | raise Exception(f"Error calling Gemini API: {str(e)}") 32 | -------------------------------------------------------------------------------- /src/mc_bench/clients/grok.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from openai import OpenAI 4 | 5 | 6 | class GrokClient: 7 | def __init__(self): 8 | self.client = OpenAI( 9 | api_key=os.environ["XAI_API_KEY"], base_url="https://api.x.ai/v1" 10 | ) 11 | 12 | def send_prompt(self, **kwargs): 13 | prompt_in_kwargs = "prompt" in kwargs 14 | messages_in_kwargs = "messages" in kwargs 15 | assert not (messages_in_kwargs and prompt_in_kwargs) 16 | assert messages_in_kwargs or prompt_in_kwargs 17 | assert "model" in kwargs 18 | 19 | # Convert single prompt to messages if needed 20 | if prompt_in_kwargs: 21 | kwargs["messages"] = [ 22 | { 23 | "role": "user", 24 | "content": kwargs.pop("prompt"), 25 | } 26 | ] 27 | 28 | try: 29 | response = self.client.chat.completions.create(**kwargs) 30 | return response.choices[0].message.content 31 | except Exception as e: 32 | raise Exception(f"Error in Grok API call: {str(e)}") 33 | -------------------------------------------------------------------------------- /src/mc_bench/clients/mistral.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from mistralai import Mistral 4 | 5 | 6 | class MistralClient: 7 | def __init__(self): 8 | self.client = Mistral(api_key=os.environ["MISTRAL_API_KEY"]) 9 | 10 | def send_prompt(self, **kwargs): 11 | prompt_in_kwargs = "prompt" in kwargs 12 | messages_in_kwargs = "messages" in kwargs 13 | assert not (messages_in_kwargs and prompt_in_kwargs) 14 | assert messages_in_kwargs or prompt_in_kwargs 15 | assert "model" in kwargs 16 | 17 | if prompt_in_kwargs: 18 | kwargs["messages"] = [ 19 | { 20 | "role": "user", 21 | "content": kwargs.pop("prompt"), 22 | } 23 | ] 24 | try: 25 | chat_completion = self.client.chat.complete(**kwargs) 26 | return chat_completion.choices[0].message.content 27 | except Exception as e: 28 | raise Exception(f"Error calling Mistral API: {str(e)}") 29 | -------------------------------------------------------------------------------- /src/mc_bench/clients/openai.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import openai 4 | 5 | 6 | class OpenAIClient: 7 | def __init__(self): 8 | self.client = openai.OpenAI(api_key=os.environ["OPENAI_API_KEY"]) 9 | 10 | def send_prompt(self, **kwargs): 11 | prompt_in_kwargs = "prompt" in kwargs 12 | messages_in_kwargs = "messages" in kwargs 13 | assert not (messages_in_kwargs and prompt_in_kwargs) 14 | assert messages_in_kwargs or prompt_in_kwargs 15 | assert "model" in kwargs 16 | 17 | if prompt_in_kwargs: 18 | kwargs["messages"] = [ 19 | { 20 | "role": "user", 21 | "content": kwargs.pop("prompt"), 22 | } 23 | ] 24 | 25 | chat_completion = self.client.chat.completions.create(**kwargs) 26 | return chat_completion.choices[0].message.content 27 | -------------------------------------------------------------------------------- /src/mc_bench/clients/openrouter.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import openai 4 | 5 | 6 | class OpenRouterClient: 7 | def __init__(self): 8 | self.client = openai.OpenAI( 9 | base_url="https://openrouter.ai/api/v1", 10 | api_key=os.environ["OPENROUTER_API_KEY"], 11 | ) 12 | 13 | def send_prompt(self, **kwargs): 14 | prompt_in_kwargs = "prompt" in kwargs 15 | messages_in_kwargs = "messages" in kwargs 16 | assert not (messages_in_kwargs and prompt_in_kwargs) 17 | assert messages_in_kwargs or prompt_in_kwargs 18 | assert "model" in kwargs 19 | 20 | if prompt_in_kwargs: 21 | kwargs["messages"] = [ 22 | { 23 | "role": "user", 24 | "content": kwargs.pop("prompt"), 25 | } 26 | ] 27 | 28 | chat_completion = self.client.chat.completions.create(**kwargs) 29 | return chat_completion.choices[0].message.content 30 | -------------------------------------------------------------------------------- /src/mc_bench/clients/reka.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from reka.client import Reka 4 | 5 | 6 | class RekaClient: 7 | def __init__(self): 8 | self.client = Reka(api_key=os.environ["REKA_API_KEY"]) 9 | 10 | def send_prompt(self, **kwargs): 11 | prompt_in_kwargs = "prompt" in kwargs 12 | messages_in_kwargs = "messages" in kwargs 13 | assert not (messages_in_kwargs and prompt_in_kwargs) 14 | assert messages_in_kwargs or prompt_in_kwargs 15 | assert "model" in kwargs 16 | 17 | if prompt_in_kwargs: 18 | kwargs["messages"] = [ 19 | { 20 | "role": "user", 21 | "content": kwargs.pop("prompt"), 22 | } 23 | ] 24 | try: 25 | response = self.client.chat.create(**kwargs) 26 | return response.responses[0].message.content 27 | except Exception as e: 28 | raise Exception(f"Error calling Reka API: {str(e)}") 29 | -------------------------------------------------------------------------------- /src/mc_bench/clients/zhipuai.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import openai 4 | 5 | 6 | class ZhipuAIClient: 7 | def __init__(self): 8 | self.client = openai.OpenAI( 9 | api_key=os.environ["ZHIPUAI_API_KEY"], 10 | base_url="https://open.bigmodel.cn/api/paas/v4/", 11 | ) 12 | 13 | def send_prompt(self, **kwargs): 14 | prompt_in_kwargs = "prompt" in kwargs 15 | messages_in_kwargs = "messages" in kwargs 16 | assert not (messages_in_kwargs and prompt_in_kwargs) 17 | assert messages_in_kwargs or prompt_in_kwargs 18 | assert "model" in kwargs 19 | 20 | if prompt_in_kwargs: 21 | kwargs["messages"] = [ 22 | { 23 | "role": "user", 24 | "content": kwargs.pop("prompt"), 25 | } 26 | ] 27 | try: 28 | chat_completion = self.client.chat.completions.create(**kwargs) 29 | return chat_completion.choices[0].message.content 30 | except Exception as e: 31 | raise Exception(f"Error calling ZhipuaAI API: {str(e)}") 32 | -------------------------------------------------------------------------------- /src/mc_bench/config/__init__.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | 4 | 5 | class CeleryConfig: 6 | @property 7 | def broker_url(self): 8 | return os.environ["CELERY_BROKER_URL"] 9 | 10 | 11 | class PostgresConfig: 12 | @property 13 | def host(self): 14 | return os.environ["POSTGRES_HOST"] 15 | 16 | @property 17 | def database(self): 18 | return os.environ["POSTGRES_DATABASE"] 19 | 20 | @property 21 | def user(self): 22 | return os.environ["POSTGRES_USER"] 23 | 24 | @property 25 | def password(self): 26 | return os.environ["POSTGRES_PASSWORD"] 27 | 28 | @property 29 | def port(self): 30 | return int(os.environ["POSTGRES_PORT"]) 31 | 32 | @property 33 | def options(self): 34 | return json.loads(os.environ.get("POSTGRES_OPTIONS", "{}")) 35 | -------------------------------------------------------------------------------- /src/mc_bench/constants.py: -------------------------------------------------------------------------------- 1 | import enum 2 | 3 | 4 | # Any values added here must also be added to the DB via a migration 5 | # see d618a24f0bed_add_generation_state_and_run_state_.py for an example 6 | class RUN_STATE(enum.Enum): 7 | CREATED = "CREATED" 8 | IN_PROGRESS = "IN_PROGRESS" 9 | COMPLETED = "COMPLETED" 10 | FAILED = "FAILED" 11 | IN_RETRY = "IN_RETRY" 12 | 13 | 14 | class RUN_STAGE_STATE(enum.Enum): 15 | PENDING = "PENDING" 16 | ENQUEUED = "ENQUEUED" 17 | IN_PROGRESS = "IN_PROGRESS" 18 | COMPLETED = "COMPLETED" 19 | FAILED = "FAILED" 20 | IN_RETRY = "IN_RETRY" 21 | 22 | 23 | # Any values added here must also be added to the DB via a migration 24 | # see d618a24f0bed_add_generation_state_and_run_state_.py for an example 25 | class GENERATION_STATE(enum.Enum): 26 | CREATED = "CREATED" 27 | IN_PROGRESS = "IN_PROGRESS" 28 | COMPLETED = "COMPLETED" 29 | PARTIAL_FAILED = "PARTIAL_FAILED" 30 | IN_RETRY = "IN_RETRY" 31 | FAILED = "FAILED" 32 | 33 | 34 | class STAGE(enum.Enum): 35 | PROMPT_EXECUTION = "PROMPT_EXECUTION" 36 | RESPONSE_PARSING = "RESPONSE_PARSING" 37 | CODE_VALIDATION = "CODE_VALIDATION" 38 | BUILDING = "BUILDING" 39 | RENDERING_SAMPLE = "RENDERING_SAMPLE" 40 | EXPORTING_CONTENT = "EXPORTING_CONTENT" 41 | POST_PROCESSING = "POST_PROCESSING" 42 | PREPARING_SAMPLE = "PREPARING_SAMPLE" 43 | 44 | 45 | class EXPERIMENTAL_STATE(enum.Enum): 46 | DRAFT = "DRAFT" 47 | IN_REVIEW = "IN_REVIEW" 48 | RELEASED = "RELEASED" 49 | DEPRECATED = "DEPRECATED" 50 | EXPERIMENTAL = "EXPERIMENTAL" 51 | REJECTED = "REJECTED" 52 | -------------------------------------------------------------------------------- /src/mc_bench/events/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Callable, Dict, List, Type 2 | 3 | from .types import E, Event 4 | 5 | _handlers: Dict[Type[Event], List[Callable]] = {} 6 | 7 | 8 | def on_event(event_type: Type[E], fn: Callable[[E], Any]) -> None: 9 | """Register an event handler for a specific event type""" 10 | if event_type not in _handlers: 11 | _handlers[event_type] = [] 12 | _handlers[event_type].append(fn) 13 | 14 | 15 | def emit_event(event: Event) -> None: 16 | """Emit an event to all registered handlers""" 17 | event_type = type(event) 18 | if event_type in _handlers: 19 | for handler in _handlers[event_type]: 20 | handler(event) 21 | -------------------------------------------------------------------------------- /src/mc_bench/events/types/__init__.py: -------------------------------------------------------------------------------- 1 | from ._base import E, Event 2 | from .event_types import GenerationStateChanged, RunStageStateChanged, RunStateChanged 3 | 4 | __all__ = [ 5 | "E", 6 | "Event", 7 | "RunStageStateChanged", 8 | "RunStateChanged", 9 | "GenerationStateChanged", 10 | ] 11 | -------------------------------------------------------------------------------- /src/mc_bench/events/types/_base.py: -------------------------------------------------------------------------------- 1 | from typing import TypeVar 2 | 3 | from pydantic import BaseModel 4 | 5 | 6 | # Base event class that all events inherit from 7 | class Event(BaseModel): 8 | pass 9 | 10 | 11 | # Type variable for event types 12 | E = TypeVar("E", bound=Event) 13 | -------------------------------------------------------------------------------- /src/mc_bench/events/types/event_types.py: -------------------------------------------------------------------------------- 1 | from mc_bench.constants import GENERATION_STATE, RUN_STAGE_STATE, RUN_STATE 2 | 3 | from ._base import Event 4 | 5 | 6 | class RunStageStateChanged(Event): 7 | stage_id: int 8 | new_state: RUN_STAGE_STATE 9 | 10 | 11 | class RunStateChanged(Event): 12 | run_id: int 13 | new_state: RUN_STATE 14 | 15 | 16 | class GenerationStateChanged(Event): 17 | generation_id: int 18 | new_state: GENERATION_STATE 19 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/README: -------------------------------------------------------------------------------- 1 | Generic single-database configuration. 2 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mc-bench/mc-bench-backend/360838f5692597b2bbe1c2622e7ac60c13e1871b/src/mc_bench/migrations/__init__.py -------------------------------------------------------------------------------- /src/mc_bench/migrations/__main__.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | from alembic.config import CommandLine, Config 4 | 5 | 6 | def get_alembic_config(): 7 | alembic_cfg = Config() 8 | alembic_cfg.set_main_option("script_location", "mc_bench:migrations") 9 | 10 | return alembic_cfg 11 | 12 | 13 | def main(): 14 | cli = CommandLine(prog=sys.argv[0]) 15 | 16 | options = cli.parser.parse_args() 17 | if not hasattr(options, "cmd"): 18 | cli.parser.error("too few arguments") 19 | 20 | cmd, args_keys, kwargs_keys = options.cmd 21 | alembic_cfg = get_alembic_config() 22 | alembic_cfg.cmd_opts = options 23 | cmd( 24 | alembic_cfg, 25 | *[getattr(options, k, None) for k in args_keys], 26 | **dict((k, getattr(options, k, None)) for k in kwargs_keys), 27 | ) 28 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/script.py.mako: -------------------------------------------------------------------------------- 1 | """${message} 2 | 3 | Revision ID: ${up_revision} 4 | Revises: ${down_revision | comma,n} 5 | Create Date: ${create_date} 6 | 7 | """ 8 | from typing import Sequence, Union 9 | 10 | from alembic import op 11 | import sqlalchemy as sa 12 | ${imports if imports else ""} 13 | 14 | # revision identifiers, used by Alembic. 15 | revision: str = ${repr(up_revision)} 16 | down_revision: Union[str, None] = ${repr(down_revision)} 17 | branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} 18 | depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} 19 | 20 | 21 | def upgrade() -> None: 22 | ${upgrades if upgrades else "pass"} 23 | 24 | 25 | def downgrade() -> None: 26 | raise RuntimeError("Upgrades only") 27 | ${downgrades if downgrades else "pass"} 28 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/0076eb8acb15_add_external_id_to_artifact.py: -------------------------------------------------------------------------------- 1 | """Add external_id to artifact 2 | 3 | Revision ID: 0076eb8acb15 4 | Revises: 8f3ea520a73e 5 | Create Date: 2024-12-03 12:22:03.602343 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | import sqlalchemy as sa 12 | from alembic import op 13 | 14 | # revision identifiers, used by Alembic. 15 | revision: str = "0076eb8acb15" 16 | down_revision: Union[str, None] = "8f3ea520a73e" 17 | branch_labels: Union[str, Sequence[str], None] = None 18 | depends_on: Union[str, Sequence[str], None] = None 19 | 20 | 21 | def upgrade() -> None: 22 | # ### commands auto generated by Alembic - please adjust! ### 23 | op.add_column( 24 | "artifact", 25 | sa.Column( 26 | "external_id", 27 | sa.UUID(), 28 | server_default=sa.text("uuid_generate_v4()"), 29 | nullable=True, 30 | ), 31 | schema="sample", 32 | ) 33 | op.execute("""UPDATE sample.artifact set external_id = uuid_generate_v4()""") 34 | # ### end Alembic commands ### 35 | 36 | 37 | def downgrade() -> None: 38 | raise RuntimeError("Upgrades only") 39 | # ### commands auto generated by Alembic - please adjust! ### 40 | op.drop_column("artifact", "external_id", schema="sample") 41 | # ### end Alembic commands ### 42 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/019b9b3a6b6c_add_test_set_id_to_scoring_comparison.py: -------------------------------------------------------------------------------- 1 | """Add test_set_id to scoring.comparison 2 | 3 | Revision ID: 019b9b3a6b6c 4 | Revises: 8f1d29465102 5 | Create Date: 2025-03-04 14:10:38.559630 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | import sqlalchemy as sa 12 | from alembic import op 13 | 14 | # revision identifiers, used by Alembic. 15 | revision: str = "019b9b3a6b6c" 16 | down_revision: Union[str, None] = "8f1d29465102" 17 | branch_labels: Union[str, Sequence[str], None] = None 18 | depends_on: Union[str, Sequence[str], None] = None 19 | 20 | 21 | def upgrade() -> None: 22 | # ### commands auto generated by Alembic - please adjust! ### 23 | op.add_column( 24 | "comparison", 25 | sa.Column("test_set_id", sa.Integer(), nullable=True), 26 | schema="scoring", 27 | ) 28 | 29 | # Add a default value to the test_set_id column 30 | op.execute( 31 | "UPDATE scoring.comparison SET test_set_id = 2 WHERE test_set_id IS NULL" 32 | ) 33 | 34 | # Make the test_set_id column non-nullable 35 | op.alter_column( 36 | "comparison", 37 | "test_set_id", 38 | nullable=False, 39 | schema="scoring", 40 | ) 41 | 42 | op.create_foreign_key( 43 | None, 44 | "comparison", 45 | "test_set", 46 | ["test_set_id"], 47 | ["id"], 48 | source_schema="scoring", 49 | referent_schema="sample", 50 | ) 51 | # ### end Alembic commands ### 52 | 53 | 54 | def downgrade() -> None: 55 | raise RuntimeError("Upgrades only") 56 | # ### commands auto generated by Alembic - please adjust! ### 57 | op.drop_constraint(None, "comparison", schema="scoring", type_="foreignkey") 58 | op.drop_column("comparison", "test_set_id", schema="scoring") 59 | # ### end Alembic commands ### 60 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/0d6ed46a9eb3_add_deepseek_model_and_provider.py: -------------------------------------------------------------------------------- 1 | """add_deepseek_model_and_provider 2 | 3 | Revision ID: 0d6ed46a9eb3 4 | Revises: 5ca7be2d7150 5 | Create Date: 2025-01-22 15:18:20.990102 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | from alembic import op 12 | 13 | # revision identifiers, used by Alembic. 14 | revision: str = "0d6ed46a9eb3" 15 | down_revision: Union[str, None] = "5ca7be2d7150" 16 | branch_labels: Union[str, Sequence[str], None] = None 17 | depends_on: Union[str, Sequence[str], None] = None 18 | 19 | 20 | def upgrade() -> None: 21 | op.execute("""\ 22 | INSERT INTO specification.provider_class (created_by, name) VALUES ( 23 | (select id from auth.user where username = 'SYSTEM'), 24 | 'DEEPSEEK_SDK' 25 | ) ON CONFLICT (name) DO NOTHING; 26 | """) 27 | 28 | 29 | def downgrade() -> None: 30 | raise RuntimeError("Upgrades only") 31 | pass 32 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/0df207ed67f9_add_artifact_kinds.py: -------------------------------------------------------------------------------- 1 | """add artifact kinds 2 | 3 | Revision ID: 0df207ed67f9 4 | Revises: 1b9505818e11 5 | Create Date: 2024-12-01 00:40:04.923558 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | import sqlalchemy as sa 12 | from alembic import op 13 | 14 | # revision identifiers, used by Alembic. 15 | revision: str = "0df207ed67f9" 16 | down_revision: Union[str, None] = "1b9505818e11" 17 | branch_labels: Union[str, Sequence[str], None] = None 18 | depends_on: Union[str, Sequence[str], None] = None 19 | 20 | KINDS = ["BUILD_SCHEMATIC", "BUILD_GLTF"] 21 | 22 | 23 | def upgrade() -> None: 24 | for kind in KINDS: 25 | op.execute( 26 | sa.text("""\ 27 | INSERT INTO sample.artifact_kind (name) VALUES (:artifact_kind) 28 | 29 | """).bindparams(artifact_kind=kind) 30 | ) 31 | 32 | 33 | def downgrade() -> None: 34 | raise RuntimeError("Upgrades only") 35 | pass 36 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/1035a263c03c_add_role_external_id.py: -------------------------------------------------------------------------------- 1 | """Add role external id 2 | 3 | Revision ID: 1035a263c03c 4 | Revises: b3bfae61b787 5 | Create Date: 2025-02-26 16:40:23.158203 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | import sqlalchemy as sa 12 | from alembic import op 13 | 14 | # revision identifiers, used by Alembic. 15 | revision: str = "1035a263c03c" 16 | down_revision: Union[str, None] = "b3bfae61b787" 17 | branch_labels: Union[str, Sequence[str], None] = None 18 | depends_on: Union[str, Sequence[str], None] = None 19 | 20 | 21 | def upgrade() -> None: 22 | # ### commands auto generated by Alembic - please adjust! ### 23 | op.add_column( 24 | "role", 25 | sa.Column( 26 | "external_id", 27 | sa.UUID(), 28 | server_default=sa.text("uuid_generate_v4()"), 29 | nullable=False, 30 | ), 31 | schema="auth", 32 | ) 33 | # ### end Alembic commands ### 34 | 35 | 36 | def downgrade() -> None: 37 | raise RuntimeError("Upgrades only") 38 | # ### commands auto generated by Alembic - please adjust! ### 39 | op.drop_column("role", "external_id", schema="auth") 40 | # ### end Alembic commands ### 41 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/163167df595b_support_render_stage.py: -------------------------------------------------------------------------------- 1 | """Support render stage 2 | 3 | Revision ID: 163167df595b 4 | Revises: ae55800708f6 5 | Create Date: 2025-01-14 13:31:32.278244 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | import sqlalchemy as sa 12 | from alembic import op 13 | 14 | # revision identifiers, used by Alembic. 15 | revision: str = "163167df595b" 16 | down_revision: Union[str, None] = "ae55800708f6" 17 | branch_labels: Union[str, Sequence[str], None] = None 18 | depends_on: Union[str, Sequence[str], None] = None 19 | 20 | STAGES = [ 21 | "RENDERING_SAMPLE", 22 | ] 23 | 24 | ARTIFACT_KINDS = [ 25 | "RENDERED_MODEL_GLB", 26 | ] 27 | 28 | 29 | def upgrade() -> None: 30 | for stage in STAGES: 31 | op.execute( 32 | sa.text( 33 | """\ 34 | INSERT INTO specification.stage (created_by, slug) values ( 35 | (SELECT id from auth.user where username = 'SYSTEM'), 36 | :slug 37 | ) 38 | ON CONFLICT (slug) DO NOTHING 39 | """ 40 | ).bindparams(slug=stage) 41 | ) 42 | 43 | for kind in ARTIFACT_KINDS: 44 | op.execute( 45 | sa.text("""\ 46 | INSERT INTO sample.artifact_kind (name) VALUES (:artifact_kind) 47 | 48 | """).bindparams(artifact_kind=kind) 49 | ) 50 | 51 | 52 | def downgrade() -> None: 53 | raise RuntimeError("Upgrades only") 54 | pass 55 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/1b9505818e11_made_comparison_sample_id_non_nullable.py: -------------------------------------------------------------------------------- 1 | """made comparison_sample_id non nullable 2 | 3 | Revision ID: 1b9505818e11 4 | Revises: 6ef6c3fb5778 5 | Create Date: 2024-11-27 12:19:14.761622 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | import sqlalchemy as sa 12 | from alembic import op 13 | 14 | # revision identifiers, used by Alembic. 15 | revision: str = "1b9505818e11" 16 | down_revision: Union[str, None] = "6ef6c3fb5778" 17 | branch_labels: Union[str, Sequence[str], None] = None 18 | depends_on: Union[str, Sequence[str], None] = None 19 | 20 | 21 | def upgrade() -> None: 22 | # ### commands auto generated by Alembic - please adjust! ### 23 | op.alter_column( 24 | "sample", 25 | "comparison_sample_id", 26 | existing_type=sa.UUID(), 27 | nullable=False, 28 | existing_server_default=sa.text("uuid_generate_v4()"), 29 | schema="sample", 30 | ) 31 | # ### end Alembic commands ### 32 | 33 | 34 | def downgrade() -> None: 35 | raise RuntimeError("Upgrades only") 36 | # ### commands auto generated by Alembic - please adjust! ### 37 | op.alter_column( 38 | "sample", 39 | "comparison_sample_id", 40 | existing_type=sa.UUID(), 41 | nullable=True, 42 | existing_server_default=sa.text("uuid_generate_v4()"), 43 | schema="sample", 44 | ) 45 | # ### end Alembic commands ### 46 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/1eef5895e9f2_add_code_validation_stage.py: -------------------------------------------------------------------------------- 1 | """Add code validation stage 2 | 3 | Revision ID: 1eef5895e9f2 4 | Revises: 688482827ae4 5 | Create Date: 2024-12-16 16:26:35.638632 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | import sqlalchemy as sa 12 | from alembic import op 13 | 14 | # revision identifiers, used by Alembic. 15 | revision: str = "1eef5895e9f2" 16 | down_revision: Union[str, None] = "688482827ae4" 17 | branch_labels: Union[str, Sequence[str], None] = None 18 | depends_on: Union[str, Sequence[str], None] = None 19 | 20 | STAGES = [ 21 | "CODE_VALIDATION", 22 | ] 23 | 24 | 25 | def upgrade() -> None: 26 | for stage in STAGES: 27 | op.execute( 28 | sa.text( 29 | """\ 30 | INSERT INTO specification.stage (created_by, slug) values ( 31 | (SELECT id from auth.user where username = 'SYSTEM'), 32 | :slug 33 | ) 34 | ON CONFLICT (slug) DO NOTHING 35 | """ 36 | ).bindparams(slug=stage) 37 | ) 38 | 39 | 40 | def downgrade() -> None: 41 | raise RuntimeError("Upgrades only") 42 | pass 43 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/1f5438c5d056_add_active_field_to_prompt_table.py: -------------------------------------------------------------------------------- 1 | """add active field to prompt table 2 | 3 | Revision ID: 1f5438c5d056 4 | Revises: 38dedff6cd15 5 | Create Date: 2024-11-18 02:16:37.401070 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | import sqlalchemy as sa 12 | from alembic import op 13 | 14 | # revision identifiers, used by Alembic. 15 | revision: str = "1f5438c5d056" 16 | down_revision: Union[str, None] = "38dedff6cd15" 17 | branch_labels: Union[str, Sequence[str], None] = None 18 | depends_on: Union[str, Sequence[str], None] = None 19 | 20 | 21 | def upgrade() -> None: 22 | # ### commands auto generated by Alembic - please adjust! ### 23 | op.add_column( 24 | "prompt", 25 | sa.Column("active", sa.Boolean(), nullable=True), 26 | schema="specification", 27 | ) 28 | # ### end Alembic commands ### 29 | 30 | 31 | def downgrade() -> None: 32 | raise RuntimeError("Upgrades only") 33 | # ### commands auto generated by Alembic - please adjust! ### 34 | op.drop_column("prompt", "active", schema="specification") 35 | # ### end Alembic commands ### 36 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/3491851af8f5_set_model_name_to_not_nullable.py: -------------------------------------------------------------------------------- 1 | """Set model.name to not nullable 2 | 3 | Revision ID: 3491851af8f5 4 | Revises: 0a61b7e99cdc 5 | Create Date: 2025-02-25 00:39:00.998736 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | import sqlalchemy as sa 12 | from alembic import op 13 | 14 | # revision identifiers, used by Alembic. 15 | revision: str = "3491851af8f5" 16 | down_revision: Union[str, None] = "0a61b7e99cdc" 17 | branch_labels: Union[str, Sequence[str], None] = None 18 | depends_on: Union[str, Sequence[str], None] = None 19 | 20 | 21 | def upgrade() -> None: 22 | # ### commands auto generated by Alembic - please adjust! ### 23 | op.alter_column( 24 | "model", 25 | "name", 26 | existing_type=sa.VARCHAR(), 27 | nullable=False, 28 | schema="specification", 29 | ) 30 | # ### end Alembic commands ### 31 | 32 | 33 | def downgrade() -> None: 34 | raise RuntimeError("Upgrades only") 35 | # ### commands auto generated by Alembic - please adjust! ### 36 | op.alter_column( 37 | "model", 38 | "name", 39 | existing_type=sa.VARCHAR(), 40 | nullable=True, 41 | schema="specification", 42 | ) 43 | # ### end Alembic commands ### 44 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/38dedff6cd15_fix_permissions.py: -------------------------------------------------------------------------------- 1 | """Fix permissions 2 | 3 | Revision ID: 38dedff6cd15 4 | Revises: 4ce0397c4cf7 5 | Create Date: 2024-11-18 01:21:43.435893 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | from alembic import op 12 | 13 | # revision identifiers, used by Alembic. 14 | revision: str = "38dedff6cd15" 15 | down_revision: Union[str, None] = "4ce0397c4cf7" 16 | branch_labels: Union[str, Sequence[str], None] = None 17 | depends_on: Union[str, Sequence[str], None] = None 18 | 19 | 20 | def upgrade() -> None: 21 | op.execute("""\ 22 | ALTER DEFAULT PRIVILEGES FOR ROLE "mc-bench-admin" IN SCHEMA auth GRANT SELECT, UPDATE, INSERT ON TABLES TO "api"; 23 | GRANT UPDATE ON ALL TABLES IN SCHEMA auth TO api; 24 | 25 | ALTER DEFAULT PRIVILEGES FOR ROLE "mc-bench-admin" IN SCHEMA specification GRANT SELECT, UPDATE, INSERT ON TABLES TO "admin-api"; 26 | GRANT UPDATE ON ALL TABLES IN SCHEMA specification TO "admin-api"; 27 | """) 28 | pass 29 | 30 | 31 | def downgrade() -> None: 32 | raise RuntimeError("Upgrades only") 33 | pass 34 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/3cb7f6ca1320_add_sorting_strategy_config.py: -------------------------------------------------------------------------------- 1 | """Add sorting strategy config 2 | 3 | Revision ID: 3cb7f6ca1320 4 | Revises: 1eeb63dd99cc 5 | Create Date: 2025-03-11 16:01:21.966090 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | from alembic import op 12 | 13 | # revision identifiers, used by Alembic. 14 | revision: str = "3cb7f6ca1320" 15 | down_revision: Union[str, None] = "1eeb63dd99cc" 16 | branch_labels: Union[str, Sequence[str], None] = None 17 | depends_on: Union[str, Sequence[str], None] = None 18 | 19 | 20 | def upgrade() -> None: 21 | op.execute( 22 | """ 23 | INSERT INTO specification.scheduler_control (key, value, description) 24 | VALUES ('RUN_SORTING_STRATEGY', '"CREATED_ASC"', 'Defines how runs are sorted when enqueuing: CREATED_ASC, CREATED_DESC, or RANDOM') 25 | """ 26 | ) 27 | 28 | 29 | def downgrade() -> None: 30 | raise RuntimeError("Upgrades only") 31 | pass 32 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/4085c38e19e8_add_provider_class_rows.py: -------------------------------------------------------------------------------- 1 | """Add provider_class rows 2 | 3 | Revision ID: 4085c38e19e8 4 | Revises: b097774ba21f 5 | Create Date: 2024-11-19 22:46:00.704472 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | from alembic import op 12 | 13 | # revision identifiers, used by Alembic. 14 | revision: str = "4085c38e19e8" 15 | down_revision: Union[str, None] = "b097774ba21f" 16 | branch_labels: Union[str, Sequence[str], None] = None 17 | depends_on: Union[str, Sequence[str], None] = None 18 | 19 | 20 | def upgrade() -> None: 21 | op.execute("""\ 22 | INSERT INTO specification.provider_class (created_by, name) VALUES ( 23 | (select id from auth.user where username = 'SYSTEM'), 24 | 'OPEN_AI_SDK' 25 | ) ON CONFLICT (name) DO NOTHING; 26 | 27 | INSERT INTO specification.provider_class (created_by, name) VALUES ( 28 | (select id from auth.user where username = 'SYSTEM'), 29 | 'ANTHROPIC_SDK' 30 | ) ON CONFLICT (name) DO NOTHING; 31 | 32 | INSERT INTO specification.provider_class (created_by, name) VALUES ( 33 | (select id from auth.user where username = 'SYSTEM'), 34 | 'OPENROUTER_SDK' 35 | ) ON CONFLICT (name) DO NOTHING; 36 | """) 37 | 38 | 39 | def downgrade() -> None: 40 | raise RuntimeError("Upgrades only") 41 | pass 42 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/473407e9d86e_add_calculate_score_to_tag.py: -------------------------------------------------------------------------------- 1 | """add_calculate_score_to_tag 2 | 3 | Revision ID: 473407e9d86e 4 | Revises: 3cb7f6ca1320 5 | Create Date: 2025-03-11 18:46:07.619455 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | import sqlalchemy as sa 12 | from alembic import op 13 | 14 | # revision identifiers, used by Alembic. 15 | revision: str = "473407e9d86e" 16 | down_revision: Union[str, None] = "3cb7f6ca1320" 17 | branch_labels: Union[str, Sequence[str], None] = None 18 | depends_on: Union[str, Sequence[str], None] = None 19 | 20 | 21 | def upgrade() -> None: 22 | # ### commands auto generated by Alembic - please adjust! ### 23 | op.add_column( 24 | "tag", 25 | sa.Column( 26 | "calculate_score", 27 | sa.Boolean(), 28 | server_default=sa.text("true"), 29 | nullable=False, 30 | ), 31 | schema="specification", 32 | ) 33 | # ### end Alembic commands ### 34 | 35 | 36 | def downgrade() -> None: 37 | raise RuntimeError("Upgrades only") 38 | # ### commands auto generated by Alembic - please adjust! ### 39 | op.drop_column("tag", "calculate_score", schema="specification") 40 | # ### end Alembic commands ### 41 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/4d220a984a72_add_metrics.py: -------------------------------------------------------------------------------- 1 | """Add metrics 2 | 3 | Revision ID: 4d220a984a72 4 | Revises: 9d12f9191c09 5 | Create Date: 2024-11-25 15:26:29.841864 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | import sqlalchemy as sa 12 | from alembic import op 13 | 14 | # revision identifiers, used by Alembic. 15 | revision: str = "4d220a984a72" 16 | down_revision: Union[str, None] = "9d12f9191c09" 17 | branch_labels: Union[str, Sequence[str], None] = None 18 | depends_on: Union[str, Sequence[str], None] = None 19 | 20 | METRICS = [ 21 | { 22 | "name": "INSTRUCTION_FOLLOWING", 23 | "description": "How well did the model follow the instructions", 24 | }, 25 | { 26 | "name": "UNQUALIFIED_BETTER", 27 | "description": "Which build is better", 28 | }, 29 | ] 30 | 31 | 32 | def upgrade() -> None: 33 | for metric in METRICS: 34 | op.execute( 35 | sa.text("""\ 36 | INSERT INTO scoring.metric (created_by, name, description) VALUES ((SELECT id FROM auth.user WHERE username = 'SYSTEM'), :name, :description) ON CONFLICT (name) DO NOTHING 37 | """).bindparams( 38 | name=metric["name"], 39 | description=metric["description"], 40 | ) 41 | ) 42 | 43 | 44 | def downgrade() -> None: 45 | raise RuntimeError("Upgrades only") 46 | pass 47 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/4e797f372395_add_role_permission_table.py: -------------------------------------------------------------------------------- 1 | """add role_permission table 2 | 3 | Revision ID: 4e797f372395 4 | Revises: 7242824efbe7 5 | Create Date: 2024-11-17 22:57:10.709682 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | import sqlalchemy as sa 12 | from alembic import op 13 | 14 | # revision identifiers, used by Alembic. 15 | revision: str = "4e797f372395" 16 | down_revision: Union[str, None] = "7242824efbe7" 17 | branch_labels: Union[str, Sequence[str], None] = None 18 | depends_on: Union[str, Sequence[str], None] = None 19 | 20 | 21 | def upgrade() -> None: 22 | # ### commands auto generated by Alembic - please adjust! ### 23 | op.create_table( 24 | "role_permission", 25 | sa.Column("id", sa.BigInteger(), autoincrement=True, nullable=False), 26 | sa.Column( 27 | "created", sa.TIMESTAMP(), server_default=sa.text("now()"), nullable=False 28 | ), 29 | sa.Column("created_by", sa.Integer(), nullable=False), 30 | sa.Column("role_id", sa.Integer(), nullable=False), 31 | sa.Column("permission_id", sa.Integer(), nullable=False), 32 | sa.ForeignKeyConstraint( 33 | ["created_by"], 34 | ["auth.user.id"], 35 | ), 36 | sa.ForeignKeyConstraint( 37 | ["permission_id"], 38 | ["auth.permission.id"], 39 | ), 40 | sa.ForeignKeyConstraint( 41 | ["role_id"], 42 | ["auth.role.id"], 43 | ), 44 | sa.PrimaryKeyConstraint("id"), 45 | sa.UniqueConstraint("role_id", "permission_id"), 46 | schema="auth", 47 | ) 48 | # ### end Alembic commands ### 49 | 50 | 51 | def downgrade() -> None: 52 | raise RuntimeError("Upgrades only") 53 | # ### commands auto generated by Alembic - please adjust! ### 54 | op.drop_table("role_permission", schema="auth") 55 | # ### end Alembic commands ### 56 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/503daea838dc_add_artifact_kinds.py: -------------------------------------------------------------------------------- 1 | """add artifact kinds 2 | 3 | Revision ID: 503daea838dc 4 | Revises: a4cbcbdc4a91 5 | Create Date: 2024-12-04 14:14:55.460701 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | import sqlalchemy as sa 12 | from alembic import op 13 | 14 | # revision identifiers, used by Alembic. 15 | revision: str = "503daea838dc" 16 | down_revision: Union[str, None] = "a4cbcbdc4a91" 17 | branch_labels: Union[str, Sequence[str], None] = None 18 | depends_on: Union[str, Sequence[str], None] = None 19 | 20 | 21 | KINDS = [ 22 | "NBT_STRUCTURE_FILE", 23 | "PROMPT", 24 | "ORIGINAL_BUILD_SCRIPT_JS", 25 | "ORIGINAL_BUILD_SCRIPT_PY", 26 | "RAW_RESPONSE", 27 | "BUILD_SCHEMATIC", 28 | "BUILD_COMMAND_LIST", 29 | "BUILD_SUMMARY", 30 | "COMMAND_LIST_BUILD_SCRIPT_JS", 31 | "COMMAND_LIST_BUILD_SCRIPT_PY", 32 | ] 33 | 34 | 35 | def upgrade() -> None: 36 | for kind in KINDS: 37 | op.execute( 38 | sa.text( 39 | """INSERT INTO sample.artifact_kind (name) VALUES (:artifact_kind) ON CONFLICT DO NOTHING""" 40 | ).bindparams(artifact_kind=kind) 41 | ) 42 | 43 | 44 | def downgrade() -> None: 45 | raise RuntimeError("Upgrades only") 46 | pass 47 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/5ca7be2d7150_add_new_artifact_kinds_for_comparison_.py: -------------------------------------------------------------------------------- 1 | """Add new artifact kinds for comparison artifacts 2 | 3 | Revision ID: 5ca7be2d7150 4 | Revises: dfa4bbbc994d 5 | Create Date: 2025-01-16 22:42:24.515955 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | import sqlalchemy as sa 12 | from alembic import op 13 | 14 | # revision identifiers, used by Alembic. 15 | revision: str = "5ca7be2d7150" 16 | down_revision: Union[str, None] = "dfa4bbbc994d" 17 | branch_labels: Union[str, Sequence[str], None] = None 18 | depends_on: Union[str, Sequence[str], None] = None 19 | 20 | 21 | KINDS = ["RENDERED_MODEL_GLB_COMPARISON_SAMPLE"] 22 | 23 | 24 | def upgrade() -> None: 25 | for kind in KINDS: 26 | op.execute( 27 | sa.text("""\ 28 | INSERT INTO sample.artifact_kind (name) VALUES (:artifact_kind) 29 | 30 | """).bindparams(artifact_kind=kind) 31 | ) 32 | 33 | 34 | def downgrade() -> None: 35 | raise RuntimeError("Upgrades only") 36 | pass 37 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/688482827ae4_change_openai_and_gemini_provider_class.py: -------------------------------------------------------------------------------- 1 | """change OpenAI and Gemini provider class 2 | 3 | Revision ID: 688482827ae4 4 | Revises: 9037d32770e9 5 | Create Date: 2024-12-13 18:11:51.318675 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | from alembic import op 12 | 13 | # revision identifiers, used by Alembic. 14 | revision: str = "688482827ae4" 15 | down_revision: Union[str, None] = "9037d32770e9" 16 | branch_labels: Union[str, Sequence[str], None] = None 17 | depends_on: Union[str, Sequence[str], None] = None 18 | 19 | 20 | def upgrade() -> None: 21 | op.execute("""\ 22 | INSERT INTO specification.provider_class (created_by, name) VALUES ( 23 | (select id from auth.user where username = 'SYSTEM'), 24 | 'OPENAI_SDK' 25 | ) ON CONFLICT (name) DO NOTHING; 26 | 27 | INSERT INTO specification.provider_class (created_by, name) VALUES ( 28 | (select id from auth.user where username = 'SYSTEM'), 29 | 'GEMINI_SDK' 30 | ) ON CONFLICT (name) DO NOTHING; 31 | 32 | UPDATE specification.provider 33 | SET provider_class = 'OPENAI_SDK' 34 | WHERE provider_class = 'OPEN_AI_SDK'; 35 | 36 | DELETE FROM specification.provider_class 37 | WHERE name = 'OPEN_AI_SDK'; 38 | """) 39 | 40 | 41 | def downgrade() -> None: 42 | raise RuntimeError("Upgrades only") 43 | pass 44 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/6c98791fa0b4_add_task_id_and_heartbeat_to_run_stage.py: -------------------------------------------------------------------------------- 1 | """add_task_id_and_heartbeat_to_run_stage 2 | 3 | Revision ID: 6c98791fa0b4 4 | Revises: eb378aad6fd8 5 | Create Date: 2025-03-10 14:15:16.311304 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | import sqlalchemy as sa 12 | from alembic import op 13 | 14 | # revision identifiers, used by Alembic. 15 | revision: str = "6c98791fa0b4" 16 | down_revision: Union[str, None] = "eb378aad6fd8" 17 | branch_labels: Union[str, Sequence[str], None] = None 18 | depends_on: Union[str, Sequence[str], None] = None 19 | 20 | 21 | def upgrade() -> None: 22 | # ### commands auto generated by Alembic - please adjust! ### 23 | op.add_column( 24 | "run_stage", 25 | sa.Column( 26 | "last_modified", 27 | sa.TIMESTAMP(), 28 | server_default=sa.text("now()"), 29 | nullable=True, 30 | ), 31 | schema="specification", 32 | ) 33 | op.add_column( 34 | "run_stage", 35 | sa.Column("task_id", sa.String(length=255), nullable=True), 36 | schema="specification", 37 | ) 38 | op.add_column( 39 | "run_stage", 40 | sa.Column("heartbeat", sa.TIMESTAMP(), nullable=True), 41 | schema="specification", 42 | ) 43 | # ### end Alembic commands ### 44 | 45 | 46 | def downgrade() -> None: 47 | raise RuntimeError("Upgrades only") 48 | # ### commands auto generated by Alembic - please adjust! ### 49 | op.drop_column("run_stage", "heartbeat", schema="specification") 50 | op.drop_column("run_stage", "task_id", schema="specification") 51 | op.drop_column("run_stage", "last_modified", schema="specification") 52 | # ### end Alembic commands ### 53 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/6d48eca377ff_add_proper_roles_for_writing_to_sample_.py: -------------------------------------------------------------------------------- 1 | """Add proper roles for writing to sample schema 2 | 3 | Revision ID: 6d48eca377ff 4 | Revises: e1181f674ab9 5 | Create Date: 2024-11-24 17:13:33.047217 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | from alembic import op 12 | 13 | # revision identifiers, used by Alembic. 14 | revision: str = "6d48eca377ff" 15 | down_revision: Union[str, None] = "e1181f674ab9" 16 | branch_labels: Union[str, Sequence[str], None] = None 17 | depends_on: Union[str, Sequence[str], None] = None 18 | 19 | 20 | def upgrade() -> None: 21 | op.execute("""\ 22 | ALTER DEFAULT PRIVILEGES FOR ROLE "mc-bench-admin" IN SCHEMA sample GRANT SELECT, INSERT, UPDATE ON TABLES TO "admin-worker"; 23 | ALTER DEFAULT PRIVILEGES FOR ROLE "mc-bench-admin" IN SCHEMA sample GRANT SELECT, INSERT, UPDATE ON TABLES TO "admin-api"; 24 | 25 | GRANT SELECT, INSERT, UPDATE ON ALL TABLES IN SCHEMA sample TO "admin-worker"; 26 | GRANT SELECT, INSERT, UPDATE ON ALL TABLES IN SCHEMA sample TO "admin-api"; 27 | """) 28 | 29 | 30 | def downgrade() -> None: 31 | raise RuntimeError("Upgrades only") 32 | pass 33 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/6e4874c96cbc_added_external_id_to_provider_class.py: -------------------------------------------------------------------------------- 1 | """Added external_id to provider_class 2 | 3 | Revision ID: 6e4874c96cbc 4 | Revises: 4085c38e19e8 5 | Create Date: 2024-11-19 22:51:19.701241 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | import sqlalchemy as sa 12 | from alembic import op 13 | 14 | # revision identifiers, used by Alembic. 15 | revision: str = "6e4874c96cbc" 16 | down_revision: Union[str, None] = "4085c38e19e8" 17 | branch_labels: Union[str, Sequence[str], None] = None 18 | depends_on: Union[str, Sequence[str], None] = None 19 | 20 | 21 | def upgrade() -> None: 22 | # ### commands auto generated by Alembic - please adjust! ### 23 | op.add_column( 24 | "provider_class", 25 | sa.Column( 26 | "external_id", 27 | sa.UUID(), 28 | server_default=sa.text("uuid_generate_v4()"), 29 | nullable=False, 30 | ), 31 | schema="specification", 32 | ) 33 | # ### end Alembic commands ### 34 | 35 | 36 | def downgrade() -> None: 37 | raise RuntimeError("Upgrades only") 38 | # ### commands auto generated by Alembic - please adjust! ### 39 | op.drop_column("provider_class", "external_id", schema="specification") 40 | # ### end Alembic commands ### 41 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/6ef6c3fb5778_added_a_comparison_sample_id_to_sample_.py: -------------------------------------------------------------------------------- 1 | """Added a comparison_sample_id to sample.sample for use with comparison workflows to ensure de-identified samples 2 | 3 | Revision ID: 6ef6c3fb5778 4 | Revises: 8877773aa6b6 5 | Create Date: 2024-11-27 12:16:17.791879 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | import sqlalchemy as sa 12 | from alembic import op 13 | 14 | # revision identifiers, used by Alembic. 15 | revision: str = "6ef6c3fb5778" 16 | down_revision: Union[str, None] = "8877773aa6b6" 17 | branch_labels: Union[str, Sequence[str], None] = None 18 | depends_on: Union[str, Sequence[str], None] = None 19 | 20 | 21 | def upgrade() -> None: 22 | # ### commands auto generated by Alembic - please adjust! ### 23 | op.add_column( 24 | "sample", 25 | sa.Column( 26 | "comparison_sample_id", 27 | sa.UUID(), 28 | server_default=sa.text("uuid_generate_v4()"), 29 | nullable=True, 30 | ), 31 | schema="sample", 32 | ) 33 | op.execute("UPDATE sample.sample set comparison_sample_id = uuid_generate_v4()") 34 | # ### end Alembic commands ### 35 | 36 | 37 | def downgrade() -> None: 38 | raise RuntimeError("Upgrades only") 39 | # ### commands auto generated by Alembic - please adjust! ### 40 | op.drop_column("sample", "comparison_sample_id", schema="sample") 41 | # ### end Alembic commands ### 42 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/70abece80ae7_fix_api_db_permissions.py: -------------------------------------------------------------------------------- 1 | """Fix API DB Permissions 2 | 3 | Revision ID: 70abece80ae7 4 | Revises: 2066aab9985b 5 | Create Date: 2025-02-11 16:38:00.321908 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | import sqlalchemy as sa 12 | from alembic import op 13 | 14 | # revision identifiers, used by Alembic. 15 | revision: str = "70abece80ae7" 16 | down_revision: Union[str, None] = "2066aab9985b" 17 | branch_labels: Union[str, Sequence[str], None] = None 18 | depends_on: Union[str, Sequence[str], None] = None 19 | 20 | 21 | def upgrade() -> None: 22 | op.execute( 23 | sa.text("""\ 24 | GRANT DELETE ON auth.auth_provider_email_hash TO "api"; 25 | """) 26 | ) 27 | 28 | 29 | def downgrade() -> None: 30 | raise RuntimeError("Upgrades only") 31 | pass 32 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/7a050a3306c3_add_session_and_identification_to_.py: -------------------------------------------------------------------------------- 1 | """add_session_and_identification_to_comparison 2 | 3 | Revision ID: 7a050a3306c3 4 | Revises: 019b9b3a6b6c 5 | Create Date: 2025-03-05 17:56:36.047729 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | import sqlalchemy as sa 12 | from alembic import op 13 | 14 | # revision identifiers, used by Alembic. 15 | revision: str = "7a050a3306c3" 16 | down_revision: Union[str, None] = "019b9b3a6b6c" 17 | branch_labels: Union[str, Sequence[str], None] = None 18 | depends_on: Union[str, Sequence[str], None] = None 19 | 20 | 21 | def upgrade() -> None: 22 | # ### commands auto generated by Alembic - please adjust! ### 23 | op.add_column( 24 | "comparison", 25 | sa.Column("session_id", sa.String(length=36), nullable=True), 26 | schema="scoring", 27 | ) 28 | op.add_column( 29 | "comparison", 30 | sa.Column("identification_id", sa.String(length=36), nullable=True), 31 | schema="scoring", 32 | ) 33 | # ### end Alembic commands ### 34 | 35 | 36 | def downgrade() -> None: 37 | raise RuntimeError("Upgrades only") 38 | # ### commands auto generated by Alembic - please adjust! ### 39 | op.drop_column("comparison", "identification_id", schema="scoring") 40 | op.drop_column("comparison", "session_id", schema="scoring") 41 | # ### end Alembic commands ### 42 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/848301f47914_add_run_states.py: -------------------------------------------------------------------------------- 1 | """add run states 2 | 3 | Revision ID: 848301f47914 4 | Revises: bcaf6ca2021c 5 | Create Date: 2024-12-05 12:00:17.466948 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | import sqlalchemy as sa 12 | from alembic import op 13 | 14 | # revision identifiers, used by Alembic. 15 | revision: str = "848301f47914" 16 | down_revision: Union[str, None] = "bcaf6ca2021c" 17 | branch_labels: Union[str, Sequence[str], None] = None 18 | depends_on: Union[str, Sequence[str], None] = None 19 | 20 | RUN_STATES = [ 21 | "CONTENT_EXPORT_COMPLETE", 22 | "CONTENT_EXPORT_FAILED", 23 | ] 24 | 25 | 26 | def upgrade() -> None: 27 | for run_state in RUN_STATES: 28 | op.execute( 29 | sa.text("""\ 30 | INSERT INTO specification.run_state (created_by, slug) VALUES ( 31 | (SELECT ID FROM auth."user" where username = 'SYSTEM'), 32 | :slug 33 | ) ON CONFLICT (slug) DO NOTHING 34 | 35 | """).bindparams(slug=run_state) 36 | ) 37 | 38 | 39 | def downgrade() -> None: 40 | raise RuntimeError("Upgrades only") 41 | pass 42 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/8863c7c9ffd4_add_default_test_set_id_to_generation.py: -------------------------------------------------------------------------------- 1 | """Add default_test_set_id to generation 2 | 3 | Revision ID: 8863c7c9ffd4 4 | Revises: 36407e97cfac 5 | Create Date: 2025-03-06 21:30:19.997387 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | import sqlalchemy as sa 12 | from alembic import op 13 | 14 | # revision identifiers, used by Alembic. 15 | revision: str = "8863c7c9ffd4" 16 | down_revision: Union[str, None] = "36407e97cfac" 17 | branch_labels: Union[str, Sequence[str], None] = None 18 | depends_on: Union[str, Sequence[str], None] = None 19 | 20 | 21 | def upgrade() -> None: 22 | # ### commands auto generated by Alembic - please adjust! ### 23 | op.add_column( 24 | "generation", 25 | sa.Column("default_test_set_id", sa.BigInteger(), nullable=True), 26 | schema="specification", 27 | ) 28 | op.create_foreign_key( 29 | None, 30 | "generation", 31 | "test_set", 32 | ["default_test_set_id"], 33 | ["id"], 34 | source_schema="specification", 35 | referent_schema="sample", 36 | ) 37 | # ### end Alembic commands ### 38 | 39 | 40 | def downgrade() -> None: 41 | raise RuntimeError("Upgrades only") 42 | # ### commands auto generated by Alembic - please adjust! ### 43 | op.drop_constraint(None, "generation", schema="specification", type_="foreignkey") 44 | op.drop_column("generation", "default_test_set_id", schema="specification") 45 | # ### end Alembic commands ### 46 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/8f3ea520a73e_add_sample_id_column_to_sample_artifact.py: -------------------------------------------------------------------------------- 1 | """Add sample_id column to sample.artifact 2 | 3 | Revision ID: 8f3ea520a73e 4 | Revises: 0df207ed67f9 5 | Create Date: 2024-12-01 01:19:42.936762 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | import sqlalchemy as sa 12 | from alembic import op 13 | 14 | # revision identifiers, used by Alembic. 15 | revision: str = "8f3ea520a73e" 16 | down_revision: Union[str, None] = "0df207ed67f9" 17 | branch_labels: Union[str, Sequence[str], None] = None 18 | depends_on: Union[str, Sequence[str], None] = None 19 | 20 | 21 | def upgrade() -> None: 22 | # ### commands auto generated by Alembic - please adjust! ### 23 | op.add_column( 24 | "artifact", sa.Column("sample_id", sa.Integer(), nullable=True), schema="sample" 25 | ) 26 | op.create_foreign_key( 27 | None, 28 | "artifact", 29 | "sample", 30 | ["sample_id"], 31 | ["id"], 32 | source_schema="sample", 33 | referent_schema="sample", 34 | ) 35 | # ### end Alembic commands ### 36 | 37 | 38 | def downgrade() -> None: 39 | raise RuntimeError("Upgrades only") 40 | # ### commands auto generated by Alembic - please adjust! ### 41 | op.drop_constraint(None, "artifact", schema="sample", type_="foreignkey") 42 | op.drop_column("artifact", "sample_id", schema="sample") 43 | # ### end Alembic commands ### 44 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/8fff682fdaa3_add_new_log_action.py: -------------------------------------------------------------------------------- 1 | """Add new log action 2 | 3 | Revision ID: 8fff682fdaa3 4 | Revises: 3491851af8f5 5 | Create Date: 2025-02-25 11:14:25.441110 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | import sqlalchemy as sa 12 | from alembic import op 13 | 14 | # revision identifiers, used by Alembic. 15 | revision: str = "8fff682fdaa3" 16 | down_revision: Union[str, None] = "3491851af8f5" 17 | branch_labels: Union[str, Sequence[str], None] = None 18 | depends_on: Union[str, Sequence[str], None] = None 19 | 20 | 21 | LOG_ACTIONS = [ 22 | "EXPERIMENTAL_STATE_PROPOSAL", 23 | "EXPERIMENTAL_STATE_APPROVAL", 24 | "EXPERIMENTAL_STATE_REJECTION", 25 | "PROMPT_OBSERVATION", 26 | "MODEL_OBSERVATION", 27 | "TEMPLATE_OBSERVATION", 28 | ] 29 | 30 | 31 | def upgrade() -> None: 32 | for log_action in LOG_ACTIONS: 33 | op.execute( 34 | sa.text( 35 | """INSERT INTO research.log_action (name) VALUES (:log_action) ON CONFLICT DO NOTHING""" 36 | ).bindparams(log_action=log_action) 37 | ) 38 | 39 | 40 | def downgrade() -> None: 41 | raise RuntimeError("Upgrades only") 42 | pass 43 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/9037d32770e9_fix_permissions_for_new_tables.py: -------------------------------------------------------------------------------- 1 | """fix permissions for new tables 2 | 3 | Revision ID: 9037d32770e9 4 | Revises: f5fda8b15a78 5 | Create Date: 2024-12-10 16:45:32.259956 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | from alembic import op 12 | 13 | # revision identifiers, used by Alembic. 14 | revision: str = "9037d32770e9" 15 | down_revision: Union[str, None] = "f5fda8b15a78" 16 | branch_labels: Union[str, Sequence[str], None] = None 17 | depends_on: Union[str, Sequence[str], None] = None 18 | 19 | 20 | def upgrade() -> None: 21 | op.execute("""\ 22 | ALTER DEFAULT PRIVILEGES FOR ROLE "mc-bench-admin" IN SCHEMA specification GRANT SELECT, INSERT, UPDATE ON TABLES TO "admin-worker"; 23 | GRANT SELECT, INSERT, UPDATE ON ALL TABLES IN SCHEMA specification TO "admin-worker"; 24 | """) 25 | 26 | 27 | def downgrade() -> None: 28 | raise RuntimeError("Upgrades only") 29 | pass 30 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/95ab1bc41af5_insert_github_as_auth_provider.py: -------------------------------------------------------------------------------- 1 | """Insert github as auth provider 2 | 3 | Revision ID: 95ab1bc41af5 4 | Revises: d0abd80553ec 5 | Create Date: 2024-11-15 19:46:00.828244 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | import sqlalchemy as sa 12 | from alembic import op 13 | 14 | # revision identifiers, used by Alembic. 15 | revision: str = "95ab1bc41af5" 16 | down_revision: Union[str, None] = "d0abd80553ec" 17 | branch_labels: Union[str, Sequence[str], None] = None 18 | depends_on: Union[str, Sequence[str], None] = None 19 | 20 | 21 | def upgrade() -> None: 22 | op.execute( 23 | sa.text("""\ 24 | INSERT INTO auth.user (username) values ('SYSTEM') 25 | """) 26 | ) 27 | 28 | op.execute( 29 | sa.text("""\ 30 | INSERT INTO auth.auth_provider (name, created_by) 31 | VALUES ( 32 | 'github', 33 | (SELECT id FROM auth.user WHERE username = 'SYSTEM') 34 | ) 35 | """) 36 | ) 37 | 38 | 39 | def downgrade() -> None: 40 | raise RuntimeError("Upgrades only") 41 | pass 42 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/9b7eed150ea2_make_user_id_nullable_in_comparison.py: -------------------------------------------------------------------------------- 1 | """make_user_id_nullable_in_comparison 2 | 3 | Revision ID: 9b7eed150ea2 4 | Revises: 5fdfa056d511 5 | Create Date: 2025-03-05 22:28:49.659840 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | import sqlalchemy as sa 12 | from alembic import op 13 | 14 | # revision identifiers, used by Alembic. 15 | revision: str = "9b7eed150ea2" 16 | down_revision: Union[str, None] = "5fdfa056d511" 17 | branch_labels: Union[str, Sequence[str], None] = None 18 | depends_on: Union[str, Sequence[str], None] = None 19 | 20 | 21 | def upgrade() -> None: 22 | # ### commands auto generated by Alembic - please adjust! ### 23 | op.alter_column( 24 | "comparison", 25 | "user_id", 26 | existing_type=sa.BIGINT(), 27 | nullable=True, 28 | schema="scoring", 29 | ) 30 | # ### end Alembic commands ### 31 | 32 | 33 | def downgrade() -> None: 34 | raise RuntimeError("Upgrades only") 35 | # ### commands auto generated by Alembic - please adjust! ### 36 | op.alter_column( 37 | "comparison", 38 | "user_id", 39 | existing_type=sa.BIGINT(), 40 | nullable=False, 41 | schema="scoring", 42 | ) 43 | # ### end Alembic commands ### 44 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mc-bench/mc-bench-backend/360838f5692597b2bbe1c2622e7ac60c13e1871b/src/mc_bench/migrations/versions/__init__.py -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/a0bb11b95fff_add_default_experimental_state_to_.py: -------------------------------------------------------------------------------- 1 | """Add default experimental state to existing assets 2 | 3 | Revision ID: a0bb11b95fff 4 | Revises: 5f1a3b5bd548 5 | Create Date: 2025-02-26 15:32:53.401649 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | from alembic import op 12 | 13 | # revision identifiers, used by Alembic. 14 | revision: str = "a0bb11b95fff" 15 | down_revision: Union[str, None] = "5f1a3b5bd548" 16 | branch_labels: Union[str, Sequence[str], None] = None 17 | depends_on: Union[str, Sequence[str], None] = None 18 | 19 | 20 | def upgrade() -> None: 21 | op.execute("""\ 22 | UPDATE specification.model 23 | SET experimental_state_id = ( SELECT id FROM research.experimental_state WHERE name = 'EXPERIMENTAL' ) 24 | WHERE experimental_state_id IS NULL 25 | """) 26 | 27 | op.execute("""\ 28 | UPDATE specification.prompt 29 | SET experimental_state_id = ( SELECT id FROM research.experimental_state WHERE name = 'EXPERIMENTAL' ) 30 | WHERE experimental_state_id IS NULL 31 | """) 32 | 33 | op.execute("""\ 34 | UPDATE specification.template 35 | SET experimental_state_id = ( SELECT id FROM research.experimental_state WHERE name = 'EXPERIMENTAL' ) 36 | WHERE experimental_state_id IS NULL 37 | """) 38 | 39 | op.execute("""\ 40 | UPDATE sample.sample 41 | SET experimental_state_id = ( SELECT id FROM research.experimental_state WHERE name = 'EXPERIMENTAL' ) 42 | WHERE experimental_state_id IS NULL 43 | """) 44 | 45 | 46 | def downgrade() -> None: 47 | raise RuntimeError("Upgrades only") 48 | pass 49 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/a4bee97b3630_ensure_auth_user_role_has_proper_unique_.py: -------------------------------------------------------------------------------- 1 | """ensure auth.user_role has proper unique constraint 2 | 3 | Revision ID: a4bee97b3630 4 | Revises: 7a30bccaa129 5 | Create Date: 2024-11-23 14:10:46.719025 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | from alembic import op 12 | 13 | # revision identifiers, used by Alembic. 14 | revision: str = "a4bee97b3630" 15 | down_revision: Union[str, None] = "7a30bccaa129" 16 | branch_labels: Union[str, Sequence[str], None] = None 17 | depends_on: Union[str, Sequence[str], None] = None 18 | 19 | 20 | def upgrade() -> None: 21 | # ### commands auto generated by Alembic - please adjust! ### 22 | op.create_unique_constraint( 23 | None, "user_role", ["user_id", "role_id"], schema="auth" 24 | ) 25 | # ### end Alembic commands ### 26 | 27 | 28 | def downgrade() -> None: 29 | raise RuntimeError("Upgrades only") 30 | # ### commands auto generated by Alembic - please adjust! ### 31 | op.drop_constraint(None, "user_role", schema="auth", type_="unique") 32 | # ### end Alembic commands ### 33 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/a4cbcbdc4a91_add_make_artifact_external_id_not_.py: -------------------------------------------------------------------------------- 1 | """Add make artifact.external_id not nullable 2 | 3 | Revision ID: a4cbcbdc4a91 4 | Revises: 0076eb8acb15 5 | Create Date: 2024-12-03 12:23:15.453734 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | import sqlalchemy as sa 12 | from alembic import op 13 | 14 | # revision identifiers, used by Alembic. 15 | revision: str = "a4cbcbdc4a91" 16 | down_revision: Union[str, None] = "0076eb8acb15" 17 | branch_labels: Union[str, Sequence[str], None] = None 18 | depends_on: Union[str, Sequence[str], None] = None 19 | 20 | 21 | def upgrade() -> None: 22 | # ### commands auto generated by Alembic - please adjust! ### 23 | op.alter_column( 24 | "artifact", 25 | "external_id", 26 | existing_type=sa.UUID(), 27 | nullable=False, 28 | existing_server_default=sa.text("uuid_generate_v4()"), 29 | schema="sample", 30 | ) 31 | # ### end Alembic commands ### 32 | 33 | 34 | def downgrade() -> None: 35 | raise RuntimeError("Upgrades only") 36 | # ### commands auto generated by Alembic - please adjust! ### 37 | op.alter_column( 38 | "artifact", 39 | "external_id", 40 | existing_type=sa.UUID(), 41 | nullable=True, 42 | existing_server_default=sa.text("uuid_generate_v4()"), 43 | schema="sample", 44 | ) 45 | # ### end Alembic commands ### 46 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/ab2c29f9cc0c_insert_google_as_auth_provider.py: -------------------------------------------------------------------------------- 1 | """insert_google_as_auth_provider 2 | 3 | Revision ID: ab2c29f9cc0c 4 | Revises: 8b6615c73a4f 5 | Create Date: 2025-01-27 17:31:03.414851 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | import sqlalchemy as sa 12 | from alembic import op 13 | 14 | # revision identifiers, used by Alembic. 15 | revision: str = "ab2c29f9cc0c" 16 | down_revision: Union[str, None] = "8b6615c73a4f" 17 | branch_labels: Union[str, Sequence[str], None] = None 18 | depends_on: Union[str, Sequence[str], None] = None 19 | 20 | 21 | def upgrade() -> None: 22 | op.execute( 23 | sa.text("""\ 24 | INSERT INTO auth.auth_provider (name, created_by) 25 | VALUES ('google', (SELECT id FROM auth.user WHERE username = 'SYSTEM')) 26 | """) 27 | ) 28 | pass 29 | 30 | 31 | def downgrade() -> None: 32 | raise RuntimeError("Upgrades only") 33 | pass 34 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/ac88d1e18c73_add_build_size_to_prompt.py: -------------------------------------------------------------------------------- 1 | """Add build_size to prompt 2 | 3 | Revision ID: ac88d1e18c73 4 | Revises: 41cc0a28bcde 5 | Create Date: 2025-03-01 09:09:07.998737 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | import sqlalchemy as sa 12 | from alembic import op 13 | 14 | # revision identifiers, used by Alembic. 15 | revision: str = "ac88d1e18c73" 16 | down_revision: Union[str, None] = "41cc0a28bcde" 17 | branch_labels: Union[str, Sequence[str], None] = None 18 | depends_on: Union[str, Sequence[str], None] = None 19 | 20 | 21 | def upgrade() -> None: 22 | # ### commands auto generated by Alembic - please adjust! ### 23 | op.add_column( 24 | "prompt", 25 | sa.Column("build_size", sa.String(), nullable=True), 26 | schema="specification", 27 | ) 28 | # ### end Alembic commands ### 29 | 30 | 31 | def downgrade() -> None: 32 | raise RuntimeError("Upgrades only") 33 | # ### commands auto generated by Alembic - please adjust! ### 34 | op.drop_column("prompt", "build_size", schema="specification") 35 | # ### end Alembic commands ### 36 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/ae55800708f6_add_grok_model_and_provider.py: -------------------------------------------------------------------------------- 1 | """add_grok_model_and_provider 2 | 3 | Revision ID: ae55800708f6 4 | Revises: d9dbc81fd67d 5 | Create Date: 2025-01-03 18:04:48.996163 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | from alembic import op 12 | 13 | # revision identifiers, used by Alembic. 14 | revision: str = "ae55800708f6" 15 | down_revision: Union[str, None] = "d9dbc81fd67d" 16 | branch_labels: Union[str, Sequence[str], None] = None 17 | depends_on: Union[str, Sequence[str], None] = None 18 | 19 | 20 | def upgrade() -> None: 21 | op.execute("""\ 22 | INSERT INTO specification.provider_class (created_by, name) VALUES ( 23 | (select id from auth.user where username = 'SYSTEM'), 24 | 'GROK_SDK' 25 | ) ON CONFLICT (name) DO NOTHING; 26 | """) 27 | 28 | 29 | def downgrade() -> None: 30 | raise RuntimeError("Upgrades only") 31 | pass 32 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/b118428bde0b_add_processed_comparison_table.py: -------------------------------------------------------------------------------- 1 | """add_processed_comparison_table 2 | 3 | Revision ID: b118428bde0b 4 | Revises: 47fc21ce40a3 5 | Create Date: 2025-03-06 01:05:23.273091 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | import sqlalchemy as sa 12 | from alembic import op 13 | 14 | # revision identifiers, used by Alembic. 15 | revision: str = "b118428bde0b" 16 | down_revision: Union[str, None] = "47fc21ce40a3" 17 | branch_labels: Union[str, Sequence[str], None] = None 18 | depends_on: Union[str, Sequence[str], None] = None 19 | 20 | 21 | def upgrade() -> None: 22 | # ### commands auto generated by Alembic - please adjust! ### 23 | op.create_table( 24 | "processed_comparison", 25 | sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), 26 | sa.Column( 27 | "created", sa.TIMESTAMP(), server_default=sa.text("now()"), nullable=False 28 | ), 29 | sa.Column("comparison_id", sa.Integer(), nullable=False), 30 | sa.ForeignKeyConstraint( 31 | ["comparison_id"], 32 | ["scoring.comparison.id"], 33 | ), 34 | sa.PrimaryKeyConstraint("id"), 35 | sa.UniqueConstraint("comparison_id", name="unique_processed_comparison"), 36 | schema="scoring", 37 | ) 38 | # ### end Alembic commands ### 39 | 40 | 41 | def downgrade() -> None: 42 | raise RuntimeError("Upgrades only") 43 | # ### commands auto generated by Alembic - please adjust! ### 44 | op.drop_table("processed_comparison", schema="scoring") 45 | # ### end Alembic commands ### 46 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/b241f88ec24a_add_external_id_to_experimental_state_.py: -------------------------------------------------------------------------------- 1 | """Add external_id to experimental_state table 2 | 3 | Revision ID: b241f88ec24a 4 | Revises: 00f736acf066 5 | Create Date: 2025-02-25 14:03:28.604663 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | import sqlalchemy as sa 12 | from alembic import op 13 | 14 | # revision identifiers, used by Alembic. 15 | revision: str = "b241f88ec24a" 16 | down_revision: Union[str, None] = "00f736acf066" 17 | branch_labels: Union[str, Sequence[str], None] = None 18 | depends_on: Union[str, Sequence[str], None] = None 19 | 20 | 21 | def upgrade() -> None: 22 | # ### commands auto generated by Alembic - please adjust! ### 23 | op.add_column( 24 | "experimental_state", 25 | sa.Column( 26 | "external_id", 27 | sa.UUID(), 28 | server_default=sa.text("uuid_generate_v4()"), 29 | nullable=False, 30 | ), 31 | schema="research", 32 | ) 33 | # ### end Alembic commands ### 34 | 35 | 36 | def downgrade() -> None: 37 | raise RuntimeError("Upgrades only") 38 | # ### commands auto generated by Alembic - please adjust! ### 39 | op.drop_column("experimental_state", "external_id", schema="research") 40 | # ### end Alembic commands ### 41 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/bcaf6ca2021c_add_artifact_kinds.py: -------------------------------------------------------------------------------- 1 | """add artifact kinds 2 | 3 | Revision ID: bcaf6ca2021c 4 | Revises: f17051270c7b 5 | Create Date: 2024-12-04 17:46:29.811067 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | import sqlalchemy as sa 12 | from alembic import op 13 | 14 | # revision identifiers, used by Alembic. 15 | revision: str = "bcaf6ca2021c" 16 | down_revision: Union[str, None] = "f17051270c7b" 17 | branch_labels: Union[str, Sequence[str], None] = None 18 | depends_on: Union[str, Sequence[str], None] = None 19 | 20 | KINDS = [ 21 | "NORTHSIDE_CAPTURE_PNG", 22 | "EASTSIDE_CAPTURE_PNG", 23 | "SOUTHSIDE_CAPTURE_PNG", 24 | "WESTSIDE_CAPTURE_PNG", 25 | "BUILD_CINEMATIC_MP4", 26 | ] 27 | 28 | 29 | def upgrade() -> None: 30 | for kind in KINDS: 31 | op.execute( 32 | sa.text( 33 | """INSERT INTO sample.artifact_kind (name) VALUES (:artifact_kind) ON CONFLICT DO NOTHING""" 34 | ).bindparams(artifact_kind=kind) 35 | ) 36 | 37 | 38 | def downgrade() -> None: 39 | raise RuntimeError("Upgrades only") 40 | pass 41 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/c26cc0577ee4_add_log_actions_not_kinds_active_states.py: -------------------------------------------------------------------------------- 1 | """Add log actions, not kinds, active states 2 | 3 | Revision ID: c26cc0577ee4 4 | Revises: 91d6a6a71bf0 5 | Create Date: 2025-01-20 00:15:34.576701 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | import sqlalchemy as sa 12 | from alembic import op 13 | 14 | # revision identifiers, used by Alembic. 15 | revision: str = "c26cc0577ee4" 16 | down_revision: Union[str, None] = "91d6a6a71bf0" 17 | branch_labels: Union[str, Sequence[str], None] = None 18 | depends_on: Union[str, Sequence[str], None] = None 19 | 20 | 21 | SAMPLE_APPROVAL_STATES = [ 22 | "APPROVED", 23 | "REJECTED", 24 | ] 25 | 26 | LOG_ACTIONS = [ 27 | "SAMPLE_REJECTION", 28 | "SAMPLE_APPROVAL", 29 | "SAMPLE_OBSERVATION", 30 | ] 31 | 32 | NOTE_KINDS = [ 33 | "JUSTIFICATION", 34 | "OBSERVATION", 35 | ] 36 | 37 | 38 | def upgrade() -> None: 39 | for sample_approval_state in SAMPLE_APPROVAL_STATES: 40 | op.execute( 41 | sa.text( 42 | """INSERT INTO scoring.sample_approval_state (name) VALUES (:sample_approval_state) ON CONFLICT DO NOTHING""" 43 | ).bindparams(sample_approval_state=sample_approval_state) 44 | ) 45 | 46 | for log_action in LOG_ACTIONS: 47 | op.execute( 48 | sa.text( 49 | """INSERT INTO research.log_action (name) VALUES (:log_action) ON CONFLICT DO NOTHING""" 50 | ).bindparams(log_action=log_action) 51 | ) 52 | 53 | for note_kind in NOTE_KINDS: 54 | op.execute( 55 | sa.text( 56 | """INSERT INTO research.note_kind (name) VALUES (:note_kind) ON CONFLICT DO NOTHING""" 57 | ).bindparams(note_kind=note_kind) 58 | ) 59 | 60 | 61 | def downgrade() -> None: 62 | raise RuntimeError("Upgrades only") 63 | pass 64 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/d0abd80553ec_fix_user_table.py: -------------------------------------------------------------------------------- 1 | """Fix user table 2 | 3 | Revision ID: d0abd80553ec 4 | Revises: 2c198d5ab7b5 5 | Create Date: 2024-11-15 19:42:23.345134 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | import sqlalchemy as sa 12 | from alembic import op 13 | 14 | # revision identifiers, used by Alembic. 15 | revision: str = "d0abd80553ec" 16 | down_revision: Union[str, None] = "2c198d5ab7b5" 17 | branch_labels: Union[str, Sequence[str], None] = None 18 | depends_on: Union[str, Sequence[str], None] = None 19 | 20 | 21 | def upgrade() -> None: 22 | # ### commands auto generated by Alembic - please adjust! ### 23 | op.drop_constraint( 24 | "user_created_by_fkey", "user", schema="auth", type_="foreignkey" 25 | ) 26 | op.drop_column("user", "created_by", schema="auth") 27 | # ### end Alembic commands ### 28 | 29 | 30 | def downgrade() -> None: 31 | raise RuntimeError("Upgrades only") 32 | # ### commands auto generated by Alembic - please adjust! ### 33 | op.add_column( 34 | "user", 35 | sa.Column("created_by", sa.INTEGER(), autoincrement=False, nullable=False), 36 | schema="auth", 37 | ) 38 | op.create_foreign_key( 39 | "user_created_by_fkey", 40 | "user", 41 | "user", 42 | ["created_by"], 43 | ["id"], 44 | source_schema="auth", 45 | referent_schema="auth", 46 | ) 47 | # ### end Alembic commands ### 48 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/d75b6128c146_initial_schema_creation.py: -------------------------------------------------------------------------------- 1 | """Initial Schema Creation 2 | 3 | Revision ID: d75b6128c146 4 | Revises: 5 | Create Date: 2024-11-12 00:21:01.744673 6 | 7 | """ 8 | 9 | import textwrap 10 | from typing import Sequence, Union 11 | 12 | from alembic import op 13 | 14 | # revision identifiers, used by Alembic. 15 | revision: str = "d75b6128c146" 16 | down_revision: Union[str, None] = None 17 | branch_labels: Union[str, Sequence[str], None] = None 18 | depends_on: Union[str, Sequence[str], None] = None 19 | 20 | 21 | def upgrade() -> None: 22 | op.execute( 23 | textwrap.dedent("""\ 24 | CREATE SCHEMA IF NOT EXISTS auth; 25 | CREATE SCHEMA IF NOT EXISTS sample; 26 | CREATE SCHEMA IF NOT EXISTS scoring; 27 | CREATE SCHEMA IF NOT EXISTS specification; 28 | """) 29 | ) 30 | 31 | 32 | def downgrade() -> None: 33 | raise RuntimeError("Upgrades only") 34 | pass 35 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/db79a9c13551_add_new_providers.py: -------------------------------------------------------------------------------- 1 | """Add new providers 2 | 3 | Revision ID: db79a9c13551 4 | Revises: 514a2c809b29 5 | Create Date: 2025-02-18 07:49:13.060909 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | from alembic import op 12 | 13 | # revision identifiers, used by Alembic. 14 | revision: str = "db79a9c13551" 15 | down_revision: Union[str, None] = "514a2c809b29" 16 | branch_labels: Union[str, Sequence[str], None] = None 17 | depends_on: Union[str, Sequence[str], None] = None 18 | 19 | 20 | def upgrade() -> None: 21 | op.execute("""\ 22 | INSERT INTO specification.provider_class (created_by, name) VALUES ( 23 | (select id from auth.user where username = 'SYSTEM'), 24 | 'ALIBABA_SDK' 25 | ) ON CONFLICT (name) DO NOTHING; 26 | """) 27 | 28 | op.execute("""\ 29 | INSERT INTO specification.provider_class (created_by, name) VALUES ( 30 | (select id from auth.user where username = 'SYSTEM'), 31 | 'MISTRAL_SDK' 32 | ) ON CONFLICT (name) DO NOTHING; 33 | """) 34 | 35 | op.execute("""\ 36 | INSERT INTO specification.provider_class (created_by, name) VALUES ( 37 | (select id from auth.user where username = 'SYSTEM'), 38 | 'ZHIPUAI_SDK' 39 | ) ON CONFLICT (name) DO NOTHING; 40 | """) 41 | 42 | op.execute("""\ 43 | INSERT INTO specification.provider_class (created_by, name) VALUES ( 44 | (select id from auth.user where username = 'SYSTEM'), 45 | 'REKA_SDK' 46 | ) ON CONFLICT (name) DO NOTHING; 47 | """) 48 | 49 | 50 | def downgrade() -> None: 51 | raise RuntimeError("Upgrades only") 52 | pass 53 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/dfa4bbbc994d_make_template_minecraft_version_not_.py: -------------------------------------------------------------------------------- 1 | """Make template.minecraft_version not nullable 2 | 3 | Revision ID: dfa4bbbc994d 4 | Revises: f1b7837b000d 5 | Create Date: 2025-01-15 22:39:24.146392 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | import sqlalchemy as sa 12 | from alembic import op 13 | 14 | # revision identifiers, used by Alembic. 15 | revision: str = "dfa4bbbc994d" 16 | down_revision: Union[str, None] = "f1b7837b000d" 17 | branch_labels: Union[str, Sequence[str], None] = None 18 | depends_on: Union[str, Sequence[str], None] = None 19 | 20 | 21 | def upgrade() -> None: 22 | # ### commands auto generated by Alembic - please adjust! ### 23 | op.alter_column( 24 | "template", 25 | "minecraft_version", 26 | existing_type=sa.VARCHAR(), 27 | nullable=False, 28 | schema="specification", 29 | ) 30 | # ### end Alembic commands ### 31 | 32 | 33 | def downgrade() -> None: 34 | raise RuntimeError("Upgrades only") 35 | # ### commands auto generated by Alembic - please adjust! ### 36 | op.alter_column( 37 | "template", 38 | "minecraft_version", 39 | existing_type=sa.VARCHAR(), 40 | nullable=True, 41 | schema="specification", 42 | ) 43 | # ### end Alembic commands ### 44 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/e1181f674ab9_add_additional_run_state_states.py: -------------------------------------------------------------------------------- 1 | """Add additional run_state states 2 | 3 | Revision ID: e1181f674ab9 4 | Revises: ab2b7d5586e0 5 | Create Date: 2024-11-24 13:01:06.227978 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | import sqlalchemy as sa 12 | from alembic import op 13 | 14 | # revision identifiers, used by Alembic. 15 | revision: str = "e1181f674ab9" 16 | down_revision: Union[str, None] = "ab2b7d5586e0" 17 | branch_labels: Union[str, Sequence[str], None] = None 18 | depends_on: Union[str, Sequence[str], None] = None 19 | 20 | RUN_STATES = [ 21 | "PROMPT_PROCESSING_ENQUEUED", 22 | "PROMPT_PROCESSING_COMPLETED", 23 | "PROMPT_PROCESSING_FAILED", 24 | ] 25 | 26 | 27 | def upgrade() -> None: 28 | for run_state in RUN_STATES: 29 | op.execute( 30 | sa.text("""\ 31 | INSERT INTO specification.run_state (created_by, slug) VALUES ( 32 | (SELECT ID FROM auth."user" where username = 'SYSTEM'), 33 | :slug 34 | ) ON CONFLICT (slug) DO NOTHING 35 | 36 | """).bindparams(slug=run_state) 37 | ) 38 | 39 | 40 | def downgrade() -> None: 41 | raise RuntimeError("Upgrades only") 42 | pass 43 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/eb378aad6fd8_add_enqueued_run_stage_state.py: -------------------------------------------------------------------------------- 1 | """add_enqueued_run_stage_state 2 | 3 | Revision ID: eb378aad6fd8 4 | Revises: e69015b27346 5 | Create Date: 2025-03-10 12:25:25.233461 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | import sqlalchemy as sa 12 | from alembic import op 13 | 14 | # revision identifiers, used by Alembic. 15 | revision: str = "eb378aad6fd8" 16 | down_revision: Union[str, None] = "e69015b27346" 17 | branch_labels: Union[str, Sequence[str], None] = None 18 | depends_on: Union[str, Sequence[str], None] = None 19 | 20 | 21 | def upgrade() -> None: 22 | # Add ENQUEUED state to run_stage_state table 23 | op.execute( 24 | sa.text( 25 | """\ 26 | INSERT INTO specification.run_stage_state (created_by, slug) values ( 27 | (SELECT id from auth.user where username = 'SYSTEM'), 28 | :slug 29 | ) 30 | ON CONFLICT (slug) DO NOTHING 31 | """ 32 | ).bindparams(slug="ENQUEUED") 33 | ) 34 | 35 | 36 | def downgrade() -> None: 37 | raise RuntimeError("Upgrades only") 38 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/f0186a991fa2_initial_extension_setup.py: -------------------------------------------------------------------------------- 1 | """Initial Extension Setup 2 | 3 | Revision ID: f0186a991fa2 4 | Revises: 1ddb45880af9 5 | Create Date: 2024-11-12 00:53:36.411788 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | from alembic import op 12 | 13 | # revision identifiers, used by Alembic. 14 | revision: str = "f0186a991fa2" 15 | down_revision: Union[str, None] = "1ddb45880af9" 16 | branch_labels: Union[str, Sequence[str], None] = None 17 | depends_on: Union[str, Sequence[str], None] = None 18 | 19 | 20 | def upgrade() -> None: 21 | op.execute('CREATE EXTENSION IF NOT EXISTS "uuid-ossp"') 22 | 23 | 24 | def downgrade() -> None: 25 | raise RuntimeError("Upgrades only") 26 | pass 27 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/f17051270c7b_add_artifact_kinds.py: -------------------------------------------------------------------------------- 1 | """add artifact kinds 2 | 3 | Revision ID: f17051270c7b 4 | Revises: 503daea838dc 5 | Create Date: 2024-12-04 15:50:58.458347 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | import sqlalchemy as sa 12 | from alembic import op 13 | 14 | # revision identifiers, used by Alembic. 15 | revision: str = "f17051270c7b" 16 | down_revision: Union[str, None] = "503daea838dc" 17 | branch_labels: Union[str, Sequence[str], None] = None 18 | depends_on: Union[str, Sequence[str], None] = None 19 | 20 | KINDS = [ 21 | "CONTENT_EXPORT_BUILD_SCRIPT_JS", 22 | "CONTENT_EXPORT_BUILD_SCRIPT_PY", 23 | ] 24 | 25 | 26 | def upgrade() -> None: 27 | for kind in KINDS: 28 | op.execute( 29 | sa.text( 30 | """INSERT INTO sample.artifact_kind (name) VALUES (:artifact_kind) ON CONFLICT DO NOTHING""" 31 | ).bindparams(artifact_kind=kind) 32 | ) 33 | 34 | 35 | def downgrade() -> None: 36 | raise RuntimeError("Upgrades only") 37 | pass 38 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/f1b7837b000d_add_minecraft_version_to_template.py: -------------------------------------------------------------------------------- 1 | """Add minecraft_version to template 2 | 3 | Revision ID: f1b7837b000d 4 | Revises: 163167df595b 5 | Create Date: 2025-01-15 22:37:33.119661 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | import sqlalchemy as sa 12 | from alembic import op 13 | 14 | # revision identifiers, used by Alembic. 15 | revision: str = "f1b7837b000d" 16 | down_revision: Union[str, None] = "163167df595b" 17 | branch_labels: Union[str, Sequence[str], None] = None 18 | depends_on: Union[str, Sequence[str], None] = None 19 | 20 | 21 | def upgrade() -> None: 22 | # ### commands auto generated by Alembic - please adjust! ### 23 | op.add_column( 24 | "template", 25 | sa.Column("minecraft_version", sa.String(), nullable=True), 26 | schema="specification", 27 | ) 28 | op.execute("UPDATE specification.template SET minecraft_version = '1.21.1'") 29 | # ### end Alembic commands ### 30 | 31 | 32 | def downgrade() -> None: 33 | raise RuntimeError("Upgrades only") 34 | # ### commands auto generated by Alembic - please adjust! ### 35 | op.drop_column("template", "minecraft_version", schema="specification") 36 | # ### end Alembic commands ### 37 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/f37ddbc4738c_fix_research_note_deleted_and_research_.py: -------------------------------------------------------------------------------- 1 | """Fix research.note.deleted and research.note.deleted_by nullability 2 | 3 | Revision ID: f37ddbc4738c 4 | Revises: 2dd0e918c173 5 | Create Date: 2025-01-20 16:32:27.944103 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | import sqlalchemy as sa 12 | from alembic import op 13 | from sqlalchemy.dialects import postgresql 14 | 15 | # revision identifiers, used by Alembic. 16 | revision: str = "f37ddbc4738c" 17 | down_revision: Union[str, None] = "2dd0e918c173" 18 | branch_labels: Union[str, Sequence[str], None] = None 19 | depends_on: Union[str, Sequence[str], None] = None 20 | 21 | 22 | def upgrade() -> None: 23 | # ### commands auto generated by Alembic - please adjust! ### 24 | op.alter_column( 25 | "note", 26 | "deleted", 27 | existing_type=postgresql.TIMESTAMP(), 28 | nullable=True, 29 | server_default=None, 30 | existing_server_default=sa.text("now()"), 31 | schema="research", 32 | ) 33 | op.alter_column( 34 | "note", 35 | "deleted_by", 36 | existing_type=sa.INTEGER(), 37 | nullable=True, 38 | schema="research", 39 | ) 40 | # ### end Alembic commands ### 41 | 42 | 43 | def downgrade() -> None: 44 | raise RuntimeError("Upgrades only") 45 | # ### commands auto generated by Alembic - please adjust! ### 46 | op.alter_column( 47 | "note", 48 | "deleted_by", 49 | existing_type=sa.INTEGER(), 50 | nullable=False, 51 | schema="research", 52 | ) 53 | op.alter_column( 54 | "note", 55 | "deleted", 56 | existing_type=postgresql.TIMESTAMP(), 57 | nullable=False, 58 | existing_server_default=sa.text("now()"), 59 | schema="research", 60 | ) 61 | # ### end Alembic commands ### 62 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/f5fda8b15a78_add_run_stage_stage_slug_for_.py: -------------------------------------------------------------------------------- 1 | """add run_stage.stage_slug for polymorphism 2 | 3 | Revision ID: f5fda8b15a78 4 | Revises: 665d4b9d70a6 5 | Create Date: 2024-12-09 02:24:05.746839 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | import sqlalchemy as sa 12 | from alembic import op 13 | 14 | # revision identifiers, used by Alembic. 15 | revision: str = "f5fda8b15a78" 16 | down_revision: Union[str, None] = "665d4b9d70a6" 17 | branch_labels: Union[str, Sequence[str], None] = None 18 | depends_on: Union[str, Sequence[str], None] = None 19 | 20 | 21 | def upgrade() -> None: 22 | # ### commands auto generated by Alembic - please adjust! ### 23 | op.add_column( 24 | "run_stage", 25 | sa.Column("stage_slug", sa.String(length=255), nullable=False), 26 | schema="specification", 27 | ) 28 | op.create_foreign_key( 29 | None, 30 | "run_stage", 31 | "stage", 32 | ["stage_slug"], 33 | ["slug"], 34 | source_schema="specification", 35 | referent_schema="specification", 36 | ) 37 | # ### end Alembic commands ### 38 | 39 | 40 | def downgrade() -> None: 41 | raise RuntimeError("Upgrades only") 42 | # ### commands auto generated by Alembic - please adjust! ### 43 | op.drop_constraint(None, "run_stage", schema="specification", type_="foreignkey") 44 | op.drop_column("run_stage", "stage_slug", schema="specification") 45 | # ### end Alembic commands ### 46 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/f67f7ce9da43_implement_cascade_delete_on_provider_.py: -------------------------------------------------------------------------------- 1 | """Implement cascade delete on provider->model 2 | 3 | Revision ID: f67f7ce9da43 4 | Revises: a0225a967e80 5 | Create Date: 2024-11-20 13:18:19.291661 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | from alembic import op 12 | 13 | # revision identifiers, used by Alembic. 14 | revision: str = "f67f7ce9da43" 15 | down_revision: Union[str, None] = "a0225a967e80" 16 | branch_labels: Union[str, Sequence[str], None] = None 17 | depends_on: Union[str, Sequence[str], None] = None 18 | 19 | 20 | def upgrade() -> None: 21 | # ### commands auto generated by Alembic - please adjust! ### 22 | op.drop_constraint( 23 | "provider_model_id_fkey", "provider", schema="specification", type_="foreignkey" 24 | ) 25 | # op.execute("""\ 26 | # ALTER TABLE specification.provider 27 | # ADD CONSTRAINT provider_model_id_fkey 28 | # FOREIGN KEY (model_id) 29 | # REFERENCES specification.model(id) 30 | # ON DELETE CASCADE; 31 | # """) 32 | op.create_foreign_key( 33 | None, 34 | "provider", 35 | "model", 36 | ["model_id"], 37 | ["id"], 38 | source_schema="specification", 39 | referent_schema="specification", 40 | ondelete="CASCADE", 41 | ) 42 | # ### end Alembic commands ### 43 | 44 | 45 | def downgrade() -> None: 46 | raise RuntimeError("Upgrades only") 47 | # ### commands auto generated by Alembic - please adjust! ### 48 | op.drop_constraint(None, "provider", schema="specification", type_="foreignkey") 49 | op.create_foreign_key( 50 | "provider_model_id_fkey", 51 | "provider", 52 | "model", 53 | ["model_id"], 54 | ["id"], 55 | source_schema="specification", 56 | referent_schema="specification", 57 | ) 58 | # ### end Alembic commands ### 59 | -------------------------------------------------------------------------------- /src/mc_bench/migrations/versions/ffb71aaa0034_added_active_field_to_model_table.py: -------------------------------------------------------------------------------- 1 | """Added active field to model table 2 | 3 | Revision ID: ffb71aaa0034 4 | Revises: 6e4874c96cbc 5 | Create Date: 2024-11-20 10:51:30.073357 6 | 7 | """ 8 | 9 | from typing import Sequence, Union 10 | 11 | import sqlalchemy as sa 12 | from alembic import op 13 | 14 | # revision identifiers, used by Alembic. 15 | revision: str = "ffb71aaa0034" 16 | down_revision: Union[str, None] = "6e4874c96cbc" 17 | branch_labels: Union[str, Sequence[str], None] = None 18 | depends_on: Union[str, Sequence[str], None] = None 19 | 20 | 21 | def upgrade() -> None: 22 | # ### commands auto generated by Alembic - please adjust! ### 23 | op.add_column( 24 | "model", 25 | sa.Column("active", sa.Boolean(), nullable=True), 26 | schema="specification", 27 | ) 28 | # ### end Alembic commands ### 29 | 30 | 31 | def downgrade() -> None: 32 | raise RuntimeError("Upgrades only") 33 | # ### commands auto generated by Alembic - please adjust! ### 34 | op.drop_column("model", "active", schema="specification") 35 | # ### end Alembic commands ### 36 | -------------------------------------------------------------------------------- /src/mc_bench/minecraft/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /src/mc_bench/minecraft/rendering/cache.py: -------------------------------------------------------------------------------- 1 | import abc 2 | 3 | 4 | class AbstractTextureCache(abc.ABC): 5 | @abc.abstractmethod 6 | def get_texture(self, name: str): 7 | pass 8 | 9 | @abc.abstractmethod 10 | def put_texture(self, name: str, image_data: bytes): 11 | pass 12 | -------------------------------------------------------------------------------- /src/mc_bench/minecraft/server/__main__.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | 4 | import mc_bench.minecraft.server 5 | from mc_bench.minecraft.server import wait_for_server 6 | from mc_bench.util.logging import get_logger 7 | 8 | logger = get_logger(__name__) 9 | 10 | SERVER_IMAGE = os.environ.get( 11 | "MINECRAFT_SERVER_IMAGE", 12 | "registry.digitalocean.com/mcbench/gameservers:minecraft-1.21.1-latest", 13 | ) 14 | 15 | 16 | def get_parser(): 17 | parser = argparse.ArgumentParser() 18 | parser.add_argument("--port", type=int, default=25565) 19 | parser.add_argument("--name", type=str, default="local-minecraft-server") 20 | parser.add_argument("--replace", action="store_true") 21 | parser.add_argument("--version", type=str, default="1.21.1") 22 | return parser 23 | 24 | 25 | def main(options): 26 | network = mc_bench.minecraft.server.create_network(options.name, exists_ok=True) 27 | 28 | result = mc_bench.minecraft.server.start_server( 29 | image=f"registry.digitalocean.com/mcbench/gameservers:minecraft-{options.version}-latest", 30 | network_name=network, 31 | suffix=options.name, 32 | ports={ 33 | "25565/tcp": options.port, 34 | }, 35 | replace=options.replace, 36 | ) 37 | logger.info("Waiting for server to start...") 38 | wait_for_server(result.id) 39 | logger.info("Server started...") 40 | 41 | logger.info("Server started", result=result) 42 | 43 | 44 | if __name__ == "__main__": 45 | parser = get_parser() 46 | options = parser.parse_args() 47 | main(options) 48 | -------------------------------------------------------------------------------- /src/mc_bench/models/__init__.py: -------------------------------------------------------------------------------- 1 | from . import ( 2 | experimental_state, 3 | log, 4 | model, 5 | prompt, 6 | provider, 7 | run, 8 | scheduler_control, 9 | template, 10 | user, 11 | ) 12 | 13 | __all__ = [ 14 | "model", 15 | "prompt", 16 | "provider", 17 | "run", 18 | "template", 19 | "user", 20 | "log", 21 | "experimental_state", 22 | "scheduler_control", 23 | ] 24 | -------------------------------------------------------------------------------- /src/mc_bench/models/_base.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy.orm import declarative_base 2 | 3 | from mc_bench.schema.postgres import metadata 4 | 5 | Base = declarative_base( 6 | metadata=metadata, 7 | ) 8 | -------------------------------------------------------------------------------- /src/mc_bench/models/experimental_state.py: -------------------------------------------------------------------------------- 1 | from typing import Dict 2 | 3 | from sqlalchemy import select 4 | 5 | import mc_bench.schema.postgres as schema 6 | from mc_bench.constants import EXPERIMENTAL_STATE 7 | 8 | from ._base import Base 9 | 10 | _experimental_state_cache: Dict[EXPERIMENTAL_STATE, int] = {} 11 | 12 | 13 | def experimental_state_id_for(db, state: EXPERIMENTAL_STATE): 14 | if state not in _experimental_state_cache: 15 | _experimental_state_cache[state] = db.scalar( 16 | select(ExperimentalState.id).where(ExperimentalState.name == state.value) 17 | ) 18 | return _experimental_state_cache[state] 19 | 20 | 21 | class ExperimentalState(Base): 22 | __table__ = schema.research.experimental_state 23 | 24 | def to_dict(self): 25 | return { 26 | "id": self.external_id, 27 | "name": self.name, 28 | } 29 | -------------------------------------------------------------------------------- /src/mc_bench/models/provider/README.md: -------------------------------------------------------------------------------- 1 | # Adding new provider clases 2 | 3 | 1. First implement the class and decide on the `provider_class` key. Set the `polymorphic_identity` to that key. 4 | 5 | ```python 6 | 7 | from ._base import Provider 8 | 9 | 10 | class MyNewProvider(Provider): 11 | 12 | __mapper_args__ = {'polymorphic_identity': 'MY_NEW_PROVIDER'} 13 | ``` 14 | 15 | 2. Create a database migration (see e.g. [4085c38e19e8_add_provider_class_rows.py](../../migrations/versions/4085c38e19e8_add_provider_class_rows.py)) 16 | -------------------------------------------------------------------------------- /src/mc_bench/models/provider/__init__.py: -------------------------------------------------------------------------------- 1 | from ._base import Provider 2 | from .alibaba import AlibabaProvider 3 | from .anthropic import AnthropicProvider 4 | from .deepseek import DeepSeekProvider 5 | from .gemini import GeminiProvider 6 | from .grok import GrokProvider 7 | from .mistral import MistralProvider 8 | from .openai import OpenAIProvider 9 | from .openrouter import OpenRouterProvider 10 | from .reka import RekaProvider 11 | from .zhipuai import ZhipuAIProvider 12 | 13 | __all__ = [ 14 | "Provider", 15 | "AlibabaProvider", 16 | "AnthropicProvider", 17 | "DeepSeekProvider", 18 | "GeminiProvider", 19 | "GrokProvider", 20 | "MistralProvider", 21 | "OpenAIProvider", 22 | "OpenRouterProvider", 23 | "RekaProvider", 24 | "ZhipuAIProvider", 25 | ] 26 | -------------------------------------------------------------------------------- /src/mc_bench/models/provider/_base.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from sqlalchemy.orm import Mapped, relationship 4 | 5 | import mc_bench.schema.postgres as schema 6 | 7 | from .._base import Base 8 | 9 | 10 | class Provider(Base): 11 | __table__ = schema.specification.provider 12 | 13 | model: Mapped["Model"] = relationship( # noqa: F821 14 | "Model", lazy="joined", back_populates="providers" 15 | ) 16 | 17 | def to_dict(self): 18 | return { 19 | "id": self.external_id, 20 | "name": self.name, 21 | "provider_class": self.provider_class, 22 | "config": json.loads(self.config) 23 | if isinstance(self.config, str) 24 | else self.config, 25 | "is_default": bool(self.is_default), 26 | } 27 | 28 | __mapper_args__ = { 29 | "polymorphic_on": "provider_class", 30 | } 31 | 32 | def execute_prompt(self, prompt): 33 | client = self.get_client() 34 | kwargs = ( 35 | json.loads(self.config) if isinstance(self.config, str) else self.config 36 | ).copy() 37 | kwargs["prompt"] = prompt 38 | return client.send_prompt(**kwargs) 39 | -------------------------------------------------------------------------------- /src/mc_bench/models/provider/alibaba.py: -------------------------------------------------------------------------------- 1 | from ._base import Provider 2 | 3 | 4 | class AlibabaProvider(Provider): 5 | __mapper_args__ = {"polymorphic_identity": "ALIBABA_SDK"} 6 | 7 | def get_client(self): 8 | from mc_bench.clients.alibaba_cloud import AlibabaCloudClient 9 | 10 | return AlibabaCloudClient() 11 | -------------------------------------------------------------------------------- /src/mc_bench/models/provider/anthropic.py: -------------------------------------------------------------------------------- 1 | from ._base import Provider 2 | 3 | 4 | class AnthropicProvider(Provider): 5 | __mapper_args__ = {"polymorphic_identity": "ANTHROPIC_SDK"} 6 | 7 | def get_client(self): 8 | from mc_bench.clients.anthropic import AnthropicClient 9 | 10 | return AnthropicClient() 11 | -------------------------------------------------------------------------------- /src/mc_bench/models/provider/deepseek.py: -------------------------------------------------------------------------------- 1 | from ._base import Provider 2 | 3 | 4 | class DeepSeekProvider(Provider): 5 | __mapper_args__ = {"polymorphic_identity": "DEEPSEEK_SDK"} 6 | 7 | def get_client(self): 8 | from mc_bench.clients.deepseek import DeepSeekClient 9 | 10 | return DeepSeekClient() 11 | -------------------------------------------------------------------------------- /src/mc_bench/models/provider/gemini.py: -------------------------------------------------------------------------------- 1 | from ._base import Provider 2 | 3 | 4 | class GeminiProvider(Provider): 5 | __mapper_args__ = {"polymorphic_identity": "GEMINI_SDK"} 6 | 7 | def get_client(self): 8 | from mc_bench.clients.gemini import GeminiClient 9 | 10 | return GeminiClient() 11 | -------------------------------------------------------------------------------- /src/mc_bench/models/provider/grok.py: -------------------------------------------------------------------------------- 1 | from ._base import Provider 2 | 3 | 4 | class GrokProvider(Provider): 5 | __mapper_args__ = {"polymorphic_identity": "GROK_SDK"} 6 | 7 | def get_client(self): 8 | from mc_bench.clients.grok import GrokClient 9 | 10 | return GrokClient() 11 | -------------------------------------------------------------------------------- /src/mc_bench/models/provider/mistral.py: -------------------------------------------------------------------------------- 1 | from ._base import Provider 2 | 3 | 4 | class MistralProvider(Provider): 5 | __mapper_args__ = {"polymorphic_identity": "MISTRAL_SDK"} 6 | 7 | def get_client(self): 8 | from mc_bench.clients.mistral import MistralClient 9 | 10 | return MistralClient() 11 | -------------------------------------------------------------------------------- /src/mc_bench/models/provider/openai.py: -------------------------------------------------------------------------------- 1 | from ._base import Provider 2 | 3 | 4 | class OpenAIProvider(Provider): 5 | __mapper_args__ = {"polymorphic_identity": "OPENAI_SDK"} 6 | 7 | def get_client(self): 8 | from mc_bench.clients.openai import OpenAIClient 9 | 10 | return OpenAIClient() 11 | -------------------------------------------------------------------------------- /src/mc_bench/models/provider/openrouter.py: -------------------------------------------------------------------------------- 1 | from ._base import Provider 2 | 3 | 4 | class OpenRouterProvider(Provider): 5 | __mapper_args__ = {"polymorphic_identity": "OPENROUTER_SDK"} 6 | 7 | def get_client(self): 8 | from mc_bench.clients.openrouter import OpenRouterClient 9 | 10 | return OpenRouterClient() 11 | -------------------------------------------------------------------------------- /src/mc_bench/models/provider/reka.py: -------------------------------------------------------------------------------- 1 | from ._base import Provider 2 | 3 | 4 | class RekaProvider(Provider): 5 | __mapper_args__ = {"polymorphic_identity": "REKA_SDK"} 6 | 7 | def get_client(self): 8 | from mc_bench.clients.reka import RekaClient 9 | 10 | return RekaClient() 11 | -------------------------------------------------------------------------------- /src/mc_bench/models/provider/zhipuai.py: -------------------------------------------------------------------------------- 1 | from ._base import Provider 2 | 3 | 4 | class ZhipuAIProvider(Provider): 5 | __mapper_args__ = {"polymorphic_identity": "ZHIPUAI_SDK"} 6 | 7 | def get_client(self): 8 | from mc_bench.clients.zhipuai import ZhipuAIClient 9 | 10 | return ZhipuAIClient() 11 | -------------------------------------------------------------------------------- /src/mc_bench/schema/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mc-bench/mc-bench-backend/360838f5692597b2bbe1c2622e7ac60c13e1871b/src/mc_bench/schema/__init__.py -------------------------------------------------------------------------------- /src/mc_bench/schema/object_store/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mc-bench/mc-bench-backend/360838f5692597b2bbe1c2622e7ac60c13e1871b/src/mc_bench/schema/object_store/__init__.py -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/__init__.py: -------------------------------------------------------------------------------- 1 | from . import auth, research, sample, scoring, specification 2 | from ._metadata import metadata 3 | 4 | __all__ = [ 5 | "auth", 6 | "metadata", 7 | "sample", 8 | "specification", 9 | "scoring", 10 | "research", 11 | ] 12 | -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/_metadata.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import MetaData 2 | 3 | metadata = MetaData() 4 | -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/auth/__init__.py: -------------------------------------------------------------------------------- 1 | from ._auth_provider import auth_provider 2 | from ._auth_provider_email_hash import auth_provider_email_hash 3 | from ._permission import permission 4 | from ._role import role 5 | from ._role_permission import role_permission 6 | from ._user import user 7 | from ._user_identification_token import user_identification_token 8 | from ._user_role import user_role 9 | 10 | __all__ = [ 11 | "auth_provider", 12 | "auth_provider_email_hash", 13 | "user", 14 | "permission", 15 | "role", 16 | "role_permission", 17 | "user_role", 18 | "user_identification_token", 19 | ] 20 | -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/auth/_auth_provider.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import ( 2 | TIMESTAMP, 3 | BigInteger, 4 | Column, 5 | ForeignKey, 6 | Integer, 7 | String, 8 | Table, 9 | func, 10 | ) 11 | 12 | from .._metadata import metadata 13 | 14 | auth_provider = Table( 15 | "auth_provider", 16 | metadata, 17 | Column("id", BigInteger, primary_key=True, autoincrement=True), 18 | Column( 19 | "created", TIMESTAMP(timezone=False), server_default=func.now(), nullable=False 20 | ), 21 | Column("created_by", Integer, ForeignKey("auth.user.id"), nullable=False), 22 | Column("last_modified", TIMESTAMP(timezone=False), nullable=True), 23 | Column("last_modified_by", Integer, ForeignKey("auth.user.id"), nullable=True), 24 | Column("name", String, nullable=False, unique=True), 25 | schema="auth", 26 | ) 27 | -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/auth/_auth_provider_email_hash.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import ( 2 | TIMESTAMP, 3 | BigInteger, 4 | Column, 5 | ForeignKey, 6 | Integer, 7 | String, 8 | Table, 9 | UniqueConstraint, 10 | func, 11 | ) 12 | 13 | from .._metadata import metadata 14 | 15 | auth_provider_email_hash = Table( 16 | "auth_provider_email_hash", 17 | metadata, 18 | Column("id", BigInteger, primary_key=True, autoincrement=True), 19 | Column( 20 | "created", TIMESTAMP(timezone=False), server_default=func.now(), nullable=False 21 | ), 22 | Column( 23 | "auth_provider_id", Integer, ForeignKey("auth.auth_provider.id"), nullable=False 24 | ), 25 | Column("auth_provider_user_id", String, nullable=False), 26 | Column("user_id", Integer, ForeignKey("auth.user.id"), nullable=False), 27 | Column("email_hash", String, nullable=False), 28 | # We never want to permit a duplicate email hash from the same auth provider 29 | UniqueConstraint("auth_provider_id", "email_hash"), 30 | schema="auth", 31 | ) 32 | -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/auth/_permission.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import ( 2 | TIMESTAMP, 3 | BigInteger, 4 | Column, 5 | ForeignKey, 6 | Integer, 7 | String, 8 | Table, 9 | func, 10 | ) 11 | 12 | from .._metadata import metadata 13 | 14 | permission = Table( 15 | "permission", 16 | metadata, 17 | Column("id", BigInteger, primary_key=True, autoincrement=True), 18 | Column( 19 | "created", TIMESTAMP(timezone=False), server_default=func.now(), nullable=False 20 | ), 21 | Column("created_by", Integer, ForeignKey("auth.user.id"), nullable=False), 22 | Column("name", String(64), unique=True, nullable=False), 23 | schema="auth", 24 | ) 25 | -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/auth/_role.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import ( 2 | TIMESTAMP, 3 | UUID, 4 | BigInteger, 5 | Column, 6 | ForeignKey, 7 | Integer, 8 | String, 9 | Table, 10 | func, 11 | text, 12 | ) 13 | 14 | from .._metadata import metadata 15 | 16 | role = Table( 17 | "role", 18 | metadata, 19 | Column("id", BigInteger, primary_key=True, autoincrement=True), 20 | Column( 21 | "external_id", UUID, nullable=False, server_default=text("uuid_generate_v4()") 22 | ), 23 | Column( 24 | "created", TIMESTAMP(timezone=False), server_default=func.now(), nullable=False 25 | ), 26 | Column("created_by", Integer, ForeignKey("auth.user.id"), nullable=False), 27 | Column("last_modified", TIMESTAMP(timezone=False), nullable=True), 28 | Column("last_modified_by", Integer, ForeignKey("auth.user.id"), nullable=True), 29 | Column("name", String, unique=True, nullable=False), 30 | schema="auth", 31 | ) 32 | -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/auth/_role_permission.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import ( 2 | TIMESTAMP, 3 | BigInteger, 4 | Column, 5 | ForeignKey, 6 | Integer, 7 | Table, 8 | UniqueConstraint, 9 | func, 10 | ) 11 | 12 | from .._metadata import metadata 13 | 14 | role_permission = Table( 15 | "role_permission", 16 | metadata, 17 | Column("id", BigInteger, primary_key=True, autoincrement=True), 18 | Column( 19 | "created", TIMESTAMP(timezone=False), server_default=func.now(), nullable=False 20 | ), 21 | Column("created_by", Integer, ForeignKey("auth.user.id"), nullable=False), 22 | Column("role_id", Integer, ForeignKey("auth.role.id"), nullable=False), 23 | Column("permission_id", Integer, ForeignKey("auth.permission.id"), nullable=False), 24 | UniqueConstraint("role_id", "permission_id"), 25 | schema="auth", 26 | ) 27 | -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/auth/_user.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import ( 2 | TIMESTAMP, 3 | UUID, 4 | BigInteger, 5 | Column, 6 | ForeignKey, 7 | Integer, 8 | String, 9 | Table, 10 | func, 11 | text, 12 | ) 13 | 14 | from .._metadata import metadata 15 | 16 | user = Table( 17 | "user", 18 | metadata, 19 | Column("id", BigInteger, primary_key=True, autoincrement=True), 20 | Column( 21 | "created", TIMESTAMP(timezone=False), server_default=func.now(), nullable=False 22 | ), 23 | Column("last_modified", TIMESTAMP(timezone=False), nullable=True), 24 | Column("last_modified_by", Integer, ForeignKey("auth.user.id"), nullable=True), 25 | Column( 26 | "external_id", UUID, nullable=False, server_default=text("uuid_generate_v4()") 27 | ), 28 | Column("username", String(64), nullable=True, unique=True, index=True), 29 | Column("username_normalized", String(64), nullable=True, unique=True, index=True), 30 | Column("display_username", String(64), nullable=True, unique=True, index=True), 31 | Column( 32 | "canonical_identification_token_id", 33 | Integer, 34 | ForeignKey("auth.user_identification_token.id"), 35 | nullable=True, 36 | ), 37 | schema="auth", 38 | ) 39 | -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/auth/_user_identification_token.py: -------------------------------------------------------------------------------- 1 | """ """ 2 | 3 | from sqlalchemy import ( 4 | TIMESTAMP, 5 | UUID, 6 | Column, 7 | ForeignKey, 8 | Integer, 9 | Table, 10 | UniqueConstraint, 11 | func, 12 | text, 13 | ) 14 | 15 | from .._metadata import metadata 16 | 17 | user_identification_token = Table( 18 | "user_identification_token", 19 | metadata, 20 | Column("id", Integer, primary_key=True, autoincrement=True), 21 | Column("token", UUID, nullable=False, server_default=text("uuid_generate_v4()")), 22 | Column("user_id", ForeignKey("auth.user.id"), nullable=True), 23 | Column( 24 | "created_at", 25 | TIMESTAMP(timezone=False), 26 | server_default=func.now(), 27 | nullable=False, 28 | ), 29 | Column( 30 | "last_used_at", 31 | TIMESTAMP(timezone=False), 32 | server_default=func.now(), 33 | nullable=False, 34 | ), 35 | UniqueConstraint("token", "user_id", name="uq_token_user_id"), 36 | schema="auth", 37 | ) 38 | -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/auth/_user_role.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import ( 2 | TIMESTAMP, 3 | BigInteger, 4 | Column, 5 | ForeignKey, 6 | Integer, 7 | Table, 8 | UniqueConstraint, 9 | func, 10 | ) 11 | 12 | from .._metadata import metadata 13 | 14 | user_role = Table( 15 | "user_role", 16 | metadata, 17 | Column("id", BigInteger, primary_key=True, autoincrement=True), 18 | Column( 19 | "created", TIMESTAMP(timezone=False), server_default=func.now(), nullable=False 20 | ), 21 | Column("created_by", Integer, ForeignKey("auth.user.id"), nullable=False), 22 | Column("user_id", Integer, ForeignKey("auth.user.id"), nullable=False), 23 | Column("role_id", Integer, ForeignKey("auth.role.id"), nullable=False), 24 | UniqueConstraint("user_id", "role_id"), 25 | schema="auth", 26 | ) 27 | -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/research/__init__.py: -------------------------------------------------------------------------------- 1 | from ._experimental_state import experimental_state 2 | from ._log import log 3 | from ._log_action import log_action 4 | from ._model_experimental_state_proposal import model_experimental_state_proposal 5 | from ._model_log import model_log 6 | from ._note import note 7 | from ._note_kind import note_kind 8 | from ._prompt_experimental_state_proposal import prompt_experimental_state_proposal 9 | from ._prompt_log import prompt_log 10 | from ._sample_log import sample_log 11 | from ._template_experimental_state_proposal import template_experimental_state_proposal 12 | from ._template_log import template_log 13 | 14 | __all__ = [ 15 | "experimental_state", 16 | "log", 17 | "log_action", 18 | "note", 19 | "note_kind", 20 | "model_log", 21 | "prompt_log", 22 | "sample_log", 23 | "template_log", 24 | "prompt_experimental_state_proposal", 25 | "model_experimental_state_proposal", 26 | "template_experimental_state_proposal", 27 | ] 28 | -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/research/_experimental_state.py: -------------------------------------------------------------------------------- 1 | """Valid experimental states 2 | 3 | - DRAFT - initial state, thing is not ready for use 4 | - IN_REVIEW - thing is ready for use, but not yet approved 5 | - RELEASED - thing is ready for use 6 | - DEPRECATED - thing is no longer for use 7 | - EXPERIMENTAL - thing is experimental and not ready for use 8 | - REJECTED - thing is rejected and not ready for use 9 | 10 | """ 11 | 12 | from sqlalchemy import TIMESTAMP, UUID, Column, Integer, String, Table, func, text 13 | 14 | from .._metadata import metadata 15 | 16 | experimental_state = Table( 17 | "experimental_state", 18 | metadata, 19 | Column("id", Integer, primary_key=True, autoincrement=True), 20 | Column( 21 | "external_id", UUID, nullable=False, server_default=text("uuid_generate_v4()") 22 | ), 23 | Column( 24 | "created", TIMESTAMP(timezone=False), server_default=func.now(), nullable=True 25 | ), 26 | Column("name", String, unique=True, nullable=False), 27 | schema="research", 28 | ) 29 | -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/research/_log.py: -------------------------------------------------------------------------------- 1 | """A log is an entry indicating an observation or an action 2 | 3 | A log has an action and a note. 4 | """ 5 | 6 | from sqlalchemy import ( 7 | TIMESTAMP, 8 | UUID, 9 | BigInteger, 10 | Column, 11 | ForeignKey, 12 | Integer, 13 | String, 14 | Table, 15 | func, 16 | text, 17 | ) 18 | 19 | from .._metadata import metadata 20 | 21 | log = Table( 22 | "log", 23 | metadata, 24 | Column("id", BigInteger, primary_key=True, autoincrement=True), 25 | Column( 26 | "created", TIMESTAMP(timezone=False), server_default=func.now(), nullable=False 27 | ), 28 | Column("created_by", Integer, ForeignKey("auth.user.id"), nullable=False), 29 | Column( 30 | "external_id", UUID, nullable=False, server_default=text("uuid_generate_v4()") 31 | ), 32 | Column( 33 | "action_slug", String, ForeignKey("research.log_action.name"), nullable=False 34 | ), 35 | Column("note_id", Integer, ForeignKey("research.note.id"), nullable=False), 36 | schema="research", 37 | comment=__doc__.strip(), 38 | ) 39 | -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/research/_log_action.py: -------------------------------------------------------------------------------- 1 | """A log action is a reason for making a log entry 2 | 3 | A log action has a name. 4 | """ 5 | 6 | from sqlalchemy import Column, Integer, String, Table 7 | 8 | from .._metadata import metadata 9 | 10 | log_action = Table( 11 | "log_action", 12 | metadata, 13 | Column("id", Integer, primary_key=True, autoincrement=True), 14 | Column("name", String, unique=True, nullable=False), 15 | comment=__doc__.strip(), 16 | schema="research", 17 | ) 18 | -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/research/_model_experimental_state_proposal.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import ( 2 | TIMESTAMP, 3 | UUID, 4 | Boolean, 5 | Column, 6 | ForeignKey, 7 | Integer, 8 | Table, 9 | func, 10 | text, 11 | ) 12 | 13 | from .._metadata import metadata 14 | 15 | model_experimental_state_proposal = Table( 16 | "model_experimental_state_proposal", 17 | metadata, 18 | Column("id", Integer, primary_key=True), 19 | Column( 20 | "external_id", UUID, nullable=False, server_default=text("uuid_generate_v4()") 21 | ), 22 | Column( 23 | "created", TIMESTAMP(timezone=False), server_default=func.now(), nullable=False 24 | ), 25 | Column("created_by", Integer, ForeignKey("auth.user.id"), nullable=False), 26 | Column("model_id", Integer, ForeignKey("specification.model.id"), nullable=False), 27 | Column( 28 | "new_experiment_state_id", 29 | Integer, 30 | ForeignKey("research.experimental_state.id"), 31 | nullable=False, 32 | ), 33 | Column("log_id", Integer, ForeignKey("research.log.id"), nullable=True), 34 | Column("accepted", Boolean, nullable=True), 35 | Column("accepted_at", TIMESTAMP(timezone=False), nullable=True), 36 | Column("accepted_by", Integer, ForeignKey("auth.user.id"), nullable=True), 37 | Column("accepted_log_id", Integer, ForeignKey("research.log.id"), nullable=True), 38 | Column("rejected", Boolean, nullable=True), 39 | Column("rejected_at", TIMESTAMP(timezone=False), nullable=True), 40 | Column("rejected_by", Integer, ForeignKey("auth.user.id"), nullable=True), 41 | Column("rejected_log_id", Integer, ForeignKey("research.log.id"), nullable=True), 42 | schema="research", 43 | ) 44 | -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/research/_model_log.py: -------------------------------------------------------------------------------- 1 | """A bridge table between a model and a log""" 2 | 3 | from sqlalchemy import Column, ForeignKey, Integer, Table 4 | 5 | from .._metadata import metadata 6 | 7 | model_log = Table( 8 | "model_log", 9 | metadata, 10 | Column("model_id", Integer, ForeignKey("specification.model.id"), nullable=False), 11 | Column("log_id", Integer, ForeignKey("research.log.id"), nullable=False), 12 | schema="research", 13 | ) 14 | -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/research/_note.py: -------------------------------------------------------------------------------- 1 | """A note is a narrative entry by a person or SYSTEM 2 | 3 | A note has a kind, content, and a created timestamp. 4 | """ 5 | 6 | from sqlalchemy import ( 7 | TIMESTAMP, 8 | UUID, 9 | BigInteger, 10 | Column, 11 | ForeignKey, 12 | Integer, 13 | String, 14 | Table, 15 | func, 16 | text, 17 | ) 18 | 19 | from .._metadata import metadata 20 | 21 | note = Table( 22 | "note", 23 | metadata, 24 | Column("id", BigInteger, primary_key=True, autoincrement=True), 25 | Column( 26 | "created", TIMESTAMP(timezone=False), server_default=func.now(), nullable=False 27 | ), 28 | Column("created_by", Integer, ForeignKey("auth.user.id"), nullable=False), 29 | Column("deleted", TIMESTAMP(timezone=False), nullable=True), 30 | Column("deleted_by", Integer, ForeignKey("auth.user.id"), nullable=True), 31 | Column( 32 | "external_id", UUID, nullable=False, server_default=text("uuid_generate_v4()") 33 | ), 34 | Column("kind_slug", String, ForeignKey("research.note_kind.name"), nullable=False), 35 | Column("content", String, nullable=False), 36 | schema="research", 37 | comment=__doc__.strip(), 38 | ) 39 | -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/research/_note_kind.py: -------------------------------------------------------------------------------- 1 | """A note kind is a type of note, e.g. observation, justification, hypothesis, etc. 2 | 3 | A note kind has a name. 4 | """ 5 | 6 | from sqlalchemy import Column, Integer, String, Table 7 | 8 | from .._metadata import metadata 9 | 10 | note_kind = Table( 11 | "note_kind", 12 | metadata, 13 | Column("id", Integer, primary_key=True, autoincrement=True), 14 | Column("name", String, unique=True, nullable=False), 15 | schema="research", 16 | comment=__doc__.strip(), 17 | ) 18 | -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/research/_prompt_experimental_state_proposal.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import ( 2 | TIMESTAMP, 3 | UUID, 4 | Boolean, 5 | Column, 6 | ForeignKey, 7 | Integer, 8 | Table, 9 | func, 10 | text, 11 | ) 12 | 13 | from .._metadata import metadata 14 | 15 | prompt_experimental_state_proposal = Table( 16 | "prompt_experimental_state_proposal", 17 | metadata, 18 | Column("id", Integer, primary_key=True), 19 | Column( 20 | "external_id", UUID, nullable=False, server_default=text("uuid_generate_v4()") 21 | ), 22 | Column( 23 | "created", TIMESTAMP(timezone=False), server_default=func.now(), nullable=False 24 | ), 25 | Column("created_by", Integer, ForeignKey("auth.user.id"), nullable=False), 26 | Column("prompt_id", Integer, ForeignKey("specification.prompt.id"), nullable=False), 27 | Column( 28 | "new_experiment_state_id", 29 | Integer, 30 | ForeignKey("research.experimental_state.id"), 31 | nullable=False, 32 | ), 33 | Column("log_id", Integer, ForeignKey("research.log.id"), nullable=True), 34 | Column("accepted", Boolean, nullable=True), 35 | Column("accepted_at", TIMESTAMP(timezone=False), nullable=True), 36 | Column("accepted_by", Integer, ForeignKey("auth.user.id"), nullable=True), 37 | Column("accepted_log_id", Integer, ForeignKey("research.log.id"), nullable=True), 38 | Column("rejected", Boolean, nullable=True), 39 | Column("rejected_at", TIMESTAMP(timezone=False), nullable=True), 40 | Column("rejected_by", Integer, ForeignKey("auth.user.id"), nullable=True), 41 | Column("rejected_log_id", Integer, ForeignKey("research.log.id"), nullable=True), 42 | schema="research", 43 | ) 44 | -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/research/_prompt_log.py: -------------------------------------------------------------------------------- 1 | """A bridge table between a prompt and a log""" 2 | 3 | from sqlalchemy import Column, ForeignKey, Integer, Table 4 | 5 | from .._metadata import metadata 6 | 7 | prompt_log = Table( 8 | "prompt_log", 9 | metadata, 10 | Column("prompt_id", Integer, ForeignKey("specification.prompt.id"), nullable=False), 11 | Column("log_id", Integer, ForeignKey("research.log.id"), nullable=False), 12 | schema="research", 13 | ) 14 | -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/research/_sample_log.py: -------------------------------------------------------------------------------- 1 | """A bridge table between a sample and a log""" 2 | 3 | from sqlalchemy import Column, ForeignKey, Integer, Table 4 | 5 | from .._metadata import metadata 6 | 7 | sample_log = Table( 8 | "sample_log", 9 | metadata, 10 | Column("sample_id", Integer, ForeignKey("sample.sample.id"), nullable=False), 11 | Column("log_id", Integer, ForeignKey("research.log.id"), nullable=False), 12 | schema="research", 13 | ) 14 | -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/research/_template_experimental_state_proposal.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import ( 2 | TIMESTAMP, 3 | UUID, 4 | Boolean, 5 | Column, 6 | ForeignKey, 7 | Integer, 8 | Table, 9 | func, 10 | text, 11 | ) 12 | 13 | from .._metadata import metadata 14 | 15 | template_experimental_state_proposal = Table( 16 | "template_experimental_state_proposal", 17 | metadata, 18 | Column("id", Integer, primary_key=True), 19 | Column( 20 | "external_id", UUID, nullable=False, server_default=text("uuid_generate_v4()") 21 | ), 22 | Column( 23 | "created", TIMESTAMP(timezone=False), server_default=func.now(), nullable=False 24 | ), 25 | Column("created_by", Integer, ForeignKey("auth.user.id"), nullable=False), 26 | Column( 27 | "template_id", Integer, ForeignKey("specification.template.id"), nullable=False 28 | ), 29 | Column( 30 | "new_experiment_state_id", 31 | Integer, 32 | ForeignKey("research.experimental_state.id"), 33 | nullable=False, 34 | ), 35 | Column("log_id", Integer, ForeignKey("research.log.id"), nullable=True), 36 | Column("accepted", Boolean, nullable=True), 37 | Column("accepted_at", TIMESTAMP(timezone=False), nullable=True), 38 | Column("accepted_by", Integer, ForeignKey("auth.user.id"), nullable=True), 39 | Column("accepted_log_id", Integer, ForeignKey("research.log.id"), nullable=True), 40 | Column("rejected", Boolean, nullable=True), 41 | Column("rejected_at", TIMESTAMP(timezone=False), nullable=True), 42 | Column("rejected_by", Integer, ForeignKey("auth.user.id"), nullable=True), 43 | Column("rejected_log_id", Integer, ForeignKey("research.log.id"), nullable=True), 44 | schema="research", 45 | ) 46 | -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/research/_template_log.py: -------------------------------------------------------------------------------- 1 | """A bridge table between a template and a log""" 2 | 3 | from sqlalchemy import Column, ForeignKey, Integer, Table 4 | 5 | from .._metadata import metadata 6 | 7 | template_log = Table( 8 | "template_log", 9 | metadata, 10 | Column( 11 | "template_id", Integer, ForeignKey("specification.template.id"), nullable=False 12 | ), 13 | Column("log_id", Integer, ForeignKey("research.log.id"), nullable=False), 14 | schema="research", 15 | ) 16 | -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/sample/__init__.py: -------------------------------------------------------------------------------- 1 | from ._artifact import artifact 2 | from ._artifact_kind import artifact_kind 3 | from ._sample import sample 4 | from ._test_set import test_set 5 | 6 | __all__ = [ 7 | "artifact", 8 | "artifact_kind", 9 | "sample", 10 | "test_set", 11 | ] 12 | -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/sample/_artifact.py: -------------------------------------------------------------------------------- 1 | """ """ 2 | 3 | from sqlalchemy import ( 4 | TIMESTAMP, 5 | UUID, 6 | Column, 7 | ForeignKey, 8 | Index, 9 | Integer, 10 | String, 11 | Table, 12 | func, 13 | text, 14 | ) 15 | 16 | from .._metadata import metadata 17 | 18 | artifact = Table( 19 | "artifact", 20 | metadata, 21 | Column("id", Integer, primary_key=True, autoincrement=True), 22 | Column( 23 | "created", TIMESTAMP(timezone=False), server_default=func.now(), nullable=True 24 | ), 25 | Column( 26 | "artifact_kind_id", 27 | Integer, 28 | ForeignKey("sample.artifact_kind.id"), 29 | nullable=False, 30 | ), 31 | Column( 32 | "run_id", 33 | Integer, 34 | ForeignKey("specification.run.id"), 35 | nullable=False, 36 | ), 37 | Column( 38 | "sample_id", 39 | Integer, 40 | ForeignKey("sample.sample.id"), 41 | nullable=True, 42 | ), 43 | Column("bucket", String, unique=False, nullable=False), 44 | Column("key", String, unique=False, nullable=False), 45 | Column( 46 | "external_id", UUID, nullable=False, server_default=text("uuid_generate_v4()") 47 | ), 48 | # Add index for (sample_id, artifact_kind_id) 49 | Index("ix_artifact_sample_id_kind_id", "sample_id", "artifact_kind_id"), 50 | schema="sample", 51 | ) 52 | -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/sample/_artifact_kind.py: -------------------------------------------------------------------------------- 1 | """ """ 2 | 3 | from sqlalchemy import TIMESTAMP, Column, Integer, String, Table, func 4 | 5 | from .._metadata import metadata 6 | 7 | artifact_kind = Table( 8 | "artifact_kind", 9 | metadata, 10 | Column("id", Integer, primary_key=True, autoincrement=True), 11 | Column( 12 | "created", TIMESTAMP(timezone=False), server_default=func.now(), nullable=True 13 | ), 14 | Column("name", String, unique=True, nullable=False), 15 | schema="sample", 16 | ) 17 | -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/sample/_test_set.py: -------------------------------------------------------------------------------- 1 | """A test set is a collection of samples that can be voted on.""" 2 | 3 | from sqlalchemy import ( 4 | TIMESTAMP, 5 | UUID, 6 | Column, 7 | ForeignKey, 8 | Integer, 9 | String, 10 | Table, 11 | func, 12 | text, 13 | ) 14 | 15 | from .._metadata import metadata 16 | 17 | test_set = Table( 18 | "test_set", 19 | metadata, 20 | Column("id", Integer, primary_key=True, autoincrement=True), 21 | Column( 22 | "created", TIMESTAMP(timezone=False), server_default=func.now(), nullable=False 23 | ), 24 | Column("created_by", Integer, ForeignKey("auth.user.id"), nullable=False), 25 | Column( 26 | "external_id", UUID, nullable=False, server_default=text("uuid_generate_v4()") 27 | ), 28 | Column("name", String, unique=True, nullable=False), 29 | Column("description", String, nullable=False), 30 | comment=__doc__.strip(), 31 | schema="sample", 32 | ) 33 | -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/scoring/__init__.py: -------------------------------------------------------------------------------- 1 | from ._comparison import comparison 2 | from ._comparison_rank import comparison_rank 3 | from ._metric import metric 4 | from ._model_leaderboard import model_leaderboard 5 | from ._processed_comparison import processed_comparison 6 | from ._prompt_leaderboard import prompt_leaderboard 7 | from ._sample_approval_state import sample_approval_state 8 | from ._sample_leaderboard import sample_leaderboard 9 | 10 | __all__ = [ 11 | "comparison", 12 | "comparison_rank", 13 | "metric", 14 | "model_leaderboard", 15 | "processed_comparison", 16 | "prompt_leaderboard", 17 | "sample_approval_state", 18 | "sample_leaderboard", 19 | ] 20 | -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/scoring/_comparison.py: -------------------------------------------------------------------------------- 1 | """ """ 2 | 3 | from sqlalchemy import ( 4 | TIMESTAMP, 5 | UUID, 6 | Column, 7 | ForeignKey, 8 | Index, 9 | Integer, 10 | Table, 11 | func, 12 | text, 13 | ) 14 | 15 | from .._metadata import metadata 16 | 17 | comparison = Table( 18 | "comparison", 19 | metadata, 20 | Column("id", Integer, primary_key=True, autoincrement=True), 21 | Column( 22 | "created", TIMESTAMP(timezone=False), server_default=func.now(), nullable=True 23 | ), 24 | Column("user_id", ForeignKey("auth.user.id"), nullable=True), 25 | Column( 26 | "comparison_id", UUID, nullable=False, server_default=text("uuid_generate_v4()") 27 | ), 28 | Column("metric_id", Integer, ForeignKey("scoring.metric.id"), nullable=False), 29 | Column("test_set_id", Integer, ForeignKey("sample.test_set.id"), nullable=False), 30 | Column("session_id", UUID, nullable=True), 31 | Column( 32 | "identification_token_id", 33 | Integer, 34 | ForeignKey("auth.user_identification_token.id"), 35 | nullable=True, 36 | ), 37 | # Add indexes for comparison table 38 | Index("ix_comparison_comparison_id", "comparison_id"), 39 | Index("ix_comparison_metric_test_set", "metric_id", "test_set_id"), 40 | schema="scoring", 41 | ) 42 | -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/scoring/_comparison_rank.py: -------------------------------------------------------------------------------- 1 | """ 2 | Table for storing sample rankings within comparisons. 3 | This allows for ties and multi-sample comparisons beyond simple pairwise comparisons. 4 | """ 5 | 6 | from sqlalchemy import ( 7 | TIMESTAMP, 8 | Column, 9 | ForeignKey, 10 | Index, 11 | Integer, 12 | Table, 13 | UniqueConstraint, 14 | func, 15 | ) 16 | 17 | from .._metadata import metadata 18 | 19 | comparison_rank = Table( 20 | "comparison_rank", 21 | metadata, 22 | Column("id", Integer, primary_key=True, autoincrement=True), 23 | Column( 24 | "comparison_id", Integer, ForeignKey("scoring.comparison.id"), nullable=False 25 | ), 26 | Column("sample_id", Integer, ForeignKey("sample.sample.id"), nullable=False), 27 | # Rank 1 = best, higher numbers = worse rank 28 | Column("rank", Integer, nullable=False), 29 | # Add created timestamp for audit trail 30 | Column( 31 | "created", TIMESTAMP(timezone=False), server_default=func.now(), nullable=True 32 | ), 33 | # Ensure a sample can only have one rank per comparison 34 | UniqueConstraint( 35 | "comparison_id", "sample_id", name="unique_sample_rank_per_comparison" 36 | ), 37 | # Add index for sample_id to speed up lookups 38 | Index("ix_comparison_rank_sample_id", "sample_id"), 39 | schema="scoring", 40 | ) 41 | -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/scoring/_metric.py: -------------------------------------------------------------------------------- 1 | """ """ 2 | 3 | from sqlalchemy import ( 4 | TIMESTAMP, 5 | UUID, 6 | BigInteger, 7 | Column, 8 | ForeignKey, 9 | Integer, 10 | String, 11 | Table, 12 | func, 13 | text, 14 | ) 15 | 16 | from .._metadata import metadata 17 | 18 | metric = Table( 19 | "metric", 20 | metadata, 21 | Column("id", BigInteger, primary_key=True, autoincrement=True), 22 | Column( 23 | "created", TIMESTAMP(timezone=False), server_default=func.now(), nullable=False 24 | ), 25 | Column("created_by", Integer, ForeignKey("auth.user.id"), nullable=False), 26 | Column("last_modified", TIMESTAMP(timezone=False), nullable=True), 27 | Column("last_modified_by", Integer, ForeignKey("auth.user.id"), nullable=True), 28 | Column( 29 | "external_id", UUID, nullable=False, server_default=text("uuid_generate_v4()") 30 | ), 31 | Column("name", String(), unique=True, nullable=False), 32 | Column("description", String(), nullable=False), 33 | schema="scoring", 34 | ) 35 | -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/scoring/_processed_comparison.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tracks which comparisons have been processed for ELO calculations. 3 | This prevents duplicate processing and helps maintain data integrity. 4 | """ 5 | 6 | from sqlalchemy import ( 7 | TIMESTAMP, 8 | Column, 9 | ForeignKey, 10 | Integer, 11 | Table, 12 | UniqueConstraint, 13 | func, 14 | ) 15 | 16 | from .._metadata import metadata 17 | 18 | processed_comparison = Table( 19 | "processed_comparison", 20 | metadata, 21 | Column("id", Integer, primary_key=True, autoincrement=True), 22 | Column( 23 | "created", TIMESTAMP(timezone=False), server_default=func.now(), nullable=False 24 | ), 25 | Column( 26 | "comparison_id", Integer, ForeignKey("scoring.comparison.id"), nullable=False 27 | ), 28 | # Ensure each comparison is only processed once 29 | UniqueConstraint("comparison_id", name="unique_processed_comparison"), 30 | schema="scoring", 31 | ) 32 | -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/scoring/_sample_approval_state.py: -------------------------------------------------------------------------------- 1 | """Valid sample approval states""" 2 | 3 | from sqlalchemy import TIMESTAMP, Column, Integer, String, Table, func 4 | 5 | from .._metadata import metadata 6 | 7 | sample_approval_state = Table( 8 | "sample_approval_state", 9 | metadata, 10 | Column("id", Integer, primary_key=True, autoincrement=True), 11 | Column( 12 | "created", TIMESTAMP(timezone=False), server_default=func.now(), nullable=True 13 | ), 14 | Column("name", String, unique=True, nullable=False), 15 | schema="scoring", 16 | ) 17 | -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/specification/__init__.py: -------------------------------------------------------------------------------- 1 | from ._generation import generation 2 | from ._generation_state import generation_state 3 | from ._model import model 4 | from ._prompt import prompt 5 | from ._prompt_tag import prompt_tag 6 | from ._provider import provider 7 | from ._provider_class import provider_class 8 | from ._run import run 9 | from ._run_stage import run_stage 10 | from ._run_stage_state import run_stage_state 11 | from ._run_state import run_state 12 | from ._scheduler_control import scheduler_control 13 | from ._stage import stage 14 | from ._tag import tag 15 | from ._template import template 16 | 17 | __all__ = [ 18 | "run", 19 | "run_state", 20 | "prompt", 21 | "prompt_tag", 22 | "generation", 23 | "generation_state", 24 | "model", 25 | "provider", 26 | "provider_class", 27 | "template", 28 | "run_stage", 29 | "run_stage_state", 30 | "scheduler_control", 31 | "stage", 32 | "tag", 33 | ] 34 | -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/specification/_generation.py: -------------------------------------------------------------------------------- 1 | """ """ 2 | 3 | from sqlalchemy import ( 4 | TIMESTAMP, 5 | UUID, 6 | BigInteger, 7 | Column, 8 | ForeignKey, 9 | Integer, 10 | String, 11 | Table, 12 | func, 13 | text, 14 | ) 15 | 16 | from .._metadata import metadata 17 | 18 | generation = Table( 19 | "generation", 20 | metadata, 21 | Column("id", BigInteger, primary_key=True, autoincrement=True), 22 | Column( 23 | "created", TIMESTAMP(timezone=False), server_default=func.now(), nullable=True 24 | ), 25 | Column("created_by", Integer, ForeignKey("auth.user.id"), nullable=False), 26 | Column( 27 | "external_id", UUID, nullable=False, server_default=text("uuid_generate_v4()") 28 | ), 29 | Column("name", String, nullable=False), 30 | Column("description", String, nullable=False), 31 | Column( 32 | "state_id", 33 | BigInteger, 34 | ForeignKey("specification.generation_state.id"), 35 | nullable=False, 36 | ), 37 | Column( 38 | "default_test_set_id", 39 | BigInteger, 40 | ForeignKey("sample.test_set.id"), 41 | nullable=True, 42 | ), 43 | schema="specification", 44 | ) 45 | -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/specification/_generation_state.py: -------------------------------------------------------------------------------- 1 | """ """ 2 | 3 | from sqlalchemy import ( 4 | TIMESTAMP, 5 | UUID, 6 | BigInteger, 7 | Column, 8 | ForeignKey, 9 | Integer, 10 | String, 11 | Table, 12 | func, 13 | text, 14 | ) 15 | 16 | from .._metadata import metadata 17 | 18 | generation_state = Table( 19 | "generation_state", 20 | metadata, 21 | Column("id", BigInteger, primary_key=True, autoincrement=True), 22 | Column( 23 | "created", TIMESTAMP(timezone=False), server_default=func.now(), nullable=False 24 | ), 25 | Column("created_by", Integer, ForeignKey("auth.user.id"), nullable=False), 26 | Column("last_modified", TIMESTAMP(timezone=False), nullable=True), 27 | Column("last_modified_by", Integer, ForeignKey("auth.user.id"), nullable=True), 28 | Column( 29 | "external_id", UUID, nullable=False, server_default=text("uuid_generate_v4()") 30 | ), 31 | Column("slug", String, unique=True, nullable=False), 32 | schema="specification", 33 | ) 34 | -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/specification/_model.py: -------------------------------------------------------------------------------- 1 | """ 2 | A reference table of model names, such as "NousResearch/Hermes-3-Llama-3.1-8B" (from hugging face) or "claude-3-5-sonnet-20240620". 3 | 4 | While some systems may have specific variants, we will mediate any of these differences by recording timestamps and other 5 | metadata for how the model was run, accessed, or called. 6 | 7 | Unless it becomes necessary, we will generally consider this model name the system under test. 8 | """ 9 | 10 | from sqlalchemy import ( 11 | TIMESTAMP, 12 | UUID, 13 | BigInteger, 14 | Boolean, 15 | Column, 16 | ForeignKey, 17 | Integer, 18 | String, 19 | Table, 20 | func, 21 | text, 22 | ) 23 | 24 | from .._metadata import metadata 25 | 26 | model = Table( 27 | "model", 28 | metadata, 29 | Column("id", BigInteger, primary_key=True, autoincrement=True), 30 | Column( 31 | "created", TIMESTAMP(timezone=False), server_default=func.now(), nullable=False 32 | ), 33 | Column("created_by", Integer, ForeignKey("auth.user.id"), nullable=False), 34 | Column("last_modified", TIMESTAMP(timezone=False), nullable=True), 35 | Column("last_modified_by", Integer, ForeignKey("auth.user.id"), nullable=True), 36 | Column( 37 | "external_id", UUID, nullable=False, server_default=text("uuid_generate_v4()") 38 | ), 39 | Column("slug", String, unique=True, nullable=False), 40 | Column("name", String, unique=True, nullable=False), 41 | Column("active", Boolean, nullable=True), 42 | Column( 43 | "experimental_state_id", 44 | Integer, 45 | ForeignKey("research.experimental_state.id"), 46 | nullable=True, 47 | ), 48 | comment=__doc__.strip(), 49 | schema="specification", 50 | ) 51 | -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/specification/_prompt.py: -------------------------------------------------------------------------------- 1 | """ 2 | A prompt. This will include a template id (which should be the same across every given run). 3 | This table is append only. If we choose to use a different template we should make a new prompt row. 4 | """ 5 | 6 | from sqlalchemy import ( 7 | TIMESTAMP, 8 | UUID, 9 | BigInteger, 10 | Boolean, 11 | Column, 12 | ForeignKey, 13 | Integer, 14 | String, 15 | Table, 16 | func, 17 | text, 18 | ) 19 | 20 | from .._metadata import metadata 21 | 22 | prompt = Table( 23 | "prompt", 24 | metadata, 25 | Column("id", BigInteger, primary_key=True, autoincrement=True), 26 | Column( 27 | "created", TIMESTAMP(timezone=False), server_default=func.now(), nullable=False 28 | ), 29 | Column("created_by", Integer, ForeignKey("auth.user.id"), nullable=False), 30 | Column("last_modified", TIMESTAMP(timezone=False), nullable=True), 31 | Column("last_modified_by", Integer, ForeignKey("auth.user.id"), nullable=True), 32 | Column( 33 | "external_id", UUID, nullable=False, server_default=text("uuid_generate_v4()") 34 | ), 35 | Column("name", String, unique=True, nullable=False), 36 | Column("active", Boolean, nullable=True), 37 | Column("build_specification", String, nullable=False), 38 | Column("build_size", String, nullable=True), 39 | Column( 40 | "experimental_state_id", 41 | Integer, 42 | ForeignKey("research.experimental_state.id"), 43 | nullable=True, 44 | ), 45 | comment=__doc__.strip(), 46 | schema="specification", 47 | ) 48 | -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/specification/_prompt_tag.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import ( 2 | TIMESTAMP, 3 | BigInteger, 4 | Column, 5 | ForeignKey, 6 | Integer, 7 | Table, 8 | UniqueConstraint, 9 | func, 10 | ) 11 | 12 | from .._metadata import metadata 13 | 14 | prompt_tag = Table( 15 | "prompt_tag", 16 | metadata, 17 | Column("id", BigInteger, primary_key=True, autoincrement=True), 18 | Column( 19 | "created", TIMESTAMP(timezone=False), server_default=func.now(), nullable=False 20 | ), 21 | Column("created_by", Integer, ForeignKey("auth.user.id"), nullable=False), 22 | Column("prompt_id", Integer, ForeignKey("specification.prompt.id"), nullable=False), 23 | Column("tag_id", Integer, ForeignKey("specification.tag.id"), nullable=False), 24 | UniqueConstraint("prompt_id", "tag_id"), 25 | schema="specification", 26 | ) 27 | -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/specification/_provider.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import ( 2 | JSON, 3 | TIMESTAMP, 4 | UUID, 5 | BigInteger, 6 | Boolean, 7 | Column, 8 | ForeignKey, 9 | Integer, 10 | String, 11 | Table, 12 | func, 13 | text, 14 | ) 15 | 16 | from .._metadata import metadata 17 | 18 | provider = Table( 19 | "provider", 20 | metadata, 21 | Column("id", BigInteger, primary_key=True, autoincrement=True), 22 | Column( 23 | "created", TIMESTAMP(timezone=False), server_default=func.now(), nullable=False 24 | ), 25 | Column("created_by", Integer, ForeignKey("auth.user.id"), nullable=False), 26 | Column("last_modified", TIMESTAMP(timezone=False), nullable=True), 27 | Column("last_modified_by", Integer, ForeignKey("auth.user.id"), nullable=True), 28 | Column( 29 | "external_id", UUID, nullable=False, server_default=text("uuid_generate_v4()") 30 | ), 31 | Column( 32 | "model_id", 33 | BigInteger, 34 | ForeignKey("specification.model.id", ondelete="CASCADE"), 35 | nullable=False, 36 | ), 37 | Column( 38 | "name", 39 | String, 40 | nullable=False, 41 | ), 42 | Column( 43 | "provider_class", 44 | String, 45 | ForeignKey("specification.provider_class.name"), 46 | nullable=False, 47 | ), 48 | Column("config", JSON, nullable=False), 49 | Column("is_default", Boolean, nullable=True), 50 | schema="specification", 51 | ) 52 | -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/specification/_provider_class.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import ( 2 | JSON, 3 | TIMESTAMP, 4 | UUID, 5 | BigInteger, 6 | Column, 7 | ForeignKey, 8 | Integer, 9 | String, 10 | Table, 11 | func, 12 | text, 13 | ) 14 | 15 | from .._metadata import metadata 16 | 17 | provider_class = Table( 18 | "provider_class", 19 | metadata, 20 | Column("id", BigInteger, primary_key=True, autoincrement=True), 21 | Column( 22 | "created", TIMESTAMP(timezone=False), server_default=func.now(), nullable=False 23 | ), 24 | Column("created_by", Integer, ForeignKey("auth.user.id"), nullable=False), 25 | Column("last_modified", TIMESTAMP(timezone=False), nullable=True), 26 | Column("last_modified_by", Integer, ForeignKey("auth.user.id"), nullable=True), 27 | Column( 28 | "external_id", UUID, nullable=False, server_default=text("uuid_generate_v4()") 29 | ), 30 | Column("name", String, unique=True, nullable=False), 31 | Column("default_config", JSON, nullable=False, server_default=text("'{}'::jsonb")), 32 | schema="specification", 33 | ) 34 | -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/specification/_run.py: -------------------------------------------------------------------------------- 1 | """ """ 2 | 3 | from sqlalchemy import ( 4 | TIMESTAMP, 5 | UUID, 6 | BigInteger, 7 | Column, 8 | ForeignKey, 9 | Index, 10 | Integer, 11 | Table, 12 | func, 13 | text, 14 | ) 15 | 16 | from .._metadata import metadata 17 | 18 | run = Table( 19 | "run", 20 | metadata, 21 | Column("id", BigInteger, primary_key=True, autoincrement=True), 22 | Column( 23 | "created", TIMESTAMP(timezone=False), server_default=func.now(), nullable=True 24 | ), 25 | Column("created_by", Integer, ForeignKey("auth.user.id"), nullable=False), 26 | Column("last_modified", TIMESTAMP(timezone=False), nullable=True), 27 | Column("last_modified_by", Integer, ForeignKey("auth.user.id"), nullable=True), 28 | Column( 29 | "external_id", UUID, nullable=False, server_default=text("uuid_generate_v4()") 30 | ), 31 | Column("template_id", Integer, ForeignKey("specification.template.id")), 32 | Column("prompt_id", Integer, ForeignKey("specification.prompt.id")), 33 | Column("model_id", Integer, ForeignKey("specification.model.id")), 34 | Column("state_id", Integer, ForeignKey("specification.run_state.id")), 35 | # While we can imagine most runs will occur as the result of a generation 36 | # it may become true that we execute individual runs that are not a part of the 37 | # generation concept 38 | Column( 39 | "generation_id", 40 | Integer, 41 | ForeignKey("specification.generation.id"), 42 | nullable=True, 43 | ), 44 | # Add index for model_id and prompt_id to speed up queries 45 | Index("ix_run_model_id_prompt_id", "model_id", "prompt_id"), 46 | schema="specification", 47 | ) 48 | -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/specification/_run_stage_state.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import ( 2 | TIMESTAMP, 3 | UUID, 4 | BigInteger, 5 | Column, 6 | ForeignKey, 7 | Integer, 8 | String, 9 | Table, 10 | func, 11 | text, 12 | ) 13 | 14 | from .._metadata import metadata 15 | 16 | # e.g. QUEUED, RUNNNIG, DONE 17 | run_stage_state = Table( 18 | "run_stage_state", 19 | metadata, 20 | Column("id", BigInteger, primary_key=True, autoincrement=True), 21 | Column( 22 | "created", TIMESTAMP(timezone=False), server_default=func.now(), nullable=False 23 | ), 24 | Column("created_by", Integer, ForeignKey("auth.user.id"), nullable=False), 25 | Column("last_modified", TIMESTAMP(timezone=False), nullable=True), 26 | Column("last_modified_by", Integer, ForeignKey("auth.user.id"), nullable=True), 27 | Column( 28 | "external_id", UUID, nullable=False, server_default=text("uuid_generate_v4()") 29 | ), 30 | Column("slug", String, unique=True, nullable=False), 31 | schema="specification", 32 | ) 33 | -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/specification/_run_state.py: -------------------------------------------------------------------------------- 1 | """ """ 2 | 3 | from sqlalchemy import ( 4 | TIMESTAMP, 5 | UUID, 6 | BigInteger, 7 | Column, 8 | ForeignKey, 9 | Integer, 10 | String, 11 | Table, 12 | func, 13 | text, 14 | ) 15 | 16 | from .._metadata import metadata 17 | 18 | # e.g. QUEUED, RUNNNIG, DONE 19 | run_state = Table( 20 | "run_state", 21 | metadata, 22 | Column("id", BigInteger, primary_key=True, autoincrement=True), 23 | Column( 24 | "created", TIMESTAMP(timezone=False), server_default=func.now(), nullable=False 25 | ), 26 | Column("created_by", Integer, ForeignKey("auth.user.id"), nullable=False), 27 | Column("last_modified", TIMESTAMP(timezone=False), nullable=True), 28 | Column("last_modified_by", Integer, ForeignKey("auth.user.id"), nullable=True), 29 | Column( 30 | "external_id", UUID, nullable=False, server_default=text("uuid_generate_v4()") 31 | ), 32 | Column("slug", String, unique=True, nullable=False), 33 | schema="specification", 34 | ) 35 | -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/specification/_scheduler_control.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import TIMESTAMP, Column, Index, String, Table, func 2 | from sqlalchemy.dialects.postgresql import UUID 3 | 4 | from mc_bench.schema.postgres._metadata import metadata 5 | 6 | scheduler_control = Table( 7 | "scheduler_control", 8 | metadata, 9 | Column( 10 | "id", 11 | UUID(as_uuid=True), 12 | primary_key=True, 13 | server_default="uuid_generate_v4()", 14 | comment="Primary key for the scheduler control value", 15 | ), 16 | Column( 17 | "key", 18 | String, 19 | nullable=False, 20 | unique=True, 21 | comment="The control key (e.g., MAX_QUEUED_TASKS, SCHEDULER_MODE)", 22 | ), 23 | Column( 24 | "value", 25 | String, 26 | nullable=False, 27 | comment="JSON serialized value for the control setting", 28 | ), 29 | Column( 30 | "description", 31 | String, 32 | nullable=True, 33 | comment="Description of what this control value does", 34 | ), 35 | Column( 36 | "created", 37 | TIMESTAMP(timezone=False), 38 | server_default=func.now(), 39 | nullable=False, 40 | ), 41 | Column( 42 | "last_modified", 43 | TIMESTAMP(timezone=False), 44 | nullable=True, 45 | ), 46 | schema="specification", 47 | ) 48 | 49 | # Create an index on the key field for faster lookups 50 | Index("idx_scheduler_control_key", scheduler_control.c.key) 51 | -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/specification/_stage.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import ( 2 | TIMESTAMP, 3 | UUID, 4 | BigInteger, 5 | Column, 6 | ForeignKey, 7 | Integer, 8 | String, 9 | Table, 10 | func, 11 | text, 12 | ) 13 | 14 | from .._metadata import metadata 15 | 16 | # e.g. QUEUED, RUNNNIG, DONE 17 | stage = Table( 18 | "stage", 19 | metadata, 20 | Column("id", BigInteger, primary_key=True, autoincrement=True), 21 | Column( 22 | "created", TIMESTAMP(timezone=False), server_default=func.now(), nullable=False 23 | ), 24 | Column("created_by", Integer, ForeignKey("auth.user.id"), nullable=False), 25 | Column("last_modified", TIMESTAMP(timezone=False), nullable=True), 26 | Column("last_modified_by", Integer, ForeignKey("auth.user.id"), nullable=True), 27 | Column( 28 | "external_id", UUID, nullable=False, server_default=text("uuid_generate_v4()") 29 | ), 30 | Column("slug", String, unique=True, nullable=False), 31 | schema="specification", 32 | ) 33 | -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/specification/_tag.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy import ( 2 | TIMESTAMP, 3 | UUID, 4 | BigInteger, 5 | Boolean, 6 | Column, 7 | ForeignKey, 8 | Integer, 9 | String, 10 | Table, 11 | func, 12 | text, 13 | ) 14 | 15 | from .._metadata import metadata 16 | 17 | tag = Table( 18 | "tag", 19 | metadata, 20 | Column("id", BigInteger, primary_key=True, autoincrement=True), 21 | Column( 22 | "external_id", UUID, nullable=False, server_default=text("uuid_generate_v4()") 23 | ), 24 | Column( 25 | "created", TIMESTAMP(timezone=False), server_default=func.now(), nullable=False 26 | ), 27 | Column("created_by", Integer, ForeignKey("auth.user.id"), nullable=False), 28 | Column("name", String(64), unique=True, nullable=False), 29 | Column("calculate_score", Boolean, nullable=False, server_default=text("true")), 30 | schema="specification", 31 | ) 32 | -------------------------------------------------------------------------------- /src/mc_bench/schema/postgres/specification/_template.py: -------------------------------------------------------------------------------- 1 | """ """ 2 | 3 | from sqlalchemy import ( 4 | TIMESTAMP, 5 | UUID, 6 | BigInteger, 7 | Boolean, 8 | Column, 9 | ForeignKey, 10 | Integer, 11 | String, 12 | Table, 13 | func, 14 | text, 15 | ) 16 | 17 | from .._metadata import metadata 18 | 19 | template = Table( 20 | "template", 21 | metadata, 22 | Column("id", BigInteger, primary_key=True, autoincrement=True), 23 | Column( 24 | "created", TIMESTAMP(timezone=False), server_default=func.now(), nullable=False 25 | ), 26 | Column("created_by", Integer, ForeignKey("auth.user.id"), nullable=False), 27 | Column("last_modified", TIMESTAMP(timezone=False), nullable=True), 28 | Column("last_modified_by", Integer, ForeignKey("auth.user.id"), nullable=True), 29 | Column( 30 | "external_id", UUID, nullable=False, server_default=text("uuid_generate_v4()") 31 | ), 32 | Column("name", String, unique=True, nullable=False), 33 | Column("description", String, unique=False, nullable=True), 34 | Column("content", String, nullable=False), 35 | Column("active", Boolean, nullable=True), 36 | Column("frozen", Boolean, nullable=True), 37 | Column("minecraft_version", String, nullable=False), 38 | Column( 39 | "experimental_state_id", 40 | Integer, 41 | ForeignKey("research.experimental_state.id"), 42 | nullable=True, 43 | ), 44 | schema="specification", 45 | ) 46 | -------------------------------------------------------------------------------- /src/mc_bench/server/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mc-bench/mc-bench-backend/360838f5692597b2bbe1c2622e7ac60c13e1871b/src/mc_bench/server/__init__.py -------------------------------------------------------------------------------- /src/mc_bench/util/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mc-bench/mc-bench-backend/360838f5692597b2bbe1c2622e7ac60c13e1871b/src/mc_bench/util/__init__.py -------------------------------------------------------------------------------- /src/mc_bench/util/cache.py: -------------------------------------------------------------------------------- 1 | """ 2 | Caching utility functions for MC-Bench. 3 | """ 4 | 5 | from datetime import datetime, timedelta 6 | from functools import wraps 7 | from typing import Any, Callable, Dict 8 | 9 | from .logging import get_logger 10 | 11 | logger = get_logger(__name__) 12 | 13 | # Process-level cache with expiry time 14 | _cache: Dict[str, Dict[str, Any]] = {} 15 | 16 | 17 | def timed_cache(hours: int = 12): 18 | """ 19 | Function decorator that caches the result with a specified expiry time. 20 | 21 | Args: 22 | hours: Number of hours to keep the cache valid 23 | 24 | Returns: 25 | Decorator function that wraps the original function 26 | """ 27 | 28 | def decorator(func: Callable): 29 | @wraps(func) 30 | def wrapper(*args, **kwargs): 31 | # Create a cache key based on the function name 32 | cache_key = func.__name__ 33 | 34 | # Check if we have a valid cache entry 35 | if cache_key in _cache: 36 | entry = _cache[cache_key] 37 | if datetime.now() < entry["expiry"]: 38 | return entry["data"] 39 | 40 | # No valid cache, call the function 41 | data = func(*args, **kwargs) 42 | 43 | # Store the result with expiry time 44 | _cache[cache_key] = { 45 | "data": data, 46 | "expiry": datetime.now() + timedelta(hours=hours), 47 | } 48 | 49 | logger.info(f"Cached {cache_key} for {hours} hours") 50 | return data 51 | 52 | return wrapper 53 | 54 | return decorator 55 | -------------------------------------------------------------------------------- /src/mc_bench/util/docker.py: -------------------------------------------------------------------------------- 1 | import queue 2 | import threading 3 | 4 | import docker 5 | 6 | 7 | class ContainerStopped: 8 | def __init__(self, status): 9 | self._status = status 10 | 11 | def errored(self): 12 | return self._status["StatusCode"] != 0 13 | 14 | @property 15 | def status_code(self): 16 | return self._status["StatusCode"] 17 | 18 | 19 | class LogItem: 20 | def __init__(self, container_id, log_line): 21 | self.container_id = container_id 22 | self.log_line = log_line 23 | 24 | 25 | def watch_container(queue, container_id): 26 | docker_client = docker.from_env() 27 | container = docker_client.containers.get(container_id) 28 | queue.put(ContainerStopped(container.wait())) 29 | 30 | 31 | def stream_logs(queue, container_id): 32 | docker_client = docker.from_env() 33 | container = docker_client.containers.get(container_id) 34 | for log_line in container.logs(stream=True): 35 | queue.put((container_id, log_line)) 36 | 37 | 38 | def wait_for_containers(container_ids): 39 | item_queue = queue.Queue() 40 | 41 | for container_id in container_ids: 42 | for target in [watch_container, stream_logs]: 43 | thread = threading.Thread( 44 | target=target, args=(item_queue, container_id), daemon=True 45 | ) 46 | thread.start() 47 | 48 | while True: 49 | queue_item = item_queue.get() 50 | if isinstance(queue_item, ContainerStopped): 51 | if queue_item.errored(): 52 | raise RuntimeError( 53 | f"{container_id} container exited with non-zero status: {queue_item.status_code}" 54 | ) 55 | else: 56 | break 57 | else: 58 | yield LogItem(queue_item[0], queue_item[1]) 59 | -------------------------------------------------------------------------------- /src/mc_bench/util/redis.py: -------------------------------------------------------------------------------- 1 | import os 2 | from functools import lru_cache 3 | 4 | from redis import ConnectionPool, SSLConnection, StrictRedis 5 | 6 | 7 | class RedisDatabase: 8 | CELERY = 0 9 | CACHE = 1 10 | COMPARISON = 2 11 | MINECRAFT_SERVER_REGISTRY = 3 12 | 13 | 14 | @lru_cache 15 | def get_redis_pool(database: int, **kwargs) -> ConnectionPool: 16 | kwargs["host"] = kwargs.get("host", os.environ.get("REDIS_HOST", "localhost")) 17 | kwargs["port"] = kwargs.get("port", os.environ.get("REDIS_PORT", 6379)) 18 | 19 | kwargs["db"] = database 20 | 21 | if os.environ.get("REDIS_USE_AUTH", "true") == "true": 22 | kwargs["password"] = os.environ["REDIS_PASSWORD"] 23 | kwargs["username"] = os.environ["REDIS_USERNAME"] 24 | 25 | if os.environ.get("REDIS_USE_SSL", "true") == "true": 26 | kwargs["connection_class"] = SSLConnection 27 | kwargs["ssl_cert_reqs"] = "none" 28 | 29 | return ConnectionPool(**kwargs) 30 | 31 | 32 | def get_redis_client(database: int = 0, **kwargs) -> StrictRedis: 33 | pool = get_redis_pool(database, **kwargs) 34 | return StrictRedis(connection_pool=pool) 35 | 36 | 37 | def get_redis_database(database): 38 | def wrapper(): 39 | redis = get_redis_client(database=database) 40 | try: 41 | yield redis 42 | finally: 43 | redis.close() 44 | 45 | return wrapper 46 | -------------------------------------------------------------------------------- /src/mc_bench/util/uuid.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | import uuid 3 | 4 | 5 | def uuid_from_ints(first_id: int, second_id: int) -> str: 6 | """This is an aligned implementation with a sql function. Do not change.""" 7 | 8 | # Create hash of the two IDs concatenated with comma 9 | hash_input = f"{first_id},{second_id}".encode("utf-8") 10 | hex_string = hashlib.sha256(hash_input).hexdigest() 11 | 12 | # Format the hash as a UUID string 13 | uuid_str = f"{hex_string[:8]}-{hex_string[8:12]}-{hex_string[12:16]}-{hex_string[16:20]}-{hex_string[20:32]}" 14 | 15 | return uuid.UUID(uuid_str) 16 | -------------------------------------------------------------------------------- /src/mc_bench/worker/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mc-bench/mc-bench-backend/360838f5692597b2bbe1c2622e7ac60c13e1871b/src/mc_bench/worker/__init__.py -------------------------------------------------------------------------------- /tests/test_export.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from mc_bench.apps.server_worker.tasks.run import get_frames_per_command 4 | 5 | 6 | @pytest.mark.parametrize( 7 | "num_commands, expected", 8 | [(4500, 1), (4501, 2), (9000, 2), (9001, 3)], 9 | ) 10 | def test_get_frames_per_command(num_commands, expected): 11 | assert get_frames_per_command(num_commands) == expected 12 | --------------------------------------------------------------------------------