├── .github └── workflows │ └── python3.9-publish.yml ├── .gitignore ├── LICENSE ├── MANIFEST.in ├── README.MD ├── assets └── image.png ├── docs ├── installation.md └── quickguide.md ├── examples ├── 0.prepare │ └── run.sh ├── 1.import │ └── run.sh ├── 2.deploy │ ├── custom_load_pipeline │ │ ├── pipeline_loader_example.py │ │ └── run.sh │ └── setting_infer_config │ │ ├── infer_config.json │ │ └── run.sh ├── 3.evaluate │ ├── automatic │ │ └── run.sh │ ├── custom │ │ ├── custom.csv │ │ ├── custom.py │ │ └── run.sh │ └── human │ │ ├── origin.csv │ │ └── run.sh ├── 4.prune │ ├── sprase │ │ └── run.sh │ └── structure │ │ └── run.sh └── 5.quantize │ ├── int4 │ └── run.sh │ └── int8 │ └── run.sh ├── lms ├── client │ ├── __init__.py │ ├── cluster │ │ ├── __init__.py │ │ └── adm.py │ ├── common_utils.py │ ├── config.py │ ├── daemon │ │ ├── __init__.py │ │ ├── main.py │ │ ├── nvml_util.py │ │ └── serving.py │ ├── deploy_module.py │ ├── evaluate_module.py │ ├── help_constants.py │ ├── import_module.py │ ├── main.py │ ├── network_utils.py │ ├── prune_module.py │ ├── quantize_module.py │ └── validaters.py ├── runtime │ ├── __init__.py │ ├── common │ │ ├── __init__.py │ │ ├── common_utils.py │ │ ├── pipeline_loader_example.py │ │ ├── test.py │ │ └── traceback_utils.py │ ├── deploy │ │ ├── __init__.py │ │ ├── deepspeed_infer.py │ │ ├── request.py │ │ ├── request_body.json │ │ ├── serving.py │ │ └── transformers_infer.py │ ├── evaluation │ │ ├── __init__.py │ │ ├── accuracy.py │ │ ├── benchmark │ │ │ ├── __init__.py │ │ │ ├── accuracy.py │ │ │ ├── data │ │ │ │ ├── AGIEval │ │ │ │ │ └── data │ │ │ │ │ │ ├── few_shot_prompts.csv │ │ │ │ │ │ └── v1 │ │ │ │ │ │ ├── LICENSE │ │ │ │ │ │ ├── aqua-rat.jsonl │ │ │ │ │ │ ├── gaokao-biology.jsonl │ │ │ │ │ │ ├── gaokao-chemistry.jsonl │ │ │ │ │ │ ├── gaokao-chinese.jsonl │ │ │ │ │ │ ├── gaokao-english.jsonl │ │ │ │ │ │ ├── gaokao-geography.jsonl │ │ │ │ │ │ ├── gaokao-history.jsonl │ │ │ │ │ │ ├── gaokao-mathcloze.jsonl │ │ │ │ │ │ ├── gaokao-mathqa.jsonl │ │ │ │ │ │ ├── gaokao-physics.jsonl │ │ │ │ │ │ ├── gaokao-physics.jsonl.all │ │ │ │ │ │ ├── jec-qa-ca.jsonl │ │ │ │ │ │ ├── jec-qa-kd.jsonl │ │ │ │ │ │ ├── logiqa-en.jsonl │ │ │ │ │ │ ├── logiqa-zh.jsonl │ │ │ │ │ │ ├── lsat-ar.jsonl │ │ │ │ │ │ ├── lsat-lr.jsonl │ │ │ │ │ │ ├── lsat-rc.jsonl │ │ │ │ │ │ ├── math.jsonl │ │ │ │ │ │ ├── sat-en-without-passage.jsonl │ │ │ │ │ │ ├── sat-en.jsonl │ │ │ │ │ │ └── sat-math.jsonl │ │ │ │ ├── ARC │ │ │ │ │ ├── ARC-c │ │ │ │ │ │ ├── ARC-Challenge-Dev.jsonl │ │ │ │ │ │ └── ARC-Challenge-Test.jsonl │ │ │ │ │ └── ARC-e │ │ │ │ │ │ ├── ARC-Easy-Dev.jsonl │ │ │ │ │ │ └── ARC-Easy-Test.jsonl │ │ │ │ ├── BBH │ │ │ │ │ ├── data │ │ │ │ │ │ ├── README.md │ │ │ │ │ │ ├── boolean_expressions.json │ │ │ │ │ │ ├── causal_judgement.json │ │ │ │ │ │ ├── date_understanding.json │ │ │ │ │ │ ├── disambiguation_qa.json │ │ │ │ │ │ ├── dyck_languages.json │ │ │ │ │ │ ├── formal_fallacies.json │ │ │ │ │ │ ├── geometric_shapes.json │ │ │ │ │ │ ├── hyperbaton.json │ │ │ │ │ │ ├── logical_deduction_five_objects.json │ │ │ │ │ │ ├── logical_deduction_seven_objects.json │ │ │ │ │ │ ├── logical_deduction_three_objects.json │ │ │ │ │ │ ├── movie_recommendation.json │ │ │ │ │ │ ├── multistep_arithmetic_two.json │ │ │ │ │ │ ├── navigate.json │ │ │ │ │ │ ├── object_counting.json │ │ │ │ │ │ ├── penguins_in_a_table.json │ │ │ │ │ │ ├── reasoning_about_colored_objects.json │ │ │ │ │ │ ├── ruin_names.json │ │ │ │ │ │ ├── salient_translation_error_detection.json │ │ │ │ │ │ ├── snarks.json │ │ │ │ │ │ ├── sports_understanding.json │ │ │ │ │ │ ├── temporal_sequences.json │ │ │ │ │ │ ├── tracking_shuffled_objects_five_objects.json │ │ │ │ │ │ ├── tracking_shuffled_objects_seven_objects.json │ │ │ │ │ │ ├── tracking_shuffled_objects_three_objects.json │ │ │ │ │ │ ├── web_of_lies.json │ │ │ │ │ │ └── word_sorting.json │ │ │ │ │ └── lib_prompt │ │ │ │ │ │ ├── boolean_expressions.txt │ │ │ │ │ │ ├── causal_judgement.txt │ │ │ │ │ │ ├── date_understanding.txt │ │ │ │ │ │ ├── disambiguation_qa.txt │ │ │ │ │ │ ├── dyck_languages.txt │ │ │ │ │ │ ├── formal_fallacies.txt │ │ │ │ │ │ ├── geometric_shapes.txt │ │ │ │ │ │ ├── hyperbaton.txt │ │ │ │ │ │ ├── logical_deduction_five_objects.txt │ │ │ │ │ │ ├── logical_deduction_seven_objects.txt │ │ │ │ │ │ ├── logical_deduction_three_objects.txt │ │ │ │ │ │ ├── movie_recommendation.txt │ │ │ │ │ │ ├── multistep_arithmetic_two.txt │ │ │ │ │ │ ├── navigate.txt │ │ │ │ │ │ ├── object_counting.txt │ │ │ │ │ │ ├── penguins_in_a_table.txt │ │ │ │ │ │ ├── reasoning_about_colored_objects.txt │ │ │ │ │ │ ├── ruin_names.txt │ │ │ │ │ │ ├── salient_translation_error_detection.txt │ │ │ │ │ │ ├── snarks.txt │ │ │ │ │ │ ├── sports_understanding.txt │ │ │ │ │ │ ├── temporal_sequences.txt │ │ │ │ │ │ ├── tracking_shuffled_objects_five_objects.txt │ │ │ │ │ │ ├── tracking_shuffled_objects_seven_objects.txt │ │ │ │ │ │ ├── tracking_shuffled_objects_three_objects.txt │ │ │ │ │ │ ├── web_of_lies.txt │ │ │ │ │ │ └── word_sorting.txt │ │ │ │ ├── __init__.py │ │ │ │ ├── ceval │ │ │ │ │ └── formal_ceval │ │ │ │ │ │ ├── dev │ │ │ │ │ │ ├── accountant_dev.csv │ │ │ │ │ │ ├── advanced_mathematics_dev.csv │ │ │ │ │ │ ├── art_studies_dev.csv │ │ │ │ │ │ ├── basic_medicine_dev.csv │ │ │ │ │ │ ├── business_administration_dev.csv │ │ │ │ │ │ ├── chinese_language_and_literature_dev.csv │ │ │ │ │ │ ├── civil_servant_dev.csv │ │ │ │ │ │ ├── clinical_medicine_dev.csv │ │ │ │ │ │ ├── college_chemistry_dev.csv │ │ │ │ │ │ ├── college_economics_dev.csv │ │ │ │ │ │ ├── college_physics_dev.csv │ │ │ │ │ │ ├── college_programming_dev.csv │ │ │ │ │ │ ├── computer_architecture_dev.csv │ │ │ │ │ │ ├── computer_network_dev.csv │ │ │ │ │ │ ├── discrete_mathematics_dev.csv │ │ │ │ │ │ ├── education_science_dev.csv │ │ │ │ │ │ ├── electrical_engineer_dev.csv │ │ │ │ │ │ ├── environmental_impact_assessment_engineer_dev.csv │ │ │ │ │ │ ├── fire_engineer_dev.csv │ │ │ │ │ │ ├── high_school_biology_dev.csv │ │ │ │ │ │ ├── high_school_chemistry_dev.csv │ │ │ │ │ │ ├── high_school_chinese_dev.csv │ │ │ │ │ │ ├── high_school_geography_dev.csv │ │ │ │ │ │ ├── high_school_history_dev.csv │ │ │ │ │ │ ├── high_school_mathematics_dev.csv │ │ │ │ │ │ ├── high_school_physics_dev.csv │ │ │ │ │ │ ├── high_school_politics_dev.csv │ │ │ │ │ │ ├── ideological_and_moral_cultivation_dev.csv │ │ │ │ │ │ ├── law_dev.csv │ │ │ │ │ │ ├── legal_professional_dev.csv │ │ │ │ │ │ ├── logic_dev.csv │ │ │ │ │ │ ├── mao_zedong_thought_dev.csv │ │ │ │ │ │ ├── marxism_dev.csv │ │ │ │ │ │ ├── metrology_engineer_dev.csv │ │ │ │ │ │ ├── middle_school_biology_dev.csv │ │ │ │ │ │ ├── middle_school_chemistry_dev.csv │ │ │ │ │ │ ├── middle_school_geography_dev.csv │ │ │ │ │ │ ├── middle_school_history_dev.csv │ │ │ │ │ │ ├── middle_school_mathematics_dev.csv │ │ │ │ │ │ ├── middle_school_physics_dev.csv │ │ │ │ │ │ ├── middle_school_politics_dev.csv │ │ │ │ │ │ ├── modern_chinese_history_dev.csv │ │ │ │ │ │ ├── operating_system_dev.csv │ │ │ │ │ │ ├── physician_dev.csv │ │ │ │ │ │ ├── plant_protection_dev.csv │ │ │ │ │ │ ├── probability_and_statistics_dev.csv │ │ │ │ │ │ ├── professional_tour_guide_dev.csv │ │ │ │ │ │ ├── sports_science_dev.csv │ │ │ │ │ │ ├── tax_accountant_dev.csv │ │ │ │ │ │ ├── teacher_qualification_dev.csv │ │ │ │ │ │ ├── urban_and_rural_planner_dev.csv │ │ │ │ │ │ └── veterinary_medicine_dev.csv │ │ │ │ │ │ ├── test │ │ │ │ │ │ ├── accountant_test.csv │ │ │ │ │ │ ├── advanced_mathematics_test.csv │ │ │ │ │ │ ├── art_studies_test.csv │ │ │ │ │ │ ├── basic_medicine_test.csv │ │ │ │ │ │ ├── business_administration_test.csv │ │ │ │ │ │ ├── chinese_language_and_literature_test.csv │ │ │ │ │ │ ├── civil_servant_test.csv │ │ │ │ │ │ ├── clinical_medicine_test.csv │ │ │ │ │ │ ├── college_chemistry_test.csv │ │ │ │ │ │ ├── college_economics_test.csv │ │ │ │ │ │ ├── college_physics_test.csv │ │ │ │ │ │ ├── college_programming_test.csv │ │ │ │ │ │ ├── computer_architecture_test.csv │ │ │ │ │ │ ├── computer_network_test.csv │ │ │ │ │ │ ├── discrete_mathematics_test.csv │ │ │ │ │ │ ├── education_science_test.csv │ │ │ │ │ │ ├── electrical_engineer_test.csv │ │ │ │ │ │ ├── environmental_impact_assessment_engineer_test.csv │ │ │ │ │ │ ├── fire_engineer_test.csv │ │ │ │ │ │ ├── high_school_biology_test.csv │ │ │ │ │ │ ├── high_school_chemistry_test.csv │ │ │ │ │ │ ├── high_school_chinese_test.csv │ │ │ │ │ │ ├── high_school_geography_test.csv │ │ │ │ │ │ ├── high_school_history_test.csv │ │ │ │ │ │ ├── high_school_mathematics_test.csv │ │ │ │ │ │ ├── high_school_physics_test.csv │ │ │ │ │ │ ├── high_school_politics_test.csv │ │ │ │ │ │ ├── ideological_and_moral_cultivation_test.csv │ │ │ │ │ │ ├── law_test.csv │ │ │ │ │ │ ├── legal_professional_test.csv │ │ │ │ │ │ ├── logic_test.csv │ │ │ │ │ │ ├── mao_zedong_thought_test.csv │ │ │ │ │ │ ├── marxism_test.csv │ │ │ │ │ │ ├── metrology_engineer_test.csv │ │ │ │ │ │ ├── middle_school_biology_test.csv │ │ │ │ │ │ ├── middle_school_chemistry_test.csv │ │ │ │ │ │ ├── middle_school_geography_test.csv │ │ │ │ │ │ ├── middle_school_history_test.csv │ │ │ │ │ │ ├── middle_school_mathematics_test.csv │ │ │ │ │ │ ├── middle_school_physics_test.csv │ │ │ │ │ │ ├── middle_school_politics_test.csv │ │ │ │ │ │ ├── modern_chinese_history_test.csv │ │ │ │ │ │ ├── operating_system_test.csv │ │ │ │ │ │ ├── physician_test.csv │ │ │ │ │ │ ├── plant_protection_test.csv │ │ │ │ │ │ ├── probability_and_statistics_test.csv │ │ │ │ │ │ ├── professional_tour_guide_test.csv │ │ │ │ │ │ ├── sports_science_test.csv │ │ │ │ │ │ ├── tax_accountant_test.csv │ │ │ │ │ │ ├── teacher_qualification_test.csv │ │ │ │ │ │ ├── urban_and_rural_planner_test.csv │ │ │ │ │ │ └── veterinary_medicine_test.csv │ │ │ │ │ │ └── val │ │ │ │ │ │ ├── accountant_val.csv │ │ │ │ │ │ ├── advanced_mathematics_val.csv │ │ │ │ │ │ ├── art_studies_val.csv │ │ │ │ │ │ ├── basic_medicine_val.csv │ │ │ │ │ │ ├── business_administration_val.csv │ │ │ │ │ │ ├── chinese_language_and_literature_val.csv │ │ │ │ │ │ ├── civil_servant_val.csv │ │ │ │ │ │ ├── clinical_medicine_val.csv │ │ │ │ │ │ ├── college_chemistry_val.csv │ │ │ │ │ │ ├── college_economics_val.csv │ │ │ │ │ │ ├── college_physics_val.csv │ │ │ │ │ │ ├── college_programming_val.csv │ │ │ │ │ │ ├── computer_architecture_val.csv │ │ │ │ │ │ ├── computer_network_val.csv │ │ │ │ │ │ ├── discrete_mathematics_val.csv │ │ │ │ │ │ ├── education_science_val.csv │ │ │ │ │ │ ├── electrical_engineer_val.csv │ │ │ │ │ │ ├── environmental_impact_assessment_engineer_val.csv │ │ │ │ │ │ ├── fire_engineer_val.csv │ │ │ │ │ │ ├── high_school_biology_val.csv │ │ │ │ │ │ ├── high_school_chemistry_val.csv │ │ │ │ │ │ ├── high_school_chinese_val.csv │ │ │ │ │ │ ├── high_school_geography_val.csv │ │ │ │ │ │ ├── high_school_history_val.csv │ │ │ │ │ │ ├── high_school_mathematics_val.csv │ │ │ │ │ │ ├── high_school_physics_val.csv │ │ │ │ │ │ ├── high_school_politics_val.csv │ │ │ │ │ │ ├── ideological_and_moral_cultivation_val.csv │ │ │ │ │ │ ├── law_val.csv │ │ │ │ │ │ ├── legal_professional_val.csv │ │ │ │ │ │ ├── logic_val.csv │ │ │ │ │ │ ├── mao_zedong_thought_val.csv │ │ │ │ │ │ ├── marxism_val.csv │ │ │ │ │ │ ├── metrology_engineer_val.csv │ │ │ │ │ │ ├── middle_school_biology_val.csv │ │ │ │ │ │ ├── middle_school_chemistry_val.csv │ │ │ │ │ │ ├── middle_school_geography_val.csv │ │ │ │ │ │ ├── middle_school_history_val.csv │ │ │ │ │ │ ├── middle_school_mathematics_val.csv │ │ │ │ │ │ ├── middle_school_physics_val.csv │ │ │ │ │ │ ├── middle_school_politics_val.csv │ │ │ │ │ │ ├── modern_chinese_history_val.csv │ │ │ │ │ │ ├── operating_system_val.csv │ │ │ │ │ │ ├── physician_val.csv │ │ │ │ │ │ ├── plant_protection_val.csv │ │ │ │ │ │ ├── probability_and_statistics_val.csv │ │ │ │ │ │ ├── professional_tour_guide_val.csv │ │ │ │ │ │ ├── sports_science_val.csv │ │ │ │ │ │ ├── tax_accountant_val.csv │ │ │ │ │ │ ├── teacher_qualification_val.csv │ │ │ │ │ │ ├── urban_and_rural_planner_val.csv │ │ │ │ │ │ └── veterinary_medicine_val.csv │ │ │ │ ├── cmmlu │ │ │ │ │ ├── dev │ │ │ │ │ │ ├── agronomy.csv │ │ │ │ │ │ ├── anatomy.csv │ │ │ │ │ │ ├── ancient_chinese.csv │ │ │ │ │ │ ├── arts.csv │ │ │ │ │ │ ├── astronomy.csv │ │ │ │ │ │ ├── business_ethics.csv │ │ │ │ │ │ ├── chinese_civil_service_exam.csv │ │ │ │ │ │ ├── chinese_driving_rule.csv │ │ │ │ │ │ ├── chinese_food_culture.csv │ │ │ │ │ │ ├── chinese_foreign_policy.csv │ │ │ │ │ │ ├── chinese_history.csv │ │ │ │ │ │ ├── chinese_literature.csv │ │ │ │ │ │ ├── chinese_teacher_qualification.csv │ │ │ │ │ │ ├── clinical_knowledge.csv │ │ │ │ │ │ ├── college_actuarial_science.csv │ │ │ │ │ │ ├── college_education.csv │ │ │ │ │ │ ├── college_engineering_hydrology.csv │ │ │ │ │ │ ├── college_law.csv │ │ │ │ │ │ ├── college_mathematics.csv │ │ │ │ │ │ ├── college_medical_statistics.csv │ │ │ │ │ │ ├── college_medicine.csv │ │ │ │ │ │ ├── computer_science.csv │ │ │ │ │ │ ├── computer_security.csv │ │ │ │ │ │ ├── conceptual_physics.csv │ │ │ │ │ │ ├── construction_project_management.csv │ │ │ │ │ │ ├── economics.csv │ │ │ │ │ │ ├── education.csv │ │ │ │ │ │ ├── electrical_engineering.csv │ │ │ │ │ │ ├── elementary_chinese.csv │ │ │ │ │ │ ├── elementary_commonsense.csv │ │ │ │ │ │ ├── elementary_information_and_technology.csv │ │ │ │ │ │ ├── elementary_mathematics.csv │ │ │ │ │ │ ├── ethnology.csv │ │ │ │ │ │ ├── food_science.csv │ │ │ │ │ │ ├── genetics.csv │ │ │ │ │ │ ├── global_facts.csv │ │ │ │ │ │ ├── high_school_biology.csv │ │ │ │ │ │ ├── high_school_chemistry.csv │ │ │ │ │ │ ├── high_school_geography.csv │ │ │ │ │ │ ├── high_school_mathematics.csv │ │ │ │ │ │ ├── high_school_physics.csv │ │ │ │ │ │ ├── high_school_politics.csv │ │ │ │ │ │ ├── human_sexuality.csv │ │ │ │ │ │ ├── international_law.csv │ │ │ │ │ │ ├── journalism.csv │ │ │ │ │ │ ├── jurisprudence.csv │ │ │ │ │ │ ├── legal_and_moral_basis.csv │ │ │ │ │ │ ├── logical.csv │ │ │ │ │ │ ├── machine_learning.csv │ │ │ │ │ │ ├── management.csv │ │ │ │ │ │ ├── marketing.csv │ │ │ │ │ │ ├── marxist_theory.csv │ │ │ │ │ │ ├── modern_chinese.csv │ │ │ │ │ │ ├── nutrition.csv │ │ │ │ │ │ ├── philosophy.csv │ │ │ │ │ │ ├── professional_accounting.csv │ │ │ │ │ │ ├── professional_law.csv │ │ │ │ │ │ ├── professional_medicine.csv │ │ │ │ │ │ ├── professional_psychology.csv │ │ │ │ │ │ ├── public_relations.csv │ │ │ │ │ │ ├── security_study.csv │ │ │ │ │ │ ├── sociology.csv │ │ │ │ │ │ ├── sports_science.csv │ │ │ │ │ │ ├── traditional_chinese_medicine.csv │ │ │ │ │ │ ├── virology.csv │ │ │ │ │ │ ├── world_history.csv │ │ │ │ │ │ └── world_religions.csv │ │ │ │ │ └── test │ │ │ │ │ │ ├── agronomy.csv │ │ │ │ │ │ ├── anatomy.csv │ │ │ │ │ │ ├── ancient_chinese.csv │ │ │ │ │ │ ├── arts.csv │ │ │ │ │ │ ├── astronomy.csv │ │ │ │ │ │ ├── business_ethics.csv │ │ │ │ │ │ ├── chinese_civil_service_exam.csv │ │ │ │ │ │ ├── chinese_driving_rule.csv │ │ │ │ │ │ ├── chinese_food_culture.csv │ │ │ │ │ │ ├── chinese_foreign_policy.csv │ │ │ │ │ │ ├── chinese_history.csv │ │ │ │ │ │ ├── chinese_literature.csv │ │ │ │ │ │ ├── chinese_teacher_qualification.csv │ │ │ │ │ │ ├── clinical_knowledge.csv │ │ │ │ │ │ ├── college_actuarial_science.csv │ │ │ │ │ │ ├── college_education.csv │ │ │ │ │ │ ├── college_engineering_hydrology.csv │ │ │ │ │ │ ├── college_law.csv │ │ │ │ │ │ ├── college_mathematics.csv │ │ │ │ │ │ ├── college_medical_statistics.csv │ │ │ │ │ │ ├── college_medicine.csv │ │ │ │ │ │ ├── computer_science.csv │ │ │ │ │ │ ├── computer_security.csv │ │ │ │ │ │ ├── conceptual_physics.csv │ │ │ │ │ │ ├── construction_project_management.csv │ │ │ │ │ │ ├── economics.csv │ │ │ │ │ │ ├── education.csv │ │ │ │ │ │ ├── electrical_engineering.csv │ │ │ │ │ │ ├── elementary_chinese.csv │ │ │ │ │ │ ├── elementary_commonsense.csv │ │ │ │ │ │ ├── elementary_information_and_technology.csv │ │ │ │ │ │ ├── elementary_mathematics.csv │ │ │ │ │ │ ├── ethnology.csv │ │ │ │ │ │ ├── food_science.csv │ │ │ │ │ │ ├── genetics.csv │ │ │ │ │ │ ├── global_facts.csv │ │ │ │ │ │ ├── high_school_biology.csv │ │ │ │ │ │ ├── high_school_chemistry.csv │ │ │ │ │ │ ├── high_school_geography.csv │ │ │ │ │ │ ├── high_school_mathematics.csv │ │ │ │ │ │ ├── high_school_physics.csv │ │ │ │ │ │ ├── high_school_politics.csv │ │ │ │ │ │ ├── human_sexuality.csv │ │ │ │ │ │ ├── international_law.csv │ │ │ │ │ │ ├── journalism.csv │ │ │ │ │ │ ├── jurisprudence.csv │ │ │ │ │ │ ├── legal_and_moral_basis.csv │ │ │ │ │ │ ├── logical.csv │ │ │ │ │ │ ├── machine_learning.csv │ │ │ │ │ │ ├── management.csv │ │ │ │ │ │ ├── marketing.csv │ │ │ │ │ │ ├── marxist_theory.csv │ │ │ │ │ │ ├── modern_chinese.csv │ │ │ │ │ │ ├── nutrition.csv │ │ │ │ │ │ ├── philosophy.csv │ │ │ │ │ │ ├── professional_accounting.csv │ │ │ │ │ │ ├── professional_law.csv │ │ │ │ │ │ ├── professional_medicine.csv │ │ │ │ │ │ ├── professional_psychology.csv │ │ │ │ │ │ ├── public_relations.csv │ │ │ │ │ │ ├── security_study.csv │ │ │ │ │ │ ├── sociology.csv │ │ │ │ │ │ ├── sports_science.csv │ │ │ │ │ │ ├── traditional_chinese_medicine.csv │ │ │ │ │ │ ├── virology.csv │ │ │ │ │ │ ├── world_history.csv │ │ │ │ │ │ └── world_religions.csv │ │ │ │ └── mmlu │ │ │ │ │ ├── dev │ │ │ │ │ ├── abstract_algebra_dev.csv │ │ │ │ │ ├── anatomy_dev.csv │ │ │ │ │ ├── astronomy_dev.csv │ │ │ │ │ ├── business_ethics_dev.csv │ │ │ │ │ ├── clinical_knowledge_dev.csv │ │ │ │ │ ├── college_biology_dev.csv │ │ │ │ │ ├── college_chemistry_dev.csv │ │ │ │ │ ├── college_computer_science_dev.csv │ │ │ │ │ ├── college_mathematics_dev.csv │ │ │ │ │ ├── college_medicine_dev.csv │ │ │ │ │ ├── college_physics_dev.csv │ │ │ │ │ ├── computer_security_dev.csv │ │ │ │ │ ├── conceptual_physics_dev.csv │ │ │ │ │ ├── econometrics_dev.csv │ │ │ │ │ ├── electrical_engineering_dev.csv │ │ │ │ │ ├── elementary_mathematics_dev.csv │ │ │ │ │ ├── formal_logic_dev.csv │ │ │ │ │ ├── global_facts_dev.csv │ │ │ │ │ ├── high_school_biology_dev.csv │ │ │ │ │ ├── high_school_chemistry_dev.csv │ │ │ │ │ ├── high_school_computer_science_dev.csv │ │ │ │ │ ├── high_school_european_history_dev.csv │ │ │ │ │ ├── high_school_geography_dev.csv │ │ │ │ │ ├── high_school_government_and_politics_dev.csv │ │ │ │ │ ├── high_school_macroeconomics_dev.csv │ │ │ │ │ ├── high_school_mathematics_dev.csv │ │ │ │ │ ├── high_school_microeconomics_dev.csv │ │ │ │ │ ├── high_school_physics_dev.csv │ │ │ │ │ ├── high_school_psychology_dev.csv │ │ │ │ │ ├── high_school_statistics_dev.csv │ │ │ │ │ ├── high_school_us_history_dev.csv │ │ │ │ │ ├── high_school_world_history_dev.csv │ │ │ │ │ ├── human_aging_dev.csv │ │ │ │ │ ├── human_sexuality_dev.csv │ │ │ │ │ ├── international_law_dev.csv │ │ │ │ │ ├── jurisprudence_dev.csv │ │ │ │ │ ├── logical_fallacies_dev.csv │ │ │ │ │ ├── machine_learning_dev.csv │ │ │ │ │ ├── management_dev.csv │ │ │ │ │ ├── marketing_dev.csv │ │ │ │ │ ├── medical_genetics_dev.csv │ │ │ │ │ ├── miscellaneous_dev.csv │ │ │ │ │ ├── moral_disputes_dev.csv │ │ │ │ │ ├── moral_scenarios_dev.csv │ │ │ │ │ ├── nutrition_dev.csv │ │ │ │ │ ├── philosophy_dev.csv │ │ │ │ │ ├── prehistory_dev.csv │ │ │ │ │ ├── professional_accounting_dev.csv │ │ │ │ │ ├── professional_law_dev.csv │ │ │ │ │ ├── professional_medicine_dev.csv │ │ │ │ │ ├── professional_psychology_dev.csv │ │ │ │ │ ├── public_relations_dev.csv │ │ │ │ │ ├── security_studies_dev.csv │ │ │ │ │ ├── sociology_dev.csv │ │ │ │ │ ├── us_foreign_policy_dev.csv │ │ │ │ │ ├── virology_dev.csv │ │ │ │ │ └── world_religions_dev.csv │ │ │ │ │ ├── test │ │ │ │ │ ├── abstract_algebra_test.csv │ │ │ │ │ ├── anatomy_test.csv │ │ │ │ │ ├── astronomy_test.csv │ │ │ │ │ ├── business_ethics_test.csv │ │ │ │ │ ├── clinical_knowledge_test.csv │ │ │ │ │ ├── college_biology_test.csv │ │ │ │ │ ├── college_chemistry_test.csv │ │ │ │ │ ├── college_computer_science_test.csv │ │ │ │ │ ├── college_mathematics_test.csv │ │ │ │ │ ├── college_medicine_test.csv │ │ │ │ │ ├── college_physics_test.csv │ │ │ │ │ ├── computer_security_test.csv │ │ │ │ │ ├── conceptual_physics_test.csv │ │ │ │ │ ├── econometrics_test.csv │ │ │ │ │ ├── electrical_engineering_test.csv │ │ │ │ │ ├── elementary_mathematics_test.csv │ │ │ │ │ ├── formal_logic_test.csv │ │ │ │ │ ├── global_facts_test.csv │ │ │ │ │ ├── high_school_biology_test.csv │ │ │ │ │ ├── high_school_chemistry_test.csv │ │ │ │ │ ├── high_school_computer_science_test.csv │ │ │ │ │ ├── high_school_european_history_test.csv │ │ │ │ │ ├── high_school_geography_test.csv │ │ │ │ │ ├── high_school_government_and_politics_test.csv │ │ │ │ │ ├── high_school_macroeconomics_test.csv │ │ │ │ │ ├── high_school_mathematics_test.csv │ │ │ │ │ ├── high_school_microeconomics_test.csv │ │ │ │ │ ├── high_school_physics_test.csv │ │ │ │ │ ├── high_school_psychology_test.csv │ │ │ │ │ ├── high_school_statistics_test.csv │ │ │ │ │ ├── high_school_us_history_test.csv │ │ │ │ │ ├── high_school_world_history_test.csv │ │ │ │ │ ├── human_aging_test.csv │ │ │ │ │ ├── human_sexuality_test.csv │ │ │ │ │ ├── international_law_test.csv │ │ │ │ │ ├── jurisprudence_test.csv │ │ │ │ │ ├── logical_fallacies_test.csv │ │ │ │ │ ├── machine_learning_test.csv │ │ │ │ │ ├── management_test.csv │ │ │ │ │ ├── marketing_test.csv │ │ │ │ │ ├── medical_genetics_test.csv │ │ │ │ │ ├── miscellaneous_test.csv │ │ │ │ │ ├── moral_disputes_test.csv │ │ │ │ │ ├── moral_scenarios_test.csv │ │ │ │ │ ├── nutrition_test.csv │ │ │ │ │ ├── philosophy_test.csv │ │ │ │ │ ├── prehistory_test.csv │ │ │ │ │ ├── professional_accounting_test.csv │ │ │ │ │ ├── professional_law_test.csv │ │ │ │ │ ├── professional_medicine_test.csv │ │ │ │ │ ├── professional_psychology_test.csv │ │ │ │ │ ├── public_relations_test.csv │ │ │ │ │ ├── security_studies_test.csv │ │ │ │ │ ├── sociology_test.csv │ │ │ │ │ ├── us_foreign_policy_test.csv │ │ │ │ │ ├── virology_test.csv │ │ │ │ │ └── world_religions_test.csv │ │ │ │ │ └── val │ │ │ │ │ ├── abstract_algebra_val.csv │ │ │ │ │ ├── anatomy_val.csv │ │ │ │ │ ├── astronomy_val.csv │ │ │ │ │ ├── business_ethics_val.csv │ │ │ │ │ ├── clinical_knowledge_val.csv │ │ │ │ │ ├── college_biology_val.csv │ │ │ │ │ ├── college_chemistry_val.csv │ │ │ │ │ ├── college_computer_science_val.csv │ │ │ │ │ ├── college_mathematics_val.csv │ │ │ │ │ ├── college_medicine_val.csv │ │ │ │ │ ├── college_physics_val.csv │ │ │ │ │ ├── computer_security_val.csv │ │ │ │ │ ├── conceptual_physics_val.csv │ │ │ │ │ ├── econometrics_val.csv │ │ │ │ │ ├── electrical_engineering_val.csv │ │ │ │ │ ├── elementary_mathematics_val.csv │ │ │ │ │ ├── formal_logic_val.csv │ │ │ │ │ ├── global_facts_val.csv │ │ │ │ │ ├── high_school_biology_val.csv │ │ │ │ │ ├── high_school_chemistry_val.csv │ │ │ │ │ ├── high_school_computer_science_val.csv │ │ │ │ │ ├── high_school_european_history_val.csv │ │ │ │ │ ├── high_school_geography_val.csv │ │ │ │ │ ├── high_school_government_and_politics_val.csv │ │ │ │ │ ├── high_school_macroeconomics_val.csv │ │ │ │ │ ├── high_school_mathematics_val.csv │ │ │ │ │ ├── high_school_microeconomics_val.csv │ │ │ │ │ ├── high_school_physics_val.csv │ │ │ │ │ ├── high_school_psychology_val.csv │ │ │ │ │ ├── high_school_statistics_val.csv │ │ │ │ │ ├── high_school_us_history_val.csv │ │ │ │ │ ├── high_school_world_history_val.csv │ │ │ │ │ ├── human_aging_val.csv │ │ │ │ │ ├── human_sexuality_val.csv │ │ │ │ │ ├── international_law_val.csv │ │ │ │ │ ├── jurisprudence_val.csv │ │ │ │ │ ├── logical_fallacies_val.csv │ │ │ │ │ ├── machine_learning_val.csv │ │ │ │ │ ├── management_val.csv │ │ │ │ │ ├── marketing_val.csv │ │ │ │ │ ├── medical_genetics_val.csv │ │ │ │ │ ├── miscellaneous_val.csv │ │ │ │ │ ├── moral_disputes_val.csv │ │ │ │ │ ├── moral_scenarios_val.csv │ │ │ │ │ ├── nutrition_val.csv │ │ │ │ │ ├── philosophy_val.csv │ │ │ │ │ ├── prehistory_val.csv │ │ │ │ │ ├── professional_accounting_val.csv │ │ │ │ │ ├── professional_law_val.csv │ │ │ │ │ ├── professional_medicine_val.csv │ │ │ │ │ ├── professional_psychology_val.csv │ │ │ │ │ ├── public_relations_val.csv │ │ │ │ │ ├── security_studies_val.csv │ │ │ │ │ ├── sociology_val.csv │ │ │ │ │ ├── us_foreign_policy_val.csv │ │ │ │ │ ├── virology_val.csv │ │ │ │ │ └── world_religions_val.csv │ │ │ ├── dataset_loader.py │ │ │ ├── eval.py │ │ │ ├── eval_dataset.py │ │ │ ├── eval_metric.py │ │ │ ├── eval_ppl.py │ │ │ └── metric.json │ │ ├── custom │ │ │ ├── __init__.py │ │ │ ├── eval.py │ │ │ └── example.py │ │ ├── eval_module.py │ │ └── manual_eval.py │ ├── main.py │ ├── prune │ │ ├── __init__.py │ │ ├── llm_pruner │ │ │ ├── LLMPruner │ │ │ │ ├── __init__.py │ │ │ │ ├── datasets │ │ │ │ │ ├── dialogue.py │ │ │ │ │ ├── example_samples.py │ │ │ │ │ └── ppl_dataset.py │ │ │ │ ├── evaluator │ │ │ │ │ └── ppl.py │ │ │ │ ├── models │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── helper.py │ │ │ │ │ ├── hf_baichuan │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ ├── baichuan13B │ │ │ │ │ │ │ ├── configuration_baichuan.py │ │ │ │ │ │ │ └── modeling_baichuan_13B.py │ │ │ │ │ │ └── baichuan7B │ │ │ │ │ │ │ ├── configuration_baichuan.py │ │ │ │ │ │ │ └── modeling_baichuan_7B.py │ │ │ │ │ ├── hf_bloom │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ └── modeling_bloom.py │ │ │ │ │ ├── hf_chatglm │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ ├── configuration_chatglm.py │ │ │ │ │ │ ├── modeling_chatglm.py │ │ │ │ │ │ └── tokenizer_chatglm.py │ │ │ │ │ ├── hf_llama │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ └── modeling_llama.py │ │ │ │ │ └── llama │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ ├── generation.py │ │ │ │ │ │ ├── model.py │ │ │ │ │ │ └── tokenizer.py │ │ │ │ ├── peft │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── import_utils.py │ │ │ │ │ ├── mapping.py │ │ │ │ │ ├── peft_model.py │ │ │ │ │ ├── tuners │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ ├── adalora.py │ │ │ │ │ │ ├── lora.py │ │ │ │ │ │ ├── p_tuning.py │ │ │ │ │ │ ├── prefix_tuning.py │ │ │ │ │ │ └── prompt_tuning.py │ │ │ │ │ └── utils │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ ├── config.py │ │ │ │ │ │ ├── other.py │ │ │ │ │ │ └── save_and_load.py │ │ │ │ ├── pruner │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── hf_baichuan_pruner.py │ │ │ │ │ ├── hf_chatglm_pruner.py │ │ │ │ │ ├── hf_llama_pruner.py │ │ │ │ │ └── llama_pruner.py │ │ │ │ ├── templates │ │ │ │ │ └── prompts.py │ │ │ │ ├── torch_pruning │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── _helpers.py │ │ │ │ │ ├── dependency.py │ │ │ │ │ ├── importance.py │ │ │ │ │ ├── ops.py │ │ │ │ │ ├── pruner │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ ├── algorithms │ │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ │ ├── batchnorm_scale_pruner.py │ │ │ │ │ │ │ ├── group_norm_pruner.py │ │ │ │ │ │ │ ├── magnitude_based_pruner.py │ │ │ │ │ │ │ ├── metapruner.py │ │ │ │ │ │ │ └── scheduler.py │ │ │ │ │ │ └── function.py │ │ │ │ │ └── utils │ │ │ │ │ │ ├── __init__.py │ │ │ │ │ │ ├── op_counter.py │ │ │ │ │ │ └── utils.py │ │ │ │ └── utils │ │ │ │ │ ├── logger.py │ │ │ │ │ └── prompter.py │ │ │ ├── __init__.py │ │ │ ├── baichuan.py │ │ │ ├── bloom.py │ │ │ ├── entry.py │ │ │ ├── hf_prune.py │ │ │ └── util.py │ │ └── sparsegpt │ │ │ ├── __init__.py │ │ │ ├── entry.py │ │ │ ├── sparsegpt.py │ │ │ ├── test.py │ │ │ └── util.py │ └── quantize │ │ ├── __init__.py │ │ ├── bnb.py │ │ ├── gptq.py │ │ └── util.py └── web │ ├── __init__.py │ ├── bin │ └── run.sh │ ├── default_open_sota_model_metrics.json │ ├── exceptions.py │ ├── internal │ ├── __init__.py │ └── route.py │ ├── launcher.py │ ├── main.py │ ├── route.py │ ├── static │ ├── __init__.py │ ├── empty.2f588adb.png │ ├── favicon.png │ ├── index.html │ ├── quickguide.md │ ├── test.md │ ├── umi.css │ └── umi.js │ └── utils │ ├── __init__.py │ ├── asyncssh_utils.py │ ├── peewee_repo.py │ └── traceback_utils.py ├── pyproject.toml ├── setup.py └── tests ├── __init__.py ├── asyncssh_test.py ├── metrics ├── __init__.py ├── a.txt └── test_find.py ├── paramiko_test.py ├── test.py ├── test ├── __init__.py ├── test.py └── test1.py ├── test_asyncio_subprocess.py ├── test_asyncssh.py └── tqdm ├── __init__.py ├── test1.py ├── test111.py ├── test1read.py ├── test_asyncio_subprocess.py ├── test_subprocess.py └── wrapper.py /.github/workflows/python3.9-publish.yml: -------------------------------------------------------------------------------- 1 | # This workflow will upload a Python Package using Twine when a release is created 2 | # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries 3 | 4 | # This workflow uses actions that are not certified by GitHub. 5 | # They are provided by a third-party and are governed by 6 | # separate terms of service, privacy policy, and support 7 | # documentation. 8 | 9 | name: Upload Python Package 10 | 11 | on: 12 | release: 13 | types: [published] 14 | 15 | permissions: 16 | contents: read 17 | 18 | jobs: 19 | deploy: 20 | 21 | runs-on: ubuntu-latest 22 | 23 | steps: 24 | - uses: actions/checkout@v3 25 | - name: Set up Python 26 | uses: actions/setup-python@v3 27 | with: 28 | python-version: '3.9' 29 | - name: Install dependencies 30 | run: | 31 | python -m pip install --upgrade pip 32 | pip install build 33 | - name: Build package 34 | run: python setup.py install 35 | - name: Publish package 36 | uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 37 | with: 38 | user: __token__ 39 | password: ${{ secrets.PYPI_API_TOKEN }} 40 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/* 2 | !.idea/runConfigurations/ 3 | .vscode 4 | **/*.pyc 5 | **/.DS_Store 6 | build 7 | dist 8 | *.egg-info 9 | venv 10 | .cache 11 | .coverage** 12 | coverage.xml 13 | .scannerwork 14 | .dir-locals.el -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include lms/web/static/*.* 2 | include lms/web/*.json 3 | include MANIFEST.in 4 | 5 | include lms/runtime/deploy/request_body.json 6 | graft lms/runtime/evaluation/benchmark/data -------------------------------------------------------------------------------- /assets/image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataCanvasIO/LMS/56b41294c22e67313d0600d0484919edb92641ba/assets/image.png -------------------------------------------------------------------------------- /docs/installation.md: -------------------------------------------------------------------------------- 1 | Installation 2 | == 3 | 4 | # Build 5 | 6 | ## Prerequisites 7 | 8 | To build the LMS components, you will first need the following software packages. 9 | 10 | **System Package** 11 | 12 | * Python>=3.9.7 13 | * make 14 | * git 15 | 16 | **PyPI Package** 17 | 18 | * build 19 | * setuptools 20 | 21 | ## Downloading and Building 22 | 23 | ```bash 24 | git clone git@gitlab.datacanvas.com:APS-OpenSource/lms.git 25 | cd lms 26 | python setup.py bdist_wheel 27 | ``` 28 | 29 | ## Installation 30 | 31 | ```bash 32 | pip3 install dist/lms-$(VERSION)-py3-none-any.whl 33 | ``` 34 | 35 | ## Setting 36 | 37 | ```bash 38 | nohup lms_web start & 39 | ``` 40 | 41 | ```bash 42 | lms join :18080 --token=default 43 | nohup lmsd & 44 | ``` -------------------------------------------------------------------------------- /examples/0.prepare/run.sh: -------------------------------------------------------------------------------- 1 | # Let's download model from huggingface. 2 | cd ../data 3 | git clone --depth 1 https://huggingface.co/facebook/opt-125m 4 | git clone --depth 1 https://huggingface.co/bigscience/bloom-560m 5 | cd ../0.prepare 6 | -------------------------------------------------------------------------------- /examples/1.import/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # clean up the model which named opt-125m. 3 | lms del --model_name opt-125m 4 | # import it to LMS. 5 | lms import --model_path ../data/opt-125m 6 | # list model 7 | lms list 8 | -------------------------------------------------------------------------------- /examples/2.deploy/custom_load_pipeline/pipeline_loader_example.py: -------------------------------------------------------------------------------- 1 | from transformers import pipeline 2 | 3 | 4 | class PipelineLoader: 5 | @staticmethod 6 | def load(model_path: str, **kwargs): 7 | print("====================") 8 | if 'device' in kwargs: 9 | pipe = pipeline(task="text2text-generation", 10 | model=model_path, 11 | trust_remote_code=True, 12 | device=kwargs['device'], 13 | model_kwargs={"ignore_mismatched_sizes": True}) 14 | else: 15 | pipe = pipeline(task="text2text-generation", 16 | model=model_path, 17 | trust_remote_code=True, 18 | model_kwargs={"ignore_mismatched_sizes": True}) 19 | return pipe 20 | 21 | 22 | -------------------------------------------------------------------------------- /examples/2.deploy/custom_load_pipeline/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | api_key=default 4 | port=18991 5 | # deploy model 6 | lms deploy --model_name opt-125m --gpu 0 --port 18991 --api_key $api_key --infer_py pipeline_loader_example.py 7 | 8 | # test prediction 9 | curl -v --noproxy '*' --location 'http://127.0.0.1:18991/prediction' \ 10 | --header 'Api_key: default' \ 11 | --header 'Content-Type: application/json' \ 12 | --data '{ 13 | "messages": [ 14 | { 15 | "role": "User", 16 | "content": "hello" 17 | } 18 | ], 19 | "repetition_penalty": 1.2, 20 | "top_k": 40, 21 | "top_p": 0.5, 22 | "temperature": 0.7, 23 | "max_new_tokens": 100 24 | }' 25 | 26 | # undeploy model 27 | lms undeploy --model_name opt-125m 28 | -------------------------------------------------------------------------------- /examples/2.deploy/setting_infer_config/infer_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "prompt_role": { 3 | "User": "User", 4 | "Assistant": "Assistant" 5 | }, 6 | "tokenizer_kwargs": { 7 | "padding_side": "left" 8 | }, 9 | "model_kwargs": { 10 | }, 11 | "generate": { 12 | "repetition_penalty": 1.2, 13 | "top_k": 40, 14 | "top_p": 0.5, 15 | "temperature": 0.7, 16 | "max_new_tokens": 512 17 | } 18 | } -------------------------------------------------------------------------------- /examples/2.deploy/setting_infer_config/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | api_key=default 4 | port=18991 5 | 6 | lms deploy --model_name opt-125m --gpu 0 --port 18991 --api_key $api_key --infer_config infer_config.json 7 | 8 | 9 | 10 | 11 | curl -v --noproxy '*' --location 'http://127.0.0.1:18991/prediction' \ 12 | --header 'Api_key: default' \ 13 | --header 'Content-Type: application/json' \ 14 | --data '{ 15 | "messages": [ 16 | { 17 | "role": "User", 18 | "content": "hello" 19 | } 20 | ], 21 | "repetition_penalty": 1.2, 22 | "top_k": 40, 23 | "top_p": 0.5, 24 | "temperature": 0.7, 25 | "max_new_tokens": 100 26 | }' 27 | 28 | 29 | 30 | lms undeploy --model_name opt-125m 31 | -------------------------------------------------------------------------------- /examples/3.evaluate/automatic/run.sh: -------------------------------------------------------------------------------- 1 | 2 | 3 | lms eval --model_name opt-125m --task ARC,MMLU --output_path out.json 4 | 5 | -------------------------------------------------------------------------------- /examples/3.evaluate/custom/custom.csv: -------------------------------------------------------------------------------- 1 | Question,A,B,C,D,Answer 2 | 肉牛屠宰后，胴体的哪个部位肉质较好,大腿,腹,胸,小腿,A 3 | 下列鸭品种中，产蛋量最高的品种是,高邮鸭,北京鸭,樱桃谷鸭,绍鸭,D 4 | 常见家畜中输精管不形成输精管壶腹的是,马,猪,牛,羊,B 5 | 牛的发情周期平均为,16～17天,21天,28天,19～20天,B 6 | 羊胴体中，肉质较好的部位是,胸下肉,肩胛肉,后腿肉,小腿肉,C -------------------------------------------------------------------------------- /examples/3.evaluate/custom/run.sh: -------------------------------------------------------------------------------- 1 | 2 | 3 | lms eval --model_name opt-125m --task custom.py --input_path custom.csv --output_path out.json 4 | 5 | -------------------------------------------------------------------------------- /examples/3.evaluate/human/origin.csv: -------------------------------------------------------------------------------- 1 | input,Expected 2 | 肉牛屠宰后，胴体的哪个部位肉质较好,大腿 3 | 下列鸭品种中，产蛋量最高的品种是,绍鸭 4 | 常见家畜中输精管不形成输精管壶腹的是, 5 | 牛的发情周期平均为, 6 | 羊胴体中，肉质较好的部位是,后腿肉 -------------------------------------------------------------------------------- /examples/3.evaluate/human/run.sh: -------------------------------------------------------------------------------- 1 | 2 | 3 | lms eval --model_name opt-125m --task human --input_path origin.csv --output_path target.csv 4 | 5 | -------------------------------------------------------------------------------- /examples/4.prune/sprase/run.sh: -------------------------------------------------------------------------------- 1 | #purne model with sparse 2 | 3 | lms pruning sparse --model_name alaya065-ep2-fp16 --pruned_model_path alaya_sparsed --nsamples 128 --device cpu --layer_name_start blocks.3 --layer_name_stop blocks.4 4 | -------------------------------------------------------------------------------- /examples/4.prune/structure/run.sh: -------------------------------------------------------------------------------- 1 | # prune model with structure 2 | 3 | lms pruning structure --model_name bloom-560m --pruned_model_path bloom-560m_structure --nsamples 128 --device cpu --model_type bloom --block_attention_layer_start 1 --block_attention_layer_end 22 --block_mlp_layer_start 1 --block_mlp_layer_end 22 4 | -------------------------------------------------------------------------------- /examples/5.quantize/int4/run.sh: -------------------------------------------------------------------------------- 1 | # quantize model to int4 percision 2 | 3 | lms quantization --model_name opt-125m --int4 --quantized_model_path quantized_model 4 | -------------------------------------------------------------------------------- /examples/5.quantize/int8/run.sh: -------------------------------------------------------------------------------- 1 | # quantize model to int8 percision 2 | 3 | lms quantization --model_name opt-125m --int8 --quantized_model_path quantized_model 4 | -------------------------------------------------------------------------------- /lms/client/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataCanvasIO/LMS/56b41294c22e67313d0600d0484919edb92641ba/lms/client/__init__.py -------------------------------------------------------------------------------- /lms/client/cluster/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataCanvasIO/LMS/56b41294c22e67313d0600d0484919edb92641ba/lms/client/cluster/__init__.py -------------------------------------------------------------------------------- /lms/client/config.py: -------------------------------------------------------------------------------- 1 | from lms.client.common_utils import get_hostname, get_server_host, get_server_id 2 | 3 | server_host = get_server_host() 4 | 5 | hostname = get_hostname() 6 | 7 | server_id = get_server_id() 8 | -------------------------------------------------------------------------------- /lms/client/daemon/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataCanvasIO/LMS/56b41294c22e67313d0600d0484919edb92641ba/lms/client/daemon/__init__.py -------------------------------------------------------------------------------- /lms/client/daemon/main.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | from lms.client.daemon.serving import make_app 4 | 5 | 6 | def main(): 7 | parser = argparse.ArgumentParser(prog='lmsd', 8 | description=""" 9 | The lmsd is daemon server of lms. It runs on the node of the lms client, 10 | exposing metrics——gpu, cpu, number of request etc.. 11 | """) 12 | 13 | parser.add_argument( 14 | "-v", "--version", action="version", 15 | version=f"{parser.prog} version 0.1.0" 16 | ) 17 | 18 | parser.add_argument('--host', type=str, default="0.0.0.0", help='global_params help') 19 | parser.add_argument('--port', type=int, default="8082", help='global_params help') 20 | parser.add_argument('--interval', type=int, default=15, help='global_params help') 21 | parser.add_argument('--server_id', type=str, help='global_params help') 22 | 23 | args = parser.parse_args() 24 | make_app(args.interval, args.server_id).run(host='0.0.0.0', port=args.port, single_process=True) 25 | 26 | 27 | if __name__ == "__main__": 28 | main() 29 | -------------------------------------------------------------------------------- /lms/client/daemon/nvml_util.py: -------------------------------------------------------------------------------- 1 | import re 2 | import subprocess 3 | 4 | 5 | def nvml_pmon(): 6 | """ 7 | nvmlDeviceGetProcessUtilization 8 | https://www.clear.rice.edu/comp422/resources/cuda/pdf/nvml.pdf 9 | :return: 10 | """ 11 | cmd = "nvidia-smi pmon -c 1 -s mu" 12 | try: 13 | output = subprocess.check_output(cmd, shell=True).decode('utf-8').strip() 14 | # output = "# gpu pid type fb sm mem enc dec command\n# Idx # C/G MB % % % % name\n 0 25985 C 893 0 0 - - python3.7\n 0 92760 C 893 12 0 - - python3.7" 15 | # print(output) 16 | headers = None 17 | stats = {} 18 | for i, line in enumerate(output.splitlines()): 19 | if line.startswith("#"): 20 | if i == 0: 21 | headers = re.split(r"\s+", line.strip())[1:] 22 | continue 23 | values = re.split(r"\s+", line.strip()) 24 | dict0 = dict(zip(headers, values)) 25 | stats[dict0["pid"]] = dict0 26 | return stats 27 | except subprocess.CalledProcessError as e: 28 | return {} 29 | 30 | 31 | if __name__ == '__main__': 32 | print(nvml_pmon()) 33 | -------------------------------------------------------------------------------- /lms/client/help_constants.py: -------------------------------------------------------------------------------- 1 | MODEL_NAME = 'Model name which registered in the Web server' 2 | MODEL_PATH = 'Model dir' 3 | -------------------------------------------------------------------------------- /lms/client/network_utils.py: -------------------------------------------------------------------------------- 1 | import socket 2 | from contextlib import closing 3 | 4 | 5 | def is_port_available(port): 6 | # Create a socket object 7 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 8 | 9 | try: 10 | # Try binding the socket to the specified port 11 | sock.bind(('localhost', port)) 12 | return True 13 | except socket.error: 14 | # Port is in use 15 | return False 16 | finally: 17 | # Close the socket 18 | sock.close() 19 | 20 | 21 | def find_free_port(): 22 | with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: 23 | s.bind(('', 0)) 24 | s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 25 | return s.getsockname()[1] -------------------------------------------------------------------------------- /lms/client/prune_module.py: -------------------------------------------------------------------------------- 1 | from subprocess import check_call, CalledProcessError 2 | 3 | 4 | def prune(model_path, model_name, args, argv): 5 | command = ['lms_rt'] 6 | command.extend(argv[1:]) 7 | for index, segment in enumerate(command): 8 | if segment == "--model_path" or \ 9 | segment == "--model_name": 10 | command[index] = "--model_path" 11 | command[index + 1] = model_path 12 | break 13 | elif segment.startswith("--model_path") or \ 14 | segment.startswith("--model_name"): 15 | command[index] = "--model_path=" + model_path 16 | break 17 | else: 18 | continue 19 | try: 20 | check_call(" ".join(command), shell=True) 21 | print("Successfully pruned %s" % model_name) 22 | except CalledProcessError: 23 | pass 24 | -------------------------------------------------------------------------------- /lms/client/quantize_module.py: -------------------------------------------------------------------------------- 1 | from subprocess import check_call, CalledProcessError 2 | 3 | 4 | def quantize(model_path, model_name, args, argv): 5 | command = ['lms_rt'] 6 | command.extend(argv[1:]) 7 | 8 | for index, segment in enumerate(command): 9 | if segment == "--model_path" or \ 10 | segment == "--model_name": 11 | command[index] = "--model_path" 12 | command[index + 1] = model_path 13 | break 14 | elif segment.startswith("--model_path") or \ 15 | segment.startswith("--model_name"): 16 | command[index] = "--model_path=" + model_path 17 | break 18 | else: 19 | continue 20 | try: 21 | check_call(" ".join(command), shell=True) 22 | print("Successfully quantized %s" % model_name) 23 | except CalledProcessError: 24 | pass 25 | -------------------------------------------------------------------------------- /lms/client/validaters.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | 4 | 5 | class ValidateModelPath(argparse.Action): 6 | def __call__(self, parser, namespace, values, option_string=None): 7 | if not os.path.exists(values): 8 | # sys.exit("This path must be exists" % values) 9 | raise Exception("This path must be exists" % values) 10 | else: 11 | abspath = os.path.abspath(values) 12 | setattr(namespace, self.dest, abspath) 13 | 14 | 15 | class ValidatePathExists(argparse.Action): 16 | def __call__(self, parser, namespace, values, option_string=None): 17 | if not os.path.exists(values): 18 | # sys.exit("This path must be exists" % values) 19 | raise Exception("This path must be exists" % values) 20 | else: 21 | abspath = os.path.abspath(values) 22 | setattr(namespace, self.dest, abspath) 23 | -------------------------------------------------------------------------------- /lms/runtime/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataCanvasIO/LMS/56b41294c22e67313d0600d0484919edb92641ba/lms/runtime/__init__.py -------------------------------------------------------------------------------- /lms/runtime/common/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataCanvasIO/LMS/56b41294c22e67313d0600d0484919edb92641ba/lms/runtime/common/__init__.py -------------------------------------------------------------------------------- /lms/runtime/common/pipeline_loader_example.py: -------------------------------------------------------------------------------- 1 | from transformers import pipeline 2 | 3 | 4 | class PipelineLoader: 5 | @staticmethod 6 | def load(model_path: str, **kwargs): 7 | print("hahahahah") 8 | print("====================") 9 | if 'device' in kwargs: 10 | pipe = pipeline(task="text2text-generation", 11 | model=model_path, 12 | trust_remote_code=True, 13 | device=kwargs['device'], 14 | model_kwargs={"ignore_mismatched_sizes": True}) 15 | else: 16 | pipe = pipeline(task="text2text-generation", 17 | model=model_path, 18 | trust_remote_code=True, 19 | model_kwargs={"ignore_mismatched_sizes": True}) 20 | return pipe 21 | 22 | 23 | if __name__ == '__main__': 24 | PipelineLoader.load("/Users/dev/Documents/APS/models/huggingface/t5-v1_1-small", learning_type="ddd") 25 | -------------------------------------------------------------------------------- /lms/runtime/common/test.py: -------------------------------------------------------------------------------- 1 | from transformers import pipeline 2 | 3 | pipe = pipeline(task="text2text-generation", 4 | model="/Users/dev/Documents/APS/models/huggingface/t5-v1_1-small", 5 | trust_remote_code=True, 6 | model_kwargs={"ignore_mismatched_sizes": True}) 7 | print(pipe('hello')) -------------------------------------------------------------------------------- /lms/runtime/common/traceback_utils.py: -------------------------------------------------------------------------------- 1 | import traceback 2 | 3 | 4 | def get_tarceback(type, value, tb): 5 | return '\n'.join(str(x) for x in traceback.format_exception(type, value, tb)) 6 | -------------------------------------------------------------------------------- /lms/runtime/deploy/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataCanvasIO/LMS/56b41294c22e67313d0600d0484919edb92641ba/lms/runtime/deploy/__init__.py -------------------------------------------------------------------------------- /lms/runtime/deploy/request.py: -------------------------------------------------------------------------------- 1 | from itertools import count 2 | 3 | from sanic import Request, Sanic 4 | from sanic.compat import Header 5 | from sanic.models.protocol_types import TransportProtocol 6 | 7 | 8 | class PredictionCountedRequest(Request): 9 | __slots__ = () 10 | 11 | _counter = count() 12 | count = next(_counter) 13 | 14 | def __init__( 15 | self, 16 | url_bytes: bytes, 17 | headers: Header, 18 | version: str, 19 | method: str, 20 | transport: TransportProtocol, 21 | app: Sanic, 22 | head: bytes = b"", 23 | stream_id: int = 0, 24 | ): 25 | super().__init__( 26 | url_bytes, 27 | headers, 28 | version, 29 | method, 30 | transport, 31 | app, 32 | head, 33 | stream_id, 34 | ) 35 | 36 | if "/prediction" == str(url_bytes, 'UTF-8'): 37 | self.__class__._increment() 38 | if hasattr(self.app, "multiplexer"): 39 | self.app.multiplexer.state["request_count"] = self.__class__.count 40 | 41 | @classmethod 42 | def _increment(cls): 43 | cls.count = next(cls._counter) 44 | 45 | @classmethod 46 | def reset_count(cls): 47 | cls._counter = count() 48 | cls.count = next(cls._counter) 49 | -------------------------------------------------------------------------------- /lms/runtime/deploy/request_body.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "object", 3 | "properties": { 4 | "messages": { 5 | "type": "array", 6 | "items": { 7 | "type": "object", 8 | "properties": { 9 | "role": { 10 | "type": "string" 11 | }, 12 | "content": { 13 | "type": "string" 14 | } 15 | }, 16 | "required": [ 17 | "role", 18 | "content" 19 | ] 20 | } 21 | } 22 | }, 23 | "required": [ 24 | "messages" 25 | ] 26 | } -------------------------------------------------------------------------------- /lms/runtime/evaluation/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataCanvasIO/LMS/56b41294c22e67313d0600d0484919edb92641ba/lms/runtime/evaluation/__init__.py -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/__init__.py: -------------------------------------------------------------------------------- 1 | import pkg_resources 2 | 3 | pkg_resources.declare_namespace(__name__) 4 | -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/data/BBH/lib_prompt/date_understanding.txt: -------------------------------------------------------------------------------- 1 | Infer the date from context. 2 | 3 | Q: Today is Christmas Eve of 1937. What is the date 10 days ago in MM/DD/YYYY? 4 | Options: 5 | (A) 12/14/2026 6 | (B) 12/14/1950 7 | (C) 12/14/2007 8 | (D) 12/14/1937 9 | (E) 07/14/1938 10 | (F) 12/14/1988 11 | A: Let's think step by step. 12 | If today is Christmas Eve of 1937, then today's date is December 24, 1937. 10 days before today is December 14, 1937, that is 12/14/1937. So the answer is (D). 13 | 14 | Q: Tomorrow is 11/12/2019. What is the date one year ago from today in MM/DD/YYYY? 15 | Options: 16 | (A) 09/04/2018 17 | (B) 11/11/2018 18 | (C) 08/25/2018 19 | (D) 11/02/2018 20 | (E) 11/04/2018 21 | A: Let's think step by step. 22 | If tomorrow is 11/12/2019, then today is 11/11/2019. The date one year ago from today is 11/11/2018. So the answer is (B). 23 | 24 | Q: Jane and John married on Jan 2, 1958. It is their 5-year anniversary today. What is the date tomorrow in MM/DD/YYYY? 25 | Options: 26 | (A) 01/11/1961 27 | (B) 01/03/1963 28 | (C) 01/18/1961 29 | (D) 10/14/1960 30 | (E) 01/03/1982 31 | (F) 12/03/1960 32 | A: Let's think step by step. 33 | If Jane and John married on Jan 2, 1958, then and if it is their 5-year anniversary today, then today's date is Jan 2, 1963. The date tomorrow is Jan 3, 1963, that is 01/03/1963. So the answer is (B). -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/data/BBH/lib_prompt/object_counting.txt: -------------------------------------------------------------------------------- 1 | Questions that involve enumerating objects and asking the model to count them. 2 | 3 | Q: I have a blackberry, a clarinet, a nectarine, a plum, a strawberry, a banana, a flute, an orange, and a violin. How many fruits do I have? 4 | A: Let's think step by step. 5 | We first identify the fruits on the list and include their quantity in parentheses: 6 | - blackberry (1) 7 | - nectarine (1) 8 | - plum (1) 9 | - strawberry (1) 10 | - banana (1) 11 | - orange (1) 12 | Now, let's add the numbers in parentheses: 1 + 1 + 1 + 1 + 1 + 1 = 6. So the answer is 6. 13 | 14 | Q: I have an orange, a raspberry, two peaches, a blackberry, an apple, a grape, a nectarine, and three plums. How many fruits do I have? 15 | A: Let's think step by step. 16 | We first identify the fruits on the list and include their quantity in parentheses: 17 | - orange (1) 18 | - raspberry (1) 19 | - peaches (2) 20 | - blackberry (1) 21 | - apple (1) 22 | - grape (1) 23 | - nectarine (1) 24 | - plums (3) 25 | Now, let's add the numbers in parentheses: 1 + 1 + 2 + 1 + 1 + 1 + 1 + 3 = 11. So the answer is 11. 26 | 27 | Q: I have a lettuce head, a head of broccoli, an onion, a stalk of celery, two carrots, a garlic, and a yam. How many vegetables do I have? 28 | A: Let's think step by step. 29 | We first identify the vegetables on the list and include their quantity in parentheses: 30 | - lettuce (1) 31 | - broccoli (1) 32 | - onion (1) 33 | - celery (1) 34 | - carrots (2) 35 | - garlic (1) 36 | - yam (1) 37 | Now, let's add the numbers in parentheses: 1 + 1 + 1 + 1 + 2 + 1 + 1 = 8. So the answer is 8. -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/data/BBH/lib_prompt/sports_understanding.txt: -------------------------------------------------------------------------------- 1 | Determine whether an artificially constructed sentence relating to sports is plausible or not. 2 | 3 | Q: Is the following sentence plausible? "Bam Adebayo scored a reverse layup in the Western Conference Finals." 4 | A: Let's think step by step. Bam Adebayo is an American basketball player. Scoring a reverse layup in the Western Conference Finals is part of the NBA Finals. So the answer is yes. 5 | 6 | Q: Is the following sentence plausible? "Santi Cazorla scored a touchdown." 7 | A: Let's think step by step. Santi Cazorla is a soccer player. Touchdown is part of American football and rugby. So the answer is no. 8 | 9 | Q: Is the following sentence plausible? "DeMar DeRozan was called for the goal tend." 10 | A: Let's think step by step. DeMar DeRozan is an American basketball player. Goal tending is part of basketball. So the answer is yes. -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/data/__init__.py: -------------------------------------------------------------------------------- 1 | import pkg_resources 2 | 3 | pkg_resources.declare_namespace(__name__) 4 | -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/data/ceval/formal_ceval/dev/accountant_dev.csv: -------------------------------------------------------------------------------- 1 | id,question,A,B,C,D,answer,explanation 2 | 0,甲公司是国内一家上市公司。甲公司对其各子公司实行全面预算管理，并通常使用增量预算方式进行战略控制，子公司预算需要经甲公司预算管理委员会批准后执行。2015年10月，甲公司投资了一个新的项目乙(子公司)。2015年11月，甲公司启动2016年度预算编制工作，此时甲公司应要求乙公司编制____。,增量预算,零基预算,固定预算,弹性预算,B,1. 乙公司属于甲公司全新的子公司，在启动2016年度预算编制工作时没有可以参考的基数，预算编制者必须根据自己的经验和判断，加上对周围环境变化的预期来做预算。因此，乙公司应编制零基预算。 3 | 1,债务人转让全部合同义务的，下列说法不正确的是____。,须债权人同意方可进行,新债务人可主张原债务人对债权人的抗辩,债务人转移债务的，原债务人对债权人享有债权的，新债务人可以向债权人主张抵销,非专属于原债务人自身的从债务，一并转让给新债务人,C,"1. 首先，债务人将债务的全部或者部分转移给第三人的，应当经债权人同意，因此选项A正确。 4 | 2. 其次，债务人转移债务的，新债务人可以主张原债务人对债权人的抗辩；原债务人对债权入享有债权的，新债务人不得向债权人主张抵销，因此选项B正确，选项C错误。 5 | 3. 然后，债务人转移债务的，新债务人应当承担与主债务有关的从债务，但是该从债务专属于原债务人自身的除外，因此选项D正确。" 6 | 2,某公司2012年以150万元的价格进口了1台仪器；2014年6月因出现故障运往日本修理(出境时已向海关报明)，2014年10月，按海关规定的期限复运进境。此时，该仪器的国际市场价格为200万元。若经海关审定的修理费和料件费为40万元，运费为1.5万元，保险费用为2.8万元，进口关税税率为6%。该仪器复运进境时，应缴纳的进口关税为____万元。,9,3,2.4,12,C,"1. 根据海关规定，运往境外修理的机械器具、运输工具或其他货物，出境时已向海关报明，并在海关规定期限内复运进境的，应当以海关审定的境外修理费和料件费为完税价格。 7 | 2. 境外修理费和料件费为40万元，进口关税为6%，因此应缴纳的进口关税为40×6%=2.4(万元)。" 8 | 3,公开发行公司债券，证监会同意注册的决定自作出之日起一定期限内有效，发行人应当该期限内发行公司债券。该期限是____。,6个月,1年,2年,3年,C,1. 中国证监会同意注册的决定自作出之日起2年内有效，发行人应当在注册决定有效期内发行公司债券，并自主选择发行时点。 9 | 4,某集团公司在一家银行开设了许多不同的银行账户，那么，该集团公司可以要求银行在考虑其利息及透支限额时将其子公司的账户余额集中起来。下列不属于现金余额集中的好处是____。,盈余与赤字相抵,加强控制,增加投资机会,匹配,D,1. 现金余额集中的好处包括盈余与赤字相抵、加强控制和增加投资机会；匹配，是汇率风险管理中的一项原则，可能与现金余额的集中相冲突，不是现金余额集中的好处。 10 | -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/data/ceval/formal_ceval/dev/art_studies_dev.csv: -------------------------------------------------------------------------------- 1 | id,question,A,B,C,D,answer,explanation 2 | 0,王利发是老舍戏剧作品____中的主人公。,《茶馆》,《龙须沟》,《方珍珠》,《女店员》,A,1. 王利发是老舍的代表作《茶馆》中的主人公。 3 | 1,《等待戈多》的作者是____。,加缪,贝克特,阿尔比,尤奈斯库,B,1. 《等待戈多》是1952年法国剧作家贝克特创作的荒诞派戏剧作品。 4 | 2,____创作了舞蹈《马赛曲》等作品，被现代舞蹈家们奉为精神领袖，誉为“现代舞之母”。,露丝·圣丹尼丝,安娜·巴普洛娃,玛莎·格雷姆,依莎多拉·邓肯,D,1. 依莎多拉·邓肯是美国现代舞的先驱人物，被誉为“现代舞之母”，最著名的代表作有《马赛曲》、《国际歌》、《少女死亡之舞》等。 5 | 3,____创立了“叙事体戏剧”理论,布莱希特,萨特,格洛托夫斯基,贝克特,A,1. “叙事体戏剧”理论是德国剧作家布莱希特提出的一种不以感情共鸣为目的和手段的戏剧形式。 6 | 4,“伊莎多拉·邓肯”通过研究____创造了现代舞的表演风格。,奥托艺术,巴洛克艺术,古希腊艺术,拜占廷艺术,C,1. 美国女舞蹈家伊莎多拉·邓肯被公认为是西方现代舞的创始人，她把古希腊艺术中的文物、雕塑当作寻求新舞蹈理想姿态的源泉，从自然界寻找舞蹈动律的启示。 7 | -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/data/ceval/formal_ceval/dev/basic_medicine_dev.csv: -------------------------------------------------------------------------------- 1 | id,question,A,B,C,D,answer,explanation 2 | 0,下列选项中，符合原癌基因概念的是____,是正常基因,是细胞癌基因,是肿瘤抑制基因,凋亡调节基因,A,"1. 原癌基因是正常存在的基因，正常并不导致肿瘤，选项A正确。只有发生某些异常时才会使细胞发生恶性转化，成为细胞癌基因，选项B错误。 3 | 2. 原癌基因不是肿瘤抑制基因和凋亡调节基因，选项C、D错误。" 4 | 1,栓塞时常伴有DIC发生，主要见于____,脂肪栓塞,空气栓塞,血栓栓塞,羊水栓塞,D,1. 栓塞时常伴有DIC发生，主要见于羊水栓塞，选项D正确。这是因为羊水中含有多量类似于组织凝血活酶的促凝物质，进入血液循环中可激活外源性凝血系统，形成大量微血栓，又因此过程中大量凝血因子及纤维蛋白原被消耗，故纤溶系统激活导致纤溶亢进及凝血障碍，最终引起DIC。 5 | 2,急性细菌性痢疾早期的结肠病变为____,假膜性炎,浆液性炎,卡他性炎,表面化脓性炎,C,1. 急性细菌性痢疾早期病变为急性卡他性炎，后期发展成为假膜性炎。 6 | 3,下列各项，属于肿瘤的是____,错构瘤,动脉瘤,室壁瘤,骨软骨瘤,D,"1. 错构瘤用来描述受累器官的正常组织在发育过程中出现错误的组合、排列，因而导致了类瘤样的畸形，A错误；动脉瘤、室壁瘤等均不是真性肿瘤，B、C错误。 7 | 2. 骨软骨瘤是儿童期常见的良性骨肿瘤，D正确。" 8 | 4,血吸虫虫卵引起的病变主要发生在____,大肠壁和肝脏,肠系膜静脉,门静脉,肺和肠,A,1. 血吸虫虫卵引起的病变主要发生在乙状结肠壁、直肠壁和肝，也可见于回肠末段、阑尾、升结肠、肺、脑等处。 9 | -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/data/ceval/formal_ceval/dev/business_administration_dev.csv: -------------------------------------------------------------------------------- 1 | id,question,A,B,C,D,answer,explanation 2 | 0,有一项年金，前3年无流入，后5年每年年初流入500万元，假设年利率为10%，(P/A，10%，5)=3.7908，(P/S，10%，3)=0.7513，(P/S，10%，2)=0.8264，其现值为____万元。,1994,1566,1813,1523,B,"1. 首先，需要了解递延年金递延期的计算。如果是发生在第W期期末，则递延期为W－1。本题中第一次现金流入发生在第4年初(与第3年末是同一时点)，所以递延期是2年。 3 | 2. 接着，递延年金现值$P=A\times (P/A,i,n)\times (P/S,i,m)$，其中，A为年金，i为利率，m指的是递延期，n指的是连续收支期。 4 | 3. 最后，带入题目中的条件进行计算。现值$P=500\times(P/A，10\%，5)\times(P/S，10\%，2)=500\times 3.7908\times 0.8264=1566.36$。" 5 | 1,企业缴纳的耕地占用税，应在____科目核算。,应交税费,管理费用,在建工程,其他应付款,C,"1. 首先，耕地占用税是对占用耕地建房或从事其他非农业建设的单位和个人征收的税 6 | 2. 接着，根据会计准则，企业的耕地占用税，应在“在建工程”科目核算。" 7 | 2,注册会计师在确定重要性时通常选定一个基准。下列因素中，注册会计师在选择基准时不需要考虑的是____。,被审计单位的性质,以前年度审计调整的金额,基准的相对波动性,是否存在财务报表使用者特别关注的项目,B,"1. 首先，选择基准时需要考虑多个因素，包括被审计单位的性质、基准的相对波动性、是否存在财务报表使用者特别关注的项目等。 8 | 2. 选项中，(B) 以前年度审计调整的金额并不是一个影响重要性水平的因素，因为审计调整的金额并不能反映当前年度的审计风险和重要性。" 9 | 3,某投资方案，当贴现率为12%时，其净现值为22万元，当贴现率为14%时，其净现值为-11万元。该方案的内部收益率____。,大于14%,小于12%,介于12%与14%之间,无法确定,C,"1. 首先，我们需要了解净现值和内部收益率的概念。净现值是指未来资金流入现值与未来资金流出现值的差额。内部收益率是指资金流入现值总额与资金流出现值总额相等、净现值等于零时的折现率。 10 | 2. 接着，当贴现率为12%时，其净现值为22万元，说明其内部收益率应大于12%；当贴现率为14%时，其净现值率为-11万元，说明其内部收益率应小于14%，因此，其收益率应介于12%与14%之间。" 11 | 4,企业期末编制资产负债表时，下列各项应包括在“存货”项目的是____。,已作销售但购货方尚未运走的商品,委托代销商品,合同约定购入的商品,为在建工程购入的工程物资,B,"1. 首先，根据会计准则，存货应当满足以下条件：(1)是企业拥有并用于销售的物品；(2)是企业拥有并用于生产的物品；(3)是企业拥有并用于提供劳务的物品。 12 | 2. 接着，根据条件，选项(A)，(C)，(D)均不属于存货。只有选项(B)委托代销商品符合存货的定义，应该计入存货项目。" 13 | -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/data/ceval/formal_ceval/dev/chinese_language_and_literature_dev.csv: -------------------------------------------------------------------------------- 1 | id,question,A,B,C,D,answer,explanation 2 | 0,“闯关东”中的“关东”指____。,东北,西北,华北,塞外,A,1. “关东”指东北三省，”关“指山海关。 3 | 1,八股文是____。,八种文体,一种程式化的文章样式,八种修辞方式,由八个部分构成的文章,B,1. 八股文是中国古代科举考试中的一种文章体裁，是一种程式化的文章样式，要求文章结构、内容、语言都按照固定的模式来写作。 4 | 2,清末被誉为“诗界革命”一面旗帜的是____。,王国维,梁启超,谭嗣同,黄遵宪,D,1. 黄遵宪是“诗界革命”的代表人物之一，他以“旧风格含新意境”为追求目标，对当时的文学界产生了积极的影响。因此被誉为“诗界革命”一面旗帜。 5 | 3,“活”和“活儿”两个词____。,意思一样，没有区别,词性有区别，“活”是形容词，“活儿”是名词,感情色彩有强弱，“活儿”的感情色彩强,音节个数有区别，“活”是一个音节，“活儿”是两个音节,B,"1. “活”代表活着、灵活、有生命力；“活儿”指工作、营生、生意等。 6 | 2. 分析选项：“活”和“活儿”的含义并不相同，A选项错误；“活儿”的感情色彩并不一定比“活”强，C选项错误；“活”是三拼音节词，选项D错误。 7 | 3. “活”是一个形容词，而“活儿”是一个名词，二者词性不同，B选项正确。" 8 | 4,元朝政府曾经实行残酷的民族政策，把全国人民分为____四个等级。,色目人、蒙古人、汉人、南人,蒙古人、汉人、南人、色目人,蒙古人、南人、色目人、汉人,蒙古人、色目人、汉人、南人,D,1. 元朝政府曾经实行残酷的民族政策，把全国人民分为蒙古人、色目人、汉人、南人。 9 | -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/data/ceval/formal_ceval/dev/civil_servant_dev.csv: -------------------------------------------------------------------------------- 1 | id,question,A,B,C,D,answer,explanation 2 | 0,透过车轮卷起的黄土，却见山野人秋，庄稼割过，静谧中含着一些寂静，只有阳光在切割过的根茬上烁烁闪亮。____。 填入横线上最恰当的是____。,这是一段颠簸的行程，一路上景色苍凉雄浑，寂静中裸露着贫瘠,也许是久旱的缘故，这边的溪流也变成了涓涓细流，在盘踞的石缝间流动,同绿色的南方相比，这里是荒凉的，乃至荒蛮,偶见人迹，大都是荒村野店。时而会有一座小小的孤庙一闪而过,D,"1. 题干中的环境描述给人一种丰收的田园风光的感觉，同时也有一些寂静的气息。此外，由“透过车轮卷起的黄土”可知，作者是在疾驰的车里看外面的景色。 3 | 2. 选项A的”苍凉雄浑“”贫瘠“和选项C的”荒凉“”荒蛮“与题干中丰收的田园风光不符，因此排除。选项B描述的溪流与题干内容没有联系，因此也排除。选项D描述的”偶见人迹“与题干中的寂静的气息相符，同时“一闪而过”与题干中体现的车辆疾驰对应，因此是最恰当的选项。" 4 | 1,据说，在东南沿海一带，渔民在捕到螃蟹后，将螃蟹放进一个上小肚大的竹篓里面，第一只螃蟹放进去时，渔夫就用重物将口封住，当第二只、第三只放进去后，渔夫就不再盖重物了，因为，第一只即将爬出篓口的螃蟹，会被第二只、第三只螃蟹拉到篓底。如此下去，即使篓口没有盖盖子，但也没有一只蟹能够爬出去。 这个故事意在告诉我们____。,人多不一定好办事,恶性竞争必然导致两败俱伤,内讧必然内耗，团结就是力量,与人方便，自己方便,C,"1. 首先，我们需要分析故事的内容。螃蟹之间的恶性竞争导致了它们无法逃脱渔夫的捕捞。如果它们能够团结一致，互相帮助，或许就能够逃脱。 5 | 2. 分析题目选项，四个选项中明确提到“团结”的只有C项。" 6 | 2,谨慎：成就____,温和：好感,勤奋：努力,轻松：普通,好学：智慧,A,"1. 首先，分析题目中两个词的关系。谨慎的态度容易让人取得成就。因此，谨慎和成就之间是一种因果关系。 7 | 2. 接着，我们分析四个选项，选项A“温和：好感”符合因果关系，且“谨慎”和“温和”都是形容词，符合题目要求；其他选项均不符合因果关系或词性要求，因此排除。" 8 | 3,①我的奶奶是这样，我的父亲也是这样——那张画像，已经成为许多老百姓生活必需品的一部分，没有它，似乎客厅都是空的 ②如果因为认知能力的提升而将偶像抽离出记忆，那么，在那个时代里成长、生活的许多人，脑子里将空空如也，甚至不记得自己曾经活过这一回 ③卷的过程，是在收叠他个人的历史 ④有时挂旧了、破了，也会认认真真卷起来放好，我曾看别人卷过这画像，那种澄澈的眼神令人难忘 ⑤有些伟大者永远不会被人遗忘，因为那个伟大者，在那个时代，其实是一种生活，精神生活 ⑥从这个意义上说，尽管也许并不懂他，但人们心甘情愿尊他的名为圣 将以上6个句子重新排列，语序正确的是____。,②⑥⑤①④③,②⑥④③①⑤,①④③②⑥⑤,⑤②⑥①④③,D,1. 我们需要找到句子之间的逻辑关系。通过分析，我们可以发现，④句的“这画像”指的是①句的“那张画像”，两句顺序为①④，因此排除B选项；①句中包含指代词“是这样”“也是这样”，若放段首则指代不明，因此排除C选项；②中提到如果将偶像抽离出记忆，脑子里将空空如也，甚至不记得自己曾经活过这一回。由此可知，偶像是那个时代的精神生活，对应⑤中描述，因此⑤②应相邻，排除A选项。 9 | 4,相机：拍摄____,空调：降温,B超：诊断,电脑：操作,地图：交通,B,"1. 首先，分析题目中两个词的关系。相机是一种用于拍摄照片或录制视频的设备。因此题目中两个词之间是设备和功能的对应关系。 10 | 2. 其次，分析各个选项，C项和D项第二个词都不对应功能，可以排除；A项的“降温”是动宾结构，与题目中的“拍摄”并不一致，可以排除；B项的“B超”与“诊断”是设备和功能的对应关系，同时“诊断”与“拍摄”结构一致，符合要求。" 11 | -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/data/ceval/formal_ceval/dev/clinical_medicine_dev.csv: -------------------------------------------------------------------------------- 1 | id,question,A,B,C,D,answer,explanation 2 | 0,下列哪项不是急性心肌梗死患者溶栓治疗的禁忌证____,发病6小时，心电图ST段抬高＞0．2mY,有慢性严重高血压病史,可疑为主动脉夹层,近期(2～4周)有活动性内脏出血,A,"1. 溶栓治疗的禁忌证主要包括：近期(2~4 周)有活动性内脏出血、严重且未控制的高血压(>180/110 mmHg)或慢性严重高血压病史、主动脉夹层等。 3 | 2. 因此，选项B、C、D符合描述，A并不是溶栓治疗的禁忌证。" 4 | 1,结核病的细胞免疫反应中起主要作用的细胞为____,B淋巴细胞,巨噬细胞,T淋巴细胞,嗜酸性粒细胞,C,1. 结核病的免疫反应以细胞免疫为主，T淋巴细胞起主要作用。 5 | 2,Ⅱ型呼吸衰竭的实验室指标是____,$PaO_2$ 50mmHg，$PaCO_2$ 60mmHg,$PaO_2$ ＞60mmHg，$PaCO_2$ ＜50mmHg,$PaO_2$ ＜60mmHg，$PaCO_2$ ＞50mmHg,$PaO_2$ ＜55mmHg，$PaCO_2$ ＞50mmHg,C,1. Ⅱ型呼吸衰竭，即高碳酸性呼吸衰竭，血气分析特点是$PaO_2$ ＜60mmHg，同时伴有$PaCO_2$ ＞50mmHg。 6 | 3,男性，51岁。因慢性咳嗽、进行性气急5～6年，肺功能检测下列哪项不是诊断阻塞性肺气肿所需要的____,最大通气量＜80％预计,动脉血气分析,残气量／肺总量＞40％,呼气峰值流速降低,B,1. 动脉血气分析主要用来判断有无低氧血症、呼吸衰竭及酸碱平衡状况，并非诊断阻塞性肺气肿所必需。 7 | 4,女性，22岁，面色苍白、排酱油样尿2周，下列哪项不是血管内溶血检查指标____,尿隐血试验,含铁血黄素尿,血清结合珠蛋白,海因小体,D,"1. 尿潜血试验、含铁血黄素尿、血清结合珠蛋白、血浆游离血红蛋白为血管内溶血的检查项目。选项A、B、C符合。 8 | 2. 海因小体是红细胞内变性珠蛋白的包涵体，是红细胞异常形态的一种表现。" 9 | -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/data/ceval/formal_ceval/dev/college_economics_dev.csv: -------------------------------------------------------------------------------- 1 | id,question,A,B,C,D,answer,explanation 2 | 0,考虑以下小型开放经济的数据：Y=1000，C=700，G=150，I=250-1000r*。如果世界利率为5%，那么小型开放经济的净出口为____。,50,-50,150,-150,B,"1. 首先，小型开放经济的净出口公式：NX = S - I，其中NX表示净出口，S表示储蓄，I表示投资。 3 | 2. 其次，根据国民账户恒等式，Y = C + I + G + NX，可以得到S = Y - C - G。将数据代入得到S = 150。 4 | 3. 接着，根据题目中的数据计算投资，I = 250 - 1000r*，其中r*表示世界利率。将r* = 5%代入得到I = 200。 5 | 4. 最后，计算净出口。将步骤2和步骤3的结果代入净出口公式，得到NX = S - I = 150 - 200 = -50。" 6 | 1,已知一垄断企业成本函数为TC=5Q2+20Q+10，产品的需求函数为Q=140-P。利润最大化的产量为____。,10,5,3,15,A,"1. 首先，垄断企业的利润最大化产量是当边际收益MR等于边际成本MC时。因此，我们需要求出该企业的边际收益和边际成本函数。 7 | 2. 其次，求出总收益TR。$TR=P*Q=140Q-Q^2$。 8 | 3. 接着，通过总收益TR求出边际收益MR函数，即$MR=\dfrac{\Delta TR}{\Delta Q}=140-2Q$。 9 | 4. 最后，通过成本函数TC求出边际成本MC函数，即$MC=\frac{\Delta TC}{\Delta Q}=20+10Q$。 10 | 5. 根据上述计算，我们将边际收益函数和边际成本函数相等，即$140-2Q=20+10Q$，解得$Q=10$。因此，利润最大化的产量为10。" 11 | 2,如果消费与实际利率负相关，则______。,IS曲线更平坦,IS曲线更陡峭,LM曲线更平坦,LM曲线更陡峭,A,"1. 首先，我们需要了解IS-LM模型中的两条曲线：IS曲线和LM曲线。IS曲线表示货币市场和实物市场的平衡关系，LM曲线表示货币市场和利率的平衡关系。 12 | 2. 其次，我们需要知道实际利率的定义，实际利率是指扣除通货膨胀后的名义利率。 13 | 3. 最后，我们考虑IS曲线。消费与实际利率负相关说明消费减少会使得利率增加，同时将使得投资减少，因而收入减少得更多，IS曲线更平坦。" 14 | 3,如果所有产品的生产都增加了10%，且所有价格都下降了10%，会发生以下哪一种情况？____,真实GDP增加10%，名义GDP减少10%,真实GDP增加10%，名义GDP不变,真实GDP不变，名义GDP增加10%,真实GDP不变，名义GDP减少10%,B,"1. 首先，名义GDP是指按当前价格计算的GDP，真实GDP是指按不变价格计算的GDP。 15 | 2. 其次，GDP = 价格水平 × 产量。我们分析题目中的情况。所有产品的生产都增加了10%，意味着产量增加了10%；所有价格都下降了10%，意味着价格水平下降了10%。真实GDP的变动只反映生产的产量的变动，产量增加10%，真实GDP增加10%。而价格下降会导致名义GDP的下降，因此名义GDP在真实GDP的基础上下降10%， 结果基本没有发生变化。 16 | 3. 因此，根据第2步的分析，答案应该是B。" 17 | 4,如果边际技术替代率MRTSLK小于劳动与资本的价格之比，为使成本最小，该厂商应该____。,同时增加资本和劳动,同时减少资本和劳动,减少资本，增加劳动,增加资本，减少劳动,D,"1. 首先，边际技术替代率MRTSLK表示单位资本和单位劳动力之间的替代关系，即单位资本的增加可以替代多少单位劳动力。 18 | 2. 如果MRTSLK小于劳动与资本的价格之比，说明：生产中劳动替代资本的比率小于价值中劳动替代资本的比率，即劳动相对过多、资本过少。因此，厂商应该用更多的资本去替代劳动，即增加资本，减少劳动。" 19 | -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/data/ceval/formal_ceval/dev/college_programming_dev.csv: -------------------------------------------------------------------------------- 1 | id,question,A,B,C,D,answer,explanation 2 | 0,"在有n(n>1000)个元素的升序数组A中查找关键字x。查找算法的伪代码如下所示。 3 | k=0; 4 | while(klog_2n，只有 x 在开头处 n/3+2 好" 12 | 1,先序序列为a，b，c的不同二叉树的个数是____,3,5,4,6,B,1. 我们可以通过手动构建二叉树的方式来确定不同先序序列为a，b，c的二叉树个数。对于三个节点的二叉树，根据先序遍历的特点，a节点必须是根节点，那么b和c节点的分布共有5种情况：(1) b是a的左子树，c是a的右子树；(2) b是a的左子树，c是b的左子树；(3) b是a的左子树，c是b的右子树；(4) b是a的右子树，c是b的左子树；(5) b是a的右子树，c是b的右子树。 13 | 2,若森林F有15条边、25个结点，则F包含树的个数是____,8,9,10,11,C,"1. 首先，一个有n个结点的树有n-1条边，每棵树就会造成边的数量比结点数量少1 14 | 3. 接着，仅根无入分支。那么现在边的数量比结点数量少了25-15=10。 15 | 4. 所以，我们得到森林F包含10棵树。" 16 | 3,"#include 17 | #define SQR(x) x*x 18 | void main() 19 | { 20 | int a=10,k=2,m=1; 21 | a/=SQR(k+m);cout< 6).,All of the lanthanide elements react with aqueous acid to liberate hydrogen.,The atomic radii of the lanthanide elements increase across the period from La to Lu.,D 2 | A 0.217 g sample of HgO (molar mass = 217 g) reacts with excess iodide ions according to the reaction shown above. Titration of the resulting solution requires how many mL of 0.10 M HCl to reach equivalence point?,1.0 mL,10 mL,20 mL,50 mL,C 3 | "Predict the number of lines in the EPR spectrum of a solution of 13C-labelled methyl radical (13CH3•), assuming the lines do not overlap.",4,3,6,24,A 4 | "3 Cl−(aq) + 4 CrO_4^2−(aq) + 23 H+(aq) → 3 HClO2(aq) + 4 Cr3+(aq) + 10 H2O(l). In the reaction shown above, Cl−(aq) behaves as",an acid,a base,a catalyst,a reducing agent,D 5 | "Which of the following lists the hydrides of group-14 elements in order of thermal stability, from lowest to highest?",PbH4 < SnH4 < GeH4 < SiH4 < CH4,PbH4 < SnH4 < CH4 < GeH4 < SiH4,CH4 < SiH4 < GeH4 < SnH4 < PbH4,CH4 < PbH4 < GeH4 < SnH4 < SiH4,A 6 | -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/data/mmlu/dev/college_mathematics_dev.csv: -------------------------------------------------------------------------------- 1 | "Let V be the set of all real polynomials p(x). Let transformations T, S be defined on V by T:p(x) -> xp(x) and S:p(x) -> p'(x) = d/dx p(x), and interpret (ST)(p(x)) as S(T(p(x))). Which of the following is true?",ST = 0,ST = T,ST = TS,ST - TS is the identity map of V onto itself.,D 2 | "A tank initially contains a salt solution of 3 grams of salt dissolved in 100 liters of water. A salt solution containing 0.02 grams of salt per liter of water is sprayed into the tank at a rate of 4 liters per minute. The sprayed solution is continually mixed with the salt solution in the tank, and the mixture flows out of the tank at a rate of 4 liters per minute. If the mixing is instantaneous, how many grams of salt are in the tank after 100 minutes have elapsed?",2,2 - e^-2,2 + e^-2,2 + e^-4,D 3 | "Let A be a real 2x2 matrix. Which of the following statements must be true? 4 | I. All of the entries of A^2 are nonnegative. 5 | II. The determinant of A^2 is nonnegative. 6 | III. If A has two distinct eigenvalues, then A^2 has two distinct eigenvalues.",I only,II only,III only,II and III only,B 7 | "Suppose that f(1 + x) = f(x) for all real x. If f is a polynomial and f(5) = 11, then f(15/2)",-11,0,11,33/2,C 8 | "Let A be the set of all ordered pairs of integers (m, n) such that 7m + 12n = 22. What is the greatest negative number in the set B = {m + n : (m, n) \in A}?",-5,-4,-3,-2,B 9 | -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/data/mmlu/dev/college_medicine_dev.csv: -------------------------------------------------------------------------------- 1 | Glucose is transported into the muscle cell:,via protein transporters called GLUT4.,only in the presence of insulin.,via hexokinase.,via monocarbylic acid transporters.,A 2 | Which of the following is not a true statement?,Muscle glycogen is broken down enzymatically to glucose-1-phosphate,Elite endurance runners have a high proportion of Type I fibres in their leg muscles,Liver glycogen is important in the maintenance of the blood glucose concentration,Insulin promotes glucose uptake by all tissues in the body,D 3 | "In a genetic test of a newborn, a rare genetic disorder is found that has X-linked recessive transmission. Which of the following statements is likely true regarding the pedigree of this disorder?",All descendants on the maternal side will have the disorder.,Females will be approximately twice as affected as males in this family.,All daughters of an affected male will be affected.,There will be equal distribution of males and females affected.,C 4 | "A high school science teacher fills a 1 liter bottle with pure nitrogen and seals the lid. The pressure is 1.70 atm, and the room temperature is 25°C. Which two variables will both increase the pressure of the system, if all other variables are held constant?","Increasing temperature, increasing moles of gas","Increasing temperature, increasing volume","Decreasing volume, decreasing temperature","Decreasing moles of gas, increasing volume",A 5 | An expected side effect of creatine supplementation is:,muscle weakness.,gain in body mass.,muscle cramps.,loss of electrolytes.,B 6 | -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/data/mmlu/dev/college_physics_dev.csv: -------------------------------------------------------------------------------- 1 | A refracting telescope consists of two converging lenses separated by 100 cm. The eye-piece lens has a focal length of 20 cm. The angular magnification of the telescope is,4,5,6,20,A 2 | For which of the following thermodynamic processes is the increase in the internal energy of an ideal gas equal to the heat added to the gas?,Constant temperature,Constant volume,Constant pressure,Adiabatic,B 3 | "One end of a Nichrome wire of length 2L and cross-sectional area A is attached to an end of another Nichrome wire of length L and cross- sectional area 2A. If the free end of the longer wire is at an electric potential of 8.0 volts, and the free end of the shorter wire is at an electric potential of 1.0 volt, the potential at the junction of the two wires is most nearly equal to",2.4 V,3.3 V,4.5 V,5.7 V,A 4 | A refracting telescope consists of two converging lenses separated by 100 cm. The eye-piece lens has a focal length of 20 cm. The angular magnification of the telescope is,4,5,6,20,A 5 | "The muon decays with a characteristic lifetime of about 10^-6 second into an electron, a muon neutrino, and an electron antineutrino. The muon is forbidden from decaying into an electron and just a single neutrino by the law of conservation of",charge,mass,energy and momentum,lepton number,D 6 | -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/data/mmlu/dev/computer_security_dev.csv: -------------------------------------------------------------------------------- 1 | SHA-1 has a message digest of,160 bits,512 bits,628 bits,820 bits,A 2 | "_____________ can modify data on your system – so that your system doesn’t run correctly or you can no longer access specific data, or it may even ask for ransom in order to give your access.",IM – Trojans,Backdoor Trojans,Trojan-Downloader,Ransom Trojan,D 3 | What is ethical hacking?,"""Hacking"" ethics so they justify unintended selfish behavior","Hacking systems (e.g., during penetration testing) to expose vulnerabilities so they can be fixed, rather than exploited",Hacking into systems run by those whose ethics you disagree with,"A slang term for rapid software development, e.g., as part of hackathons",B 4 | Exploitation of the Heartbleed bug permits,overwriting cryptographic keys in memory,a kind of code injection,a read outside bounds of a buffer,a format string attack,C 5 | The ____________ is anything which your search engine cannot search.,Haunted web,World Wide Web,Surface web,Deep Web,D 6 | -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/data/mmlu/dev/conceptual_physics_dev.csv: -------------------------------------------------------------------------------- 1 | "Compared with the mass of a uranium atom undergoing fission, the combined masses of the products after fission are",less,more,the same,zero,A 2 | Things that are equivalent according to the equivalence principle are,space and time.,a traveling twin and a stay-at-home twin.,gravity and acceleration.,mass and energy.,C 3 | Colors in a soap bubble result from light,converted to a different frequency,deflection,interference,polarization,C 4 | A model airplane flies slower when flying into the wind and faster with wind at its back. When launched at right angles to the wind a cross wind its groundspeed compared with flying in still air is,the same,greater,less,either greater or less depending on wind speed,B 5 | Which of these three elements has the most mass per nucleon?,Hydrogen,Iron,Uranium,Same in each,A 6 | -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/data/mmlu/dev/econometrics_dev.csv: -------------------------------------------------------------------------------- 1 | "For a stationary autoregressive process, shocks will",Eventually die away,Persist indefinitely,Grow exponentially,Never occur,A 2 | "Consider the following AR(1) model with the disturbances having zero mean and unit variance 3 | 4 | yt = 0.2 + 0.4 yt-1 + ut 5 | 6 | The (unconditional) mean of y will be given by",0.2,0.4,0.5,0.33,D 7 | "Suppose that a test statistic has associated with it a p-value of 0.08. Which one of the following statements is true? 8 | 9 | (i) If the size of the test were exactly 8%, we would be indifferent between rejecting and not rejecting the null hypothesis 10 | 11 | (ii) The null would be rejected if a 10% size of test were used 12 | 13 | (iii) The null would not be rejected if a 1% size of test were used 14 | 15 | (iv) The null would be rejected if a 5% size of test were used.",(ii) and (iv) only,(i) and (iii) only,"(i), (ii), and (iii) only","(i), (ii), (iii), and (iv)",C 16 | What would be then consequences for the OLS estimator if heteroscedasticity is present in a regression model but ignored?,It will be biased,It will be inconsistent,It will be inefficient,"All of (a), (b) and (c) will be true.",C 17 | "Suppose now that a researcher wishes to use information criteria to determine the optimal lag length for a VAR. 500 observations are available for the bi-variate VAR, and the values of the determinant of the variance-covariance matrix of residuals are 0.0336, 0.0169, 0.0084, and 0.0062 for 1, 2, 3, and 4 lags respectively. What is the optimal model order according to Akaike's information criterion?",1 lag,2 lags,3 lags,4 lags,C 18 | -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/data/mmlu/dev/electrical_engineering_dev.csv: -------------------------------------------------------------------------------- 1 | "In an SR latch built from NOR gates, which condition is not allowed","S=0, R=0","S=0, R=1","S=1, R=0","S=1, R=1",D 2 | "In a 2 pole lap winding dc machine , the resistance of one conductor is 2Ω and total number of conductors is 100. Find the total resistance",200Ω,100Ω,50Ω,10Ω,C 3 | "The coil of a moving coil meter has 100 turns, is 40 mm long and 30 mm wide. The control torque is 240*10-6 N-m on full scale. If magnetic flux density is 1Wb/m2 range of meter is",1 mA.,2 mA.,3 mA.,4 mA.,B 4 | "Two long parallel conductors carry 100 A. If the conductors are separated by 20 mm, the force per meter of length of each conductor will be",100 N.,0.1 N.,1 N.,0.01 N.,B 5 | A point pole has a strength of 4π * 10^-4 weber. The force in newtons on a point pole of 4π * 1.5 * 10^-4 weber placed at a distance of 10 cm from it will be,15 N.,20 N.,7.5 N.,3.75 N.,A 6 | -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/data/mmlu/dev/elementary_mathematics_dev.csv: -------------------------------------------------------------------------------- 1 | "The population of the city where Michelle was born is 145,826. What is the value of the 5 in the number 145,826?",5 thousands,5 hundreds,5 tens,5 ones,A 2 | "Olivia used the rule ""Add 11"" to create the number pattern shown below. 10, 21, 32, 43, 54 Which statement about the number pattern is true?",The 10th number in the pattern will be an even number.,The number pattern will never have two even numbers next to each other.,The next two numbers in the pattern will be an even number then an odd number.,If the number pattern started with an odd number then the pattern would have only odd numbers in it.,B 3 | A total of 30 players will play basketball at a park. There will be exactly 5 players on each team. Which statement correctly explains how to find the number of teams needed?,Add 5 to 30 to find 35 teams.,Divide 30 by 5 to find 6 teams.,Multiply 30 and 5 to find 150 teams.,Subtract 5 from 30 to find 25 teams.,B 4 | A store sells 107 different colors of paint. They have 25 cans of each color in storage. The number of cans of paint the store has in storage can be found using the expression below. 107 × 25. How many cans of paint does the store have in storage?,749,"2,675","2,945","4,250",B 5 | Which expression is equivalent to 5 x 9?,(5 x 4) x (6 x 5),(5 x 5) + (5 x 4),(5 x 5) + (5 x 9),(5 x 9) x (6 x 9),B 6 | -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/data/mmlu/dev/formal_logic_dev.csv: -------------------------------------------------------------------------------- 1 | Select the best translation into predicate logic: No people drive on Mars.,~Pd,(∀x)(Px ∨ ~Dx),(∀x)(Px ⊃ ~Dx),~Dp,C 2 | Select the best translation into predicate logic.George borrows Hector's lawnmower. (g: George; h: Hector; l: Hector's lawnmower; Bxyx: x borrows y from z),Blgh,Bhlg,Bglh,Bghl,C 3 | " Select the best English interpretation of the given arguments in predicate logic. 4 | Dm 5 | (∀x)(Wx ⊃ ~Dx) 6 | (∀x)Wx ∨ Ag / (∃x)Ax",Marina is a dancer. Some weaklings are not dancers. Either everything is a weakling or Georgia plays volleyball. So something plays volleyball.,Marina is a dancer. No weakling is a dancer. Everything is either a weakling or plays volleyball. So something plays volleyball.,Marina is a dancer. Some weaklings are not dancers. Everything is either a weakling or plays volleyball. So something plays volleyball.,Marina is a dancer. No weakling is a dancer. Either everything is a weakling or Georgia plays volleyball. So something plays volleyball.,D 7 | " Construct a complete truth table for the following pairs of propositions. Then, using the truth tables, determine whether the statements are logically equivalent or contradictory. If neither, determine whether they are consistent or inconsistent. Justify your answers. 8 | E ⊃ (F · E) and ~E · F",Logically equivalent,Contradictory,"Neither logically equivalent nor contradictory, but consistent",Inconsistent,C 9 | " Which of the given formulas of PL is the best symbolization of the following sentence? 10 | Turtles live long lives and are happy creatures, unless they are injured.",(L • H) ≡ I,(L • H) ∨ I,L • (H ∨ I),L • (H ⊃ R),B 11 | -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/data/mmlu/dev/global_facts_dev.csv: -------------------------------------------------------------------------------- 1 | Which of the following pairs of statements are both true (as of 2019)?,People tend to be optimistic about their own future and the future of their nation or the world.,People tend to be optimistic about their own future but pessimistic about the future of their nation or the world.,People tend to be pessimistic about their own future but optimistic about the future of their nation or the world.,People tend to be pessimistic about their own future and the future of their nation or the world.,B 2 | "As of 2019, about what percentage of Americans agree that the state is run for the benefit of all the people?",31%,46%,61%,76%,B 3 | "As of 2015, since 1990 forests have ____ in Europe and have ____ in Africa and the Americas.","increased, increased","increased, decreased","decreased, increased","decreased, decreased",B 4 | "As of 2019, about what percentage of Russians say it is very important to have free media in our country without government/state censorship?",38%,53%,68%,83%,A 5 | "As of 2017, how many of the world’s 1-year-old children today have been vaccinated against some disease? *",80%,60%,40%,20%,A -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/data/mmlu/dev/high_school_biology_dev.csv: -------------------------------------------------------------------------------- 1 | Which of the following is not a way to form recombinant DNA?,Translation,Conjugation,Specialized transduction,Transformation,A 2 | A mutation in a bacterial enzyme changed a previously polar amino acid into a nonpolar amino acid. This amino acid was located at a site distant from the enzyme’s active site. How might this mutation alter the enzyme’s substrate specificity?,By changing the enzyme’s pH optimum,By changing the enzyme’s location in the cell,By changing the shape of the protein,An amino acid change away from the active site cannot alter the enzyme’s substrate specificity.,C 3 | "In animal cells, which of the following represents the most likely pathway that a secretory protein takes as it is synthesized in a cell?",Plasma membrane–Golgi apparatus–ribosome–secretory vesicle–rough ER,Ribosome–Golgi apparatus–rough ER–secretory vesicle–plasma membrane,Plasma membrane–Golgi apparatus–ribosome–secretory vesicle–rough ER,Ribosome–rough ER–Golgi apparatus–secretory vesicle–plasma membrane,D 4 | Which of the following is not known to be involved in the control of cell division?,Cyclins,Protein kinases,Checkpoints,Fibroblast cells,D 5 | Homologous structures are often cited as evidence for the process of natural selection. All of the following are examples of homologous structures EXCEPT,the wings of a bird and the wings of a bat,the flippers of a whale and the arms of a man,the pectoral fins of a porpoise and the flippers of a seal,the forelegs of an insect and the forelimbs of a dog,D 6 | -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/data/mmlu/dev/high_school_chemistry_dev.csv: -------------------------------------------------------------------------------- 1 | Which of the following is considered an acid anhydride?,HCl,H2SO3,SO2,Al(NO3)3,C 2 | Which of the following is expected to be a polar molecule?,PCl4F,BF3,CO2,Si(CH3)4,A 3 | "From the solubility rules, which of the following is true?","All chlorides, bromides, and iodides are soluble",All sulfates are soluble,All hydroxides are soluble,All ammonium-containing compounds are soluble,D 4 | "A new compound is synthesized and found to be a monoprotic acid with a molar mass of 248 g/mol. When 0.0050 mol of this acid are dissolved in 0.500 L of water, the pH is measured as 3.89. What is the pKa of this acid?",3.89,7.78,5.78,2.33,C 5 | "A solution contains 2.00 mole of acetic acid, CH3COOH, and 1.00 mole of calcium acetate, Ca(CH3COO)2. The solution is able to resist the addition of a small amount of strong acid or strong base with only minor changes in the pH of the solution. Larger quantities of strong acid or strong base can cause a significant change in pH. How many moles of nitric acid, HNO3, may be added before the pH begins to change significantly?",0.500 mole,1.00 mole,2.00 mole,3.00 mole,C 6 | -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/data/mmlu/dev/high_school_geography_dev.csv: -------------------------------------------------------------------------------- 1 | The rate of natural increase of a population is found by subtracting the,crude death rate from the crude birth date.,crude birth rate from the crude death rate.,doubling time from the crude birth rate.,fertility rate from the crude death rate.,A 2 | "During the third stage of the demographic transition model, which of the following is true?",Birth rates increase and population growth rate is less rapid.,Birth rates decline and population growth rate is less rapid.,Birth rates increase and population growth rate increases.,Birth rates decrease and population growth rate increases.,B 3 | Which of the following statements is NOT accurate regarding the services provided by local governments in the United States?,Duplication of efforts occurs often.,Social problems of the central city spill over into the surrounding residential suburbs.,Inefficiency in providing services occurs often.,One neighborhood's efforts to reduce pollution are always supported by neighboring communities.,D 4 | The practice of hiring a foreign third-party service provider to run an operation is called,outsourcing.,offshoring.,maquiladoras.,locational interdependence.,B 5 | Which one of the following items is an example of nonmaterial culture?,Dove soap,Dove candy bar,Dove symbol,A dove (bird),C 6 | -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/data/mmlu/dev/high_school_government_and_politics_dev.csv: -------------------------------------------------------------------------------- 1 | Uncertainty over the limits to presidential power is caused primarily by the fact that,the constitutional definition of those powers is broad and unspecific,most people agree that the Constitution places too many limits on presidential power,the Supreme Court consistently refuses to rule on cases concerning presidential powers,constitutional amendments have greatly increased presidential powers,A 2 | "The term ""budget deficit"" refers to the",annual increase in federal spending on the military,amount of interest on the national debt,difference between the initial budget proposals made by the president and Congress,amount the government spends in excess of its revenues,D 3 | "Which of the following cases established the precedent that a defendant must be informed of the right to remain silent, the right to a lawyer, and protection from self-incrimination?",Weeks v. United States,Betts v. Brady,Mapp v. Ohio,Miranda v. Arizona,D 4 | Which of the following statements about cabinet departments is FALSE?,They are established by the legislative branch.,Their members often don't have much influence over presidential decisions.,They cannot all be run by leaders who belong to the same political party the president does.,Not every federal agency is a cabinet department.,C 5 | Which of the following best states an argument made by James Madison in The Federalist number 10?,Honest politicians can prevent factions from developing.,Factions are more likely to occur in large republics than in small ones.,The negative effects of factionalism can be reduced by a republican government.,Free elections are the people's best defense against factionalism.,C 6 | -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/data/mmlu/dev/high_school_macroeconomics_dev.csv: -------------------------------------------------------------------------------- 1 | Which of the following is not included in the U.S. GDP?,The U.S. military opens a new base in a foreign country with 1000 U.S. personnel.,Japanese consumers buy thousands of CDs produced in the United States.,An American pop singer performs a sold-out concert in Paris.,A French theatrical production tours dozens of American cities.,C 2 | The short-run Phillips curve indicates a,direct relation between unemployment and inflation,direct relation between price and quantity demanded,inverse relation between price and quantity demanded,inverse relation between unemployment and inflation,D 3 | A federal deficit occurs when,exports exceed imports.,imports exceed exports.,federal tax collections exceed spending.,federal spending exceeds federal tax revenues.,D 4 | Holding all else equal which of the following monetary policies would be used to boost U.S. exports?,Increasing the discount rate,Increasing the reserve ratio,Buying government securities,Lowering tariffs,C 5 | Which of the following policies best describes supply-side fiscal policy?,An increase in the money supply,Increased government spending,Lower taxes on research and development of new technology,Higher taxes on household income,C 6 | -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/data/mmlu/dev/high_school_mathematics_dev.csv: -------------------------------------------------------------------------------- 1 | "Joe was in charge of lights for a dance. The red light blinks every two seconds, the yellow light every three seconds, and the blue light every five seconds. If we include the very beginning and very end of the dance, how many times during a seven minute dance will all the lights come on at the same time? (Assume that all three lights blink simultaneously at the very beginning of the dance.)",3,15,6,5,B 2 | "Five thousand dollars compounded annually at an $x\%$ interest rate takes six years to double. At the same interest rate, how many years will it take $\$300$ to grow to $\$9600$?",12,1,30,5,C 3 | "The variable $x$ varies directly as the square of $y$, and $y$ varies directly as the cube of $z$. If $x$ equals $-16$ when $z$ equals 2, what is the value of $x$ when $z$ equals $\frac{1}{2}$?",-1,16,-\frac{1}{256},\frac{1}{16},C 4 | Simplify and write the result with a rational denominator: $$\sqrt{\sqrt[3]{\sqrt{\frac{1}{729}}}}$$,\frac{3\sqrt{3}}{3},\frac{1}{3},\sqrt{3},\frac{\sqrt{3}}{3},D 5 | "Ten students take a biology test and receive the following scores: 45, 55, 50, 70, 65, 80, 40, 90, 70, 85. What is the mean of the students’ test scores?",55,60,62,65,D 6 | -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/data/mmlu/dev/high_school_microeconomics_dev.csv: -------------------------------------------------------------------------------- 1 | "In a competitive labor market for housepainters, which of the following would increase the demand for housepainters?",An effective minimum wage imposed on this labor market.,An increase in the price of gallons of paint.,An increase in the construction of new houses.,An increase in the price of mechanical painters so long as the output effect exceeds the substitution effect.,C 2 | "If the government subsidizes producers in a perfectly competitive market, then",the demand for the product will increase,the demand for the product will decrease,the consumer surplus will increase,the consumer surplus will decrease,C 3 | The concentration ratio for a monopoly is,0,5,10,100,D 4 | Which of the following is true of a price floor?,The price floor shifts the demand curve to the left.,An effective floor creates a shortage of the good.,The price floor shifts the supply curve of the good to the right.,"To be an effective floor, it must be set above the equilibrium price.",D 5 | Which of the following is necessarily a characteristic of oligopoly?,Free entry into and exit from the market,A few large producers,One producer of a good with no close substitutes,A homogenous product,B 6 | -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/data/mmlu/dev/high_school_physics_dev.csv: -------------------------------------------------------------------------------- 1 | Which of the following conditions will ensure that angular momentum is conserved? I. Conservation of linear momentum II. Zero net external force III. Zero net external torque,I and II only,I and III only,II and III only,III only,D 2 | "A pipe full of air is closed at one end. A standing wave is produced in the pipe, causing the pipe to sound a note. Which of the following is a correct statement about the wave’s properties at the closed end of the pipe?","The pressure is at a node, but the particle displacement is at an antinode.","The pressure is at an antinode, but the particle displacement is at a node.",The pressure and the particle displacement are both at nodes.,The pressure and the particle displacement are both at antinodes.,B 3 | "A photocell of work function ϕ = 2eV is connected to a resistor in series. Light of frequency f = 1 × 10^15 Hz hits a metal plate of the photocell. If the power of the light is P = 100 W, what is the current through the resistor?",2:00 AM,6:00 AM,12:00 AM,24 A,D 4 | "A microwave oven is connected to an outlet, 120 V, and draws a current of 2 amps. At what rate is energy being used by the microwave oven?",10 W,30 W,60 W,240 W,D 5 | "A point charge, Q = +1 mC, is fixed at the origin. How much work is required to move a charge, Q = +8 µC, from the point (0, 4 meters) to the point (3 meters, 0)?",3.5 J,6.0 J,22.5 J,40 J,B -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/data/mmlu/dev/human_aging_dev.csv: -------------------------------------------------------------------------------- 1 | "Which of the following persons is more likely to remain at home alone, as of 2019?",An Asian man or woman,A Hispanic man,An African American woman,A white man or woman,C 2 | The finding that adults tend to remember events from their adolescence better than from other periods in their lives is referred to as the,Adolescence advantage,Reminiscence bump,Memorial memorial,Quadratic retrieval spike,B 3 | "When older adults move to a new state after retirement, which of the following is the more likely destination?",Texas,California,Hawaii,Vermont,A 4 | Which element in tobacco smoke is responsible for cancers?,Nicotine,Tar,Carbon monoxide,Smoke particles,B 5 | "All other things being equal, which of the following persons is more likely to show osteoporosis?",An older Hispanic American woman,An older African American woman,An older Asian American woman,An older Native American woman,C -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/data/mmlu/dev/human_sexuality_dev.csv: -------------------------------------------------------------------------------- 1 | Morning sickness is typically a problem:,during the first trimester,during the second trimester,during the third trimester,all through the pregnancy,A 2 | "A woman who knows she has active herpes and untreated syphilis but continues to have sex without informing her partners of her condition has, in psychoanalytic terms:",a strong ego,a weak superego,a weak id,a strong superego,B 3 | Women's ability to have multiple orgasms is primarily due to:,the fact that they do not have a refractory period.,the response of the inner layers of the vagina.,having alternating orgasms in different locations.,the G-Spot.,A 4 | The nature of homosexual activities that occur during preadolescence include all but which of the following?,sexual intercourse,circle jerks,exhibitionism,touching each other's genitals,A 5 | The most common disorder among men who seek sexual therapy is:,premature ejaculation,inhibited ejaculation,erectile disorder,ejaculatory disorder,C 6 | -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/data/mmlu/dev/jurisprudence_dev.csv: -------------------------------------------------------------------------------- 1 | Which position does Rawls claim is the least likely to be adopted by the POP (people in the original position)?,The POP would choose equality above liberty.,The POP would opt for the 'maximin' strategy.,The POP would opt for the 'difference principle'.,The POP would reject the 'system of natural liberty.',A 2 | Functions of the law include all but which of the following?,maximizing individual freedom,providing a basis for compromise,keeping the peace,promoting the principles of the free enterprise system,D 3 | Which word best summarizes Weber's explanation of the development of formally rational law?,Authority.,Charisma.,Co-operation.,Capitalism.,D 4 | "The ________ School of jurisprudence postulates that the law is based on what is ""correct.""",Natural Law,Analytical,Historical,Sociological,A 5 | "Iverson Jewelers wrote a letter to Miller, 'We have received an exceptionally fine self winding Rolox watch which we will sell to you at a very favorable price.'",The letter is an offer to sell,A valid offer cannot be made by letter.,The letter contains a valid offer which will terminate within a reasonable time.,The letter lacks one of the essential elements of an offer.,D 6 | -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/data/mmlu/dev/logical_fallacies_dev.csv: -------------------------------------------------------------------------------- 1 | "If someone attacks the character of an opposing arguer, instead of responding to that opponent's arguments, the first person has probably committed which of the following fallacies?",tu quoque,horse laugh,argument against the person,ignoratio elenchi,C 2 | The complex question fallacy consists of,arguing something is inferior just because it doesn't do something it was never intended to do.,including more than one claim in the proposition and treating proof for one claim as proof for all the claims.,"drawing a conclusion before examining the evidence, and only considering evidence that supports that conclusion.","asking a question that includes either an unproven assumption or more than one question, thus making a straightforward yes or no answer meaningless.",D 3 | Which of the following is true of a valid categorical syllogism?,The minor premise must deny the antecedent,The major premise must affirm the consequent,The middle term must be used in at least one premise in a universal or unqualified sense,All of the above,C 4 | Arguing that what is true of the parts must be true of the whole is the fallacy of...,Division,Composition,Appeal to the person,Appeal to ignorance,B 5 | "When an arguer causes confusion during refutation because of real or feigned lack of an ability to engage in refutation, that arguer may have committed the fallacy of",poor sportsmanship,appeal to compassion,argument against the person,ignorance of refutation,D 6 | -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/data/mmlu/dev/management_dev.csv: -------------------------------------------------------------------------------- 1 | What are the two main dimensions of the Ohio Studies into leadership?,Starting position and end position,Initial environment and changed environment,Organisational structure and conditioning,Initiating structure and considerations,D 2 | Hygiene factors are associated with which writer?,Frederick Hertzberg,D.C. McClelland,Abraham Maslow,Douglas McGregor,A 3 | Which element of the cultural web forms regalia?,Symbols,Rituals and routines,Power structures,Control systems,A 4 | What characteristic is not a key feature of the 'open systems' model of management?,Morale,Innovation,Growth resource,Adaptation,A 5 | How can organisational structures that are characterised by democratic and inclusive styles of management be described?,Hierarchical,Bureaucratic,Flat,Functional,C 6 | -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/data/mmlu/dev/marketing_dev.csv: -------------------------------------------------------------------------------- 1 | _____________ is a natural outcome when combining demographic and geographic variables.,Geodemographics,Product differentiation.,ANSOFF matrix.,Brand management.,A 2 | "In an organization, the group of people tasked with buying decisions is referred to as the _______________.",Outsourcing unit.,Procurement centre.,Chief executive unit.,Decision-making unit.,D 3 | Which of the following is an assumption in Maslow's hierarchy of needs?,Needs are dependent on culture and also on social class.,Lower-level needs must be at least partially satisfied before higher needs can affect behaviour.,Needs are not prioritized or arranged in any particular order.,"Satisfied needs are motivators, and new needs emerge when current needs remain unmet.",B 4 | The single group within society that is most vulnerable to reference group influence is:,The older consumer who feels somewhat left out of things.,"The married women, many of whom feel a need for stability in their lives.",New immigrants who really want to assimilate into their new culture.,"Children, who base most of their buying decisions on outside influences.",D 5 | "Although the content and quality can be as controlled as direct mail, response rates of this medium are lower because of the lack of a personal address mechanism. This media format is known as:",Care lines.,Direct mail.,Inserts.,Door to door.,D 6 | -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/data/mmlu/dev/medical_genetics_dev.csv: -------------------------------------------------------------------------------- 1 | Large triplet repeat expansions can be detected by:,polymerase chain reaction.,single strand conformational polymorphism analysis.,Southern blotting.,Western blotting.,C 2 | DNA ligase is,an enzyme that joins fragments in normal DNA replication,an enzyme of bacterial origin which cuts DNA at defined base sequences,an enzyme that facilitates transcription of specific genes,an enzyme which limits the level to which a particular nutrient reaches,A 3 | A gene showing codominance,has both alleles independently expressed in the heterozygote,has one allele dominant to the other,has alleles tightly linked on the same chromosome,has alleles expressed at the same time in development,A 4 | Which of the following conditions does not show multifactorial inheritance?,Pyloric stenosis,Schizophrenia,Spina bifida (neural tube defects),Marfan syndrome,D 5 | The stage of meiosis in which chromosomes pair and cross over is:,prophase I,metaphase I,prophase II,metaphase II,A 6 | -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/data/mmlu/dev/miscellaneous_dev.csv: -------------------------------------------------------------------------------- 1 | How many axles does a standard automobile have?,one,two,four,eight,B 2 | What place is named in the title of the 1979 live album by rock legends Cheap Trick?,Budapest,Budokan,Bhutan,Britain,B 3 | Who is the shortest man to ever win an NBA slam dunk competition?,Anthony 'Spud' Webb,Michael 'Air' Jordan,Tyrone 'Muggsy' Bogues,Julius 'Dr J' Erving,A 4 | What is produced during photosynthesis?,hydrogen,nylon,oxygen,light,C 5 | Which of these songs was a Top 10 hit for the rock band The Police?,'Radio Ga-Ga','Ob-la-di Ob-la-da','De Do Do Do De Da Da Da','In-a-Gadda-Da-Vida',C 6 | -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/data/mmlu/dev/moral_disputes_dev.csv: -------------------------------------------------------------------------------- 1 | "According to Metz, what is wrong with consequentialist arguments against capital punishment based on African values?",It is unclear as of yet whether or not capital punishment deters harm to the community.,It is unclear as of yet whether or not capital punishment deters harm to any individuals.,Consequentialism is not supported by African values.,"Even though consequentialism is supported by African values, no consequentialist arguments framed in terms of African values have been offered.",A 2 | A fertilized ovum is also known as,a zygote.,an embryo.,a viability.,a blastocyst.,A 3 | "Which of the following is an intrinsic good, according to Singer?",being in the upper class,the ability to learn and solve problems,being more beautiful than most of the population,being shorter than average,B 4 | "Pence compares six different cases of reproduction, from natural twinning to SCNT. What conclusion does he draw from this comparison?",SCNT is not a different kind of reproduction because there are no morally relevant differences between it and other permissible means of reproduction.,"Because there is a low risk of harm for natural twinning, there will be a low risk of harm for SCNT.",both A and B,neither A nor B,A 5 | "Baron admits that the versions of the ticking bomb hypothetical she discusses are ""stunningly stupid,"" but she claims this is actually evidence of",the stupidity of most traditional philosophical examples.,a general lack of intelligence among people with advanced degrees.,the wrongness of torture.,the readiness on the part of many intelligent people to see torture as the best solution to deal with terrorism.,D -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/data/mmlu/dev/philosophy_dev.csv: -------------------------------------------------------------------------------- 1 | Psychological egoism is:,an ethical theory about how we ought to behave.,a generalization concerning the way people tend to behave.,a claim about human nature and the ways people are capable of behaving.,none of the above.,C 2 | "According to Moore’s “ideal utilitarianism,” the right action is the one that brings about the greatest amount of:",pleasure.,happiness.,good.,virtue.,C 3 | "According to d'Holbach, people always act according to _____.",free choices,dictates of the soul,necessary natural laws,undetermined will,C 4 | "Before Tolstoy's Christian conversion, what was his perspective on the meaning of life?",optimist,satisfied,nominally religious,pessimist,D 5 | "The study of reality in the broadest sense, an inquiry into the elemental nature of the universe and the things in it, is known as _____.",metaphysics,epistemology,quantum physics,axiology,A 6 | -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/data/mmlu/dev/public_relations_dev.csv: -------------------------------------------------------------------------------- 1 | What should a public relations media practitioner do if she does not know the answer to a reporter's question?,Give the reporter other information she is certain is correct.,Say that the information is 'off the record' and will be disseminated later.,Say 'I don't know' and promise to provide the information later.,"Say 'no comment,' rather than appear uninformed.",C 2 | "In issues management, what is the most proactive approach to addressing negative or misleading information posted online about your organization?",Buy domain names that could be used by opposition groups.,Post anonymous comments on blogs to combat this information.,Prepare a news release that discredits the inaccurate information.,Make policy changes to address complaints highlighted on these sites.,D 3 | Which of these statements is true of the Vatican in 2010 at the time of the accusations of child abuse cover-ups?,There was a coordinated media response.,Consistent messages were communicated.,Criticisms were taken as attacks on the Catholic Church.,The credibility of the Vatican was upheld.,C 4 | At which stage in the planning process would a situation analysis be carried out?,Defining the program,Planning the program,Taking action and implementing ideas,Evaluation of the program,A 5 | Earth Hour was a campaign launched by which organization?,Greenpeace,The UN,Oxfam,World Wildlife Fund,D 6 | -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/data/mmlu/dev/sociology_dev.csv: -------------------------------------------------------------------------------- 1 | Which of the following did the post-war welfare state of 1948 not aim to provide:,free health care and education for all,a minimum wage,full employment,universal welfare,B 2 | What does Berger (1963) describe as a metaphor for social reality?,a fairground ride,a circus,a puppet theatre,a ballet,C 3 | The shift from 'civil religion' to 'common religion' means that:,the increasing bureaucracy of the state has made religion only a marginal part of our lives,"despite the weakening of traditional authority, our everyday lives and 'common sense' remain shaped by religious beliefs and values","religious participation in collective worship may have declined, but people still practise their faiths in private","people are much more likely to discuss their religious beliefs in public, informal settings",B 4 | The term 'hegemony' refers to:,the tendency for the working class not to realize their own interests,"a dominant ideology that legitimates economic, political and cultural power",a form of dual consciousness based on ideology and everyday experiences,a mode of payment given for outstanding topiary,B 5 | Which of the following is not a problem associated with official statistics on strike action?,most strikes go unnoticed by employers and the mass media,not all industrial disputes will be reported by the employer,the definition of strikes excludes those that involve fewer than ten workers or last less than one day,it is hard to compare strikes that were measured in different ways,A 6 | -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/data/mmlu/dev/us_foreign_policy_dev.csv: -------------------------------------------------------------------------------- 1 | How did the 2008 financial crisis affect America's international reputation?,It damaged support for the US model of political economy and capitalism,It created anger at the United States for exaggerating the crisis,It increased support for American global leadership under President Obama,It reduced global use of the US dollar,A 2 | How did NSC-68 change U.S. strategy?,It globalized containment.,It militarized containment.,It called for the development of the hydrogen bomb.,All of the above,D 3 | The realm of policy decisions concerned primarily with relations between the United States and the rest of the world is known as,terrorism policy.,economic policy.,foreign policy.,international policy.,C 4 | How do Defensive Realism and Offensive Realism differ in their explanation of state behaviour?,Defensive realists place greater emphasis on the role of international institutions,Defensive realists place less emphasis on geographical factors,Offensive realists give more priority to the national interest than Defensive realists.,"Defensive realists believe states are security maximizers, while Offensive realists believe states to be power maximizers",D 5 | How did Donald Trump attack globalization in the 2016 campaign?,Globalization had made men like him too rich,"Globalization only benefited certain American states, such as New York","Liberal elites had encouraged globalization, while 'ordinary Americans' lost jobs because of it",Globalization encouraged damaging trade wars,C 6 | -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/data/mmlu/dev/virology_dev.csv: -------------------------------------------------------------------------------- 1 | Why are parvoviruses a highly impactful parasite?,Because they have no nucleic acid,They require a helper virus,Only replicate in dividing cells,Can integrate into host chromosomes,A 2 | Which of the following is a morphological characteristic of the paramyxoviruses.,Fragile viruses often visualised with RNA spewing from the inside,Elongate viruses,Icosahedral viruses with envelope,Very large viruses,A 3 | A key factor facilitating the application of nested case-control studies from the MACS was:,Data collection,Establishment of a repository of biologic specimens,Participant interest,Administration of the questionnaire by staff,B 4 | The most important goal of a behavioral intervention is:,Change in behavior,Comprehensive coverage,Effective use of behavioral theory,Sustained behavior change,D 5 | The median survival time to AIDS and death was established by following:,Seroprevalent HIV-infected individuals,Seronegatives,Seroconverters,High-risk seronegatives,C -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/data/mmlu/dev/world_religions_dev.csv: -------------------------------------------------------------------------------- 1 | What is the sign of the covenant for Jewish males?,The rainbow,Circumcision,A son,Bar mitzvah,B 2 | What is the Second Gem in Buddhism?,The Dharma,The Sangha,The Buddha,The Bodhisattva,A 3 | " In which dynasty was the ""Mandate of Heaven"" developed to legitimatize the new rulers?",Shang,Zhou,Han,Xia,B 4 | Which Japanese government promoted a kind of national cult based on the emperor and his associations with kami?,Honen,Tanaka,Tokugawa,Meiji,D 5 | How can the Upanishads be characterized?,Ritual texts,Philosophical texts,Hymns,Origin stories,B -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/data/mmlu/val/global_facts_val.csv: -------------------------------------------------------------------------------- 1 | "As of 2014, about what percentage of primary aged children in Sub-Saharan Africa were out-of-school?",10%,20%,35%,50%,B 2 | "There are two billion children in the world today (2020), aged 0 to 15 years old. How many children will there be in the year 2100 according to the United Nations?",4 billion,3 billion,2 billion,1 billion,C 3 | "Controlling for inflation and PPP-adjustment, about how much did GDP per capita increase from 1950 to 2016 in India?",by 4 fold,by 8 fold,by 16 fold,by 32 fold,A 4 | "As of 2020, about what percentage of the population in Sub-Saharan Africa, is living on less than $1.90?",10%,20%,40%,80%,C 5 | In which of the following country did the largest number of children die under the age of five years in 2017?,China,United States,Indonesia,Pakistan,D 6 | Which of the following is true about the global growth rate?,The global growth rate was four times as high 50 years ago as it is in 2020.,The global growth rate was two times as high 50 years ago as it is in 2020.,The global growth rate is two times as high as it is in 2020.,The global growth rate is four times as high today as it is in 2020.,B 7 | Most Latin American countries achieved independence,shortly after World War II,in the 1960s,around the time of World War I,in the early nineteenth century,D 8 | After the Atom bomb how many Americans support giving aid to Japan even at the cost of food shortages?,33%,50%,66%,85%,C 9 | In 1948 how many Americans believed the Kinsey Report on Sexuality in the Male to be a bad thing?,92%,64%,27%,11%,D 10 | Total global healthcare expenditure is about what percentage of global GDP as of 2017?,1%,2%,5%,10%,D -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/data/mmlu/val/management_val.csv: -------------------------------------------------------------------------------- 1 | Simon (1960) is associated with what type of decision making model?,Rational,Classical,Programmed,Administrative,D 2 | At what level of an organisation does a corporate manager operate?,Functional,Operational,Middle level,Top level,D 3 | Which one is not a recognised key skill of management?,Conceptual skills,Human skills,Technical skills,Writing skills,D 4 | Which one of the following is not a characteristic of a bureaucratic organisation?,Authority,Regulations,Command structure,Change,D 5 | To what does authority refer?,The ability to organise people,The power to command and direct,The need for order,The right to change jobs,B 6 | Power distance in national cultures is a characteristic of a model by which writer?,Geert Hofstede,David Yip,Tony Purdie,Michael Porter,A 7 | Of what is an investment decision an example?,Programmed decision,Routine decision,Management decision,Non-programmed decision,D 8 | How do you call a system that allows workers to work part of the week from home?,Piece time,Part-time,Job sharing,Telecommuting,D 9 | What are the three interpersonal roles of managers?,"Figurehead, leader and liaison","Spokesperson, leader, coordinator","Director, coordinator, disseminator","Communicator, organiser, spokesperson",A 10 | "According to Lewin, Lippet and White's 1939 experiment, which form of leadership produced the most work from participants?",Laissez-faire,Democratic,Authoritarian,A mix of laissez-faire and democratic,C 11 | Which of the following motivators is the most basic need in Maslow's hierarchy?,Safety,Belonging,Physiological,Esteem,C 12 | -------------------------------------------------------------------------------- /lms/runtime/evaluation/benchmark/metric.json: -------------------------------------------------------------------------------- 1 | { 2 | "model": "/Users/dev/Documents/APS/models/huggingface/opt-125m", 3 | "benchmarks": [ 4 | { 5 | "benchmarks_name": "ARC", 6 | "Metric": { 7 | "acc": 12.54237288135593 8 | } 9 | } 10 | ] 11 | } -------------------------------------------------------------------------------- /lms/runtime/evaluation/custom/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataCanvasIO/LMS/56b41294c22e67313d0600d0484919edb92641ba/lms/runtime/evaluation/custom/__init__.py -------------------------------------------------------------------------------- /lms/runtime/evaluation/eval_module.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | 4 | 5 | def evaluate1(model_path, task, input_path, output_path): 6 | if task == 'human': 7 | from lms.runtime.evaluation.manual_eval import manual_eval 8 | manual_eval(model_path, input_path, output_path) 9 | # save to web 10 | output = {'eval_kind': "human"} 11 | return output 12 | elif re.match(r"^.*\.py$", task): 13 | from lms.runtime.evaluation.custom.eval import do_eval 14 | accuracy_script_path = os.path.dirname(__file__) + "/accuracy.py" 15 | output = do_eval(task, input_path, model_path, output_path, accuracy_script_path) 16 | output['eval_kind'] = "custom" 17 | return output 18 | else: 19 | from lms.runtime.evaluation.benchmark.eval import do_eval, task_map 20 | tasks = task.split(",") 21 | check = any(item in task_map.keys() for item in tasks) 22 | if check is True: 23 | output = do_eval(task, model_path, output_path) 24 | output['eval_kind'] = "automatic" 25 | return output 26 | else: 27 | print("No, List1 doesn't have any elements of the List2.") 28 | -------------------------------------------------------------------------------- /lms/runtime/prune/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataCanvasIO/LMS/56b41294c22e67313d0600d0484919edb92641ba/lms/runtime/prune/__init__.py -------------------------------------------------------------------------------- /lms/runtime/prune/llm_pruner/LLMPruner/__init__.py: -------------------------------------------------------------------------------- 1 | from . import models -------------------------------------------------------------------------------- /lms/runtime/prune/llm_pruner/LLMPruner/evaluator/ppl.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import numpy as np 3 | from tqdm import tqdm 4 | 5 | from LLMPruner.datasets.ppl_dataset import get_loaders 6 | 7 | def PPLMetric(model, tokenizer, datasets, seq_len=128, batch_size = 4, device="cuda"): 8 | metric = {} 9 | for dataset in datasets: 10 | _, test_loader = get_loaders(dataset, tokenizer, seq_len=seq_len, batch_size = batch_size) 11 | ppl = llama_eval(model, test_loader, device) 12 | metric[dataset] = ppl 13 | print(metric) 14 | return metric 15 | 16 | @torch.no_grad() 17 | def llama_eval(model, test_lodaer, device): 18 | nlls = [] 19 | n_samples = 0 20 | for batch in tqdm(test_lodaer): 21 | batch = batch.to(device) 22 | output = model(batch) 23 | lm_logits = output.logits 24 | 25 | shift_logits = lm_logits[:, :-1, :].contiguous() 26 | shift_labels = batch[:, 1:].contiguous() 27 | 28 | loss_fct = torch.nn.CrossEntropyLoss(reduction="none") 29 | loss = loss_fct(shift_logits.reshape(-1, shift_logits.size(-1)), shift_labels.view(-1)) 30 | nlls.append(loss) 31 | #print(torch.cat(nlls, dim=-1).mean()) 32 | ppl = np.exp(torch.cat(nlls, dim=-1).mean().item()) 33 | return ppl.item() -------------------------------------------------------------------------------- /lms/runtime/prune/llm_pruner/LLMPruner/models/__init__.py: -------------------------------------------------------------------------------- 1 | from . import hf_baichuan 2 | from . import hf_llama 3 | from . import hf_chatglm -------------------------------------------------------------------------------- /lms/runtime/prune/llm_pruner/LLMPruner/models/helper.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | def reorder_qkv(qkv_module, head_dim, num_head): 4 | """ 5 | Reorder the index of query-key-value. 6 | The reason why we need this step: The tracing of torch.view is too complicated to trace, and thus, the index mapping of query_key_value would have some problem. 7 | The Goal of this function is to reorder the concatenated query-key-value 8 | from 9 | [q1, k1, v1, q2, k2, v2, ...] 10 | to 11 | [q1, q2, ..., k1, k2, ..., v1, v2, ...] 12 | """ 13 | #batch_size, seq_length, three_times_hidden_size = fused_qkv.shape 14 | #fused_qkv = fused_qkv.view(batch_size, seq_length, self.num_heads, 3, self.head_dim) # LLM-Pruner: Modify the split of query_key_value here 15 | #return fused_qkv[..., 0, :], fused_qkv[..., 1, :], fused_qkv[..., 2, :] 16 | 17 | weight = qkv_module.weight 18 | bias = qkv_module.bias 19 | out_features, in_features = weight.shape 20 | assert bias.shape == (out_features, ) 21 | 22 | head_dim_qkv_head = weight.view(num_head, -1, head_dim, in_features) 23 | q, k, v = head_dim_qkv_head[..., 0, :, :], head_dim_qkv_head[..., 1, :, :], head_dim_qkv_head[..., 2, :, :] 24 | new_weight = torch.cat([q, k, v], dim=0).view(out_features, in_features) 25 | qkv_module.weight = torch.nn.Parameter(new_weight) # out X in 26 | 27 | bias = bias.view(num_head, -1, head_dim) 28 | q_bias, k_bias, v_bias = bias[..., 0, :], bias[..., 1, :], bias[..., 2, :] 29 | new_bias = torch.cat([q_bias, k_bias, v_bias], dim=0).view(out_features) 30 | qkv_module.bias = torch.nn.Parameter(new_bias) # out 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /lms/runtime/prune/llm_pruner/LLMPruner/models/hf_baichuan/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataCanvasIO/LMS/56b41294c22e67313d0600d0484919edb92641ba/lms/runtime/prune/llm_pruner/LLMPruner/models/hf_baichuan/__init__.py -------------------------------------------------------------------------------- /lms/runtime/prune/llm_pruner/LLMPruner/models/hf_baichuan/baichuan13B/configuration_baichuan.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023, Baichuan Intelligent Technology. All rights reserved. 2 | 3 | from transformers.configuration_utils import PretrainedConfig 4 | 5 | class BaichuanConfig(PretrainedConfig): 6 | model_type = "baichuan" 7 | keys_to_ignore_at_inference = ["past_key_values"] 8 | 9 | def __init__( 10 | self, 11 | vocab_size=64000, 12 | hidden_size=5120, 13 | intermediate_size=13696, 14 | num_hidden_layers=40, 15 | num_attention_heads=40, 16 | hidden_act="silu", 17 | model_max_length=4096, 18 | initializer_range=0.02, 19 | rms_norm_eps=1e-6, 20 | use_cache=True, 21 | pad_token_id=0, 22 | bos_token_id=1, 23 | eos_token_id=2, 24 | tie_word_embeddings=False, 25 | gradient_checkpointing=False, 26 | **kwargs, 27 | ): 28 | self.vocab_size = vocab_size 29 | self.model_max_length = model_max_length 30 | self.hidden_size = hidden_size 31 | self.intermediate_size = intermediate_size 32 | self.num_hidden_layers = num_hidden_layers 33 | self.num_attention_heads = num_attention_heads 34 | self.hidden_act = hidden_act 35 | self.initializer_range = initializer_range 36 | self.rms_norm_eps = rms_norm_eps 37 | self.use_cache = use_cache 38 | self.gradient_checkpointing = gradient_checkpointing, 39 | super().__init__( 40 | pad_token_id=pad_token_id, 41 | bos_token_id=bos_token_id, 42 | eos_token_id=eos_token_id, 43 | tie_word_embeddings=tie_word_embeddings, 44 | **kwargs, 45 | ) 46 | -------------------------------------------------------------------------------- /lms/runtime/prune/llm_pruner/LLMPruner/models/hf_bloom/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataCanvasIO/LMS/56b41294c22e67313d0600d0484919edb92641ba/lms/runtime/prune/llm_pruner/LLMPruner/models/hf_bloom/__init__.py -------------------------------------------------------------------------------- /lms/runtime/prune/llm_pruner/LLMPruner/models/hf_chatglm/__init__.py: -------------------------------------------------------------------------------- 1 | from .modeling_chatglm import ChatGLMForConditionalGeneration, ChatGLMModel 2 | from .tokenizer_chatglm import ChatGLMTokenizer -------------------------------------------------------------------------------- /lms/runtime/prune/llm_pruner/LLMPruner/models/hf_llama/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataCanvasIO/LMS/56b41294c22e67313d0600d0484919edb92641ba/lms/runtime/prune/llm_pruner/LLMPruner/models/hf_llama/__init__.py -------------------------------------------------------------------------------- /lms/runtime/prune/llm_pruner/LLMPruner/models/llama/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # This software may be used and distributed according to the terms of the GNU General Public License version 3. 3 | 4 | from .generation import LLaMA 5 | from .model import ModelArgs, Transformer 6 | from .tokenizer import Tokenizer 7 | -------------------------------------------------------------------------------- /lms/runtime/prune/llm_pruner/LLMPruner/models/llama/tokenizer.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # This software may be used and distributed according to the terms of the GNU General Public License version 3. 3 | 4 | from sentencepiece import SentencePieceProcessor 5 | from logging import getLogger 6 | from typing import List 7 | import os 8 | 9 | 10 | logger = getLogger() 11 | 12 | class Tokenizer: 13 | def __init__(self, model_path: str): 14 | # reload tokenizer 15 | assert os.path.isfile(model_path), model_path 16 | self.sp_model = SentencePieceProcessor(model_file=model_path) 17 | logger.info(f"Reloaded SentencePiece model from {model_path}") 18 | 19 | # BOS / EOS token IDs 20 | self.n_words: int = self.sp_model.vocab_size() 21 | self.bos_id: int = self.sp_model.bos_id() 22 | self.eos_id: int = self.sp_model.eos_id() 23 | self.pad_id: int = self.sp_model.pad_id() 24 | logger.info( 25 | f"#words: {self.n_words} - BOS ID: {self.bos_id} - EOS ID: {self.eos_id}" 26 | ) 27 | assert self.sp_model.vocab_size() == self.sp_model.get_piece_size() 28 | 29 | def encode(self, s: str, bos: bool, eos: bool) -> List[int]: 30 | assert type(s) is str 31 | t = self.sp_model.encode(s) 32 | if bos: 33 | t = [self.bos_id] + t 34 | if eos: 35 | t = t + [self.eos_id] 36 | return t 37 | 38 | def decode(self, t: List[int]) -> str: 39 | return self.sp_model.decode(t) 40 | -------------------------------------------------------------------------------- /lms/runtime/prune/llm_pruner/LLMPruner/peft/import_utils.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | # Copyright 2023-present the HuggingFace Inc. team. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | import importlib 16 | 17 | 18 | def is_bnb_available(): 19 | return importlib.util.find_spec("bitsandbytes") is not None 20 | -------------------------------------------------------------------------------- /lms/runtime/prune/llm_pruner/LLMPruner/peft/tuners/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | # There's no way to ignore "F401 '...' imported but unused" warnings in this 3 | # module, but to preserve other warnings. So, don't check this module at all 4 | 5 | # coding=utf-8 6 | # Copyright 2023-present the HuggingFace Inc. team. 7 | # 8 | # Licensed under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # Unless required by applicable law or agreed to in writing, software 15 | # distributed under the License is distributed on an "AS IS" BASIS, 16 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | # See the License for the specific language governing permissions and 18 | # limitations under the License. 19 | 20 | from .lora import LoraConfig, LoraModel 21 | from .adalora import AdaLoraConfig, AdaLoraModel 22 | from .p_tuning import PromptEncoder, PromptEncoderConfig, PromptEncoderReparameterizationType 23 | from .prefix_tuning import PrefixEncoder, PrefixTuningConfig 24 | from .prompt_tuning import PromptEmbedding, PromptTuningConfig, PromptTuningInit 25 | -------------------------------------------------------------------------------- /lms/runtime/prune/llm_pruner/LLMPruner/peft/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | # There's no way to ignore "F401 '...' imported but unused" warnings in this 3 | # module, but to preserve other warnings. So, don't check this module at all 4 | 5 | # coding=utf-8 6 | # Copyright 2023-present the HuggingFace Inc. team. 7 | # 8 | # Licensed under the Apache License, Version 2.0 (the "License"); 9 | # you may not use this file except in compliance with the License. 10 | # You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # Unless required by applicable law or agreed to in writing, software 15 | # distributed under the License is distributed on an "AS IS" BASIS, 16 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | # See the License for the specific language governing permissions and 18 | # limitations under the License. 19 | 20 | from .config import PeftConfig, PeftType, PromptLearningConfig, TaskType 21 | from .other import ( 22 | TRANSFORMERS_MODELS_TO_PREFIX_TUNING_POSTPROCESS_MAPPING, 23 | TRANSFORMERS_MODELS_TO_LORA_TARGET_MODULES_MAPPING, 24 | TRANSFORMERS_MODELS_TO_ADALORA_TARGET_MODULES_MAPPING, 25 | CONFIG_NAME, 26 | WEIGHTS_NAME, 27 | _set_trainable, 28 | bloom_model_postprocess_past_key_value, 29 | prepare_model_for_int8_training, 30 | shift_tokens_right, 31 | transpose, 32 | _get_submodules, 33 | _set_adapter, 34 | _freeze_adapter, 35 | ModulesToSaveWrapper, 36 | ) 37 | from .save_and_load import get_peft_model_state_dict, set_peft_model_state_dict 38 | -------------------------------------------------------------------------------- /lms/runtime/prune/llm_pruner/LLMPruner/pruner/__init__.py: -------------------------------------------------------------------------------- 1 | from .hf_baichuan_pruner import * 2 | from .hf_llama_pruner import * 3 | from .hf_chatglm_pruner import * 4 | -------------------------------------------------------------------------------- /lms/runtime/prune/llm_pruner/LLMPruner/templates/prompts.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | prompts = [ 4 | # For these prompts, the expected answer is the natural continuation of the prompt 5 | "I believe the meaning of life is", 6 | "Simply put, the theory of relativity states that ", 7 | "Building a website can be done in 10 simple steps:\n", 8 | # Few shot prompts: https://huggingface.co/blog/few-shot-learning-gpt-neo-and-inference-api 9 | """Tweet: "I hate it when my phone battery dies." 10 | Sentiment: Negative 11 | ### 12 | Tweet: "My day has been 👍" 13 | Sentiment: Positive 14 | ### 15 | Tweet: "This is the link to the article" 16 | Sentiment: Neutral 17 | ### 18 | Tweet: "This new music video was incredibile" 19 | Sentiment:""", 20 | """Translate English to French: 21 | 22 | sea otter => loutre de mer 23 | 24 | peppermint => menthe poivrée 25 | 26 | plush girafe => girafe peluche 27 | 28 | cheese =>""", 29 | ] 30 | 31 | zh_prompts = [ 32 | "我相信生命的意义是", 33 | "简单地说，相对论指出", 34 | "描述一副山水画", 35 | ] -------------------------------------------------------------------------------- /lms/runtime/prune/llm_pruner/LLMPruner/torch_pruning/__init__.py: -------------------------------------------------------------------------------- 1 | from .dependency import * 2 | from .pruner import * 3 | from . import _helpers, utils, importance -------------------------------------------------------------------------------- /lms/runtime/prune/llm_pruner/LLMPruner/torch_pruning/pruner/__init__.py: -------------------------------------------------------------------------------- 1 | from .function import * 2 | from .algorithms import * -------------------------------------------------------------------------------- /lms/runtime/prune/llm_pruner/LLMPruner/torch_pruning/pruner/algorithms/__init__.py: -------------------------------------------------------------------------------- 1 | from .metapruner import MetaPruner 2 | from .magnitude_based_pruner import MagnitudePruner 3 | from .batchnorm_scale_pruner import BNScalePruner 4 | from .group_norm_pruner import GroupNormPruner -------------------------------------------------------------------------------- /lms/runtime/prune/llm_pruner/LLMPruner/torch_pruning/pruner/algorithms/magnitude_based_pruner.py: -------------------------------------------------------------------------------- 1 | from .metapruner import MetaPruner 2 | 3 | class MagnitudePruner(MetaPruner): 4 | pass 5 | -------------------------------------------------------------------------------- /lms/runtime/prune/llm_pruner/LLMPruner/torch_pruning/pruner/algorithms/scheduler.py: -------------------------------------------------------------------------------- 1 | 2 | def linear_scheduler(ch_sparsity_dict, steps): 3 | return [((i) / float(steps)) * ch_sparsity_dict for i in range(steps+1)] -------------------------------------------------------------------------------- /lms/runtime/prune/llm_pruner/LLMPruner/torch_pruning/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from .utils import * 2 | from .op_counter import count_ops_and_params -------------------------------------------------------------------------------- /lms/runtime/prune/llm_pruner/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | current_path = os.path.dirname(os.path.abspath(__file__)) 5 | sys.path.append(current_path) 6 | 7 | from .entry import prune 8 | 9 | 10 | def load_model(model_dir, device=None): 11 | from pathlib import Path 12 | import json 13 | import torch 14 | 15 | info = Path(model_dir, "model.json") 16 | 17 | if not info.exists(): 18 | return None, None 19 | 20 | with open(info, "r", encoding="utf-8") as f: 21 | info = json.load(f) 22 | 23 | if info["load_method"] != "pytorch": 24 | return None, None 25 | 26 | if device is None: 27 | device = "cuda:0" if torch.cuda.is_available() else "cpu" 28 | 29 | raw = torch.load(Path(model_dir, info["model_path"]), map_location=device) 30 | tokenizer = raw[info["tokenizer_key"]] 31 | model = raw[info["model_key"]] 32 | 33 | return model, tokenizer 34 | -------------------------------------------------------------------------------- /lms/runtime/prune/llm_pruner/util.py: -------------------------------------------------------------------------------- 1 | def save_json(model_path, model_key, tokenizer_key): 2 | from pathlib import Path 3 | import json 4 | save_dir = Path(model_path).parent 5 | info_path = Path(save_dir, "model.json") 6 | s = {"load_method": "pytorch", 7 | "model_path": Path(model_path).name, 8 | "model_key": model_key, 9 | "tokenizer_key": tokenizer_key} 10 | with open(info_path, "w", encoding="utf-8") as f: 11 | json.dump(s, f, indent=4) -------------------------------------------------------------------------------- /lms/runtime/prune/sparsegpt/__init__.py: -------------------------------------------------------------------------------- 1 | from .entry import prune 2 | -------------------------------------------------------------------------------- /lms/runtime/prune/sparsegpt/test.py: -------------------------------------------------------------------------------- 1 | from lms.runtime.prune.sparsegpt import prune 2 | 3 | 4 | def main(): 5 | seed = 1 6 | nsamples = 1 7 | seqlen = 1024 8 | sparsity = "2:4" 9 | model_save_path = "tmp/pruned_model" 10 | tokenizer_save_path = "tmp/pruned_model" 11 | 12 | model_path = "/Users/dev/Documents/APS/models/huggingface/opt-125m" 13 | tokenizer_path = None 14 | only_check_model = False 15 | layer_name_start = "decoder.layers.1" 16 | layer_name_stop = "decoder.layers.11.fc1" 17 | 18 | # model_path = "decapoda-research/llama-7b-hf" 19 | # only_check_model = False 20 | # layer_name_start = "layers.1" 21 | # layer_name_end = "layers.29" 22 | prune(model_path=model_path, 23 | tokenizer_path=tokenizer_path, 24 | only_check_model=only_check_model, 25 | layer_name_start=layer_name_start, 26 | layer_name_stop=layer_name_stop, 27 | seed=seed, 28 | nsamples=nsamples, 29 | seqlen=seqlen, 30 | sparsity=sparsity, 31 | model_save_path=model_save_path, 32 | tokenizer_save_path=tokenizer_save_path, 33 | device='cpu' 34 | ) 35 | 36 | if __name__ == "__main__": 37 | main() 38 | -------------------------------------------------------------------------------- /lms/runtime/quantize/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataCanvasIO/LMS/56b41294c22e67313d0600d0484919edb92641ba/lms/runtime/quantize/__init__.py -------------------------------------------------------------------------------- /lms/runtime/quantize/util.py: -------------------------------------------------------------------------------- 1 | def load_tokenizer(tokenizer_path): 2 | def try_load(clz): 3 | try: 4 | tokenizer = clz.from_pretrained(tokenizer_path) 5 | except Exception as e: 6 | print(e) 7 | print("Failed to load tokenizer, try trust_remote_code=True.") 8 | tokenizer = clz.from_pretrained(tokenizer_path, trust_remote_code=True) 9 | return tokenizer 10 | 11 | if "llama" in tokenizer_path.lower(): 12 | from transformers import LlamaTokenizer 13 | return try_load(LlamaTokenizer) 14 | else: 15 | from transformers import AutoTokenizer 16 | return try_load(AutoTokenizer) -------------------------------------------------------------------------------- /lms/web/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataCanvasIO/LMS/56b41294c22e67313d0600d0484919edb92641ba/lms/web/__init__.py -------------------------------------------------------------------------------- /lms/web/bin/run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | nohup lms_web start & 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /lms/web/exceptions.py: -------------------------------------------------------------------------------- 1 | class BusinessException(Exception): 2 | 3 | def __init__(self, message, status_code=409): 4 | self.message = message 5 | self.status_code = status_code 6 | super(BusinessException, self).__init__(message) 7 | -------------------------------------------------------------------------------- /lms/web/internal/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataCanvasIO/LMS/56b41294c22e67313d0600d0484919edb92641ba/lms/web/internal/__init__.py -------------------------------------------------------------------------------- /lms/web/static/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataCanvasIO/LMS/56b41294c22e67313d0600d0484919edb92641ba/lms/web/static/__init__.py -------------------------------------------------------------------------------- /lms/web/static/empty.2f588adb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataCanvasIO/LMS/56b41294c22e67313d0600d0484919edb92641ba/lms/web/static/empty.2f588adb.png -------------------------------------------------------------------------------- /lms/web/static/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataCanvasIO/LMS/56b41294c22e67313d0600d0484919edb92641ba/lms/web/static/favicon.png -------------------------------------------------------------------------------- /lms/web/static/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 9 | 10 | 11 | 15 | 19 | 20 | 21 |
22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /lms/web/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataCanvasIO/LMS/56b41294c22e67313d0600d0484919edb92641ba/lms/web/utils/__init__.py -------------------------------------------------------------------------------- /lms/web/utils/traceback_utils.py: -------------------------------------------------------------------------------- 1 | import traceback 2 | 3 | 4 | def get_tarceback(type, value, tb): 5 | return '\n'.join(str(x) for x in traceback.format_exception(type, value, tb)) 6 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [project] 6 | name = "dc-lms" 7 | version = "1.0.0" 8 | requires-python = ">=3.9.0" 9 | dependencies = [ 10 | #====for web 11 | "requests", 12 | "paramiko", 13 | "sanic", 14 | "peewee", 15 | "pandas", 16 | "httpx", 17 | "asyncssh", 18 | "aiofiles", 19 | #====for runtime 20 | "torch", 21 | "transformers==4.33.2", 22 | "deepspeed>=0.11.1", 23 | "sentencepiece", 24 | "protobuf~=3.20.0", 25 | "sanic", 26 | "sanic-ext", 27 | "importlib-metadata", 28 | "accelerate", 29 | "datasets", 30 | "bitsandbytes", 31 | "evaluate", 32 | "scipy", 33 | # "xformers", 34 | "scikit-learn", 35 | "fastjsonschema", 36 | "tiktoken", 37 | # "seqeval", 38 | # "optimum", 39 | # "auto-gptq", 40 | #====for client 41 | "nvidia-ml-py3", 42 | "psutil", 43 | "requests", 44 | "optimum", 45 | "auto-gptq" 46 | ] 47 | 48 | [tool.setuptools.packages.find] 49 | where = ["."] # ["."] by default 50 | include = ["lms*"] # ["*"] by default 51 | exclude = ["lms.tests*","tests*"] # empty by default 52 | namespaces = true # true by default 53 | 54 | [project.scripts] 55 | lms_web = "lms.web.main:main" 56 | lms_rt = "lms.runtime.main:main" 57 | lmsrt = "lms.runtime.main:main" 58 | lms = "lms.client.main:main" 59 | lmsd = "lms.client.daemon.main:main" 60 | 61 | [tool.setuptools] 62 | include-package-data = true -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | LMS_VERSION = '1.0.0' 4 | 5 | setup( 6 | name='dc-lms', 7 | version=LMS_VERSION, 8 | packages=find_packages(), 9 | include_package_data=True, 10 | url='https://github.com/DataCanvasIO/LMS', 11 | license='Apache License 2.0', 12 | author='datacanvas', 13 | description='', 14 | long_description=""" 15 | LMS（Large Model Serving） is an open source tool that provides large model services. 16 | See the LMS HOME https://github.com/DataCanvasIO/LMS for details. 17 | """, 18 | long_description_content_type="text/markdown", 19 | entry_points={ 20 | "console_scripts": ['lms = lms.web:main'] 21 | }, 22 | python_requires='>=3.9', 23 | classifiers=[ 24 | "License :: OSI Approved :: Apache Software License", 25 | "Programming Language :: Python :: 3", 26 | "Programming Language :: Python :: 3.9"] 27 | ) 28 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataCanvasIO/LMS/56b41294c22e67313d0600d0484919edb92641ba/tests/__init__.py -------------------------------------------------------------------------------- /tests/metrics/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataCanvasIO/LMS/56b41294c22e67313d0600d0484919edb92641ba/tests/metrics/__init__.py -------------------------------------------------------------------------------- /tests/metrics/test_find.py: -------------------------------------------------------------------------------- 1 | # from deepspeed.launcher.runner import main 2 | import re 3 | 4 | # LMS_RT_PATTERN = r'.*(lms_rt\s+deploy|deepspeed_infer\.py|transformers_infer\.py).*--model_path=.*?\s+--model_name=(?P?)\s+(--port=(?P\d*))?(?!.*>)' 5 | LMS_RT_PATTERN = r'.*(lms_rt\s+deploy|deepspeed_infer\.py|transformers_infer\.py).*--model_path=.*?\s+--model_name=(?P.*?)\s+(--port=(?P\d*))?\s*--server_id=' 6 | 7 | with open("/Users/dev/PycharmProjects/lms/tests/metrics/a.txt", "r") as f: 8 | for line in f.readlines(): 9 | line = line.strip() 10 | # print(line) 11 | m = re.match(LMS_RT_PATTERN, line) 12 | if m: 13 | print(m.group('model_name')) 14 | print(m.group('port')) 15 | print(line) 16 | -------------------------------------------------------------------------------- /tests/paramiko_test.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | 4 | import paramiko 5 | 6 | key_path = os.path.expanduser("~") + '/.ssh/id_rsa' 7 | 8 | private = paramiko.RSAKey.from_private_key_file(key_path) 9 | client = paramiko.SSHClient() 10 | client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) 11 | client.connect(hostname='172.20.51.5', username='wuchen', pkey=private) 12 | 13 | 14 | def dir_list(model_path): 15 | sftp = client.open_sftp() 16 | try: 17 | start = time.time() 18 | listdir = sftp.listdir_attr(model_path) 19 | end = time.time() 20 | print(end - start) 21 | except FileNotFoundError: 22 | raise FileNotFoundError 23 | finally: 24 | sftp.close() 25 | client.close() 26 | files = [] 27 | for f in listdir: 28 | if not f.filename.startswith('.'): 29 | files.append({"file_name": f.filename, "is_dir": 1, "last_update_time": f.st_atime * 1000}) 30 | 31 | for j in range(len(files)): 32 | for i in range(1, len(files) - j): 33 | current = files[i] 34 | prev = files[i - 1] 35 | if current['last_update_time'] > prev['last_update_time']: 36 | files[i] = prev 37 | files[i - 1] = current 38 | elif current['last_update_time'] == prev['last_update_time']: 39 | if current['file_name'] < prev['file_name']: 40 | files[i] = prev 41 | files[i - 1] = current 42 | return files 43 | 44 | 45 | start = time.time() 46 | files0 = dir_list("/home/wuchen/opt-125m") 47 | end = time.time() 48 | print(end - start) 49 | print(files0) 50 | -------------------------------------------------------------------------------- /tests/test.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | # create the top-level parser 4 | parser = argparse.ArgumentParser(prog='PROG') 5 | parser.add_argument('--foo', action='store_true', help='foo help') 6 | subparsers = parser.add_subparsers(help='sub-command help') 7 | 8 | 9 | # create the parser for the "a" command 10 | parser_a = subparsers.add_parser('a', help='a help') 11 | 12 | # parser_a.add_argument('model_path',type=str) 13 | 14 | subsub = parser_a.add_subparsers(help="sub-sub help") 15 | 16 | parser_a.add_argument() 17 | 18 | parser_a1 = subsub.add_parser("a1") 19 | parser_a1.add_argument("--pp", type=int, help="pp help", required=True) 20 | parser_a2 = subsub.add_parser("a2") 21 | parser_a2.add_argument("--tt", type=int, help="tt help", required=False) 22 | 23 | parser_a.add_argument('model_path',type=str) 24 | 25 | # create the parser for the "b" command 26 | parser_b = subparsers.add_parser('b', help='b help') 27 | parser_b.add_argument('--baz', choices='XYZ', help='baz help') 28 | 29 | parser.parse_args() 30 | print("Done") -------------------------------------------------------------------------------- /tests/test/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataCanvasIO/LMS/56b41294c22e67313d0600d0484919edb92641ba/tests/test/__init__.py -------------------------------------------------------------------------------- /tests/test/test.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | import threading 3 | 4 | process = subprocess.Popen("python3 /Users/dev/PycharmProjects/lms/lms_rt/lms/rt/test/test1.py", 5 | shell=True, 6 | stdout=subprocess.PIPE, 7 | # stderr=subprocess.STDOUT 8 | ) 9 | 10 | 11 | def print_progress(stdout): 12 | from tqdm import tqdm 13 | tqdm.write(stdout.readline().decode("utf-8")) 14 | 15 | 16 | t1 = threading.Thread(target=print_progress, args=(process.stdout,)) 17 | t1.start() 18 | t1.join() 19 | 20 | for line in iter(process.stdout.readline, b''): 21 | # print(str(line, 'UTF-8')) 22 | line 23 | 24 | out, err = process.communicate() 25 | if process.returncode == 0: 26 | print(line) 27 | else: 28 | print("====") 29 | print(process.stderr) 30 | print(out) 31 | raise Exception(str(err)) 32 | -------------------------------------------------------------------------------- /tests/test/test1.py: -------------------------------------------------------------------------------- 1 | from tqdm import tqdm 2 | import time 3 | for i in tqdm(range(1000)): 4 | # 假设我们正在进行一些耗时的操作，比如训练深度学习模型 5 | time.sleep(0.01) 6 | print("hello") 7 | raise Exception("abcd") -------------------------------------------------------------------------------- /tests/test_asyncio_subprocess.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import sys 3 | 4 | 5 | async def read_stdout(stdout): 6 | while True: 7 | line = await stdout.readline() 8 | if not line: 9 | break 10 | print(str(line, 'utf-8').replace('prompt:', '')) 11 | 12 | 13 | async def read_stderr(stderr): 14 | while True: 15 | line = await stderr.readline() 16 | if not line: 17 | break 18 | sys.stderr.write(str(line, 'utf-8')) 19 | 20 | 21 | async def write_stdin(stdin): 22 | for i in range(100): 23 | buf = f'hello {i}\n'.encode() 24 | stdin.write(buf) 25 | await stdin.drain() 26 | await asyncio.sleep(0.5) 27 | 28 | 29 | async def dd(): 30 | proc = await asyncio.create_subprocess_shell( 31 | 'python3 /Users/dev/PycharmProjects/lms/lms/runtime/deploy/infer.py --model_path=/Users/dev/Documents/APS/models/huggingface/t5-v1_1-small', 32 | stdin=asyncio.subprocess.PIPE, 33 | stdout=asyncio.subprocess.PIPE, 34 | stderr=asyncio.subprocess.PIPE 35 | ) 36 | 37 | await asyncio.gather( 38 | read_stderr(proc.stderr), 39 | read_stdout(proc.stdout), 40 | write_stdin(proc.stdin)) 41 | 42 | 43 | asyncio.run(dd()) 44 | -------------------------------------------------------------------------------- /tests/test_asyncssh.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | import asyncssh as asyncssh 4 | 5 | 6 | async def aa(): 7 | path = "/tmp/abc" 8 | data = "haha\n1234" 9 | async with asyncssh.connect('127.0.0.1', username='dev', password='123456', known_hosts=None) as conn: 10 | async with conn.start_sftp_client() as sftp: 11 | if await sftp.listdir(path): 12 | raise Exception("The path is a dir") 13 | async with await sftp.open(path, pflags_or_mode='w') as f: 14 | await f.write(data) 15 | 16 | 17 | async def bb(path, chunk_size=2048): 18 | async with asyncssh.connect('172.20.51.5', username='wuchen', password='wuchen', known_hosts=None) as conn: 19 | async with conn.start_sftp_client() as sftp: 20 | 21 | if not await sftp.isfile(path): 22 | raise Exception("The path is not a file") 23 | 24 | async with await sftp.open(path, pflags_or_mode='rb') as f: 25 | while True: 26 | data = await f.read(chunk_size) 27 | if not data: 28 | break 29 | yield data 30 | 31 | 32 | async def abc(): 33 | async for i in bb("/home/wuchen/human/target11.csv"): 34 | print(i) 35 | 36 | asyncio.run(abc()) 37 | -------------------------------------------------------------------------------- /tests/tqdm/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DataCanvasIO/LMS/56b41294c22e67313d0600d0484919edb92641ba/tests/tqdm/__init__.py -------------------------------------------------------------------------------- /tests/tqdm/test1.py: -------------------------------------------------------------------------------- 1 | from tqdm import tqdm 2 | from time import sleep 3 | import sys 4 | 5 | # sys.stderr.write("hahahah\n") 6 | # with open('/Users/dev/PycharmProjects/lms/tests/tqdm/a.txt','w') as f: 7 | # for i in tqdm(range(0, 100),file=f, desc="Text You Want"): 8 | # sleep(.1) 9 | 10 | 11 | sys.stderr.write("hahahah\n") 12 | for i in tqdm(range(0, 100), desc="Text You Want"): #file=sys.stdout, 13 | sleep(.1) 14 | sys.stderr.write("hello\n") -------------------------------------------------------------------------------- /tests/tqdm/test111.py: -------------------------------------------------------------------------------- 1 | print(ord('▍')) 2 | print("==") 3 | print(chr(ord('█')+1)) 4 | print(chr(ord('█')+2)) 5 | print(chr(ord('█')+3)) 6 | print(chr(ord('█')+4)) 7 | print(chr(ord('█')+5)) 8 | print(chr(ord('█')+6)) 9 | print(chr(ord('█')+7)) 10 | print("==") 11 | # print(int.from_bytes(b'█','little')) 12 | print(int.from_bytes(b'\x89','little')) 13 | print(int.from_bytes(b'\x96','little')) 14 | print(int.from_bytes(b'\x96','big')) 15 | print(int.from_bytes(b'\x8e','little')) 16 | print(int.from_bytes(b'\x8f','little')) 17 | 18 | print("=====") 19 | 20 | print(b'\x8e') 21 | 22 | print(len(b'\x8e')) 23 | 24 | print((8*16) +14) 25 | 26 | print(b'\x8e'.decode('cp437')) 27 | print(b'\x89'.decode('cp437')) 28 | print(b'\x96'.decode('cp437')) 29 | 30 | print(bytes((142,)).decode('cp437')) 31 | 32 | 33 | 34 | print('█') 35 | print(ord('█')) 36 | print(bytes((219,)).decode('cp437')) 37 | print(u"\u2588") -------------------------------------------------------------------------------- /tests/tqdm/test1read.py: -------------------------------------------------------------------------------- 1 | import locale 2 | 3 | from tqdm import tqdm 4 | from time import sleep 5 | import sys 6 | 7 | print(locale.getpreferredencoding(False)) 8 | 9 | sys.stderr.write("hahahah\n") 10 | with open('/Users/dev/PycharmProjects/lms/tests/tqdm/a.txt','r',encoding='UTF-8') as f: 11 | while True: 12 | b = f.read(1) 13 | try: 14 | # if b == b'\r': 15 | # print('\n') 16 | sys.stdout.write(b) 17 | except UnicodeDecodeError: 18 | # print(b) 19 | # print(b == '\x96') 20 | # sys.stderr.write('█') 21 | sys.stderr.write('▍') 22 | # sys.stderr.write('\r') 23 | 24 | # 25 | # sys.stderr.write("hahahah\n") 26 | # for i in tqdm(range(0, 100), desc="Text You Want"): #file=sys.stdout, 27 | # sleep(.1) -------------------------------------------------------------------------------- /tests/tqdm/test_subprocess.py: -------------------------------------------------------------------------------- 1 | import codecs 2 | import io 3 | import subprocess 4 | import sys 5 | import threading 6 | import time 7 | 8 | from tests.tqdm.wrapper import TextIOWrapper 9 | 10 | process = subprocess.Popen(" ".join(["python3", "/Users/dev/PycharmProjects/lms/tests/tqdm/test1.py"]), shell=True, 11 | stderr=subprocess.PIPE, 12 | # stdout=subprocess.PIPE, 13 | ) 14 | 15 | 16 | def print_progress(stderr): 17 | # stderr = io.TextIOWrapper(stderr, newline='') 18 | # while True: 19 | # b = stderr.read(1) 20 | # sys.stdout.write(b) 21 | 22 | make_decoder = codecs.getincrementaldecoder('utf-8') 23 | _decoder = make_decoder("strict") 24 | _CHUNK_SIZE = 512 25 | 26 | while True: 27 | input_chunk = stderr.read(_CHUNK_SIZE) 28 | decoded_chars = _decoder.decode(input_chunk, False) 29 | buf = '' 30 | for c in decoded_chars: 31 | if c == '\r': 32 | # sys.stderr.write('\r') 33 | sys.stderr.write(buf) 34 | sys.stderr.write('\r') 35 | buf = '' 36 | else: 37 | buf += c 38 | # sys.stdout.write(decoder.decode(b, final=True)) 39 | # sys.stdout.write(b.decode("utf-8",'strict')) 40 | 41 | 42 | t1 = threading.Thread(target=print_progress, args=(process.stderr,)) 43 | t1.start() 44 | t1.join() 45 | --------------------------------------------------------------------------------