├── .gitignore ├── LICENSE ├── README.md ├── README_ZH.md ├── data └── CHARM │ ├── README.md │ ├── README_ZH.md │ ├── few-shot-examples │ ├── Chinese_Anachronisms_Judgment_Direct.txt │ ├── Chinese_Anachronisms_Judgment_EN-CoT.txt │ ├── Chinese_Anachronisms_Judgment_XLT.txt │ ├── Chinese_Anachronisms_Judgment_ZH-CoT.txt │ ├── Chinese_Movie_and_Music_Recommendation_Direct.txt │ ├── Chinese_Movie_and_Music_Recommendation_EN-CoT.txt │ ├── Chinese_Movie_and_Music_Recommendation_XLT.txt │ ├── Chinese_Movie_and_Music_Recommendation_ZH-CoT.txt │ ├── Chinese_Natural_Language_Inference_Direct.txt │ ├── Chinese_Natural_Language_Inference_EN-CoT.txt │ ├── Chinese_Natural_Language_Inference_XLT.txt │ ├── Chinese_Natural_Language_Inference_ZH-CoT.txt │ ├── Chinese_Reading_Comprehension_Direct.txt │ ├── Chinese_Reading_Comprehension_EN-CoT.txt │ ├── Chinese_Reading_Comprehension_XLT.txt │ ├── Chinese_Reading_Comprehension_ZH-CoT.txt │ ├── Chinese_Sequence_Understanding_Direct.txt │ ├── Chinese_Sequence_Understanding_EN-CoT.txt │ ├── Chinese_Sequence_Understanding_XLT.txt │ ├── Chinese_Sequence_Understanding_ZH-CoT.txt │ ├── Chinese_Sport_Understanding_Direct.txt │ ├── Chinese_Sport_Understanding_EN-CoT.txt │ ├── Chinese_Sport_Understanding_XLT.txt │ ├── Chinese_Sport_Understanding_ZH-CoT.txt │ ├── Chinese_Time_Understanding_Direct.txt │ ├── Chinese_Time_Understanding_EN-CoT.txt │ ├── Chinese_Time_Understanding_XLT.txt │ ├── Chinese_Time_Understanding_ZH-CoT.txt │ ├── Global_Anachronisms_Judgment_Direct.txt │ ├── Global_Anachronisms_Judgment_EN-CoT.txt │ ├── Global_Anachronisms_Judgment_XLT.txt │ ├── Global_Anachronisms_Judgment_ZH-CoT.txt │ ├── Global_Movie_and_Music_Recommendation_Direct.txt │ ├── Global_Movie_and_Music_Recommendation_EN-CoT.txt │ ├── Global_Movie_and_Music_Recommendation_XLT.txt │ ├── Global_Movie_and_Music_Recommendation_ZH-CoT.txt │ ├── Global_Natural_Language_Inference_Direct.txt │ ├── Global_Natural_Language_Inference_EN-CoT.txt │ ├── Global_Natural_Language_Inference_XLT.txt │ ├── Global_Natural_Language_Inference_ZH-CoT.txt │ ├── Global_Reading_Comprehension_Direct.txt │ ├── Global_Reading_Comprehension_EN-CoT.txt │ ├── Global_Reading_Comprehension_XLT.txt │ ├── Global_Reading_Comprehension_ZH-CoT.txt │ ├── Global_Sequence_Understanding_Direct.txt │ ├── Global_Sequence_Understanding_EN-CoT.txt │ ├── Global_Sequence_Understanding_XLT.txt │ ├── Global_Sequence_Understanding_ZH-CoT.txt │ ├── Global_Sport_Understanding_Direct.txt │ ├── Global_Sport_Understanding_EN-CoT.txt │ ├── Global_Sport_Understanding_XLT.txt │ ├── Global_Sport_Understanding_ZH-CoT.txt │ ├── Global_Time_Understanding_Direct.txt │ ├── Global_Time_Understanding_EN-CoT.txt │ ├── Global_Time_Understanding_XLT.txt │ └── Global_Time_Understanding_ZH-CoT.txt │ ├── few-shot-examples_Translate-EN │ ├── Chinese_Anachronisms_Judgment_Translate-EN.txt │ ├── Chinese_Movie_and_Music_Recommendation_Translate-EN.txt │ ├── Chinese_Natural_Language_Inference_Translate-EN.txt │ ├── Chinese_Reading_Comprehension_Translate-EN.txt │ ├── Chinese_Sequence_Understanding_Translate-EN.txt │ ├── Chinese_Sport_Understanding_Translate-EN.txt │ ├── Chinese_Time_Understanding_Translate-EN.txt │ ├── Global_Anachronisms_Judgment_Translate-EN.txt │ ├── Global_Movie_and_Music_Recommendation_Translate-EN.txt │ ├── Global_Natural_Language_Inference_Translate-EN.txt │ ├── Global_Reading_Comprehension_Translate-EN.txt │ ├── Global_Sequence_Understanding_Translate-EN.txt │ ├── Global_Sport_Understanding_Translate-EN.txt │ └── Global_Time_Understanding_Translate-EN.txt │ ├── memorization │ ├── Chinese_Anachronisms_Judgment.json │ ├── Chinese_Movie_and_Music_Recommendation.json │ ├── Chinese_Sport_Understanding.json │ └── Chinese_Time_Understanding.json │ ├── memorization_Translate-EN │ ├── Chinese_Anachronisms_Judgment.json │ ├── Chinese_Movie_and_Music_Recommendation.json │ ├── Chinese_Sport_Understanding.json │ └── Chinese_Time_Understanding.json │ ├── reasoning │ ├── Chinese_Anachronisms_Judgment.json │ ├── Chinese_Movie_and_Music_Recommendation.json │ ├── Chinese_Natural_Language_Inference.json │ ├── Chinese_Reading_Comprehension.json │ ├── Chinese_Sequence_Understanding.json │ ├── Chinese_Sport_Understanding.json │ ├── Chinese_Time_Understanding.json │ ├── Global_Anachronisms_Judgment.json │ ├── Global_Movie_and_Music_Recommendation.json │ ├── Global_Natural_Language_Inference.json │ ├── Global_Reading_Comprehension.json │ ├── Global_Sequence_Understanding.json │ ├── Global_Sport_Understanding.json │ └── Global_Time_Understanding.json │ └── reasoning_Translate-EN │ ├── Chinese_Anachronisms_Judgment.json │ ├── Chinese_Movie_and_Music_Recommendation.json │ ├── Chinese_Natural_Language_Inference.json │ ├── Chinese_Reading_Comprehension.json │ ├── Chinese_Sequence_Understanding.json │ ├── Chinese_Sport_Understanding.json │ ├── Chinese_Time_Understanding.json │ ├── Global_Anachronisms_Judgment.json │ ├── Global_Movie_and_Music_Recommendation.json │ ├── Global_Natural_Language_Inference.json │ ├── Global_Reading_Comprehension.json │ ├── Global_Sequence_Understanding.json │ ├── Global_Sport_Understanding.json │ └── Global_Time_Understanding.json ├── docs ├── figure │ ├── Fig2_CHARM_construction.png │ ├── Fig_mem-independent-rea-err-example.png │ ├── Fig_mem-integrated_rea-avg_v4.png │ └── Fig_pie_chart-mem_independent_err-3LLms.png ├── findings.html ├── findings_ZH.html ├── index.html ├── index_ZH.html ├── leaderboard.html └── leaderboard_ZH.html ├── requirements.txt └── tools ├── analyze_mem_indep_rea.py ├── models_and_prompts.py ├── summarize_mem_rea.py ├── summarize_reasoning.py ├── update_html.py └── utils.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/README.md -------------------------------------------------------------------------------- /README_ZH.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/README_ZH.md -------------------------------------------------------------------------------- /data/CHARM/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/README.md -------------------------------------------------------------------------------- /data/CHARM/README_ZH.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/README_ZH.md -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Chinese_Anachronisms_Judgment_Direct.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Chinese_Anachronisms_Judgment_Direct.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Chinese_Anachronisms_Judgment_EN-CoT.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Chinese_Anachronisms_Judgment_EN-CoT.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Chinese_Anachronisms_Judgment_XLT.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Chinese_Anachronisms_Judgment_XLT.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Chinese_Anachronisms_Judgment_ZH-CoT.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Chinese_Anachronisms_Judgment_ZH-CoT.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Chinese_Movie_and_Music_Recommendation_Direct.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Chinese_Movie_and_Music_Recommendation_Direct.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Chinese_Movie_and_Music_Recommendation_EN-CoT.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Chinese_Movie_and_Music_Recommendation_EN-CoT.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Chinese_Movie_and_Music_Recommendation_XLT.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Chinese_Movie_and_Music_Recommendation_XLT.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Chinese_Movie_and_Music_Recommendation_ZH-CoT.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Chinese_Movie_and_Music_Recommendation_ZH-CoT.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Chinese_Natural_Language_Inference_Direct.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Chinese_Natural_Language_Inference_Direct.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Chinese_Natural_Language_Inference_EN-CoT.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Chinese_Natural_Language_Inference_EN-CoT.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Chinese_Natural_Language_Inference_XLT.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Chinese_Natural_Language_Inference_XLT.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Chinese_Natural_Language_Inference_ZH-CoT.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Chinese_Natural_Language_Inference_ZH-CoT.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Chinese_Reading_Comprehension_Direct.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Chinese_Reading_Comprehension_Direct.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Chinese_Reading_Comprehension_EN-CoT.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Chinese_Reading_Comprehension_EN-CoT.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Chinese_Reading_Comprehension_XLT.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Chinese_Reading_Comprehension_XLT.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Chinese_Reading_Comprehension_ZH-CoT.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Chinese_Reading_Comprehension_ZH-CoT.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Chinese_Sequence_Understanding_Direct.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Chinese_Sequence_Understanding_Direct.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Chinese_Sequence_Understanding_EN-CoT.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Chinese_Sequence_Understanding_EN-CoT.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Chinese_Sequence_Understanding_XLT.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Chinese_Sequence_Understanding_XLT.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Chinese_Sequence_Understanding_ZH-CoT.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Chinese_Sequence_Understanding_ZH-CoT.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Chinese_Sport_Understanding_Direct.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Chinese_Sport_Understanding_Direct.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Chinese_Sport_Understanding_EN-CoT.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Chinese_Sport_Understanding_EN-CoT.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Chinese_Sport_Understanding_XLT.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Chinese_Sport_Understanding_XLT.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Chinese_Sport_Understanding_ZH-CoT.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Chinese_Sport_Understanding_ZH-CoT.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Chinese_Time_Understanding_Direct.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Chinese_Time_Understanding_Direct.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Chinese_Time_Understanding_EN-CoT.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Chinese_Time_Understanding_EN-CoT.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Chinese_Time_Understanding_XLT.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Chinese_Time_Understanding_XLT.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Chinese_Time_Understanding_ZH-CoT.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Chinese_Time_Understanding_ZH-CoT.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Global_Anachronisms_Judgment_Direct.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Global_Anachronisms_Judgment_Direct.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Global_Anachronisms_Judgment_EN-CoT.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Global_Anachronisms_Judgment_EN-CoT.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Global_Anachronisms_Judgment_XLT.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Global_Anachronisms_Judgment_XLT.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Global_Anachronisms_Judgment_ZH-CoT.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Global_Anachronisms_Judgment_ZH-CoT.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Global_Movie_and_Music_Recommendation_Direct.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Global_Movie_and_Music_Recommendation_Direct.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Global_Movie_and_Music_Recommendation_EN-CoT.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Global_Movie_and_Music_Recommendation_EN-CoT.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Global_Movie_and_Music_Recommendation_XLT.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Global_Movie_and_Music_Recommendation_XLT.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Global_Movie_and_Music_Recommendation_ZH-CoT.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Global_Movie_and_Music_Recommendation_ZH-CoT.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Global_Natural_Language_Inference_Direct.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Global_Natural_Language_Inference_Direct.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Global_Natural_Language_Inference_EN-CoT.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Global_Natural_Language_Inference_EN-CoT.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Global_Natural_Language_Inference_XLT.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Global_Natural_Language_Inference_XLT.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Global_Natural_Language_Inference_ZH-CoT.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Global_Natural_Language_Inference_ZH-CoT.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Global_Reading_Comprehension_Direct.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Global_Reading_Comprehension_Direct.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Global_Reading_Comprehension_EN-CoT.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Global_Reading_Comprehension_EN-CoT.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Global_Reading_Comprehension_XLT.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Global_Reading_Comprehension_XLT.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Global_Reading_Comprehension_ZH-CoT.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Global_Reading_Comprehension_ZH-CoT.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Global_Sequence_Understanding_Direct.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Global_Sequence_Understanding_Direct.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Global_Sequence_Understanding_EN-CoT.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Global_Sequence_Understanding_EN-CoT.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Global_Sequence_Understanding_XLT.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Global_Sequence_Understanding_XLT.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Global_Sequence_Understanding_ZH-CoT.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Global_Sequence_Understanding_ZH-CoT.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Global_Sport_Understanding_Direct.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Global_Sport_Understanding_Direct.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Global_Sport_Understanding_EN-CoT.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Global_Sport_Understanding_EN-CoT.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Global_Sport_Understanding_XLT.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Global_Sport_Understanding_XLT.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Global_Sport_Understanding_ZH-CoT.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Global_Sport_Understanding_ZH-CoT.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Global_Time_Understanding_Direct.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Global_Time_Understanding_Direct.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Global_Time_Understanding_EN-CoT.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Global_Time_Understanding_EN-CoT.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Global_Time_Understanding_XLT.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Global_Time_Understanding_XLT.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples/Global_Time_Understanding_ZH-CoT.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples/Global_Time_Understanding_ZH-CoT.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples_Translate-EN/Chinese_Anachronisms_Judgment_Translate-EN.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples_Translate-EN/Chinese_Anachronisms_Judgment_Translate-EN.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples_Translate-EN/Chinese_Movie_and_Music_Recommendation_Translate-EN.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples_Translate-EN/Chinese_Movie_and_Music_Recommendation_Translate-EN.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples_Translate-EN/Chinese_Natural_Language_Inference_Translate-EN.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples_Translate-EN/Chinese_Natural_Language_Inference_Translate-EN.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples_Translate-EN/Chinese_Reading_Comprehension_Translate-EN.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples_Translate-EN/Chinese_Reading_Comprehension_Translate-EN.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples_Translate-EN/Chinese_Sequence_Understanding_Translate-EN.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples_Translate-EN/Chinese_Sequence_Understanding_Translate-EN.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples_Translate-EN/Chinese_Sport_Understanding_Translate-EN.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples_Translate-EN/Chinese_Sport_Understanding_Translate-EN.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples_Translate-EN/Chinese_Time_Understanding_Translate-EN.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples_Translate-EN/Chinese_Time_Understanding_Translate-EN.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples_Translate-EN/Global_Anachronisms_Judgment_Translate-EN.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples_Translate-EN/Global_Anachronisms_Judgment_Translate-EN.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples_Translate-EN/Global_Movie_and_Music_Recommendation_Translate-EN.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples_Translate-EN/Global_Movie_and_Music_Recommendation_Translate-EN.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples_Translate-EN/Global_Natural_Language_Inference_Translate-EN.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples_Translate-EN/Global_Natural_Language_Inference_Translate-EN.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples_Translate-EN/Global_Reading_Comprehension_Translate-EN.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples_Translate-EN/Global_Reading_Comprehension_Translate-EN.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples_Translate-EN/Global_Sequence_Understanding_Translate-EN.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples_Translate-EN/Global_Sequence_Understanding_Translate-EN.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples_Translate-EN/Global_Sport_Understanding_Translate-EN.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples_Translate-EN/Global_Sport_Understanding_Translate-EN.txt -------------------------------------------------------------------------------- /data/CHARM/few-shot-examples_Translate-EN/Global_Time_Understanding_Translate-EN.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/few-shot-examples_Translate-EN/Global_Time_Understanding_Translate-EN.txt -------------------------------------------------------------------------------- /data/CHARM/memorization/Chinese_Anachronisms_Judgment.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/memorization/Chinese_Anachronisms_Judgment.json -------------------------------------------------------------------------------- /data/CHARM/memorization/Chinese_Movie_and_Music_Recommendation.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/memorization/Chinese_Movie_and_Music_Recommendation.json -------------------------------------------------------------------------------- /data/CHARM/memorization/Chinese_Sport_Understanding.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/memorization/Chinese_Sport_Understanding.json -------------------------------------------------------------------------------- /data/CHARM/memorization/Chinese_Time_Understanding.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/memorization/Chinese_Time_Understanding.json -------------------------------------------------------------------------------- /data/CHARM/memorization_Translate-EN/Chinese_Anachronisms_Judgment.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/memorization_Translate-EN/Chinese_Anachronisms_Judgment.json -------------------------------------------------------------------------------- /data/CHARM/memorization_Translate-EN/Chinese_Movie_and_Music_Recommendation.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/memorization_Translate-EN/Chinese_Movie_and_Music_Recommendation.json -------------------------------------------------------------------------------- /data/CHARM/memorization_Translate-EN/Chinese_Sport_Understanding.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/memorization_Translate-EN/Chinese_Sport_Understanding.json -------------------------------------------------------------------------------- /data/CHARM/memorization_Translate-EN/Chinese_Time_Understanding.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/memorization_Translate-EN/Chinese_Time_Understanding.json -------------------------------------------------------------------------------- /data/CHARM/reasoning/Chinese_Anachronisms_Judgment.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/reasoning/Chinese_Anachronisms_Judgment.json -------------------------------------------------------------------------------- /data/CHARM/reasoning/Chinese_Movie_and_Music_Recommendation.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/reasoning/Chinese_Movie_and_Music_Recommendation.json -------------------------------------------------------------------------------- /data/CHARM/reasoning/Chinese_Natural_Language_Inference.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/reasoning/Chinese_Natural_Language_Inference.json -------------------------------------------------------------------------------- /data/CHARM/reasoning/Chinese_Reading_Comprehension.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/reasoning/Chinese_Reading_Comprehension.json -------------------------------------------------------------------------------- /data/CHARM/reasoning/Chinese_Sequence_Understanding.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/reasoning/Chinese_Sequence_Understanding.json -------------------------------------------------------------------------------- /data/CHARM/reasoning/Chinese_Sport_Understanding.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/reasoning/Chinese_Sport_Understanding.json -------------------------------------------------------------------------------- /data/CHARM/reasoning/Chinese_Time_Understanding.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/reasoning/Chinese_Time_Understanding.json -------------------------------------------------------------------------------- /data/CHARM/reasoning/Global_Anachronisms_Judgment.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/reasoning/Global_Anachronisms_Judgment.json -------------------------------------------------------------------------------- /data/CHARM/reasoning/Global_Movie_and_Music_Recommendation.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/reasoning/Global_Movie_and_Music_Recommendation.json -------------------------------------------------------------------------------- /data/CHARM/reasoning/Global_Natural_Language_Inference.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/reasoning/Global_Natural_Language_Inference.json -------------------------------------------------------------------------------- /data/CHARM/reasoning/Global_Reading_Comprehension.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/reasoning/Global_Reading_Comprehension.json -------------------------------------------------------------------------------- /data/CHARM/reasoning/Global_Sequence_Understanding.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/reasoning/Global_Sequence_Understanding.json -------------------------------------------------------------------------------- /data/CHARM/reasoning/Global_Sport_Understanding.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/reasoning/Global_Sport_Understanding.json -------------------------------------------------------------------------------- /data/CHARM/reasoning/Global_Time_Understanding.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/reasoning/Global_Time_Understanding.json -------------------------------------------------------------------------------- /data/CHARM/reasoning_Translate-EN/Chinese_Anachronisms_Judgment.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/reasoning_Translate-EN/Chinese_Anachronisms_Judgment.json -------------------------------------------------------------------------------- /data/CHARM/reasoning_Translate-EN/Chinese_Movie_and_Music_Recommendation.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/reasoning_Translate-EN/Chinese_Movie_and_Music_Recommendation.json -------------------------------------------------------------------------------- /data/CHARM/reasoning_Translate-EN/Chinese_Natural_Language_Inference.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/reasoning_Translate-EN/Chinese_Natural_Language_Inference.json -------------------------------------------------------------------------------- /data/CHARM/reasoning_Translate-EN/Chinese_Reading_Comprehension.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/reasoning_Translate-EN/Chinese_Reading_Comprehension.json -------------------------------------------------------------------------------- /data/CHARM/reasoning_Translate-EN/Chinese_Sequence_Understanding.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/reasoning_Translate-EN/Chinese_Sequence_Understanding.json -------------------------------------------------------------------------------- /data/CHARM/reasoning_Translate-EN/Chinese_Sport_Understanding.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/reasoning_Translate-EN/Chinese_Sport_Understanding.json -------------------------------------------------------------------------------- /data/CHARM/reasoning_Translate-EN/Chinese_Time_Understanding.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/reasoning_Translate-EN/Chinese_Time_Understanding.json -------------------------------------------------------------------------------- /data/CHARM/reasoning_Translate-EN/Global_Anachronisms_Judgment.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/reasoning_Translate-EN/Global_Anachronisms_Judgment.json -------------------------------------------------------------------------------- /data/CHARM/reasoning_Translate-EN/Global_Movie_and_Music_Recommendation.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/reasoning_Translate-EN/Global_Movie_and_Music_Recommendation.json -------------------------------------------------------------------------------- /data/CHARM/reasoning_Translate-EN/Global_Natural_Language_Inference.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/reasoning_Translate-EN/Global_Natural_Language_Inference.json -------------------------------------------------------------------------------- /data/CHARM/reasoning_Translate-EN/Global_Reading_Comprehension.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/reasoning_Translate-EN/Global_Reading_Comprehension.json -------------------------------------------------------------------------------- /data/CHARM/reasoning_Translate-EN/Global_Sequence_Understanding.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/reasoning_Translate-EN/Global_Sequence_Understanding.json -------------------------------------------------------------------------------- /data/CHARM/reasoning_Translate-EN/Global_Sport_Understanding.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/reasoning_Translate-EN/Global_Sport_Understanding.json -------------------------------------------------------------------------------- /data/CHARM/reasoning_Translate-EN/Global_Time_Understanding.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/data/CHARM/reasoning_Translate-EN/Global_Time_Understanding.json -------------------------------------------------------------------------------- /docs/figure/Fig2_CHARM_construction.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/docs/figure/Fig2_CHARM_construction.png -------------------------------------------------------------------------------- /docs/figure/Fig_mem-independent-rea-err-example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/docs/figure/Fig_mem-independent-rea-err-example.png -------------------------------------------------------------------------------- /docs/figure/Fig_mem-integrated_rea-avg_v4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/docs/figure/Fig_mem-integrated_rea-avg_v4.png -------------------------------------------------------------------------------- /docs/figure/Fig_pie_chart-mem_independent_err-3LLms.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/docs/figure/Fig_pie_chart-mem_independent_err-3LLms.png -------------------------------------------------------------------------------- /docs/findings.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/docs/findings.html -------------------------------------------------------------------------------- /docs/findings_ZH.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/docs/findings_ZH.html -------------------------------------------------------------------------------- /docs/index.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/docs/index.html -------------------------------------------------------------------------------- /docs/index_ZH.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/docs/index_ZH.html -------------------------------------------------------------------------------- /docs/leaderboard.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/docs/leaderboard.html -------------------------------------------------------------------------------- /docs/leaderboard_ZH.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/docs/leaderboard_ZH.html -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | matplotlib<=3.8.4 2 | openpyxl 3 | xlsxwriter -------------------------------------------------------------------------------- /tools/analyze_mem_indep_rea.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/tools/analyze_mem_indep_rea.py -------------------------------------------------------------------------------- /tools/models_and_prompts.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/tools/models_and_prompts.py -------------------------------------------------------------------------------- /tools/summarize_mem_rea.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/tools/summarize_mem_rea.py -------------------------------------------------------------------------------- /tools/summarize_reasoning.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/tools/summarize_reasoning.py -------------------------------------------------------------------------------- /tools/update_html.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/tools/update_html.py -------------------------------------------------------------------------------- /tools/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/CHARM/HEAD/tools/utils.py --------------------------------------------------------------------------------