├── .github └── workflows │ └── publish.yml ├── .gitignore ├── .pre-commit-config.yaml ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── NOTICE ├── README.md ├── configs ├── experiments │ ├── amazon_nova_micro_deceptive.yaml │ ├── anthr_haiku.yaml │ ├── anthr_haiku_deceptive.yaml │ ├── anthr_haiku_deceptive_NoSwap.yaml │ ├── anthr_haiku_deciles.yaml │ ├── anthr_haiku_overlap.yaml │ ├── anthr_haiku_revision.yaml │ ├── anthr_instant.yaml │ ├── anthr_sonnet.yaml │ ├── config.yaml │ ├── meta_llama.yaml │ ├── mistral_large.yaml │ ├── mistral_small.yaml │ ├── openai_gpt35.yaml │ ├── openai_gpt35_overlap.yaml │ └── test_config.yaml ├── generation_prompt │ ├── config.md │ └── sample_config.yaml ├── generation_sets │ ├── deceptive.yaml │ ├── deciles.yaml │ ├── numbers.yaml │ └── words.yaml ├── models.yaml ├── post_analysis │ ├── nova_lite.json │ └── study_config.json ├── post_hypothesis │ └── hypothesis.json └── study_to_models.yaml ├── notebooks └── Hypothesis Testing - Manuscript.ipynb ├── pyproject.toml ├── scripts ├── analysis_for_one_study.py ├── download-google-ngrams.sh ├── make-deciles.sh ├── make_hyponyms.py ├── make_percentiles.py └── save_processed_results_for_study_list.py ├── setlexsem ├── __init__.py ├── analyze │ ├── __init__.py │ ├── error_analysis.py │ ├── hypothesis_testing_utils.py │ └── visualize.py ├── constants.py ├── experiment │ ├── __init__.py │ ├── experiment.py │ ├── lmapi.py │ └── run_experiments.py ├── generate │ ├── __init__.py │ ├── generate_prompts.py │ ├── generate_sets.py │ ├── prompt.py │ ├── sample.py │ └── utils_io.py ├── prepare │ ├── __init__.py │ ├── download.py │ └── percentiles.py └── utils.py ├── setup.py └── tests ├── generate ├── test_generate_prompts.py ├── test_generate_sets.py └── test_sample.py ├── test_experiment.py ├── test_lmapi.py ├── test_prompt.py ├── test_samplers.py └── test_utils.py /.github/workflows/publish.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/.github/workflows/publish.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/.gitignore -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/.pre-commit-config.yaml -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/CODE_OF_CONDUCT.md -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/CONTRIBUTING.md -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/LICENSE -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/README.md -------------------------------------------------------------------------------- /configs/experiments/amazon_nova_micro_deceptive.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/configs/experiments/amazon_nova_micro_deceptive.yaml -------------------------------------------------------------------------------- /configs/experiments/anthr_haiku.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/configs/experiments/anthr_haiku.yaml -------------------------------------------------------------------------------- /configs/experiments/anthr_haiku_deceptive.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/configs/experiments/anthr_haiku_deceptive.yaml -------------------------------------------------------------------------------- /configs/experiments/anthr_haiku_deceptive_NoSwap.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/configs/experiments/anthr_haiku_deceptive_NoSwap.yaml -------------------------------------------------------------------------------- /configs/experiments/anthr_haiku_deciles.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/configs/experiments/anthr_haiku_deciles.yaml -------------------------------------------------------------------------------- /configs/experiments/anthr_haiku_overlap.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/configs/experiments/anthr_haiku_overlap.yaml -------------------------------------------------------------------------------- /configs/experiments/anthr_haiku_revision.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/configs/experiments/anthr_haiku_revision.yaml -------------------------------------------------------------------------------- /configs/experiments/anthr_instant.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/configs/experiments/anthr_instant.yaml -------------------------------------------------------------------------------- /configs/experiments/anthr_sonnet.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/configs/experiments/anthr_sonnet.yaml -------------------------------------------------------------------------------- /configs/experiments/config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/configs/experiments/config.yaml -------------------------------------------------------------------------------- /configs/experiments/meta_llama.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/configs/experiments/meta_llama.yaml -------------------------------------------------------------------------------- /configs/experiments/mistral_large.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/configs/experiments/mistral_large.yaml -------------------------------------------------------------------------------- /configs/experiments/mistral_small.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/configs/experiments/mistral_small.yaml -------------------------------------------------------------------------------- /configs/experiments/openai_gpt35.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/configs/experiments/openai_gpt35.yaml -------------------------------------------------------------------------------- /configs/experiments/openai_gpt35_overlap.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/configs/experiments/openai_gpt35_overlap.yaml -------------------------------------------------------------------------------- /configs/experiments/test_config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/configs/experiments/test_config.yaml -------------------------------------------------------------------------------- /configs/generation_prompt/config.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/configs/generation_prompt/config.md -------------------------------------------------------------------------------- /configs/generation_prompt/sample_config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/configs/generation_prompt/sample_config.yaml -------------------------------------------------------------------------------- /configs/generation_sets/deceptive.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/configs/generation_sets/deceptive.yaml -------------------------------------------------------------------------------- /configs/generation_sets/deciles.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/configs/generation_sets/deciles.yaml -------------------------------------------------------------------------------- /configs/generation_sets/numbers.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/configs/generation_sets/numbers.yaml -------------------------------------------------------------------------------- /configs/generation_sets/words.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/configs/generation_sets/words.yaml -------------------------------------------------------------------------------- /configs/models.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/configs/models.yaml -------------------------------------------------------------------------------- /configs/post_analysis/nova_lite.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/configs/post_analysis/nova_lite.json -------------------------------------------------------------------------------- /configs/post_analysis/study_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/configs/post_analysis/study_config.json -------------------------------------------------------------------------------- /configs/post_hypothesis/hypothesis.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/configs/post_hypothesis/hypothesis.json -------------------------------------------------------------------------------- /configs/study_to_models.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/configs/study_to_models.yaml -------------------------------------------------------------------------------- /notebooks/Hypothesis Testing - Manuscript.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/notebooks/Hypothesis Testing - Manuscript.ipynb -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/pyproject.toml -------------------------------------------------------------------------------- /scripts/analysis_for_one_study.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/scripts/analysis_for_one_study.py -------------------------------------------------------------------------------- /scripts/download-google-ngrams.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/scripts/download-google-ngrams.sh -------------------------------------------------------------------------------- /scripts/make-deciles.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/scripts/make-deciles.sh -------------------------------------------------------------------------------- /scripts/make_hyponyms.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/scripts/make_hyponyms.py -------------------------------------------------------------------------------- /scripts/make_percentiles.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/scripts/make_percentiles.py -------------------------------------------------------------------------------- /scripts/save_processed_results_for_study_list.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/scripts/save_processed_results_for_study_list.py -------------------------------------------------------------------------------- /setlexsem/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /setlexsem/analyze/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /setlexsem/analyze/error_analysis.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/setlexsem/analyze/error_analysis.py -------------------------------------------------------------------------------- /setlexsem/analyze/hypothesis_testing_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/setlexsem/analyze/hypothesis_testing_utils.py -------------------------------------------------------------------------------- /setlexsem/analyze/visualize.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/setlexsem/analyze/visualize.py -------------------------------------------------------------------------------- /setlexsem/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/setlexsem/constants.py -------------------------------------------------------------------------------- /setlexsem/experiment/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /setlexsem/experiment/experiment.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/setlexsem/experiment/experiment.py -------------------------------------------------------------------------------- /setlexsem/experiment/lmapi.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/setlexsem/experiment/lmapi.py -------------------------------------------------------------------------------- /setlexsem/experiment/run_experiments.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/setlexsem/experiment/run_experiments.py -------------------------------------------------------------------------------- /setlexsem/generate/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /setlexsem/generate/generate_prompts.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/setlexsem/generate/generate_prompts.py -------------------------------------------------------------------------------- /setlexsem/generate/generate_sets.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/setlexsem/generate/generate_sets.py -------------------------------------------------------------------------------- /setlexsem/generate/prompt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/setlexsem/generate/prompt.py -------------------------------------------------------------------------------- /setlexsem/generate/sample.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/setlexsem/generate/sample.py -------------------------------------------------------------------------------- /setlexsem/generate/utils_io.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/setlexsem/generate/utils_io.py -------------------------------------------------------------------------------- /setlexsem/prepare/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /setlexsem/prepare/download.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/setlexsem/prepare/download.py -------------------------------------------------------------------------------- /setlexsem/prepare/percentiles.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/setlexsem/prepare/percentiles.py -------------------------------------------------------------------------------- /setlexsem/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/setlexsem/utils.py -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/setup.py -------------------------------------------------------------------------------- /tests/generate/test_generate_prompts.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/tests/generate/test_generate_prompts.py -------------------------------------------------------------------------------- /tests/generate/test_generate_sets.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/tests/generate/test_generate_sets.py -------------------------------------------------------------------------------- /tests/generate/test_sample.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/tests/generate/test_sample.py -------------------------------------------------------------------------------- /tests/test_experiment.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/tests/test_experiment.py -------------------------------------------------------------------------------- /tests/test_lmapi.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/tests/test_lmapi.py -------------------------------------------------------------------------------- /tests/test_prompt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/tests/test_prompt.py -------------------------------------------------------------------------------- /tests/test_samplers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/tests/test_samplers.py -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/amazon-science/SetLexSem-Challenge/HEAD/tests/test_utils.py --------------------------------------------------------------------------------