├── .github └── workflows │ └── pylint.yml ├── .gitignore ├── .mailmap ├── LICENSE.txt ├── Makefile ├── README.md ├── analyse ├── into_fp8.py ├── pre_training_distribution │ ├── calc_kl_divergence.py │ ├── claude-4-sonnet │ │ └── evaluation │ │ │ └── direct (samples=50) │ │ │ └── response_count_results.json │ ├── direct_claude-4-sonnet.json │ ├── direct_gpt-4.1.json │ ├── gpt-4.1 │ │ └── evaluation │ │ │ └── direct (samples=50) │ │ │ └── response_count_results.json │ ├── pre_training_distribution.py │ ├── pre_training_distribution_comparison.pdf │ ├── sequence_claude-4-sonnet.json │ ├── sequence_gpt-4.1.json │ ├── state_name_distribution.json │ ├── test_cal.py │ ├── vs_claude-4-sonnet.json │ └── vs_gpt-4.1.json ├── reg-share.py ├── run_llama_70b.sh ├── run_openrouter.sh └── serve.sh ├── assets └── teaser.png ├── data ├── book.txt ├── creative_story.txt ├── gsm8k.txt ├── joke.txt ├── lcb.txt ├── math │ ├── aime │ │ ├── data-00000-of-00001.arrow │ │ ├── dataset_info.json │ │ └── state.json │ ├── amc │ │ ├── data-00000-of-00001.arrow │ │ ├── dataset_info.json │ │ └── state.json │ ├── dataset_dict.json │ ├── math │ │ ├── data-00000-of-00001.arrow │ │ ├── dataset_info.json │ │ └── state.json │ ├── minerva │ │ ├── data-00000-of-00001.arrow │ │ ├── dataset_info.json │ │ └── state.json │ └── olympiad_bench │ │ ├── data-00000-of-00001.arrow │ │ ├── dataset_info.json │ │ └── state.json ├── poem.txt ├── poem_titles.txt ├── rand_num.txt ├── safety.txt ├── safety_harmbench.txt ├── simple_qa.csv ├── simple_qa.txt ├── speech.txt ├── state_name.json ├── state_name.txt ├── state_name_CoverageQA.json ├── state_name_CoverageQA.txt ├── synthetic_negative.txt └── synthetic_negative_new.txt ├── examples ├── 01_quick_start.py ├── 02_basic_usage.py ├── 03_transforms.py ├── 04_recipes.py ├── 05_research_api.py ├── README.md ├── plan.md └── simple_api.py ├── notebooks ├── framework_demo.ipynb ├── vs_base.ipynb └── vs_with_image.ipynb ├── pyproject.toml ├── scripts ├── EXPERIMENTS.md ├── ablation │ ├── analyse_probs.py │ ├── diversity_tuning │ │ ├── run_bias.py │ │ ├── run_joke.py │ │ ├── run_poem.py │ │ ├── run_poem_gemini.py │ │ ├── run_story.py │ │ └── run_story_gemini.py │ ├── prob_def │ │ ├── run_bias_ablation.py │ │ ├── run_poem.py │ │ └── run_poem_local.py │ ├── run_min_p_ablation.py │ ├── run_num_samples_ablation.py │ ├── run_pre_training_distribution.py │ ├── run_sampling_ablation.py │ ├── run_sampling_candidates_ablation.py │ ├── run_sampling_comparison.py │ └── run_top_p_ablation.py ├── analysis │ ├── analyze_dialogue_results.py │ ├── latex │ │ ├── README.md │ │ ├── ablation │ │ │ ├── latex_figures │ │ │ │ ├── poem_temperature_plot.pdf │ │ │ │ ├── poem_temperature_plot.png │ │ │ │ ├── safety_evaluation_analysis.pdf │ │ │ │ └── safety_evaluation_analysis.png │ │ │ ├── min_p_ablation_comparison.pdf │ │ │ ├── min_p_ablation_comparison.png │ │ │ ├── model_size_ablation.py │ │ │ ├── num_samples_ablation_comparison.pdf │ │ │ ├── num_samples_ablation_comparison.png │ │ │ ├── open_ended_coverage.py │ │ │ ├── plot_creative_ablation.py │ │ │ ├── plot_min_p.py │ │ │ ├── plot_num_samples.py │ │ │ ├── plot_num_samples_inset.py │ │ │ ├── plot_safety_evaluation.py │ │ │ ├── plot_sampling_candidates.py │ │ │ ├── plot_temperature.py │ │ │ ├── plot_top_p.py │ │ │ ├── sampling_candidates_ablation_comparison.pdf │ │ │ ├── top_p_ablation_comparison.pdf │ │ │ ├── top_p_ablation_comparison.png │ │ │ ├── top_p_ablation_inset.pdf │ │ │ ├── top_p_ablation_inset.png │ │ │ ├── top_p_ablation_line_plots.pdf │ │ │ ├── top_p_ablation_line_plots.png │ │ │ ├── top_p_ablation_zoomed.pdf │ │ │ └── top_p_ablation_zoomed.png │ │ ├── appendix │ │ │ ├── book_diversity_tuning_comparison.pdf │ │ │ ├── book_diversity_tuning_comparison.png │ │ │ ├── cognitive_bias.py │ │ │ ├── joke_diversity_tuning_comparison.pdf │ │ │ ├── joke_diversity_tuning_comparison.png │ │ │ ├── plot_bias_metrics_tuning.py │ │ │ ├── plot_diversity_tuning.py │ │ │ ├── poem_diversity_tuning_comparison.pdf │ │ │ ├── poem_diversity_tuning_comparison.png │ │ │ └── prob_manipulation_line_plot.py │ │ ├── bias_prompts_ablation.py │ │ ├── calculate_bias_metrics.py │ │ ├── calculate_factual_equivalence.py │ │ ├── calculate_factual_metrics.py │ │ ├── config.py │ │ ├── coverage_analysis.py │ │ ├── generate_all_figures.py │ │ ├── generate_latex_tables.py │ │ ├── generate_plots.py │ │ ├── gsm8k_eval.py │ │ ├── intro_performance.py │ │ ├── latex_figures │ │ │ ├── safety_evaluation_analysis.pdf │ │ │ └── safety_evaluation_analysis.png │ │ ├── num_samples_ablation_comparison.pdf │ │ ├── num_samples_ablation_comparison.png │ │ ├── parse_latex_table.py │ │ ├── plot_appendix_individual.py │ │ ├── plot_unify_creativity.py │ │ ├── plot_unify_creativity_rouge.py │ │ ├── plot_unify_creativity_violin.py │ │ ├── plot_unify_creativity_w_diversity_tuning.py │ │ ├── poem │ │ │ ├── plot_diversity_comparison.py │ │ │ ├── plot_num_samples_ablation.py │ │ │ └── plot_prob_tuning.py │ │ ├── qualitative_tasks │ │ │ ├── bias_prompts_ablation_line_chart.pdf │ │ │ ├── comparison_of_different_prompt_variants.pdf │ │ │ ├── rng_direct_samples.json │ │ │ ├── rng_distribution_comparison.pdf │ │ │ ├── rng_vs_samples.json │ │ │ ├── state_name.py │ │ │ ├── synthetic_data.py │ │ │ ├── synthetic_data_combined_metrics.pdf │ │ │ ├── synthetic_data_gsm8k_similarity.json │ │ │ ├── synthetic_data_gsm8k_similarity_histogram.pdf │ │ │ ├── synthetic_data_metrics_grouped_barplot.pdf │ │ │ ├── synthetic_data_negative_gpt-4.1_diversity_quality_histogram.pdf │ │ │ ├── synthetic_data_negative_gpt-4.1_similarity_results copy.json │ │ │ ├── synthetic_data_negative_gpt-4.1_similarity_results.json │ │ │ └── synthetic_data_teaser_graph.py │ │ ├── rng.py │ │ ├── synthetic_data │ │ │ ├── amc_aime_training_file.py │ │ │ ├── gsm8k_eval.py │ │ │ ├── gsm_training_file.py │ │ │ ├── lcb_training_file.py │ │ │ ├── lcb_training_file_sequence.py │ │ │ ├── lcb_training_file_vs_cot.py │ │ │ └── lcb_training_file_vs_standard.py │ │ ├── synthetic_data_histogram.py │ │ └── synthetic_data_negative.py │ └── plot_max_diversity.py ├── data_processing │ ├── create_seed_data.py │ ├── filter_responses.py │ ├── load_harmbench_data.py │ ├── process_aime_dataset.py │ ├── process_gsm8k_dataset.py │ ├── processing │ │ ├── categorize_dad_jokes.py │ │ ├── dad_jokes_categories.summary.json │ │ ├── gen_response.py │ │ ├── process.sh │ │ ├── process_synthetic_negative.py │ │ └── transfer_positive.py │ └── push_checkpoints_to_hf.py ├── migration │ └── migrate_method_names.py └── tasks │ ├── run_bias_task_coverageqa.py │ ├── run_bias_task_general.py │ ├── run_dialogue_simulation.py │ ├── run_jokes.py │ ├── run_math_simple.py │ ├── run_negative.py │ ├── run_poem.py │ ├── run_positive_amc_aime.py │ ├── run_positive_gsm8k.py │ ├── run_positive_lcb.py │ ├── run_rng.py │ ├── run_safety.py │ ├── run_simple_qa.py │ ├── run_state_name.py │ ├── run_story.py │ └── vllm │ └── serve.sh ├── setup.py └── verbalized_sampling ├── __init__.py ├── analysis ├── dialogue_plots.py ├── evals │ ├── README.md │ ├── __init__.py │ ├── assets │ │ └── cwv3_rubrics.py │ ├── base.py │ ├── chi_square.py │ ├── creative_writing_v3.py │ ├── creativity_index.py │ ├── data_quality.py │ ├── data_quality_qa.py │ ├── diversity.py │ ├── factuality.py │ ├── joke_quality.py │ ├── length.py │ ├── ngram.py │ ├── quality.py │ └── response_count.py ├── plots.py └── plotting.py ├── api.py ├── cli.py ├── evals ├── accuracy.py ├── dialogue │ ├── __init__.py │ ├── donation.py │ └── linguistic.py └── safety.py ├── examples ├── evaluation_demo.py └── tasks_demo.py ├── llms ├── __init__.py ├── base.py ├── embed.py ├── google.py ├── litellm.py ├── openai.py ├── openrouter.py └── vllm.py ├── methods ├── __init__.py ├── factory.py ├── parser.py ├── prompt.py └── schema.py ├── pipeline.py ├── plots ├── README.md ├── __init__.py ├── base.py ├── comparison.py ├── convenience.py ├── evaluator_specific.py ├── factory.py └── plotters.py ├── selection.py ├── tasks ├── README.md ├── __init__.py ├── base.py ├── bias │ ├── __init__.py │ ├── rand_num.py │ └── state_name.py ├── creativity │ ├── __init__.py │ ├── book.py │ ├── joke.py │ ├── poem.py │ ├── speech.py │ └── story.py ├── dialogue │ ├── __init__.py │ ├── base.py │ └── persuasion.py ├── fact │ ├── __init__.py │ └── simple_qa.py ├── math │ ├── __init__.py │ └── math_task.py ├── math_eval.py ├── safety │ ├── __init__.py │ └── safety.py └── synthetic_data │ ├── __init__.py │ ├── amc_aime.py │ ├── gsm8k.py │ ├── livecodebench.py │ └── synthetic_negative.py └── utils ├── __init__.py └── data_compatibility.py /.github/workflows/pylint.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/.github/workflows/pylint.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/.gitignore -------------------------------------------------------------------------------- /.mailmap: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/.mailmap -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/LICENSE.txt -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/Makefile -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/README.md -------------------------------------------------------------------------------- /analyse/into_fp8.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/analyse/into_fp8.py -------------------------------------------------------------------------------- /analyse/pre_training_distribution/calc_kl_divergence.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/analyse/pre_training_distribution/calc_kl_divergence.py -------------------------------------------------------------------------------- /analyse/pre_training_distribution/claude-4-sonnet/evaluation/direct (samples=50)/response_count_results.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/analyse/pre_training_distribution/claude-4-sonnet/evaluation/direct (samples=50)/response_count_results.json -------------------------------------------------------------------------------- /analyse/pre_training_distribution/direct_claude-4-sonnet.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/analyse/pre_training_distribution/direct_claude-4-sonnet.json -------------------------------------------------------------------------------- /analyse/pre_training_distribution/direct_gpt-4.1.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/analyse/pre_training_distribution/direct_gpt-4.1.json -------------------------------------------------------------------------------- /analyse/pre_training_distribution/gpt-4.1/evaluation/direct (samples=50)/response_count_results.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/analyse/pre_training_distribution/gpt-4.1/evaluation/direct (samples=50)/response_count_results.json -------------------------------------------------------------------------------- /analyse/pre_training_distribution/pre_training_distribution.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/analyse/pre_training_distribution/pre_training_distribution.py -------------------------------------------------------------------------------- /analyse/pre_training_distribution/pre_training_distribution_comparison.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/analyse/pre_training_distribution/pre_training_distribution_comparison.pdf -------------------------------------------------------------------------------- /analyse/pre_training_distribution/sequence_claude-4-sonnet.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/analyse/pre_training_distribution/sequence_claude-4-sonnet.json -------------------------------------------------------------------------------- /analyse/pre_training_distribution/sequence_gpt-4.1.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/analyse/pre_training_distribution/sequence_gpt-4.1.json -------------------------------------------------------------------------------- /analyse/pre_training_distribution/state_name_distribution.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/analyse/pre_training_distribution/state_name_distribution.json -------------------------------------------------------------------------------- /analyse/pre_training_distribution/test_cal.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/analyse/pre_training_distribution/test_cal.py -------------------------------------------------------------------------------- /analyse/pre_training_distribution/vs_claude-4-sonnet.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/analyse/pre_training_distribution/vs_claude-4-sonnet.json -------------------------------------------------------------------------------- /analyse/pre_training_distribution/vs_gpt-4.1.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/analyse/pre_training_distribution/vs_gpt-4.1.json -------------------------------------------------------------------------------- /analyse/reg-share.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/analyse/reg-share.py -------------------------------------------------------------------------------- /analyse/run_llama_70b.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/analyse/run_llama_70b.sh -------------------------------------------------------------------------------- /analyse/run_openrouter.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/analyse/run_openrouter.sh -------------------------------------------------------------------------------- /analyse/serve.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/analyse/serve.sh -------------------------------------------------------------------------------- /assets/teaser.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/assets/teaser.png -------------------------------------------------------------------------------- /data/book.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/data/book.txt -------------------------------------------------------------------------------- /data/creative_story.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/data/creative_story.txt -------------------------------------------------------------------------------- /data/gsm8k.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/data/gsm8k.txt -------------------------------------------------------------------------------- /data/joke.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/data/joke.txt -------------------------------------------------------------------------------- /data/lcb.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/data/lcb.txt -------------------------------------------------------------------------------- /data/math/aime/data-00000-of-00001.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/data/math/aime/data-00000-of-00001.arrow -------------------------------------------------------------------------------- /data/math/aime/dataset_info.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/data/math/aime/dataset_info.json -------------------------------------------------------------------------------- /data/math/aime/state.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/data/math/aime/state.json -------------------------------------------------------------------------------- /data/math/amc/data-00000-of-00001.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/data/math/amc/data-00000-of-00001.arrow -------------------------------------------------------------------------------- /data/math/amc/dataset_info.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/data/math/amc/dataset_info.json -------------------------------------------------------------------------------- /data/math/amc/state.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/data/math/amc/state.json -------------------------------------------------------------------------------- /data/math/dataset_dict.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/data/math/dataset_dict.json -------------------------------------------------------------------------------- /data/math/math/data-00000-of-00001.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/data/math/math/data-00000-of-00001.arrow -------------------------------------------------------------------------------- /data/math/math/dataset_info.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/data/math/math/dataset_info.json -------------------------------------------------------------------------------- /data/math/math/state.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/data/math/math/state.json -------------------------------------------------------------------------------- /data/math/minerva/data-00000-of-00001.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/data/math/minerva/data-00000-of-00001.arrow -------------------------------------------------------------------------------- /data/math/minerva/dataset_info.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/data/math/minerva/dataset_info.json -------------------------------------------------------------------------------- /data/math/minerva/state.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/data/math/minerva/state.json -------------------------------------------------------------------------------- /data/math/olympiad_bench/data-00000-of-00001.arrow: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/data/math/olympiad_bench/data-00000-of-00001.arrow -------------------------------------------------------------------------------- /data/math/olympiad_bench/dataset_info.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/data/math/olympiad_bench/dataset_info.json -------------------------------------------------------------------------------- /data/math/olympiad_bench/state.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/data/math/olympiad_bench/state.json -------------------------------------------------------------------------------- /data/poem.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/data/poem.txt -------------------------------------------------------------------------------- /data/poem_titles.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/data/poem_titles.txt -------------------------------------------------------------------------------- /data/rand_num.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/data/rand_num.txt -------------------------------------------------------------------------------- /data/safety.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/data/safety.txt -------------------------------------------------------------------------------- /data/safety_harmbench.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/data/safety_harmbench.txt -------------------------------------------------------------------------------- /data/simple_qa.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/data/simple_qa.csv -------------------------------------------------------------------------------- /data/simple_qa.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/data/simple_qa.txt -------------------------------------------------------------------------------- /data/speech.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/data/speech.txt -------------------------------------------------------------------------------- /data/state_name.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/data/state_name.json -------------------------------------------------------------------------------- /data/state_name.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/data/state_name.txt -------------------------------------------------------------------------------- /data/state_name_CoverageQA.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/data/state_name_CoverageQA.json -------------------------------------------------------------------------------- /data/state_name_CoverageQA.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/data/state_name_CoverageQA.txt -------------------------------------------------------------------------------- /data/synthetic_negative.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/data/synthetic_negative.txt -------------------------------------------------------------------------------- /data/synthetic_negative_new.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/data/synthetic_negative_new.txt -------------------------------------------------------------------------------- /examples/01_quick_start.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/examples/01_quick_start.py -------------------------------------------------------------------------------- /examples/02_basic_usage.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/examples/02_basic_usage.py -------------------------------------------------------------------------------- /examples/03_transforms.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/examples/03_transforms.py -------------------------------------------------------------------------------- /examples/04_recipes.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/examples/04_recipes.py -------------------------------------------------------------------------------- /examples/05_research_api.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/examples/05_research_api.py -------------------------------------------------------------------------------- /examples/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/examples/README.md -------------------------------------------------------------------------------- /examples/plan.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/examples/plan.md -------------------------------------------------------------------------------- /examples/simple_api.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/examples/simple_api.py -------------------------------------------------------------------------------- /notebooks/framework_demo.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/notebooks/framework_demo.ipynb -------------------------------------------------------------------------------- /notebooks/vs_base.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/notebooks/vs_base.ipynb -------------------------------------------------------------------------------- /notebooks/vs_with_image.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/notebooks/vs_with_image.ipynb -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/pyproject.toml -------------------------------------------------------------------------------- /scripts/EXPERIMENTS.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/EXPERIMENTS.md -------------------------------------------------------------------------------- /scripts/ablation/analyse_probs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/ablation/analyse_probs.py -------------------------------------------------------------------------------- /scripts/ablation/diversity_tuning/run_bias.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/ablation/diversity_tuning/run_bias.py -------------------------------------------------------------------------------- /scripts/ablation/diversity_tuning/run_joke.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/ablation/diversity_tuning/run_joke.py -------------------------------------------------------------------------------- /scripts/ablation/diversity_tuning/run_poem.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/ablation/diversity_tuning/run_poem.py -------------------------------------------------------------------------------- /scripts/ablation/diversity_tuning/run_poem_gemini.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/ablation/diversity_tuning/run_poem_gemini.py -------------------------------------------------------------------------------- /scripts/ablation/diversity_tuning/run_story.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/ablation/diversity_tuning/run_story.py -------------------------------------------------------------------------------- /scripts/ablation/diversity_tuning/run_story_gemini.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/ablation/diversity_tuning/run_story_gemini.py -------------------------------------------------------------------------------- /scripts/ablation/prob_def/run_bias_ablation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/ablation/prob_def/run_bias_ablation.py -------------------------------------------------------------------------------- /scripts/ablation/prob_def/run_poem.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/ablation/prob_def/run_poem.py -------------------------------------------------------------------------------- /scripts/ablation/prob_def/run_poem_local.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/ablation/prob_def/run_poem_local.py -------------------------------------------------------------------------------- /scripts/ablation/run_min_p_ablation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/ablation/run_min_p_ablation.py -------------------------------------------------------------------------------- /scripts/ablation/run_num_samples_ablation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/ablation/run_num_samples_ablation.py -------------------------------------------------------------------------------- /scripts/ablation/run_pre_training_distribution.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/ablation/run_pre_training_distribution.py -------------------------------------------------------------------------------- /scripts/ablation/run_sampling_ablation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/ablation/run_sampling_ablation.py -------------------------------------------------------------------------------- /scripts/ablation/run_sampling_candidates_ablation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/ablation/run_sampling_candidates_ablation.py -------------------------------------------------------------------------------- /scripts/ablation/run_sampling_comparison.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/ablation/run_sampling_comparison.py -------------------------------------------------------------------------------- /scripts/ablation/run_top_p_ablation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/ablation/run_top_p_ablation.py -------------------------------------------------------------------------------- /scripts/analysis/analyze_dialogue_results.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/analyze_dialogue_results.py -------------------------------------------------------------------------------- /scripts/analysis/latex/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/README.md -------------------------------------------------------------------------------- /scripts/analysis/latex/ablation/latex_figures/poem_temperature_plot.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/ablation/latex_figures/poem_temperature_plot.pdf -------------------------------------------------------------------------------- /scripts/analysis/latex/ablation/latex_figures/poem_temperature_plot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/ablation/latex_figures/poem_temperature_plot.png -------------------------------------------------------------------------------- /scripts/analysis/latex/ablation/latex_figures/safety_evaluation_analysis.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/ablation/latex_figures/safety_evaluation_analysis.pdf -------------------------------------------------------------------------------- /scripts/analysis/latex/ablation/latex_figures/safety_evaluation_analysis.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/ablation/latex_figures/safety_evaluation_analysis.png -------------------------------------------------------------------------------- /scripts/analysis/latex/ablation/min_p_ablation_comparison.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/ablation/min_p_ablation_comparison.pdf -------------------------------------------------------------------------------- /scripts/analysis/latex/ablation/min_p_ablation_comparison.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/ablation/min_p_ablation_comparison.png -------------------------------------------------------------------------------- /scripts/analysis/latex/ablation/model_size_ablation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/ablation/model_size_ablation.py -------------------------------------------------------------------------------- /scripts/analysis/latex/ablation/num_samples_ablation_comparison.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/ablation/num_samples_ablation_comparison.pdf -------------------------------------------------------------------------------- /scripts/analysis/latex/ablation/num_samples_ablation_comparison.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/ablation/num_samples_ablation_comparison.png -------------------------------------------------------------------------------- /scripts/analysis/latex/ablation/open_ended_coverage.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/ablation/open_ended_coverage.py -------------------------------------------------------------------------------- /scripts/analysis/latex/ablation/plot_creative_ablation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/ablation/plot_creative_ablation.py -------------------------------------------------------------------------------- /scripts/analysis/latex/ablation/plot_min_p.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/ablation/plot_min_p.py -------------------------------------------------------------------------------- /scripts/analysis/latex/ablation/plot_num_samples.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/ablation/plot_num_samples.py -------------------------------------------------------------------------------- /scripts/analysis/latex/ablation/plot_num_samples_inset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/ablation/plot_num_samples_inset.py -------------------------------------------------------------------------------- /scripts/analysis/latex/ablation/plot_safety_evaluation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/ablation/plot_safety_evaluation.py -------------------------------------------------------------------------------- /scripts/analysis/latex/ablation/plot_sampling_candidates.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/ablation/plot_sampling_candidates.py -------------------------------------------------------------------------------- /scripts/analysis/latex/ablation/plot_temperature.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/ablation/plot_temperature.py -------------------------------------------------------------------------------- /scripts/analysis/latex/ablation/plot_top_p.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/ablation/plot_top_p.py -------------------------------------------------------------------------------- /scripts/analysis/latex/ablation/sampling_candidates_ablation_comparison.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/ablation/sampling_candidates_ablation_comparison.pdf -------------------------------------------------------------------------------- /scripts/analysis/latex/ablation/top_p_ablation_comparison.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/ablation/top_p_ablation_comparison.pdf -------------------------------------------------------------------------------- /scripts/analysis/latex/ablation/top_p_ablation_comparison.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/ablation/top_p_ablation_comparison.png -------------------------------------------------------------------------------- /scripts/analysis/latex/ablation/top_p_ablation_inset.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/ablation/top_p_ablation_inset.pdf -------------------------------------------------------------------------------- /scripts/analysis/latex/ablation/top_p_ablation_inset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/ablation/top_p_ablation_inset.png -------------------------------------------------------------------------------- /scripts/analysis/latex/ablation/top_p_ablation_line_plots.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/ablation/top_p_ablation_line_plots.pdf -------------------------------------------------------------------------------- /scripts/analysis/latex/ablation/top_p_ablation_line_plots.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/ablation/top_p_ablation_line_plots.png -------------------------------------------------------------------------------- /scripts/analysis/latex/ablation/top_p_ablation_zoomed.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/ablation/top_p_ablation_zoomed.pdf -------------------------------------------------------------------------------- /scripts/analysis/latex/ablation/top_p_ablation_zoomed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/ablation/top_p_ablation_zoomed.png -------------------------------------------------------------------------------- /scripts/analysis/latex/appendix/book_diversity_tuning_comparison.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/appendix/book_diversity_tuning_comparison.pdf -------------------------------------------------------------------------------- /scripts/analysis/latex/appendix/book_diversity_tuning_comparison.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/appendix/book_diversity_tuning_comparison.png -------------------------------------------------------------------------------- /scripts/analysis/latex/appendix/cognitive_bias.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/appendix/cognitive_bias.py -------------------------------------------------------------------------------- /scripts/analysis/latex/appendix/joke_diversity_tuning_comparison.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/appendix/joke_diversity_tuning_comparison.pdf -------------------------------------------------------------------------------- /scripts/analysis/latex/appendix/joke_diversity_tuning_comparison.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/appendix/joke_diversity_tuning_comparison.png -------------------------------------------------------------------------------- /scripts/analysis/latex/appendix/plot_bias_metrics_tuning.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/appendix/plot_bias_metrics_tuning.py -------------------------------------------------------------------------------- /scripts/analysis/latex/appendix/plot_diversity_tuning.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/appendix/plot_diversity_tuning.py -------------------------------------------------------------------------------- /scripts/analysis/latex/appendix/poem_diversity_tuning_comparison.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/appendix/poem_diversity_tuning_comparison.pdf -------------------------------------------------------------------------------- /scripts/analysis/latex/appendix/poem_diversity_tuning_comparison.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/appendix/poem_diversity_tuning_comparison.png -------------------------------------------------------------------------------- /scripts/analysis/latex/appendix/prob_manipulation_line_plot.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/appendix/prob_manipulation_line_plot.py -------------------------------------------------------------------------------- /scripts/analysis/latex/bias_prompts_ablation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/bias_prompts_ablation.py -------------------------------------------------------------------------------- /scripts/analysis/latex/calculate_bias_metrics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/calculate_bias_metrics.py -------------------------------------------------------------------------------- /scripts/analysis/latex/calculate_factual_equivalence.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/calculate_factual_equivalence.py -------------------------------------------------------------------------------- /scripts/analysis/latex/calculate_factual_metrics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/calculate_factual_metrics.py -------------------------------------------------------------------------------- /scripts/analysis/latex/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/config.py -------------------------------------------------------------------------------- /scripts/analysis/latex/coverage_analysis.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/coverage_analysis.py -------------------------------------------------------------------------------- /scripts/analysis/latex/generate_all_figures.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/generate_all_figures.py -------------------------------------------------------------------------------- /scripts/analysis/latex/generate_latex_tables.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/generate_latex_tables.py -------------------------------------------------------------------------------- /scripts/analysis/latex/generate_plots.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/generate_plots.py -------------------------------------------------------------------------------- /scripts/analysis/latex/gsm8k_eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/gsm8k_eval.py -------------------------------------------------------------------------------- /scripts/analysis/latex/intro_performance.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/intro_performance.py -------------------------------------------------------------------------------- /scripts/analysis/latex/latex_figures/safety_evaluation_analysis.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/latex_figures/safety_evaluation_analysis.pdf -------------------------------------------------------------------------------- /scripts/analysis/latex/latex_figures/safety_evaluation_analysis.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/latex_figures/safety_evaluation_analysis.png -------------------------------------------------------------------------------- /scripts/analysis/latex/num_samples_ablation_comparison.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/num_samples_ablation_comparison.pdf -------------------------------------------------------------------------------- /scripts/analysis/latex/num_samples_ablation_comparison.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/num_samples_ablation_comparison.png -------------------------------------------------------------------------------- /scripts/analysis/latex/parse_latex_table.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/parse_latex_table.py -------------------------------------------------------------------------------- /scripts/analysis/latex/plot_appendix_individual.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/plot_appendix_individual.py -------------------------------------------------------------------------------- /scripts/analysis/latex/plot_unify_creativity.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/plot_unify_creativity.py -------------------------------------------------------------------------------- /scripts/analysis/latex/plot_unify_creativity_rouge.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/plot_unify_creativity_rouge.py -------------------------------------------------------------------------------- /scripts/analysis/latex/plot_unify_creativity_violin.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/plot_unify_creativity_violin.py -------------------------------------------------------------------------------- /scripts/analysis/latex/plot_unify_creativity_w_diversity_tuning.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/plot_unify_creativity_w_diversity_tuning.py -------------------------------------------------------------------------------- /scripts/analysis/latex/poem/plot_diversity_comparison.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/poem/plot_diversity_comparison.py -------------------------------------------------------------------------------- /scripts/analysis/latex/poem/plot_num_samples_ablation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/poem/plot_num_samples_ablation.py -------------------------------------------------------------------------------- /scripts/analysis/latex/poem/plot_prob_tuning.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/poem/plot_prob_tuning.py -------------------------------------------------------------------------------- /scripts/analysis/latex/qualitative_tasks/bias_prompts_ablation_line_chart.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/qualitative_tasks/bias_prompts_ablation_line_chart.pdf -------------------------------------------------------------------------------- /scripts/analysis/latex/qualitative_tasks/comparison_of_different_prompt_variants.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/qualitative_tasks/comparison_of_different_prompt_variants.pdf -------------------------------------------------------------------------------- /scripts/analysis/latex/qualitative_tasks/rng_direct_samples.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/qualitative_tasks/rng_direct_samples.json -------------------------------------------------------------------------------- /scripts/analysis/latex/qualitative_tasks/rng_distribution_comparison.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/qualitative_tasks/rng_distribution_comparison.pdf -------------------------------------------------------------------------------- /scripts/analysis/latex/qualitative_tasks/rng_vs_samples.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/qualitative_tasks/rng_vs_samples.json -------------------------------------------------------------------------------- /scripts/analysis/latex/qualitative_tasks/state_name.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/qualitative_tasks/state_name.py -------------------------------------------------------------------------------- /scripts/analysis/latex/qualitative_tasks/synthetic_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/qualitative_tasks/synthetic_data.py -------------------------------------------------------------------------------- /scripts/analysis/latex/qualitative_tasks/synthetic_data_combined_metrics.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/qualitative_tasks/synthetic_data_combined_metrics.pdf -------------------------------------------------------------------------------- /scripts/analysis/latex/qualitative_tasks/synthetic_data_gsm8k_similarity.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/qualitative_tasks/synthetic_data_gsm8k_similarity.json -------------------------------------------------------------------------------- /scripts/analysis/latex/qualitative_tasks/synthetic_data_gsm8k_similarity_histogram.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/qualitative_tasks/synthetic_data_gsm8k_similarity_histogram.pdf -------------------------------------------------------------------------------- /scripts/analysis/latex/qualitative_tasks/synthetic_data_metrics_grouped_barplot.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/qualitative_tasks/synthetic_data_metrics_grouped_barplot.pdf -------------------------------------------------------------------------------- /scripts/analysis/latex/qualitative_tasks/synthetic_data_negative_gpt-4.1_diversity_quality_histogram.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/qualitative_tasks/synthetic_data_negative_gpt-4.1_diversity_quality_histogram.pdf -------------------------------------------------------------------------------- /scripts/analysis/latex/qualitative_tasks/synthetic_data_negative_gpt-4.1_similarity_results copy.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/qualitative_tasks/synthetic_data_negative_gpt-4.1_similarity_results copy.json -------------------------------------------------------------------------------- /scripts/analysis/latex/qualitative_tasks/synthetic_data_negative_gpt-4.1_similarity_results.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/qualitative_tasks/synthetic_data_negative_gpt-4.1_similarity_results.json -------------------------------------------------------------------------------- /scripts/analysis/latex/qualitative_tasks/synthetic_data_teaser_graph.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/qualitative_tasks/synthetic_data_teaser_graph.py -------------------------------------------------------------------------------- /scripts/analysis/latex/rng.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/rng.py -------------------------------------------------------------------------------- /scripts/analysis/latex/synthetic_data/amc_aime_training_file.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/synthetic_data/amc_aime_training_file.py -------------------------------------------------------------------------------- /scripts/analysis/latex/synthetic_data/gsm8k_eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/synthetic_data/gsm8k_eval.py -------------------------------------------------------------------------------- /scripts/analysis/latex/synthetic_data/gsm_training_file.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/synthetic_data/gsm_training_file.py -------------------------------------------------------------------------------- /scripts/analysis/latex/synthetic_data/lcb_training_file.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/synthetic_data/lcb_training_file.py -------------------------------------------------------------------------------- /scripts/analysis/latex/synthetic_data/lcb_training_file_sequence.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/synthetic_data/lcb_training_file_sequence.py -------------------------------------------------------------------------------- /scripts/analysis/latex/synthetic_data/lcb_training_file_vs_cot.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/synthetic_data/lcb_training_file_vs_cot.py -------------------------------------------------------------------------------- /scripts/analysis/latex/synthetic_data/lcb_training_file_vs_standard.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/synthetic_data/lcb_training_file_vs_standard.py -------------------------------------------------------------------------------- /scripts/analysis/latex/synthetic_data_histogram.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/synthetic_data_histogram.py -------------------------------------------------------------------------------- /scripts/analysis/latex/synthetic_data_negative.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/latex/synthetic_data_negative.py -------------------------------------------------------------------------------- /scripts/analysis/plot_max_diversity.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/analysis/plot_max_diversity.py -------------------------------------------------------------------------------- /scripts/data_processing/create_seed_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/data_processing/create_seed_data.py -------------------------------------------------------------------------------- /scripts/data_processing/filter_responses.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/data_processing/filter_responses.py -------------------------------------------------------------------------------- /scripts/data_processing/load_harmbench_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/data_processing/load_harmbench_data.py -------------------------------------------------------------------------------- /scripts/data_processing/process_aime_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/data_processing/process_aime_dataset.py -------------------------------------------------------------------------------- /scripts/data_processing/process_gsm8k_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/data_processing/process_gsm8k_dataset.py -------------------------------------------------------------------------------- /scripts/data_processing/processing/categorize_dad_jokes.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/data_processing/processing/categorize_dad_jokes.py -------------------------------------------------------------------------------- /scripts/data_processing/processing/dad_jokes_categories.summary.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/data_processing/processing/dad_jokes_categories.summary.json -------------------------------------------------------------------------------- /scripts/data_processing/processing/gen_response.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/data_processing/processing/gen_response.py -------------------------------------------------------------------------------- /scripts/data_processing/processing/process.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/data_processing/processing/process.sh -------------------------------------------------------------------------------- /scripts/data_processing/processing/process_synthetic_negative.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/data_processing/processing/process_synthetic_negative.py -------------------------------------------------------------------------------- /scripts/data_processing/processing/transfer_positive.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/data_processing/processing/transfer_positive.py -------------------------------------------------------------------------------- /scripts/data_processing/push_checkpoints_to_hf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/data_processing/push_checkpoints_to_hf.py -------------------------------------------------------------------------------- /scripts/migration/migrate_method_names.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/migration/migrate_method_names.py -------------------------------------------------------------------------------- /scripts/tasks/run_bias_task_coverageqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/tasks/run_bias_task_coverageqa.py -------------------------------------------------------------------------------- /scripts/tasks/run_bias_task_general.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/tasks/run_bias_task_general.py -------------------------------------------------------------------------------- /scripts/tasks/run_dialogue_simulation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/tasks/run_dialogue_simulation.py -------------------------------------------------------------------------------- /scripts/tasks/run_jokes.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/tasks/run_jokes.py -------------------------------------------------------------------------------- /scripts/tasks/run_math_simple.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/tasks/run_math_simple.py -------------------------------------------------------------------------------- /scripts/tasks/run_negative.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/tasks/run_negative.py -------------------------------------------------------------------------------- /scripts/tasks/run_poem.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/tasks/run_poem.py -------------------------------------------------------------------------------- /scripts/tasks/run_positive_amc_aime.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/tasks/run_positive_amc_aime.py -------------------------------------------------------------------------------- /scripts/tasks/run_positive_gsm8k.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/tasks/run_positive_gsm8k.py -------------------------------------------------------------------------------- /scripts/tasks/run_positive_lcb.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/tasks/run_positive_lcb.py -------------------------------------------------------------------------------- /scripts/tasks/run_rng.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/tasks/run_rng.py -------------------------------------------------------------------------------- /scripts/tasks/run_safety.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/tasks/run_safety.py -------------------------------------------------------------------------------- /scripts/tasks/run_simple_qa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/tasks/run_simple_qa.py -------------------------------------------------------------------------------- /scripts/tasks/run_state_name.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/tasks/run_state_name.py -------------------------------------------------------------------------------- /scripts/tasks/run_story.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/tasks/run_story.py -------------------------------------------------------------------------------- /scripts/tasks/vllm/serve.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/scripts/tasks/vllm/serve.sh -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/setup.py -------------------------------------------------------------------------------- /verbalized_sampling/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/__init__.py -------------------------------------------------------------------------------- /verbalized_sampling/analysis/dialogue_plots.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/analysis/dialogue_plots.py -------------------------------------------------------------------------------- /verbalized_sampling/analysis/evals/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/analysis/evals/README.md -------------------------------------------------------------------------------- /verbalized_sampling/analysis/evals/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/analysis/evals/__init__.py -------------------------------------------------------------------------------- /verbalized_sampling/analysis/evals/assets/cwv3_rubrics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/analysis/evals/assets/cwv3_rubrics.py -------------------------------------------------------------------------------- /verbalized_sampling/analysis/evals/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/analysis/evals/base.py -------------------------------------------------------------------------------- /verbalized_sampling/analysis/evals/chi_square.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/analysis/evals/chi_square.py -------------------------------------------------------------------------------- /verbalized_sampling/analysis/evals/creative_writing_v3.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/analysis/evals/creative_writing_v3.py -------------------------------------------------------------------------------- /verbalized_sampling/analysis/evals/creativity_index.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/analysis/evals/creativity_index.py -------------------------------------------------------------------------------- /verbalized_sampling/analysis/evals/data_quality.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/analysis/evals/data_quality.py -------------------------------------------------------------------------------- /verbalized_sampling/analysis/evals/data_quality_qa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/analysis/evals/data_quality_qa.py -------------------------------------------------------------------------------- /verbalized_sampling/analysis/evals/diversity.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/analysis/evals/diversity.py -------------------------------------------------------------------------------- /verbalized_sampling/analysis/evals/factuality.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/analysis/evals/factuality.py -------------------------------------------------------------------------------- /verbalized_sampling/analysis/evals/joke_quality.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/analysis/evals/joke_quality.py -------------------------------------------------------------------------------- /verbalized_sampling/analysis/evals/length.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/analysis/evals/length.py -------------------------------------------------------------------------------- /verbalized_sampling/analysis/evals/ngram.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/analysis/evals/ngram.py -------------------------------------------------------------------------------- /verbalized_sampling/analysis/evals/quality.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/analysis/evals/quality.py -------------------------------------------------------------------------------- /verbalized_sampling/analysis/evals/response_count.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/analysis/evals/response_count.py -------------------------------------------------------------------------------- /verbalized_sampling/analysis/plots.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/analysis/plots.py -------------------------------------------------------------------------------- /verbalized_sampling/analysis/plotting.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/analysis/plotting.py -------------------------------------------------------------------------------- /verbalized_sampling/api.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/api.py -------------------------------------------------------------------------------- /verbalized_sampling/cli.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/cli.py -------------------------------------------------------------------------------- /verbalized_sampling/evals/accuracy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/evals/accuracy.py -------------------------------------------------------------------------------- /verbalized_sampling/evals/dialogue/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/evals/dialogue/__init__.py -------------------------------------------------------------------------------- /verbalized_sampling/evals/dialogue/donation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/evals/dialogue/donation.py -------------------------------------------------------------------------------- /verbalized_sampling/evals/dialogue/linguistic.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/evals/dialogue/linguistic.py -------------------------------------------------------------------------------- /verbalized_sampling/evals/safety.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/evals/safety.py -------------------------------------------------------------------------------- /verbalized_sampling/examples/evaluation_demo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/examples/evaluation_demo.py -------------------------------------------------------------------------------- /verbalized_sampling/examples/tasks_demo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/examples/tasks_demo.py -------------------------------------------------------------------------------- /verbalized_sampling/llms/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/llms/__init__.py -------------------------------------------------------------------------------- /verbalized_sampling/llms/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/llms/base.py -------------------------------------------------------------------------------- /verbalized_sampling/llms/embed.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/llms/embed.py -------------------------------------------------------------------------------- /verbalized_sampling/llms/google.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/llms/google.py -------------------------------------------------------------------------------- /verbalized_sampling/llms/litellm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/llms/litellm.py -------------------------------------------------------------------------------- /verbalized_sampling/llms/openai.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/llms/openai.py -------------------------------------------------------------------------------- /verbalized_sampling/llms/openrouter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/llms/openrouter.py -------------------------------------------------------------------------------- /verbalized_sampling/llms/vllm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/llms/vllm.py -------------------------------------------------------------------------------- /verbalized_sampling/methods/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/methods/__init__.py -------------------------------------------------------------------------------- /verbalized_sampling/methods/factory.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/methods/factory.py -------------------------------------------------------------------------------- /verbalized_sampling/methods/parser.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/methods/parser.py -------------------------------------------------------------------------------- /verbalized_sampling/methods/prompt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/methods/prompt.py -------------------------------------------------------------------------------- /verbalized_sampling/methods/schema.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/methods/schema.py -------------------------------------------------------------------------------- /verbalized_sampling/pipeline.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/pipeline.py -------------------------------------------------------------------------------- /verbalized_sampling/plots/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/plots/README.md -------------------------------------------------------------------------------- /verbalized_sampling/plots/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/plots/__init__.py -------------------------------------------------------------------------------- /verbalized_sampling/plots/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/plots/base.py -------------------------------------------------------------------------------- /verbalized_sampling/plots/comparison.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/plots/comparison.py -------------------------------------------------------------------------------- /verbalized_sampling/plots/convenience.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/plots/convenience.py -------------------------------------------------------------------------------- /verbalized_sampling/plots/evaluator_specific.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/plots/evaluator_specific.py -------------------------------------------------------------------------------- /verbalized_sampling/plots/factory.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/plots/factory.py -------------------------------------------------------------------------------- /verbalized_sampling/plots/plotters.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/plots/plotters.py -------------------------------------------------------------------------------- /verbalized_sampling/selection.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/selection.py -------------------------------------------------------------------------------- /verbalized_sampling/tasks/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/tasks/README.md -------------------------------------------------------------------------------- /verbalized_sampling/tasks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/tasks/__init__.py -------------------------------------------------------------------------------- /verbalized_sampling/tasks/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/tasks/base.py -------------------------------------------------------------------------------- /verbalized_sampling/tasks/bias/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/tasks/bias/__init__.py -------------------------------------------------------------------------------- /verbalized_sampling/tasks/bias/rand_num.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/tasks/bias/rand_num.py -------------------------------------------------------------------------------- /verbalized_sampling/tasks/bias/state_name.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/tasks/bias/state_name.py -------------------------------------------------------------------------------- /verbalized_sampling/tasks/creativity/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/tasks/creativity/__init__.py -------------------------------------------------------------------------------- /verbalized_sampling/tasks/creativity/book.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/tasks/creativity/book.py -------------------------------------------------------------------------------- /verbalized_sampling/tasks/creativity/joke.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/tasks/creativity/joke.py -------------------------------------------------------------------------------- /verbalized_sampling/tasks/creativity/poem.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/tasks/creativity/poem.py -------------------------------------------------------------------------------- /verbalized_sampling/tasks/creativity/speech.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/tasks/creativity/speech.py -------------------------------------------------------------------------------- /verbalized_sampling/tasks/creativity/story.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/tasks/creativity/story.py -------------------------------------------------------------------------------- /verbalized_sampling/tasks/dialogue/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/tasks/dialogue/__init__.py -------------------------------------------------------------------------------- /verbalized_sampling/tasks/dialogue/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/tasks/dialogue/base.py -------------------------------------------------------------------------------- /verbalized_sampling/tasks/dialogue/persuasion.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/tasks/dialogue/persuasion.py -------------------------------------------------------------------------------- /verbalized_sampling/tasks/fact/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/tasks/fact/__init__.py -------------------------------------------------------------------------------- /verbalized_sampling/tasks/fact/simple_qa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/tasks/fact/simple_qa.py -------------------------------------------------------------------------------- /verbalized_sampling/tasks/math/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/tasks/math/__init__.py -------------------------------------------------------------------------------- /verbalized_sampling/tasks/math/math_task.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/tasks/math/math_task.py -------------------------------------------------------------------------------- /verbalized_sampling/tasks/math_eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/tasks/math_eval.py -------------------------------------------------------------------------------- /verbalized_sampling/tasks/safety/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/tasks/safety/__init__.py -------------------------------------------------------------------------------- /verbalized_sampling/tasks/safety/safety.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/tasks/safety/safety.py -------------------------------------------------------------------------------- /verbalized_sampling/tasks/synthetic_data/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/tasks/synthetic_data/__init__.py -------------------------------------------------------------------------------- /verbalized_sampling/tasks/synthetic_data/amc_aime.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/tasks/synthetic_data/amc_aime.py -------------------------------------------------------------------------------- /verbalized_sampling/tasks/synthetic_data/gsm8k.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/tasks/synthetic_data/gsm8k.py -------------------------------------------------------------------------------- /verbalized_sampling/tasks/synthetic_data/livecodebench.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/tasks/synthetic_data/livecodebench.py -------------------------------------------------------------------------------- /verbalized_sampling/tasks/synthetic_data/synthetic_negative.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/tasks/synthetic_data/synthetic_negative.py -------------------------------------------------------------------------------- /verbalized_sampling/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/utils/__init__.py -------------------------------------------------------------------------------- /verbalized_sampling/utils/data_compatibility.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CHATS-lab/verbalized-sampling/HEAD/verbalized_sampling/utils/data_compatibility.py --------------------------------------------------------------------------------