├── README.md ├── calculate_auroc_tpr.py ├── exp.py ├── exp_utils ├── __init__.py ├── levenshtein.pyx ├── mersenne.js └── mersenne.py ├── img ├── main_table.png └── pareto_figure.png ├── lm_eval ├── __init__.py ├── arguments.py ├── base.py ├── evaluator.py ├── generation.py ├── tasks │ ├── __init__.py │ ├── apps.py │ ├── code_evaluation_prompts.py │ ├── codexglue_code_to_text.py │ ├── codexglue_text_to_text.py │ ├── conala.py │ ├── concode.py │ ├── custom_metrics │ │ ├── __init__.py │ │ ├── codexglue_code_to_text_bleu.py │ │ ├── multiple_metrics │ │ │ ├── __init__.py │ │ │ ├── containerized_eval.py │ │ │ ├── eval_cpp.py │ │ │ ├── eval_cs.py │ │ │ ├── eval_dlang.py │ │ │ ├── eval_go.py │ │ │ ├── eval_java.py │ │ │ ├── eval_javascript.py │ │ │ ├── eval_julia.py │ │ │ ├── eval_lua.py │ │ │ ├── eval_php.py │ │ │ ├── eval_pl.py │ │ │ ├── eval_python.py │ │ │ ├── eval_r.py │ │ │ ├── eval_racket.py │ │ │ ├── eval_ruby.py │ │ │ ├── eval_rust.py │ │ │ ├── eval_scala.py │ │ │ ├── eval_sh.py │ │ │ ├── eval_swift.py │ │ │ ├── eval_ts.py │ │ │ ├── evaluation.py │ │ │ ├── generic_eval.py │ │ │ ├── libeval.py │ │ │ ├── safe_subprocess │ │ │ │ ├── .gitignore │ │ │ │ ├── __init__.py │ │ │ │ ├── evil_programs │ │ │ │ │ ├── block_on_inputs.py │ │ │ │ │ ├── close_outputs.py │ │ │ │ │ ├── fork_bomb.py │ │ │ │ │ ├── fork_once.py │ │ │ │ │ ├── sleep_forever.py │ │ │ │ │ └── unbounded_output.py │ │ │ │ └── module_test.py │ │ │ └── single_experiment_pass_k.py │ │ └── pal_metric │ │ │ ├── pal_code_exec.py │ │ │ └── python_executor.py │ ├── ds1000.py │ ├── few_shot_examples │ │ ├── codexglue_text_to_text_few_shot_prompts.json │ │ ├── conala_few_shot_prompts.json │ │ ├── concode_few_shot_prompts.json │ │ └── gsm8k_few_shot_prompts.json │ ├── gsm.py │ ├── humaneval.py │ ├── mbpp.py │ └── multiple.py └── utils.py ├── main.py ├── scripts └── main │ ├── run_exp_detection.sh │ ├── run_exp_detection_human.sh │ ├── run_exp_generation.sh │ ├── run_sweet_detection.sh │ ├── run_sweet_detection_human.sh │ ├── run_sweet_generation.sh │ ├── run_wllm_detection.sh │ ├── run_wllm_detection_human.sh │ └── run_wllm_generation.sh ├── sweet.py └── watermark.py /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/README.md -------------------------------------------------------------------------------- /calculate_auroc_tpr.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/calculate_auroc_tpr.py -------------------------------------------------------------------------------- /exp.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/exp.py -------------------------------------------------------------------------------- /exp_utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/exp_utils/__init__.py -------------------------------------------------------------------------------- /exp_utils/levenshtein.pyx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/exp_utils/levenshtein.pyx -------------------------------------------------------------------------------- /exp_utils/mersenne.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/exp_utils/mersenne.js -------------------------------------------------------------------------------- /exp_utils/mersenne.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/exp_utils/mersenne.py -------------------------------------------------------------------------------- /img/main_table.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/img/main_table.png -------------------------------------------------------------------------------- /img/pareto_figure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/img/pareto_figure.png -------------------------------------------------------------------------------- /lm_eval/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lm_eval/arguments.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/arguments.py -------------------------------------------------------------------------------- /lm_eval/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/base.py -------------------------------------------------------------------------------- /lm_eval/evaluator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/evaluator.py -------------------------------------------------------------------------------- /lm_eval/generation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/generation.py -------------------------------------------------------------------------------- /lm_eval/tasks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/__init__.py -------------------------------------------------------------------------------- /lm_eval/tasks/apps.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/apps.py -------------------------------------------------------------------------------- /lm_eval/tasks/code_evaluation_prompts.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/code_evaluation_prompts.py -------------------------------------------------------------------------------- /lm_eval/tasks/codexglue_code_to_text.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/codexglue_code_to_text.py -------------------------------------------------------------------------------- /lm_eval/tasks/codexglue_text_to_text.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/codexglue_text_to_text.py -------------------------------------------------------------------------------- /lm_eval/tasks/conala.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/conala.py -------------------------------------------------------------------------------- /lm_eval/tasks/concode.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/concode.py -------------------------------------------------------------------------------- /lm_eval/tasks/custom_metrics/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lm_eval/tasks/custom_metrics/codexglue_code_to_text_bleu.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/custom_metrics/codexglue_code_to_text_bleu.py -------------------------------------------------------------------------------- /lm_eval/tasks/custom_metrics/multiple_metrics/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lm_eval/tasks/custom_metrics/multiple_metrics/containerized_eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/custom_metrics/multiple_metrics/containerized_eval.py -------------------------------------------------------------------------------- /lm_eval/tasks/custom_metrics/multiple_metrics/eval_cpp.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/custom_metrics/multiple_metrics/eval_cpp.py -------------------------------------------------------------------------------- /lm_eval/tasks/custom_metrics/multiple_metrics/eval_cs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/custom_metrics/multiple_metrics/eval_cs.py -------------------------------------------------------------------------------- /lm_eval/tasks/custom_metrics/multiple_metrics/eval_dlang.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/custom_metrics/multiple_metrics/eval_dlang.py -------------------------------------------------------------------------------- /lm_eval/tasks/custom_metrics/multiple_metrics/eval_go.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/custom_metrics/multiple_metrics/eval_go.py -------------------------------------------------------------------------------- /lm_eval/tasks/custom_metrics/multiple_metrics/eval_java.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/custom_metrics/multiple_metrics/eval_java.py -------------------------------------------------------------------------------- /lm_eval/tasks/custom_metrics/multiple_metrics/eval_javascript.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/custom_metrics/multiple_metrics/eval_javascript.py -------------------------------------------------------------------------------- /lm_eval/tasks/custom_metrics/multiple_metrics/eval_julia.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/custom_metrics/multiple_metrics/eval_julia.py -------------------------------------------------------------------------------- /lm_eval/tasks/custom_metrics/multiple_metrics/eval_lua.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/custom_metrics/multiple_metrics/eval_lua.py -------------------------------------------------------------------------------- /lm_eval/tasks/custom_metrics/multiple_metrics/eval_php.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/custom_metrics/multiple_metrics/eval_php.py -------------------------------------------------------------------------------- /lm_eval/tasks/custom_metrics/multiple_metrics/eval_pl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/custom_metrics/multiple_metrics/eval_pl.py -------------------------------------------------------------------------------- /lm_eval/tasks/custom_metrics/multiple_metrics/eval_python.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/custom_metrics/multiple_metrics/eval_python.py -------------------------------------------------------------------------------- /lm_eval/tasks/custom_metrics/multiple_metrics/eval_r.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/custom_metrics/multiple_metrics/eval_r.py -------------------------------------------------------------------------------- /lm_eval/tasks/custom_metrics/multiple_metrics/eval_racket.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/custom_metrics/multiple_metrics/eval_racket.py -------------------------------------------------------------------------------- /lm_eval/tasks/custom_metrics/multiple_metrics/eval_ruby.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/custom_metrics/multiple_metrics/eval_ruby.py -------------------------------------------------------------------------------- /lm_eval/tasks/custom_metrics/multiple_metrics/eval_rust.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/custom_metrics/multiple_metrics/eval_rust.py -------------------------------------------------------------------------------- /lm_eval/tasks/custom_metrics/multiple_metrics/eval_scala.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/custom_metrics/multiple_metrics/eval_scala.py -------------------------------------------------------------------------------- /lm_eval/tasks/custom_metrics/multiple_metrics/eval_sh.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/custom_metrics/multiple_metrics/eval_sh.py -------------------------------------------------------------------------------- /lm_eval/tasks/custom_metrics/multiple_metrics/eval_swift.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/custom_metrics/multiple_metrics/eval_swift.py -------------------------------------------------------------------------------- /lm_eval/tasks/custom_metrics/multiple_metrics/eval_ts.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/custom_metrics/multiple_metrics/eval_ts.py -------------------------------------------------------------------------------- /lm_eval/tasks/custom_metrics/multiple_metrics/evaluation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/custom_metrics/multiple_metrics/evaluation.py -------------------------------------------------------------------------------- /lm_eval/tasks/custom_metrics/multiple_metrics/generic_eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/custom_metrics/multiple_metrics/generic_eval.py -------------------------------------------------------------------------------- /lm_eval/tasks/custom_metrics/multiple_metrics/libeval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/custom_metrics/multiple_metrics/libeval.py -------------------------------------------------------------------------------- /lm_eval/tasks/custom_metrics/multiple_metrics/safe_subprocess/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/custom_metrics/multiple_metrics/safe_subprocess/.gitignore -------------------------------------------------------------------------------- /lm_eval/tasks/custom_metrics/multiple_metrics/safe_subprocess/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/custom_metrics/multiple_metrics/safe_subprocess/__init__.py -------------------------------------------------------------------------------- /lm_eval/tasks/custom_metrics/multiple_metrics/safe_subprocess/evil_programs/block_on_inputs.py: -------------------------------------------------------------------------------- 1 | while True: 2 | input() 3 | -------------------------------------------------------------------------------- /lm_eval/tasks/custom_metrics/multiple_metrics/safe_subprocess/evil_programs/close_outputs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/custom_metrics/multiple_metrics/safe_subprocess/evil_programs/close_outputs.py -------------------------------------------------------------------------------- /lm_eval/tasks/custom_metrics/multiple_metrics/safe_subprocess/evil_programs/fork_bomb.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | while True: 4 | os.fork() 5 | -------------------------------------------------------------------------------- /lm_eval/tasks/custom_metrics/multiple_metrics/safe_subprocess/evil_programs/fork_once.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/custom_metrics/multiple_metrics/safe_subprocess/evil_programs/fork_once.py -------------------------------------------------------------------------------- /lm_eval/tasks/custom_metrics/multiple_metrics/safe_subprocess/evil_programs/sleep_forever.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/custom_metrics/multiple_metrics/safe_subprocess/evil_programs/sleep_forever.py -------------------------------------------------------------------------------- /lm_eval/tasks/custom_metrics/multiple_metrics/safe_subprocess/evil_programs/unbounded_output.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/custom_metrics/multiple_metrics/safe_subprocess/evil_programs/unbounded_output.py -------------------------------------------------------------------------------- /lm_eval/tasks/custom_metrics/multiple_metrics/safe_subprocess/module_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/custom_metrics/multiple_metrics/safe_subprocess/module_test.py -------------------------------------------------------------------------------- /lm_eval/tasks/custom_metrics/multiple_metrics/single_experiment_pass_k.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/custom_metrics/multiple_metrics/single_experiment_pass_k.py -------------------------------------------------------------------------------- /lm_eval/tasks/custom_metrics/pal_metric/pal_code_exec.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/custom_metrics/pal_metric/pal_code_exec.py -------------------------------------------------------------------------------- /lm_eval/tasks/custom_metrics/pal_metric/python_executor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/custom_metrics/pal_metric/python_executor.py -------------------------------------------------------------------------------- /lm_eval/tasks/ds1000.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/ds1000.py -------------------------------------------------------------------------------- /lm_eval/tasks/few_shot_examples/codexglue_text_to_text_few_shot_prompts.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/few_shot_examples/codexglue_text_to_text_few_shot_prompts.json -------------------------------------------------------------------------------- /lm_eval/tasks/few_shot_examples/conala_few_shot_prompts.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/few_shot_examples/conala_few_shot_prompts.json -------------------------------------------------------------------------------- /lm_eval/tasks/few_shot_examples/concode_few_shot_prompts.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/few_shot_examples/concode_few_shot_prompts.json -------------------------------------------------------------------------------- /lm_eval/tasks/few_shot_examples/gsm8k_few_shot_prompts.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/few_shot_examples/gsm8k_few_shot_prompts.json -------------------------------------------------------------------------------- /lm_eval/tasks/gsm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/gsm.py -------------------------------------------------------------------------------- /lm_eval/tasks/humaneval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/humaneval.py -------------------------------------------------------------------------------- /lm_eval/tasks/mbpp.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/mbpp.py -------------------------------------------------------------------------------- /lm_eval/tasks/multiple.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/tasks/multiple.py -------------------------------------------------------------------------------- /lm_eval/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/lm_eval/utils.py -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/main.py -------------------------------------------------------------------------------- /scripts/main/run_exp_detection.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/scripts/main/run_exp_detection.sh -------------------------------------------------------------------------------- /scripts/main/run_exp_detection_human.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/scripts/main/run_exp_detection_human.sh -------------------------------------------------------------------------------- /scripts/main/run_exp_generation.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/scripts/main/run_exp_generation.sh -------------------------------------------------------------------------------- /scripts/main/run_sweet_detection.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/scripts/main/run_sweet_detection.sh -------------------------------------------------------------------------------- /scripts/main/run_sweet_detection_human.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/scripts/main/run_sweet_detection_human.sh -------------------------------------------------------------------------------- /scripts/main/run_sweet_generation.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/scripts/main/run_sweet_generation.sh -------------------------------------------------------------------------------- /scripts/main/run_wllm_detection.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/scripts/main/run_wllm_detection.sh -------------------------------------------------------------------------------- /scripts/main/run_wllm_detection_human.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/scripts/main/run_wllm_detection_human.sh -------------------------------------------------------------------------------- /scripts/main/run_wllm_generation.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/scripts/main/run_wllm_generation.sh -------------------------------------------------------------------------------- /sweet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/sweet.py -------------------------------------------------------------------------------- /watermark.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hongcheki/sweet-watermark/HEAD/watermark.py --------------------------------------------------------------------------------