├── requirements.txt ├── README.md ├── openai_generate ├── README.md ├── generate_openai.py ├── detect_gptzero.py ├── detect_openai.py ├── regenerate_gpt4.py ├── regenerate_gpt35.py ├── utils.py ├── regenerate_gpt3.py ├── my_detector_whitebox.ipynb ├── my_detector_gpt35or4.ipynb └── my_detector_gpt3.ipynb ├── LICENSE ├── get_data ├── natural_abstract.py └── reddit.py ├── open_source_models ├── generate_gptneox.py ├── generate_gptllama.py ├── regenerate_gptneox.py └── regenerate_gptllama.py ├── .gitignore └── DNA-GPT-dist.py /requirements.txt: -------------------------------------------------------------------------------- 1 | rouge-score 2 | transformers>=4.28.1 3 | openai 4 | sklearn 5 | torch 6 | nltk 7 | spacy 8 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # DNA-GPT 2 | Official release for DNA-GPT: https://arxiv.org/abs/2305.17359 3 | 4 | Run DNA-GPT-dist.py to launch a gradio demo 5 | -------------------------------------------------------------------------------- /openai_generate/README.md: -------------------------------------------------------------------------------- 1 | Special Credits to: https://github.com/martiansideofthemoon/ai-detection-paraphrases/tree/main/dipper_paraphrases 2 | TBD -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Xianjun (Nolan) Yang 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /get_data/natural_abstract.py: -------------------------------------------------------------------------------- 1 | import requests, json 2 | from bs4 import BeautifulSoup 3 | from datetime import datetime 4 | 5 | titles = [] 6 | abstracts = [] 7 | for subject in ['earth-and-environmental', 'health', 'biological', 'physical']: 8 | url = 'https://www.nature.com/subjects/' + subject + '-sciences/ncomms' # biological, physical 9 | # url = 'https://www.nature.com/subjects/biological-sciences/ncomms?searchType=journalSearch&sort=PubDate&page=2' 10 | # Send an HTTP GET request to the URL and get the content 11 | response = requests.get(url) 12 | html_content = response.text 13 | # Parse the HTML content using BeautifulSoup 14 | soup = BeautifulSoup(html_content, 'html.parser') 15 | # Find all article links by searching for 'a' tags with itemprop='url' 16 | article_links = soup.find_all('a', itemprop='url') 17 | # Extract the href attribute (URL) of each link and store in a list 18 | paper_links = [link['href'] for link in article_links] 19 | print('number of links:', len(paper_links)) 20 | for paper_id in paper_links: 21 | url = 'https://www.nature.com' + paper_id 22 | # Send a request to the URL and get the content 23 | response = requests.get(url) 24 | soup = BeautifulSoup(response.content, 'html.parser') 25 | # Extract the title 26 | title = soup.find('h1', {'class': 'c-article-title'}).text.strip() 27 | # Extract the abstract (update the class name if needed after inspecting the HTML structure) 28 | abstract_div = soup.find('div', {'class': 'c-article-section__content'}) 29 | abstract = abstract_div.text.strip() if abstract_div else '' 30 | if len( abstract ) > 1000: 31 | titles.append(title) 32 | abstracts.append(abstract) 33 | if len(titles) % 50 == 0: 34 | break 35 | 36 | with open( 'natural_abstract.json', 'w') as f: 37 | json.dump({'title': titles, 'abstract': abstracts}, f) -------------------------------------------------------------------------------- /openai_generate/generate_openai.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import numpy as np 3 | import nltk 4 | from nltk.tokenize import sent_tokenize 5 | import tqdm 6 | import json 7 | import os 8 | import torch, openai 9 | import random 10 | from utils import get_openai_response, get_chatgpt_qa_response, get_gpt4_qa_response 11 | 12 | openai_key = "" 13 | openai.api_key = openai_key 14 | 15 | parser = argparse.ArgumentParser() 16 | parser.add_argument('--dataset', default="reddit_eli5.jsonl") 17 | parser.add_argument('--model', default="chatgpt") # "gpt-3.5-turbo", "gpt-4-0314", 18 | parser.add_argument('--max_new_tokens', default=300, type=int) 19 | parser.add_argument('--total_instances', default=200, type=int) 20 | parser.add_argument('--temperature', default= 0.5, type=float) 21 | args = parser.parse_args() 22 | 23 | with open(args.dataset, "r") as f: 24 | data = [json.loads(x) for x in f.read().strip().split("\n") ] 25 | len(data) 26 | 27 | output_file = f"results/{args.model}_" + str(args.total_instances) +"_len.jsonl" 28 | 29 | random.seed(43) 30 | if os.path.exists(output_file): 31 | with open(output_file, "r") as f: 32 | num_curr_outputs = len(f.read().strip().split("\n")) 33 | else: 34 | num_curr_outputs = 0 35 | 36 | print("Skipping {} instances".format(num_curr_outputs)) 37 | data = data[num_curr_outputs:args.total_instances] 38 | print("Total instances: {}".format(len(data))) 39 | 40 | if args.model == "davinci_003": 41 | openai_fn = get_openai_response 42 | elif args.model == "chatgpt": 43 | openai_fn = get_chatgpt_qa_response 44 | elif args.model == "gpt4": 45 | openai_fn = get_gpt4_qa_response 46 | else: 47 | raise NotImplementedError 48 | 49 | outputs = [] 50 | for idx, dd in tqdm.tqdm(enumerate(data), total= len(data) ): # 51 | question = 'Answer the following question in 180-300 words: ' + dd['question'] 52 | long_answer = dd['human_answer'] 53 | 54 | gen_text = openai_fn(prompt_text=question, temperature=args.temperature, max_tokens=args.max_new_tokens) 55 | outputs.append(json.dumps({ 56 | "question": question, 57 | "gold_completion": long_answer, 58 | "gen_completion": gen_text 59 | })) 60 | 61 | with open(output_file, "a") as f: 62 | f.write("\n".join(outputs) + "\n") 63 | outputs = [] 64 | 65 | with open(output_file, "a") as f: 66 | f.write("\n".join(outputs) + "\n") 67 | outputs = [] 68 | -------------------------------------------------------------------------------- /openai_generate/detect_gptzero.py: -------------------------------------------------------------------------------- 1 | import os, json, tqdm 2 | import requests 3 | 4 | api_key = '' 5 | 6 | class GPTZeroAPI: 7 | def __init__(self, api_key): 8 | self.api_key = api_key 9 | self.base_url = 'https://api.gptzero.me/v2/predict' 10 | def text_predict(self, document): 11 | url = f'{self.base_url}/text' 12 | headers = { 13 | 'accept': 'application/json', 14 | 'X-Api-Key': self.api_key, 15 | 'Content-Type': 'application/json' 16 | } 17 | data = { 18 | 'document': document 19 | } 20 | response = requests.post(url, headers=headers, json=data) 21 | return response.json() 22 | def file_predict(self, file_path): 23 | url = f'{self.base_url}/files' 24 | headers = { 25 | 'accept': 'application/json', 26 | 'X-Api-Key': self.api_key 27 | } 28 | files = { 29 | 'files': (os.path.basename(file_path), open(file_path, 'rb')) 30 | } 31 | response = requests.post(url, headers=headers, files=files) 32 | return response.json() 33 | # Credits for this code go to https://github.com/Haste171/gptzero 34 | output_file = 'results/detect_gptzero_200.jsonl' 35 | with open( 'results/chatgpt_200_len.jsonl', 'r') as f: 36 | new_data = [json.loads(x) for x in f.read().strip().split("\n")] 37 | print( 'Total instances: {}'.format( len( new_data))) 38 | 39 | gptzero = GPTZeroAPI(api_key) 40 | 41 | if os.path.exists(output_file): 42 | with open(output_file, "r") as f: 43 | num_curr_outputs = len(f.read().strip().split("\n")) 44 | else: 45 | num_curr_outputs = 0 46 | 47 | print("Skipping {} instances".format(num_curr_outputs)) 48 | data = new_data[num_curr_outputs:] 49 | print("Total instances: {}".format(len(data))) 50 | 51 | outputs = [] 52 | for i, instance in tqdm.tqdm(enumerate(new_data), total=len(new_data)): 53 | gold_gen = instance['gold_completion'] 54 | gen_completion = instance['gen_completion'] 55 | 56 | gold_gen_prob = gptzero.text_predict(gold_gen)['documents'][0]['completely_generated_prob'] 57 | gen_completion_prob = gptzero.text_predict(gen_completion)['documents'][0]['completely_generated_prob'] 58 | 59 | new_data[i]['gold_gen_prob'] = gold_gen_prob 60 | new_data[i]['gen_completion_prob'] = gen_completion_prob 61 | 62 | outputs.append(json.dumps( new_data[i])) 63 | 64 | with open(output_file, "a") as f: 65 | f.write("\n".join(outputs) + "\n") 66 | outputs = [] -------------------------------------------------------------------------------- /open_source_models/generate_gptneox.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import tqdm 3 | import json 4 | import os 5 | import torch 6 | import random 7 | from transformers import GPTNeoXForCausalLM, GPTNeoXTokenizerFast 8 | import torch 9 | 10 | tokenizer = GPTNeoXTokenizerFast.from_pretrained("EleutherAI/gpt-neox-20b") 11 | 12 | device = torch.device("cuda:1") 13 | model = GPTNeoXForCausalLM.from_pretrained("EleutherAI/gpt-neox-20b").half().to(device) #.half() to use FP16 14 | 15 | def gptx_generate(prompt, max_length): 16 | inputs = tokenizer(prompt, return_tensors="pt") 17 | gen_tokens = model.generate( 18 | input_ids = inputs["input_ids"].to(device), 19 | attention_mask=inputs["attention_mask"].to(device), 20 | do_sample=True, 21 | temperature=0.7, 22 | max_length=max_length, 23 | num_return_sequences = 1, 24 | ) 25 | gen_text = tokenizer.batch_decode(gen_tokens)[0] 26 | return gen_text[len(prompt):] 27 | 28 | parser = argparse.ArgumentParser() 29 | parser.add_argument('--dataset', default="reddit_eli5.jsonl") 30 | parser.add_argument('--model', default="gptx") 31 | parser.add_argument('--max_new_tokens', default=300, type=int) 32 | parser.add_argument('--total_instances', default= 200, type=int) 33 | args = parser.parse_args() 34 | 35 | with open( args.dataset, "r") as f: 36 | data = [json.loads(x) for x in f.read().strip().split("\n") ] 37 | len(data) 38 | 39 | output_file = f"results/{args.model}_200_len.jsonl" 40 | 41 | random.seed(43) 42 | if os.path.exists(output_file): 43 | with open(output_file, "r") as f: 44 | num_curr_outputs = len(f.read().strip().split("\n")) 45 | else: 46 | num_curr_outputs = 0 47 | 48 | print("Skipping {} instances".format(num_curr_outputs)) 49 | data = data[num_curr_outputs:] 50 | print("Total instances: {}".format(len(data))) 51 | 52 | 53 | outputs = [] 54 | for idx, dd in tqdm.tqdm(enumerate(data), total= len(data) ): # 55 | question = dd['question'] 56 | prefix = "Answer the following question in around 200 words: " #dd['prefix'] 57 | gold_completion = dd['human_answer'] 58 | 59 | prompt_input = question + prefix 60 | 61 | gen_text = gptx_generate(prompt_input, max_length=args.max_new_tokens) 62 | 63 | outputs.append(json.dumps({ 64 | "question": question, 65 | "prefix": prefix, 66 | "gold_completion": gold_completion, 67 | "gen_completion": gen_text 68 | })) 69 | 70 | with open(output_file, "a") as f: 71 | f.write("\n".join(outputs) + "\n") 72 | outputs = [] 73 | 74 | with open(output_file, "a") as f: 75 | f.write("\n".join(outputs) + "\n") 76 | outputs = [] 77 | -------------------------------------------------------------------------------- /open_source_models/generate_gptllama.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import tqdm 3 | import json 4 | import os 5 | import torch 6 | import random 7 | from transformers import LlamaForCausalLM, LlamaTokenizer 8 | import torch 9 | 10 | path = ""# path to llama model .hf file 11 | tokenizer = LlamaTokenizer.from_pretrained( path ) 12 | 13 | device = torch.device("cuda:3") 14 | model = LlamaForCausalLM.from_pretrained( path ).half().to(device) #.half() to use FP16 15 | model.eval() 16 | 17 | def gptx_generate(prompt, max_length): 18 | inputs = tokenizer(prompt, return_tensors="pt") 19 | gen_tokens = model.generate( 20 | input_ids = inputs["input_ids"].to(device), 21 | attention_mask=inputs["attention_mask"].to(device), 22 | do_sample=True, 23 | temperature=0.7, 24 | max_length=max_length, 25 | num_return_sequences = 1, 26 | ) 27 | gen_text = tokenizer.batch_decode(gen_tokens, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] 28 | return gen_text[len(prompt):] 29 | 30 | parser = argparse.ArgumentParser() 31 | parser.add_argument('--dataset', default="xsum_150_raw.jsonl") 32 | parser.add_argument('--output_dir', default="open-generation-data") 33 | parser.add_argument('--model', default="gptllama") 34 | parser.add_argument('--max_new_tokens', default=300, type=int) 35 | parser.add_argument('--total_instances', default= 200, type=int) 36 | args = parser.parse_args() 37 | 38 | with open( args.dataset, "r") as f: 39 | data = [json.loads(x) for x in f.read().strip().split("\n") ] 40 | len(data) 41 | 42 | output_file = f"results/{args.model}_200_len.jsonl" 43 | 44 | random.seed(43) 45 | if os.path.exists(output_file): 46 | with open(output_file, "r") as f: 47 | num_curr_outputs = len(f.read().strip().split("\n")) 48 | else: 49 | num_curr_outputs = 0 50 | 51 | print("Skipping {} instances".format(num_curr_outputs)) 52 | data = data[num_curr_outputs:] 53 | print("Total instances: {}".format(len(data))) 54 | 55 | 56 | outputs = [] 57 | for idx, dd in tqdm.tqdm(enumerate(data), total= len(data) ): # 58 | question = dd['question'] 59 | prefix = dd['prefix'] 60 | prompt_input = question + prefix 61 | 62 | gen_text = gptx_generate(prompt_input, max_length=args.max_new_tokens) 63 | 64 | outputs.append(json.dumps({ 65 | "question": question, 66 | "prefix": prefix, 67 | "gold_completion": dd['gold_completion'], 68 | "gen_completion": gen_text 69 | })) 70 | 71 | with open(output_file, "a") as f: 72 | f.write("\n".join(outputs) + "\n") 73 | outputs = [] 74 | 75 | with open(output_file, "a") as f: 76 | f.write("\n".join(outputs) + "\n") 77 | outputs = [] 78 | -------------------------------------------------------------------------------- /get_data/reddit.py: -------------------------------------------------------------------------------- 1 | import praw 2 | import datetime 3 | import random, json 4 | from tqdm import tqdm 5 | from collections import defaultdict 6 | 7 | # Replace these values with your own Reddit API application credentials 8 | client_id = '' # 'YOUR_CLIENT_ID' 9 | client_secret = '' # 'YOUR_CLIENT_SECRET' 10 | user_agent = '' # 'YOUR_USER_AGENT' 11 | reddit = praw.Reddit(client_id=client_id, client_secret=client_secret, user_agent=user_agent) 12 | 13 | # Define the time range 14 | start_time = datetime.datetime(2022, 1, 1) 15 | end_time = datetime.datetime(2023, 3, 31) 16 | 17 | # Define the domains and number of posts to scrape 18 | domains = ['biology', 'physics', 'chemistry', 'economics', 'law', 'technology'] 19 | num_posts = 500 20 | 21 | # Function to filter posts within the time range 22 | def in_time_range(post, start_time, end_time): 23 | post_time = datetime.datetime.fromtimestamp(post.created_utc) 24 | return start_time <= post_time <= end_time 25 | 26 | # Scrape questions and their longest answers 27 | questions_and_answers = defaultdict(list) 28 | 29 | for domain in domains: 30 | print(f"Scraping '{domain}' domain...") 31 | domain_questions = [] 32 | count = 0 33 | for submission in reddit.subreddit('explainlikeimfive').search(f"flair:{domain}", sort='top', time_filter='year', limit=None): 34 | count += 1 35 | if count % 100 == 0: 36 | print(f"Scraped {count} posts...") 37 | 38 | if in_time_range(submission, start_time, end_time): 39 | domain_questions.append(submission) 40 | 41 | random.shuffle(domain_questions) 42 | domain_questions = domain_questions[:num_posts] 43 | 44 | count2 = 0 45 | for question in tqdm(domain_questions): 46 | question.comment_sort = 'best' 47 | question.comments.replace_more(limit=None) 48 | 49 | longest_answer = None 50 | longest_length = 0 51 | 52 | for comment in question.comments.list(): 53 | if not comment.author or comment.author == "AutoModerator" or comment.author == "[deleted]": 54 | continue 55 | 56 | comment_length = len(comment.body) 57 | if comment_length > 1200 and comment_length < 5000 and comment.body[:100]!='**Please read this entire message**\n\n---\n\nYour comment has been removed for the following reason(s):' and comment.body[:100] != '**Please read this entire message**\r\n\r\n---\r\n\r\nYour comment has been removed for the following reason': 58 | 59 | longest_answer = comment.body 60 | questions_and_answers[domain].append((question.title, longest_answer)) 61 | break 62 | 63 | print("Scraping completed.") 64 | # Save results to a JSON file 65 | output_filename = "reddit_eli5.json" 66 | 67 | with open(output_filename, "w", encoding="utf-8") as output_file: 68 | json.dump(questions_and_answers, output_file, ensure_ascii=False, indent=4) 69 | 70 | print(f"Results saved to {output_filename}.") -------------------------------------------------------------------------------- /openai_generate/detect_openai.py: -------------------------------------------------------------------------------- 1 | import json, openai 2 | import argparse 3 | import numpy as np 4 | import nltk 5 | from nltk.tokenize import sent_tokenize 6 | import tqdm 7 | import json 8 | import os 9 | import torch, openai 10 | import random 11 | 12 | openai_key = "" 13 | openai.api_key = openai_key 14 | 15 | parser = argparse.ArgumentParser() 16 | parser.add_argument('--dataset', default="nature_paper.json") 17 | parser.add_argument('--model', default="gpt35") # "gpt-3.5-turbo", "gpt-4-0314", 18 | parser.add_argument('--min_characters', default=1000, type=int) 19 | parser.add_argument('--total_instances', default=200, type=int) 20 | args = parser.parse_args() 21 | 22 | output_file = f"results/openai_detect_{args.model}.jsonl" 23 | 24 | print("Loading model {}".format(args.model)) 25 | with open( f'results/chatgpt_200_len.jsonl', 'r') as f: 26 | data = [json.loads(x) for x in f.read().strip().split("\n")] 27 | len(data) 28 | random.seed(43) 29 | 30 | if os.path.exists(output_file): 31 | with open(output_file, "r") as f: 32 | num_curr_outputs = len(f.read().strip().split("\n")) 33 | else: 34 | num_curr_outputs = 0 35 | 36 | print("Skipping {} instances".format(num_curr_outputs)) 37 | data = data[num_curr_outputs:args.total_instances] 38 | print("Total instances: {}".format(len(data))) 39 | 40 | def openai_detect(prompt): 41 | response = openai.Completion.create(engine="model-detect-v2", 42 | prompt=prompt, 43 | max_tokens=1, 44 | temperature=1, 45 | top_p=1, 46 | n=1, 47 | logprobs=5, 48 | stop="\n", 49 | stream=False) 50 | top_logprobs = response["choices"][0]["logprobs"]["top_logprobs"][0] 51 | 52 | if "\"" in top_logprobs: 53 | quote_logprob = np.exp(top_logprobs["\""]) 54 | elif "!" in top_logprobs: 55 | quote_logprob = 1.0 - np.exp(top_logprobs["!"]) 56 | else: 57 | print("No quote or exclamation mark found in top logprobs") 58 | quote_logprob = 0.5 59 | return quote_logprob, response 60 | 61 | outputs = [] 62 | for idx, dd in tqdm.tqdm(enumerate(data), total= len(data) ): # 63 | gen_completion = dd['gen_completion']#['choices'][0]['message']['content'] 64 | gold_gen = dd['gold_completion'] 65 | if len(gen_completion) <= args.min_characters or len( gold_gen ) <= args.min_characters: 66 | continue 67 | 68 | prompt_gen = gen_completion + "<|disc_score|>" 69 | prompt_gold = gold_gen + "<|disc_score|>" 70 | 71 | quote_logprob_gen, response_gen = openai_detect(prompt_gen) 72 | quote_logprob_gold, response_gold = openai_detect(prompt_gold) 73 | 74 | outputs.append(json.dumps({ 75 | "question": dd['question'], 76 | "gold_gen_prob": quote_logprob_gold, 77 | "gold_response": response_gold, 78 | "gen_completion_prob": quote_logprob_gen, 79 | "openai_response": response_gen, 80 | })) 81 | 82 | with open(output_file, "a") as f: 83 | f.write("\n".join(outputs) + "\n") 84 | outputs = [] 85 | 86 | with open(output_file, "a") as f: 87 | f.write("\n".join(outputs) + "\n") 88 | outputs = [] -------------------------------------------------------------------------------- /openai_generate/regenerate_gpt4.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import tqdm 3 | import json 4 | import os 5 | import openai 6 | import random 7 | 8 | openai_key = "" 9 | openai.api_key = openai_key 10 | 11 | parser = argparse.ArgumentParser() 12 | parser.add_argument('--dataset', default="results/gpt4_200_len.jsonl") 13 | parser.add_argument('--model', default= "gpt-4-0314") # gpt-3.5-turbo, gpt-4-0314 14 | parser.add_argument('--max_new_tokens', default=300, type=int) 15 | parser.add_argument('--regen_number', default=10, type=int) 16 | parser.add_argument('--temperature', default=0.7, type=float) 17 | parser.add_argument('--truncate_ratio', default=0.7, type=float) 18 | args = parser.parse_args() 19 | 20 | print('load model ...', args.model) 21 | with open(args.dataset, "r") as f: 22 | data = [json.loads(x) for x in f.read().strip().split("\n")] 23 | 24 | random.seed(43) 25 | 26 | output_file = f"results/regen_{args.model}_10_" + str(args.truncate_ratio) + ".jsonl" 27 | 28 | if os.path.exists(output_file): 29 | with open(output_file, "r") as f: 30 | num_curr_outputs = len(f.read().strip().split("\n")) 31 | else: 32 | num_curr_outputs = 0 33 | 34 | print("Skipping {} instances".format(num_curr_outputs)) 35 | print('len(data): ', len(data)) 36 | data = data[num_curr_outputs:] 37 | 38 | outputs = [] 39 | for idx, dd in tqdm.tqdm(enumerate(data), total = len(data) ): # len(data) 40 | gold_completion = dd['gold_completion'] 41 | human_prefix_prompt = gold_completion[ :int( args.truncate_ratio*len(gold_completion) ) ] 42 | 43 | gen_completion = dd['gen_completion'] 44 | machine_prefix_prompt = gen_completion[ :int( args.truncate_ratio*len(gen_completion) ) ] 45 | 46 | human_gen_text = openai.ChatCompletion.create(model= args.model, 47 | messages=[{"role": "system", "content": "You are a helpful assistant that continues the passage from the sentences provided."}, 48 | {"role": "user", "content": dd['question']}, 49 | {"role": "assistant", "content": human_prefix_prompt}, 50 | ], 51 | temperature= args.temperature, 52 | max_tokens = args.max_new_tokens, 53 | n= args.regen_number) 54 | 55 | machine_gen_text = openai.ChatCompletion.create(model= args.model, 56 | messages=[{"role": "system", "content": "You are a helpful assistant that continues the passage from the sentences provided."}, 57 | {"role": "user", "content": dd['question']}, 58 | {"role": "assistant", "content": machine_prefix_prompt}, 59 | ], 60 | temperature= args.temperature, 61 | max_tokens = args.max_new_tokens, 62 | n= args.regen_number) 63 | 64 | outputs.append( json.dumps( { 'question': dd['question'], 65 | 'human_gen_truncate': gold_completion[ int( args.truncate_ratio*len(gold_completion)): ], 66 | 'machine_gen_truncate': gen_completion[ int( args.truncate_ratio*len(gen_completion) ): ], 67 | "human_gen_text": human_gen_text, 68 | "machine_gen_text": machine_gen_text }) ) 69 | 70 | with open(output_file, "a") as f: 71 | f.write("\n".join(outputs) + "\n") 72 | outputs = [] 73 | 74 | with open(output_file, "a") as f: 75 | f.write("\n".join(outputs) + "\n") 76 | outputs = [] -------------------------------------------------------------------------------- /openai_generate/regenerate_gpt35.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import tqdm 3 | import json 4 | import os 5 | import openai 6 | import random 7 | 8 | openai_key = "" 9 | openai.api_key = openai_key 10 | 11 | parser = argparse.ArgumentParser() 12 | parser.add_argument('--dataset', default="results/chatgpt_200_len.jsonl") 13 | parser.add_argument('--model', default= "gpt-3.5-turbo") # gpt-3.5-turbo, gpt-4-0314 14 | parser.add_argument('--max_new_tokens', default=300, type=int) 15 | parser.add_argument('--regen_number', default=10, type=int) 16 | parser.add_argument('--truncate_ratio', default=0.5, type=float) 17 | parser.add_argument('--temperature', default= 0.7, type=float) 18 | args = parser.parse_args() 19 | 20 | print('load model ...', args.model) 21 | with open(args.dataset, "r") as f: 22 | data = [json.loads(x) for x in f.read().strip().split("\n")] 23 | 24 | random.seed(43) 25 | output_file = f"results/wop_regen_gpt35_" + str(args.truncate_ratio) + ".jsonl" 26 | 27 | if os.path.exists(output_file): 28 | with open(output_file, "r") as f: 29 | num_curr_outputs = len(f.read().strip().split("\n")) 30 | else: 31 | num_curr_outputs = 0 32 | 33 | print("Skipping {} instances".format(num_curr_outputs)) 34 | print('len(data): ', len(data)) 35 | data = data[num_curr_outputs:] 36 | 37 | outputs = [] 38 | for idx, dd in tqdm.tqdm(enumerate(data), total = len(data) ): # len(data) 39 | gold_completion = dd['gold_completion'] 40 | human_prefix_prompt = gold_completion[ :int( args.truncate_ratio*len(gold_completion) ) ] 41 | 42 | gen_completion = dd['gen_completion'] 43 | machine_prefix_prompt = gen_completion[ :int( args.truncate_ratio*len(gen_completion) ) ] 44 | 45 | human_gen_text = openai.ChatCompletion.create(model= args.model, 46 | messages=[{"role": "system", "content": "You are a helpful assistant that continues the passage from the sentences provided."}, 47 | # {"role": "user", "content": dd['question']}, # mask out to simulate no golden prompt 48 | {"role": "assistant", "content": human_prefix_prompt}, 49 | ], 50 | temperature= args.temperature, 51 | max_tokens = args.max_new_tokens, 52 | n= args.regen_number) 53 | 54 | machine_gen_text = openai.ChatCompletion.create(model= args.model, 55 | messages=[{"role": "system", "content": "You are a helpful assistant that continues the passage from the sentences provided."}, 56 | # {"role": "user", "content": dd['question']}, # mask out to simulate no golden prompt 57 | {"role": "assistant", "content": machine_prefix_prompt}, 58 | ], 59 | temperature= args.temperature, 60 | max_tokens = args.max_new_tokens, 61 | n= args.regen_number) 62 | 63 | outputs.append( json.dumps( { 'question': dd['question'], 64 | 'human_gen_truncate': gold_completion[ int( args.truncate_ratio*len(gold_completion)): ], 65 | 'machine_gen_truncate': gen_completion[ int( args.truncate_ratio*len(gen_completion) ): ], 66 | "human_gen_text": human_gen_text, 67 | "machine_gen_text": machine_gen_text }) ) 68 | 69 | with open(output_file, "a") as f: 70 | f.write("\n".join(outputs) + "\n") 71 | outputs = [] 72 | 73 | with open(output_file, "a") as f: 74 | f.write("\n".join(outputs) + "\n") 75 | outputs = [] -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # pdm 105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 106 | #pdm.lock 107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 108 | # in version control. 109 | # https://pdm.fming.dev/#use-with-ide 110 | .pdm.toml 111 | 112 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 113 | __pypackages__/ 114 | 115 | # Celery stuff 116 | celerybeat-schedule 117 | celerybeat.pid 118 | 119 | # SageMath parsed files 120 | *.sage.py 121 | 122 | # Environments 123 | .env 124 | .venv 125 | env/ 126 | venv/ 127 | ENV/ 128 | env.bak/ 129 | venv.bak/ 130 | 131 | # Spyder project settings 132 | .spyderproject 133 | .spyproject 134 | 135 | # Rope project settings 136 | .ropeproject 137 | 138 | # mkdocs documentation 139 | /site 140 | 141 | # mypy 142 | .mypy_cache/ 143 | .dmypy.json 144 | dmypy.json 145 | 146 | # Pyre type checker 147 | .pyre/ 148 | 149 | # pytype static type analyzer 150 | .pytype/ 151 | 152 | # Cython debug symbols 153 | cython_debug/ 154 | 155 | # PyCharm 156 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 157 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 158 | # and can be added to the global gitignore or merged into this file. For a more nuclear 159 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 160 | #.idea/ 161 | -------------------------------------------------------------------------------- /openai_generate/utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import openai 3 | 4 | import re, six 5 | import spacy 6 | from nltk.stem.porter import PorterStemmer 7 | from rouge_score.rouge_scorer import _create_ngrams, _score_ngrams 8 | 9 | PorterStemmer = PorterStemmer() 10 | nlp = spacy.load('en_core_web_sm') 11 | stopwords = nlp.Defaults.stop_words 12 | 13 | def get_openai_response(prompt: str, max_tokens = 150, temperature = 0.7, top_p = 1, n = 1, logprobs = 1, stop = None, echo = True): 14 | response = openai.Completion.create(engine="text-davinci-003", 15 | prompt=prompt, 16 | max_tokens=max_tokens, 17 | temperature = temperature, 18 | top_p=top_p, 19 | n=n, 20 | logprobs=logprobs, 21 | stop=stop, 22 | echo=echo) 23 | output = response['choices'][0]['text'] 24 | assert output.startswith(prompt) 25 | gen_text = output[len(prompt):].strip() 26 | return gen_text 27 | 28 | def get_davinci003_response(prompt: str, max_tokens = 150, temperature = 0.7, top_p = 1, n = 1, logprobs = 1, stop = None, echo = True): 29 | response = openai.Completion.create(engine="text-davinci-003", 30 | prompt=prompt, 31 | max_tokens=max_tokens, 32 | temperature = temperature, 33 | top_p=top_p, 34 | n=n, 35 | logprobs=logprobs, 36 | stop=stop, 37 | echo=echo) 38 | # output = response['choices'][0]['text'] 39 | # assert output.startswith(prompt) 40 | # gen_text = output[len(prompt):].strip() 41 | return response 42 | 43 | def get_chatgpt_qa_response(prompt_text, temperature = 0.7, max_tokens=1000): 44 | messages = [{"role":"system", "content": "You are a helpful assistant that answers the question provided."}, 45 | {"role":"user", "content": prompt_text}] 46 | response = openai.ChatCompletion.create( 47 | model = "gpt-3.5-turbo", 48 | messages = messages, 49 | temperature = temperature, 50 | max_tokens = max_tokens 51 | ) 52 | return response['choices'][0]['message']['content'] 53 | 54 | def get_gpt4_qa_response(prompt_text, temperature = 0.7, max_tokens=1000): 55 | messages = [{"role":"system", "content": "You are a helpful assistant that answers the question provided."}, 56 | {"role":"user", "content": prompt_text}] 57 | response = openai.ChatCompletion.create( 58 | model = "gpt-4-0314", 59 | messages = messages, 60 | temperature = temperature, 61 | max_tokens = max_tokens 62 | ) 63 | return response['choices'][0]['message']['content'] 64 | 65 | def get_gpt4_completion_response(prompt_text, max_tokens): 66 | messages = [{"role":"system", "content": "You are a helpful assistant that continues the passage from the sentences provided."}, 67 | {"role":"user", "content": prompt_text}] 68 | response = openai.ChatCompletion.create( 69 | model = "gpt-4-0314", 70 | messages = messages, 71 | temperature = 0.7, 72 | max_tokens = max_tokens 73 | ) 74 | return response['choices'][0]['message']['content'] 75 | 76 | def get_chatgpt_completion_response(prompt_text, max_tokens): 77 | messages = [{"role":"system", "content": "You are a helpful assistant that continues the passage from the sentences provided."}, 78 | {"role":"user", "content": prompt_text}] 79 | response = openai.ChatCompletion.create( 80 | model = "gpt-3.5-turbo", 81 | messages = messages, 82 | temperature = 0.7, 83 | max_tokens = max_tokens 84 | ) 85 | return response['choices'][0]['message']['content'] 86 | 87 | def tokenize(text, stemmer, stopwords=[]): 88 | """Tokenize input text into a list of tokens. 89 | 90 | This approach aims to replicate the approach taken by Chin-Yew Lin in 91 | the original ROUGE implementation. 92 | 93 | Args: 94 | text: A text blob to tokenize. 95 | stemmer: An optional stemmer. 96 | 97 | Returns: 98 | A list of string tokens extracted from input text. 99 | """ 100 | 101 | # Convert everything to lowercase. 102 | text = text.lower() 103 | # Replace any non-alpha-numeric characters with spaces. 104 | text = re.sub(r"[^a-z0-9]+", " ", six.ensure_str(text)) 105 | 106 | tokens = re.split(r"\s+", text) 107 | if stemmer: 108 | # Only stem words more than 3 characters long. 109 | tokens = [stemmer.stem(x) if len(x) > 3 else x for x in tokens if x not in stopwords] 110 | 111 | # One final check to drop any empty or invalid tokens. 112 | tokens = [x for x in tokens if re.match(r"^[a-z0-9]+$", six.ensure_str(x))] 113 | 114 | return tokens 115 | -------------------------------------------------------------------------------- /openai_generate/regenerate_gpt3.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import numpy as np 3 | import nltk 4 | from nltk.tokenize import sent_tokenize 5 | import tqdm 6 | import json 7 | import os 8 | import torch, openai 9 | import random, time 10 | from utils import get_davinci003_response 11 | 12 | openai_key = "" 13 | openai.api_key = openai_key 14 | 15 | 16 | parser = argparse.ArgumentParser() 17 | parser.add_argument('--dataset', default="results/davinci_003_200_len.jsonl") 18 | parser.add_argument('--output_dir', default="results") 19 | parser.add_argument('--model', default="text-davinci-003") # "gpt-3.5-turbo", "gpt-4-0314", 20 | parser.add_argument('--max_new_tokens', default=300, type=int) 21 | parser.add_argument('--total_instances', default=200, type=int) 22 | parser.add_argument('--longest_charachter', default=1200, type=int) 23 | parser.add_argument('--truncate_ratio', default=0.7, type=float) 24 | parser.add_argument('--regen_number', default=20, type=int) 25 | args = parser.parse_args() 26 | 27 | print("Loading model {}".format(args.model)) 28 | # load saved json file 29 | with open( args.dataset, 'r') as f: 30 | data = [json.loads(x) for x in f.read().strip().split("\n")] 31 | 32 | output_file = f"results/regen_davinci003_20_" + str( args.truncate_ratio ) +".jsonl" 33 | random.seed(43) 34 | 35 | if os.path.exists(output_file): 36 | with open(output_file, "r") as f: 37 | num_curr_outputs = len(f.read().strip().split("\n")) 38 | else: 39 | num_curr_outputs = 0 40 | 41 | print("Skipping {} instances".format(num_curr_outputs)) 42 | data = data[num_curr_outputs:] 43 | print("Total instances: {}".format(len(data))) 44 | 45 | outputs = [] 46 | for idx, dd in tqdm.tqdm(enumerate(data), total= len(data) ): # 47 | question = dd['question'] 48 | #prefix = "Continues the passage from the sentences provided in 180-300 words." 49 | gold_gen = dd['gold_completion'] 50 | gen_completion = dd['gen_completion'] 51 | 52 | prompt_gold_text = question + '\n' + gold_gen[ :int( args.truncate_ratio * len(gold_gen)) ] 53 | gold_gen_regen = get_davinci003_response( prompt = prompt_gold_text, 54 | max_tokens = args.max_new_tokens, 55 | n= args.regen_number, 56 | logprobs=5, 57 | echo=False ) 58 | time.sleep(1) 59 | original_human_response = get_davinci003_response( prompt = question + '\n' + gold_gen, 60 | max_tokens = 0, 61 | n= 1, 62 | logprobs=5, 63 | echo=True) 64 | 65 | original_human_response_truncate = get_davinci003_response( prompt = prompt_gold_text, 66 | max_tokens = 0, 67 | n= 1, 68 | logprobs=5, 69 | echo=True) 70 | 71 | prompt_gen_completion = question + '\n' + gen_completion[: int( args.truncate_ratio * len(gen_completion)) ] 72 | gen_completion_regen = get_davinci003_response( prompt = prompt_gen_completion, 73 | max_tokens = args.max_new_tokens, 74 | n= args.regen_number, 75 | logprobs=5, 76 | echo=False) 77 | 78 | original_gen_response = get_davinci003_response( prompt = question + '\n' + gen_completion, 79 | max_tokens = 0, 80 | n = 1, 81 | logprobs=5, 82 | echo=True) 83 | 84 | original_gen_response_truncate = get_davinci003_response( prompt = prompt_gen_completion, 85 | max_tokens = 0, 86 | n = 1, 87 | logprobs=5, 88 | echo=True) 89 | 90 | outputs.append(json.dumps({ 91 | "question": question, 92 | "gold_gen_truncate": gold_gen[int( args.truncate_ratio*len(gold_gen)): ], 93 | "gen_completion_truncate": gen_completion[int( args.truncate_ratio*len(gen_completion)): ], 94 | "gold_gen_regen": gold_gen_regen, 95 | "original_human_response": original_human_response, 96 | "gen_completion_regen": gen_completion_regen, 97 | "original_gen_response": original_gen_response, 98 | "original_human_response_truncate": original_human_response_truncate, 99 | "original_gen_response_truncate": original_gen_response_truncate, 100 | })) 101 | 102 | with open(output_file, "a") as f: 103 | f.write("\n".join(outputs) + "\n") 104 | outputs = [] 105 | 106 | with open(output_file, "a") as f: 107 | f.write("\n".join(outputs) + "\n") 108 | outputs = [] 109 | -------------------------------------------------------------------------------- /openai_generate/my_detector_whitebox.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "from rouge_score.rouge_scorer import _create_ngrams\n", 10 | "from nltk.stem.porter import PorterStemmer\n", 11 | "import spacy, six, json, openai\n", 12 | "from utils import tokenize\n", 13 | "import numpy as np\n", 14 | "import numpy as np\n", 15 | "import matplotlib.pyplot as plt\n", 16 | "from sklearn.metrics import roc_curve, auc\n", 17 | "\n", 18 | "PorterStemmer = PorterStemmer()\n", 19 | "nlp = spacy.load('en_core_web_sm')" 20 | ] 21 | }, 22 | { 23 | "cell_type": "code", 24 | "execution_count": null, 25 | "metadata": {}, 26 | "outputs": [], 27 | "source": [ 28 | "\n", 29 | "def plot_roc_curve(human_scores, gpt_scores):\n", 30 | " # Data\n", 31 | " A = human_scores\n", 32 | " B = gpt_scores\n", 33 | " # Combine scores and true labels\n", 34 | " scores = A + B\n", 35 | " labels = [0] * len(A) + [1] * len(B)\n", 36 | " # Calculate ROC curve\n", 37 | " fpr, tpr, thresholds = roc_curve(labels, scores)\n", 38 | " # Calculate AUC (Area Under Curve)\n", 39 | " roc_auc = auc(fpr, tpr)\n", 40 | " # Plot ROC curve\n", 41 | " plt.figure()\n", 42 | " plt.plot(fpr, tpr, color='darkorange', lw=2, label='ROC curve (area = %0.4f)' % roc_auc)\n", 43 | " plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')\n", 44 | " plt.xlim([0.0, 1.0])\n", 45 | " plt.ylim([0.0, 1.05])\n", 46 | " plt.xlabel('False Positive Rate')\n", 47 | " plt.ylabel('True Positive Rate')\n", 48 | " plt.title('ROC curve: Open-gen w/ GPT3.5-Reddit w prompts' )\n", 49 | " plt.legend(loc=\"lower right\")\n", 50 | " plt.show()\n", 51 | " # what is the TPR for FPR = 0.1?\n", 52 | " for idx, fpr_ in enumerate(fpr):\n", 53 | " if fpr_ > 0.01:\n", 54 | " print(f\"TPR at 1% FPR: {tpr[idx]:.4f}\")\n", 55 | " break" 56 | ] 57 | }, 58 | { 59 | "cell_type": "code", 60 | "execution_count": null, 61 | "metadata": {}, 62 | "outputs": [], 63 | "source": [ 64 | "with open('results/regen_davinci003_20_0.7.jsonl', \"r\") as f:\n", 65 | " regen_davinci003_20_05 = [json.loads(x) for x in f.read().strip().split(\"\\n\") ]\n", 66 | "len(regen_davinci003_20_05)" 67 | ] 68 | }, 69 | { 70 | "cell_type": "code", 71 | "execution_count": null, 72 | "metadata": {}, 73 | "outputs": [], 74 | "source": [ 75 | "def get_ratio_avgk(instance, num_samples=20):\n", 76 | " truncate_len = len( instance['original_human_response_truncate']['choices'][0]['logprobs']['token_logprobs'] )\n", 77 | " orignal_prob = instance['original_human_response']['choices'][0]['logprobs']['token_logprobs'][truncate_len:]\n", 78 | " orignal_logprob = np.mean(orignal_prob) # / len(orignal_prob)\n", 79 | " regen_probs = [ sum( instance['gold_gen_regen']['choices'][i]['logprobs']['token_logprobs']) / \\\n", 80 | " len( instance['gold_gen_regen']['choices'][i]['logprobs']['token_logprobs'] ) for i in range(num_samples) \\\n", 81 | " if len( instance['gold_gen_regen']['choices'][i]['logprobs']['token_logprobs'] ) != 0 ]\n", 82 | " regen_logprobs_avg_20 = np.mean(regen_probs) \n", 83 | " original_th = orignal_logprob - regen_logprobs_avg_20\n", 84 | "\n", 85 | " truncate_gen_len = len( instance['original_gen_response_truncate']['choices'][0]['logprobs']['token_logprobs'] )\n", 86 | " gen_prob = instance['original_gen_response']['choices'][0]['logprobs']['token_logprobs'][truncate_gen_len:]\n", 87 | " gen_logprob = np.mean(gen_prob) # \n", 88 | " gen_regen_probs = [ sum( instance['gen_completion_regen']['choices'][i]['logprobs']['token_logprobs']) / \\\n", 89 | " len( instance['gen_completion_regen']['choices'][i]['logprobs']['token_logprobs'] ) for i in range(num_samples) \\\n", 90 | " if len( instance['gen_completion_regen']['choices'][i]['logprobs']['token_logprobs'] ) != 0 ]\n", 91 | " gen_regen_logprobs_avg_20 = np.mean(gen_regen_probs) \n", 92 | " gen_th = gen_logprob - gen_regen_logprobs_avg_20\n", 93 | "\n", 94 | " return original_th, gen_th" 95 | ] 96 | }, 97 | { 98 | "cell_type": "code", 99 | "execution_count": null, 100 | "metadata": {}, 101 | "outputs": [], 102 | "source": [ 103 | "########################## with questions ##########################\n", 104 | "human_scores, gpt_scores = [], []\n", 105 | "for idx, instance in enumerate( regen_davinci003_20_05 ):\n", 106 | " original_th, gen_th = get_ratio_avgk(instance, num_samples = 20)\n", 107 | " human_scores.append( original_th )\n", 108 | " gpt_scores.append( gen_th )" 109 | ] 110 | }, 111 | { 112 | "cell_type": "code", 113 | "execution_count": null, 114 | "metadata": {}, 115 | "outputs": [], 116 | "source": [ 117 | "# remove nan\n", 118 | "human_scores = [x for x in human_scores if str(x) != 'nan']\n", 119 | "gpt_scores = [x for x in gpt_scores if str(x) != 'nan']" 120 | ] 121 | }, 122 | { 123 | "cell_type": "code", 124 | "execution_count": null, 125 | "metadata": {}, 126 | "outputs": [], 127 | "source": [ 128 | "# plot and give different colors\n", 129 | "import matplotlib.pyplot as plt\n", 130 | "plt.figure(figsize=(10, 5))\n", 131 | "plt.plot( human_scores , label='human')\n", 132 | "plt.plot( gpt_scores, label='gpt')\n", 133 | "plt.legend()\n", 134 | "plt.show()" 135 | ] 136 | }, 137 | { 138 | "cell_type": "code", 139 | "execution_count": null, 140 | "metadata": {}, 141 | "outputs": [], 142 | "source": [ 143 | "plot_roc_curve( human_scores, gpt_scores )" 144 | ] 145 | } 146 | ], 147 | "metadata": { 148 | "kernelspec": { 149 | "display_name": "vpt", 150 | "language": "python", 151 | "name": "python3" 152 | }, 153 | "language_info": { 154 | "codemirror_mode": { 155 | "name": "ipython", 156 | "version": 3 157 | }, 158 | "file_extension": ".py", 159 | "mimetype": "text/x-python", 160 | "name": "python", 161 | "nbconvert_exporter": "python", 162 | "pygments_lexer": "ipython3", 163 | "version": "3.8.15" 164 | }, 165 | "orig_nbformat": 4, 166 | "vscode": { 167 | "interpreter": { 168 | "hash": "015bfb409bf441c0a66e03b2de1c9b891435fcbf36ed1d1e9d7c8167e73e6b62" 169 | } 170 | } 171 | }, 172 | "nbformat": 4, 173 | "nbformat_minor": 2 174 | } 175 | -------------------------------------------------------------------------------- /open_source_models/regenerate_gptneox.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import tqdm 3 | import json, math 4 | import os 5 | import torch 6 | import random 7 | from torch.nn.functional import softmax 8 | from transformers import GPTNeoXForCausalLM, GPTNeoXTokenizerFast 9 | import torch 10 | 11 | def gptx_generate(prompt, max_length, echo=False): 12 | inputs = tokenizer(prompt, return_tensors="pt").to(device) 13 | with torch.no_grad(): 14 | gen_tokens = model.generate( 15 | input_ids = inputs["input_ids"], 16 | attention_mask=inputs["attention_mask"], 17 | do_sample=True, 18 | temperature=0.7, 19 | max_length= max_length, 20 | num_return_sequences = 1, 21 | ) 22 | gen_text = tokenizer.batch_decode(gen_tokens)[0] 23 | if echo: 24 | return gen_text, gen_text.startswith(prompt) 25 | else: 26 | return gen_text[len(prompt):], gen_text.startswith(prompt) 27 | 28 | def gptx_prob(prompt, gen_text, echo=False): 29 | inputs = tokenizer(prompt + gen_text, return_tensors="pt").to(device) 30 | prompt_inputs = tokenizer(prompt, return_tensors="pt") 31 | with torch.no_grad(): 32 | outputs = model( 33 | input_ids = inputs["input_ids"], 34 | attention_mask= inputs["attention_mask"], 35 | ) 36 | logits = outputs.logits 37 | probabilities = softmax(logits, dim=-1) 38 | # Get the probability of each token in the input text 39 | indices = inputs['input_ids'][0][1:].unsqueeze(-1) # Add an extra dimension to match the probabilities tensor 40 | token_probabilities = torch.gather( probabilities[0, :, :], 1, indices).squeeze().tolist() 41 | token_probabilities = [0] + token_probabilities # add dummy probability for the first token 42 | log_probabilities = [math.log(x) if x != 0 else -100 for x in token_probabilities ] 43 | if echo: 44 | return log_probabilities 45 | else: 46 | return log_probabilities[ prompt_inputs['input_ids'].size(1): ] 47 | 48 | parser = argparse.ArgumentParser() 49 | parser.add_argument('--dataset', default="results_reddit/gptx_200_len.jsonl") 50 | parser.add_argument('--model', default="gptx") 51 | parser.add_argument('--max_new_tokens', default=300, type=int) 52 | parser.add_argument('--longest_charachter', default=1200, type=int) 53 | parser.add_argument('--truncate_ratio', default=0.5, type=float) 54 | parser.add_argument('--regen_number', default=20, type=int) 55 | parser.add_argument('--device', default=0, type=int) 56 | args = parser.parse_args() 57 | 58 | print("Loading model {}".format(args.model)) 59 | device = torch.device("cuda:" + str( args.device )) 60 | tokenizer = GPTNeoXTokenizerFast.from_pretrained("EleutherAI/gpt-neox-20b") 61 | model = GPTNeoXForCausalLM.from_pretrained("EleutherAI/gpt-neox-20b").half().to(device) #.half() to use FP16 62 | model.eval() # Set the model to evaluation mode 63 | 64 | # load saved json file 65 | with open( args.dataset, 'r') as f: 66 | data = [json.loads(x) for x in f.read().strip().split("\n")] 67 | 68 | output_file = f"results_reddit/wop_regen_gptx_20" + str(args.truncate_ratio) + ".jsonl" 69 | random.seed(43) 70 | 71 | if os.path.exists(output_file): 72 | with open(output_file, "r") as f: 73 | num_curr_outputs = len(f.read().strip().split("\n")) 74 | else: 75 | num_curr_outputs = 0 76 | print("Skipping {} instances".format(num_curr_outputs)) 77 | data = data[num_curr_outputs:] 78 | print("Total instances: {}".format(len(data))) 79 | 80 | outputs = [] 81 | for idx, dd in tqdm.tqdm(enumerate(data), total= len(data) ): # 82 | prefix = dd['prefix'] 83 | question = prefix 84 | 85 | gold_gen = dd['gold_completion'] 86 | gen_completion = dd['gen_completion'] 87 | 88 | prompt_gold_text = question + '\n' + gold_gen[ :int( args.truncate_ratio * len(gold_gen)) ] 89 | # regen from gptx_generate and gptx_prob 90 | gold_gen_regen = [] 91 | for i in range(args.regen_number): 92 | truth = False 93 | count = 0 94 | while not truth: 95 | if count > 1 and count < 10: 96 | print('regen') 97 | elif count > 10: 98 | break 99 | gen_text, truth = gptx_generate(prompt_gold_text, args.max_new_tokens, echo=False) 100 | count += 1 101 | gptx_probabilities = gptx_prob( prompt_gold_text, gen_text, echo=False) 102 | gold_gen_regen.append( {'gen_text': gen_text, 'gptx_probabilities': gptx_probabilities} ) 103 | 104 | original_human_response = gptx_prob( question + '\n', 105 | gold_gen, 106 | echo=True) 107 | 108 | original_human_response_truncate = gptx_prob( prompt_gold_text, 109 | '', 110 | echo=True) 111 | 112 | prompt_gen_completion = question + '\n' + gen_completion[: int( args.truncate_ratio * len(gen_completion)) ] 113 | gen_completion_regen = [] 114 | for i in range(args.regen_number): 115 | truth = False 116 | count = 0 117 | while not truth: 118 | if count > 1 and count < 10: 119 | print('regen') 120 | elif count > 10: 121 | break 122 | gen_text, truth = gptx_generate(prompt_gen_completion, args.max_new_tokens, echo=False) 123 | count += 1 124 | gptx_probabilities = gptx_prob( prompt_gen_completion, gen_text, echo=False) 125 | gen_completion_regen.append( {'gen_text': gen_text, 'gptx_probabilities': gptx_probabilities} ) 126 | 127 | original_gen_response = gptx_prob( question + '\n', 128 | gen_completion, 129 | echo=True) 130 | 131 | original_gen_response_truncate = gptx_prob( prompt_gen_completion, 132 | '', 133 | echo=True) 134 | 135 | 136 | outputs.append(json.dumps({ 137 | "question": question, 138 | "gold_gen_truncate": gold_gen[int( args.truncate_ratio*len(gold_gen)): ], 139 | "gen_completion_truncate": gen_completion[int( args.truncate_ratio*len(gen_completion)): ], 140 | "gold_gen_regen": gold_gen_regen, 141 | "original_human_response": original_human_response, 142 | "gen_completion_regen": gen_completion_regen, 143 | "original_gen_response": original_gen_response, 144 | "original_human_response_truncate": original_human_response_truncate, 145 | "original_gen_response_truncate": original_gen_response_truncate, 146 | })) 147 | 148 | with open(output_file, "a") as f: 149 | f.write("\n".join(outputs) + "\n") 150 | outputs = [] 151 | 152 | with open(output_file, "a") as f: 153 | f.write("\n".join(outputs) + "\n") 154 | outputs = [] 155 | -------------------------------------------------------------------------------- /open_source_models/regenerate_gptllama.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import numpy as np 3 | import nltk 4 | from nltk.tokenize import sent_tokenize 5 | import tqdm 6 | import json, math 7 | import os 8 | import torch, openai 9 | import random 10 | from torch.nn.functional import softmax 11 | 12 | from transformers import LlamaForCausalLM, LlamaTokenizer 13 | import torch 14 | 15 | path = "" # path to the model 16 | 17 | tokenizer = LlamaTokenizer.from_pretrained( path ) 18 | 19 | def gptx_generate(prompt, max_length, echo=False): 20 | inputs = tokenizer(prompt, return_tensors="pt").to(device) 21 | with torch.no_grad(): # torch.no_grad() or torch.cuda.amp.autocast(): 22 | gen_tokens = model.generate( 23 | input_ids = inputs["input_ids"], 24 | attention_mask=inputs["attention_mask"], 25 | do_sample=True, 26 | temperature=0.7, 27 | max_length= max_length, 28 | num_return_sequences = 1, 29 | ) 30 | gen_text = tokenizer.batch_decode(gen_tokens, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] 31 | if echo: 32 | return gen_text, gen_text.startswith(prompt) 33 | else: 34 | return gen_text[len(prompt):], gen_text.startswith(prompt) 35 | 36 | def gptx_prob(prompt, gen_text, echo=False): 37 | inputs = tokenizer(prompt + gen_text, return_tensors="pt").to(device) 38 | prompt_inputs = tokenizer(prompt, return_tensors="pt") 39 | with torch.no_grad(): # torch.no_grad() or torch.cuda.amp.autocast(): 40 | outputs = model( 41 | input_ids = inputs["input_ids"], 42 | attention_mask= inputs["attention_mask"], 43 | ) 44 | logits = outputs.logits 45 | probabilities = softmax(logits, dim=-1) 46 | indices = inputs['input_ids'][0][1:].unsqueeze(-1) # Add an extra dimension to match the probabilities tensor 47 | token_prob = torch.gather(probabilities[0, :, :], dim=1, index=indices).squeeze().tolist() 48 | # convert token_probabilities to log_probabilities, for inf value, use -10000, in one line 49 | log_probabilities = [math.log(x) if x != 0 else -100 for x in token_prob ] 50 | # Get the probability of each token in the input text 51 | if echo: 52 | return log_probabilities 53 | else: 54 | return log_probabilities[ prompt_inputs['input_ids'].size(1) - 1: ] # remove the start token 55 | 56 | parser = argparse.ArgumentParser() 57 | parser.add_argument('--dataset', default="xsum_truncate/gptllama_150_len.jsonl") 58 | parser.add_argument('--model', default="gptllama") # "gpt-3.5-turbo", "gpt-4-0314", 59 | parser.add_argument('--max_new_tokens', default=300, type=int) 60 | parser.add_argument('--longest_charachter', default=1200, type=int) 61 | parser.add_argument('--truncate_ratio', default=0.5, type=float) 62 | parser.add_argument('--regen_number', default=10, type=int) 63 | parser.add_argument('--device', default=0, type=int) 64 | args = parser.parse_args() 65 | 66 | print("Loading model {}".format(args.model)) 67 | device = torch.device("cuda:" + str( args.device )) 68 | model = LlamaForCausalLM.from_pretrained( path ).half().to(device) #.half() to use FP16 69 | model.eval() 70 | 71 | # load saved json file 72 | with open( args.dataset, 'r') as f: 73 | data = [json.loads(x) for x in f.read().strip().split("\n")] 74 | 75 | output_file = f"xsum_truncate/wop_regen_gptllama_10_" + str(args.truncate_ratio) + ".jsonl" 76 | random.seed(43) 77 | 78 | data = data[ :] 79 | print("Total instances: {}".format(len(data))) 80 | 81 | outputs = [] 82 | for idx, dd in tqdm.tqdm(enumerate(data), total= len(data) ): # 83 | #prefix = dd['prefix'] 84 | question = dd['question'] # no prefix for reddit 85 | #question = prefix +# question # for xsum 86 | #question = "Continues the following sentences in around 200 words: " # for xsum 87 | gold_gen = dd['gold_completion'] 88 | gen_completion = dd['gen_completion'] 89 | 90 | prompt_gold_text = question + '\n' + gold_gen[ :int( args.truncate_ratio * len(gold_gen)) ] 91 | # regen from gptx_generate and gptx_prob 92 | gold_gen_regen = [] 93 | for i in range(args.regen_number): 94 | truth = False 95 | count = 0 96 | while not truth: 97 | if count > 1 and count < 10: 98 | print('regen') 99 | elif count > 10: 100 | break 101 | gen_text, truth = gptx_generate(prompt_gold_text, args.max_new_tokens, echo=False) 102 | count += 1 103 | gptx_probabilities = gptx_prob( prompt_gold_text, gen_text, echo=False) 104 | gold_gen_regen.append( {'gen_text': gen_text, 'gptx_probabilities': gptx_probabilities} ) 105 | 106 | original_human_response = gptx_prob( question + '\n', 107 | gold_gen, 108 | echo=True) 109 | 110 | original_human_response_truncate = gptx_prob( prompt_gold_text, 111 | '', 112 | echo=True) 113 | 114 | prompt_gen_completion = question + '\n' + gen_completion[: int( args.truncate_ratio * len(gen_completion)) ] 115 | gen_completion_regen = [] 116 | for i in range(args.regen_number): 117 | truth = False 118 | count = 0 119 | while not truth: 120 | if count > 1 and count < 10: 121 | print('regen') 122 | elif count > 10: 123 | break 124 | gen_text, truth = gptx_generate(prompt_gen_completion, args.max_new_tokens, echo=False) 125 | count += 1 126 | gptx_probabilities = gptx_prob( prompt_gen_completion, gen_text, echo=False) 127 | gen_completion_regen.append( {'gen_text': gen_text, 'gptx_probabilities': gptx_probabilities} ) 128 | 129 | original_gen_response = gptx_prob( question + '\n', 130 | gen_completion, 131 | echo=True) 132 | 133 | original_gen_response_truncate = gptx_prob( prompt_gen_completion, 134 | '', 135 | echo=True) 136 | 137 | 138 | outputs.append(json.dumps({ 139 | "question": question, 140 | "gold_gen_truncate": gold_gen[int( args.truncate_ratio*len(gold_gen)): ], 141 | "gen_completion_truncate": gen_completion[int( args.truncate_ratio*len(gen_completion)): ], 142 | "gold_gen_regen": gold_gen_regen, 143 | "original_human_response": original_human_response, 144 | "gen_completion_regen": gen_completion_regen, 145 | "original_gen_response": original_gen_response, 146 | "original_human_response_truncate": original_human_response_truncate, 147 | "original_gen_response_truncate": original_gen_response_truncate, 148 | })) 149 | 150 | with open(output_file, "a") as f: 151 | f.write("\n".join(outputs) + "\n") 152 | outputs = [] 153 | 154 | with open(output_file, "a") as f: 155 | f.write("\n".join(outputs) + "\n") 156 | outputs = [] 157 | -------------------------------------------------------------------------------- /openai_generate/my_detector_gpt35or4.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "from rouge_score.rouge_scorer import _create_ngrams\n", 10 | "from nltk.stem.porter import PorterStemmer\n", 11 | "import spacy, six, json\n", 12 | "from utils import tokenize\n", 13 | "import numpy as np\n", 14 | "import matplotlib.pyplot as plt\n", 15 | "from sklearn.metrics import roc_curve, auc\n", 16 | "\n", 17 | "PorterStemmer = PorterStemmer()\n", 18 | "nlp = spacy.load('en_core_web_sm')" 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": null, 24 | "metadata": {}, 25 | "outputs": [], 26 | "source": [ 27 | "def plot_roc_curve(human_scores, gpt_scores):\n", 28 | " # Data\n", 29 | " A = human_scores\n", 30 | " B = gpt_scores\n", 31 | " # Combine scores and true labels\n", 32 | " scores = A + B\n", 33 | " labels = [0] * len(A) + [1] * len(B)\n", 34 | " # Calculate ROC curve\n", 35 | " fpr, tpr, thresholds = roc_curve(labels, scores)\n", 36 | " # Calculate AUC (Area Under Curve)\n", 37 | " roc_auc = auc(fpr, tpr)\n", 38 | " # Plot ROC curve\n", 39 | " plt.figure()\n", 40 | " plt.plot(fpr, tpr, color='darkorange', lw=2, label='ROC curve (area = %0.4f)' % roc_auc)\n", 41 | " plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')\n", 42 | " plt.xlim([0.0, 1.0])\n", 43 | " plt.ylim([0.0, 1.05])\n", 44 | " plt.xlabel('False Positive Rate')\n", 45 | " plt.ylabel('True Positive Rate')\n", 46 | " plt.title('ROC curve: Open-gen w/ GPT3.5-Reddit w prompts' )\n", 47 | " plt.legend(loc=\"lower right\")\n", 48 | " plt.show()\n", 49 | " # what is the TPR for FPR = 0.1?\n", 50 | " for idx, fpr_ in enumerate(fpr):\n", 51 | " if fpr_ > 0.01:\n", 52 | " print(f\"TPR at 1% FPR: {tpr[idx]:.4f}\")\n", 53 | " break\n", 54 | " return roc_auc, tpr[idx]" 55 | ] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "execution_count": null, 60 | "metadata": {}, 61 | "outputs": [], 62 | "source": [ 63 | "def get_score_ngrams(target_ngrams, prediction_ngrams):\n", 64 | " intersection_ngrams_count = 0\n", 65 | " ngram_dict = {}\n", 66 | " for ngram in six.iterkeys(target_ngrams):\n", 67 | " intersection_ngrams_count += min(target_ngrams[ngram],\n", 68 | " prediction_ngrams[ngram])\n", 69 | " ngram_dict[ngram] = min(target_ngrams[ngram], prediction_ngrams[ngram])\n", 70 | " target_ngrams_count = sum(target_ngrams.values()) # prediction_ngrams\n", 71 | " return intersection_ngrams_count / max(target_ngrams_count, 1), ngram_dict\n", 72 | "\n", 73 | "\n", 74 | "def get_ngram_info(article_tokens, summary_tokens, _ngram):\n", 75 | " article_ngram = _create_ngrams( article_tokens , _ngram)\n", 76 | " summary_ngram = _create_ngrams( summary_tokens , _ngram)\n", 77 | " ngram_score, ngram_dict = get_score_ngrams( article_ngram, summary_ngram) \n", 78 | " return ngram_score, ngram_dict, sum( ngram_dict.values() )" 79 | ] 80 | }, 81 | { 82 | "cell_type": "code", 83 | "execution_count": null, 84 | "metadata": {}, 85 | "outputs": [], 86 | "source": [ 87 | "with open('results/regen_gpt-3.5-turbo_20_0.7.jsonl', \"r\") as f:\n", 88 | " gpt35_on4 = [json.loads(x) for x in f.read().strip().split(\"\\n\") ]\n", 89 | "len(gpt35_on4)" 90 | ] 91 | }, 92 | { 93 | "cell_type": "code", 94 | "execution_count": null, 95 | "metadata": {}, 96 | "outputs": [], 97 | "source": [ 98 | "gpt35_on4[0].keys()" 99 | ] 100 | }, 101 | { 102 | "cell_type": "code", 103 | "execution_count": null, 104 | "metadata": {}, 105 | "outputs": [], 106 | "source": [ 107 | "len( gpt35_on4[0]['human_gen_text']['choices'] )" 108 | ] 109 | }, 110 | { 111 | "cell_type": "code", 112 | "execution_count": null, 113 | "metadata": {}, 114 | "outputs": [], 115 | "source": [ 116 | "########################## with questions ##########################\n", 117 | "ngram_overlap_count_wq = []\n", 118 | "for idx, instance in enumerate( gpt35_on4 ):\n", 119 | " human_07 = gpt35_on4[idx][\"human_gen_truncate\"] # gold_gen_truncate, gold_gen_07\n", 120 | " gpt_07 = instance['machine_gen_truncate'] # gen_completion_truncate, gen_completion_07\n", 121 | " \n", 122 | " human_tokens = tokenize( human_07, stemmer=PorterStemmer)\n", 123 | " gpt_tokens = tokenize( gpt_07, stemmer=PorterStemmer)\n", 124 | " ########################################\n", 125 | " human_half = gpt35_on4[idx]['human_gen_text']['choices']\n", 126 | " gpt_half = instance['machine_gen_text']['choices']\n", 127 | " temp = []\n", 128 | " for i in range(20): # len(human_half)\n", 129 | " temp1 = {}\n", 130 | " temp2 = {}\n", 131 | " human_generate_tokens = tokenize(human_half[i]['message']['content'], stemmer=PorterStemmer)\n", 132 | " gpt_generate_tokens = tokenize(gpt_half[i]['message']['content'], stemmer=PorterStemmer )\n", 133 | " if len(human_generate_tokens) == 0 or len(gpt_generate_tokens) == 0:\n", 134 | " continue\n", 135 | " for _ngram in range(1, 25):\n", 136 | " ngram_score, ngram_dict, overlap_count = get_ngram_info(human_tokens, human_generate_tokens, _ngram)\n", 137 | " temp1['human_truncate_ngram_{}_score'.format(_ngram)] = ngram_score / len(human_generate_tokens)\n", 138 | " temp1['human_truncate_ngram_{}_count'.format(_ngram)] = overlap_count\n", 139 | "\n", 140 | " ngram_score, ngram_dict, overlap_count = get_ngram_info(gpt_tokens, gpt_generate_tokens, _ngram)\n", 141 | " temp2['gpt_truncate_ngram_{}_score'.format(_ngram)] = ngram_score / len(gpt_generate_tokens)\n", 142 | " temp2['gpt_truncate_ngram_{}_count'.format(_ngram)] = overlap_count\n", 143 | " temp.append({'human':temp1, 'machine':temp2})\n", 144 | "\n", 145 | " ngram_overlap_count_wq.append(temp)" 146 | ] 147 | }, 148 | { 149 | "cell_type": "code", 150 | "execution_count": null, 151 | "metadata": {}, 152 | "outputs": [], 153 | "source": [ 154 | "def N_gram_detector(ngram_n_ratio):\n", 155 | " score = 0\n", 156 | " non_zero = []\n", 157 | " \n", 158 | " for idx, key in enumerate(ngram_n_ratio):\n", 159 | " if idx in range(3) and 'score' in key or 'ratio' in key:\n", 160 | " score += 0. * ngram_n_ratio[ key ]\n", 161 | " continue\n", 162 | " if 'score' in key or 'ratio' in key:\n", 163 | " score += (idx+1) * np.log((idx+1)) * ngram_n_ratio[ key ]\n", 164 | " if ngram_n_ratio[ key ] != 0:\n", 165 | " non_zero.append( idx+1 )\n", 166 | " return score/ (sum( non_zero ) + 1e-8)\n", 167 | "\n", 168 | "human_scores = []\n", 169 | "gpt_scores = []\n", 170 | "\n", 171 | "for instance in ngram_overlap_count_wq:\n", 172 | " human_score = []\n", 173 | " gpt_score = []\n", 174 | "\n", 175 | " for i in range(len(instance)):\n", 176 | " human_score.append( N_gram_detector(instance[i]['human'] ) )\n", 177 | " gpt_score.append( N_gram_detector(instance[i]['machine'] ) )\n", 178 | "\n", 179 | " human_scores.append( sum(human_score) )\n", 180 | " gpt_scores.append( sum(gpt_score) ) " 181 | ] 182 | }, 183 | { 184 | "cell_type": "code", 185 | "execution_count": null, 186 | "metadata": {}, 187 | "outputs": [], 188 | "source": [ 189 | "# plot and give different colors\n", 190 | "import matplotlib.pyplot as plt\n", 191 | "#human_scores, gpt_scores = human_scores[:40], gpt_scores[:40]\n", 192 | "plt.figure(figsize=(10, 5))\n", 193 | "plt.plot(human_scores, label='human')\n", 194 | "plt.plot(gpt_scores, label='gpt')\n", 195 | "plt.legend()\n", 196 | "plt.show()" 197 | ] 198 | }, 199 | { 200 | "cell_type": "code", 201 | "execution_count": null, 202 | "metadata": {}, 203 | "outputs": [], 204 | "source": [ 205 | "plot_roc_curve( human_scores, gpt_scores )" 206 | ] 207 | }, 208 | { 209 | "cell_type": "code", 210 | "execution_count": null, 211 | "metadata": {}, 212 | "outputs": [], 213 | "source": [ 214 | "###### openai detector ######\n", 215 | "with open( f'results/openai_detect_gpt35.jsonl', 'r') as f:\n", 216 | " openai_detect_gpt = [json.loads(x) for x in f.read().strip().split(\"\\n\")]\n", 217 | "len(openai_detect_gpt)" 218 | ] 219 | }, 220 | { 221 | "cell_type": "code", 222 | "execution_count": null, 223 | "metadata": {}, 224 | "outputs": [], 225 | "source": [ 226 | "# plot and give different colors\n", 227 | "import matplotlib.pyplot as plt\n", 228 | "human_scores2 = [x['gold_gen_prob'] for x in openai_detect_gpt]\n", 229 | "gpt_scores2 = [x['gen_completion_prob'] for x in openai_detect_gpt]\n", 230 | "\n", 231 | "plt.figure(figsize=(10, 5))\n", 232 | "plt.plot(human_scores2, label='human')\n", 233 | "plt.plot(gpt_scores2, label='gpt')\n", 234 | "plt.legend()\n", 235 | "plt.show()" 236 | ] 237 | }, 238 | { 239 | "cell_type": "code", 240 | "execution_count": null, 241 | "metadata": {}, 242 | "outputs": [], 243 | "source": [ 244 | "plot_roc_curve(human_scores2, gpt_scores2)" 245 | ] 246 | }, 247 | { 248 | "cell_type": "code", 249 | "execution_count": null, 250 | "metadata": {}, 251 | "outputs": [], 252 | "source": [ 253 | "###### gptzero detector ######\n", 254 | "with open( f'results/detect_gptzero_200.jsonl', 'r') as f:\n", 255 | " gptzero_detect_gpt = [json.loads(x) for x in f.read().strip().split(\"\\n\")]\n", 256 | "len(gptzero_detect_gpt)" 257 | ] 258 | }, 259 | { 260 | "cell_type": "code", 261 | "execution_count": null, 262 | "metadata": {}, 263 | "outputs": [], 264 | "source": [ 265 | "# plot and give different colors\n", 266 | "import matplotlib.pyplot as plt\n", 267 | "human_scores2 = [x['gold_gen_prob'] for x in gptzero_detect_gpt]\n", 268 | "gpt_scores2 = [x['gen_completion_prob'] for x in gptzero_detect_gpt]\n", 269 | "\n", 270 | "plt.figure(figsize=(10, 5))\n", 271 | "plt.plot(human_scores2, label='human')\n", 272 | "plt.plot(gpt_scores2, label='gpt')\n", 273 | "plt.legend()\n", 274 | "plt.show()" 275 | ] 276 | }, 277 | { 278 | "cell_type": "code", 279 | "execution_count": null, 280 | "metadata": {}, 281 | "outputs": [], 282 | "source": [ 283 | "plot_roc_curve(human_scores2, gpt_scores2)" 284 | ] 285 | } 286 | ], 287 | "metadata": { 288 | "kernelspec": { 289 | "display_name": "vpt", 290 | "language": "python", 291 | "name": "python3" 292 | }, 293 | "language_info": { 294 | "codemirror_mode": { 295 | "name": "ipython", 296 | "version": 3 297 | }, 298 | "file_extension": ".py", 299 | "mimetype": "text/x-python", 300 | "name": "python", 301 | "nbconvert_exporter": "python", 302 | "pygments_lexer": "ipython3", 303 | "version": "3.8.15" 304 | }, 305 | "orig_nbformat": 4, 306 | "vscode": { 307 | "interpreter": { 308 | "hash": "015bfb409bf441c0a66e03b2de1c9b891435fcbf36ed1d1e9d7c8167e73e6b62" 309 | } 310 | } 311 | }, 312 | "nbformat": 4, 313 | "nbformat_minor": 2 314 | } 315 | -------------------------------------------------------------------------------- /openai_generate/my_detector_gpt3.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "from rouge_score.rouge_scorer import _create_ngrams\n", 10 | "from nltk.stem.porter import PorterStemmer\n", 11 | "import spacy, six, json\n", 12 | "from utils import tokenize\n", 13 | "import numpy as np\n", 14 | "import matplotlib.pyplot as plt\n", 15 | "from sklearn.metrics import roc_curve, auc\n", 16 | "\n", 17 | "PorterStemmer = PorterStemmer()\n", 18 | "nlp = spacy.load('en_core_web_sm')" 19 | ] 20 | }, 21 | { 22 | "cell_type": "code", 23 | "execution_count": null, 24 | "metadata": {}, 25 | "outputs": [], 26 | "source": [ 27 | "\n", 28 | "def plot_roc_curve(human_scores, gpt_scores):\n", 29 | " # Data\n", 30 | " A = human_scores\n", 31 | " B = gpt_scores\n", 32 | " # Combine scores and true labels\n", 33 | " scores = A + B\n", 34 | " labels = [0] * len(A) + [1] * len(B)\n", 35 | " # Calculate ROC curve\n", 36 | " fpr, tpr, thresholds = roc_curve(labels, scores)\n", 37 | " # Calculate AUC (Area Under Curve)\n", 38 | " roc_auc = auc(fpr, tpr)\n", 39 | " # Plot ROC curve\n", 40 | " plt.figure()\n", 41 | " plt.plot(fpr, tpr, color='darkorange', lw=2, label='ROC curve (area = %0.4f)' % roc_auc)\n", 42 | " plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')\n", 43 | " plt.xlim([0.0, 1.0])\n", 44 | " plt.ylim([0.0, 1.05])\n", 45 | " plt.xlabel('False Positive Rate')\n", 46 | " plt.ylabel('True Positive Rate')\n", 47 | " plt.title('ROC curve: Open-gen w/ GPT3.5-Reddit w prompts' )\n", 48 | " plt.legend(loc=\"lower right\")\n", 49 | " plt.show()\n", 50 | " # what is the TPR for FPR = 0.1?\n", 51 | " for idx, fpr_ in enumerate(fpr):\n", 52 | " if fpr_ > 0.01:\n", 53 | " print(f\"TPR at 1% FPR: {tpr[idx]:.4f}\")\n", 54 | " break" 55 | ] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "execution_count": null, 60 | "metadata": {}, 61 | "outputs": [], 62 | "source": [ 63 | "def get_score_ngrams(target_ngrams, prediction_ngrams):\n", 64 | " intersection_ngrams_count = 0\n", 65 | " ngram_dict = {}\n", 66 | " for ngram in six.iterkeys(target_ngrams):\n", 67 | " intersection_ngrams_count += min(target_ngrams[ngram],\n", 68 | " prediction_ngrams[ngram])\n", 69 | " ngram_dict[ngram] = min(target_ngrams[ngram], prediction_ngrams[ngram])\n", 70 | " target_ngrams_count = sum(target_ngrams.values()) # prediction_ngrams\n", 71 | " return intersection_ngrams_count / max(target_ngrams_count, 1), ngram_dict\n", 72 | "\n", 73 | "\n", 74 | "def get_ngram_info(article_tokens, summary_tokens, _ngram):\n", 75 | " article_ngram = _create_ngrams( article_tokens , _ngram)\n", 76 | " summary_ngram = _create_ngrams( summary_tokens , _ngram)\n", 77 | " ngram_score, ngram_dict = get_score_ngrams( article_ngram, summary_ngram) \n", 78 | " return ngram_score, ngram_dict, sum( ngram_dict.values() )" 79 | ] 80 | }, 81 | { 82 | "cell_type": "code", 83 | "execution_count": null, 84 | "metadata": {}, 85 | "outputs": [], 86 | "source": [ 87 | "with open('results/regen_davinci003_20_0.5.jsonl', \"r\") as f:\n", 88 | " wop_regen_davinci003_20_05 = [json.loads(x) for x in f.read().strip().split(\"\\n\") ]\n", 89 | "len(wop_regen_davinci003_20_05)" 90 | ] 91 | }, 92 | { 93 | "cell_type": "code", 94 | "execution_count": null, 95 | "metadata": {}, 96 | "outputs": [], 97 | "source": [ 98 | "wop_regen_davinci003_20_05[0].keys()" 99 | ] 100 | }, 101 | { 102 | "cell_type": "code", 103 | "execution_count": null, 104 | "metadata": {}, 105 | "outputs": [], 106 | "source": [ 107 | "len( wop_regen_davinci003_20_05[0]['gen_completion_regen']['choices'])#['choices'][1]['text']" 108 | ] 109 | }, 110 | { 111 | "cell_type": "code", 112 | "execution_count": null, 113 | "metadata": {}, 114 | "outputs": [], 115 | "source": [ 116 | "########################## with questions ##########################\n", 117 | "ngram_overlap_count_wq = []\n", 118 | "for idx, instance in enumerate( wop_regen_davinci003_20_05):\n", 119 | " human_07 = instance[\"gold_gen_truncate\"] # gold_gen_truncate, human_gen_truncate\n", 120 | " gpt_07 = instance['gen_completion_truncate'] # gen_completion_truncate, machine_gen_truncate\n", 121 | " \n", 122 | " human_tokens = tokenize( human_07, stemmer=PorterStemmer)\n", 123 | " gpt_tokens = tokenize( gpt_07, stemmer=PorterStemmer)\n", 124 | " ########################################\n", 125 | " human_half = instance['gold_gen_regen']['choices']\n", 126 | " gpt_half = instance['gen_completion_regen']['choices']\n", 127 | " temp = []\n", 128 | " for i in range(20): # len(human_half)\n", 129 | " temp1 = {}\n", 130 | " temp2 = {}\n", 131 | " human_generate_tokens = tokenize(human_half[i]['text'], stemmer=PorterStemmer) # ['message']['content'] for chatgpt, ['text'] for gpt3\n", 132 | " gpt_generate_tokens = tokenize(gpt_half[i]['text'], stemmer=PorterStemmer )\n", 133 | " if len(human_generate_tokens) == 0 or len(gpt_generate_tokens) == 0:\n", 134 | " continue\n", 135 | "\n", 136 | " for _ngram in range(1, 25):\n", 137 | " ngram_score, ngram_dict, overlap_count = get_ngram_info(human_tokens, human_generate_tokens, _ngram)\n", 138 | " temp1['human_truncate_ngram_{}_score'.format(_ngram)] = ngram_score / len(human_generate_tokens)\n", 139 | " temp1['human_truncate_ngram_{}_count'.format(_ngram)] = overlap_count\n", 140 | "\n", 141 | " ngram_score, ngram_dict, overlap_count = get_ngram_info(gpt_tokens, gpt_generate_tokens, _ngram)\n", 142 | " temp2['gpt_truncate_ngram_{}_score'.format(_ngram)] = ngram_score / len(gpt_generate_tokens)\n", 143 | " temp2['gpt_truncate_ngram_{}_count'.format(_ngram)] = overlap_count\n", 144 | " temp.append({'human':temp1, 'machine':temp2})\n", 145 | "\n", 146 | " ngram_overlap_count_wq.append(temp)" 147 | ] 148 | }, 149 | { 150 | "cell_type": "code", 151 | "execution_count": null, 152 | "metadata": {}, 153 | "outputs": [], 154 | "source": [ 155 | "def N_gram_detector(ngram_n_ratio):\n", 156 | " score = 0\n", 157 | " non_zero = []\n", 158 | " \n", 159 | " for idx, key in enumerate(ngram_n_ratio):\n", 160 | " if idx in range(3) and 'score' in key or 'ratio' in key:\n", 161 | " score += 0. * ngram_n_ratio[ key ]\n", 162 | " continue\n", 163 | " if 'score' in key or 'ratio' in key:\n", 164 | " score += (idx+1) * np.log((idx+1)) * ngram_n_ratio[ key ]\n", 165 | " if ngram_n_ratio[ key ] != 0:\n", 166 | " non_zero.append( idx+1 ) \n", 167 | " return score/ (sum( non_zero ) + 1e-8)\n", 168 | "\n", 169 | "human_scores = []\n", 170 | "gpt_scores = []\n", 171 | "\n", 172 | "for instance in ngram_overlap_count_wq:\n", 173 | " human_score = []\n", 174 | " gpt_score = []\n", 175 | "\n", 176 | " for i in range(len(instance)):\n", 177 | " human_score.append( N_gram_detector(instance[i]['human'] ) )\n", 178 | " gpt_score.append( N_gram_detector(instance[i]['machine'] ) )\n", 179 | "\n", 180 | " human_scores.append( sum(human_score) )\n", 181 | " gpt_scores.append( sum(gpt_score) )" 182 | ] 183 | }, 184 | { 185 | "cell_type": "code", 186 | "execution_count": null, 187 | "metadata": {}, 188 | "outputs": [], 189 | "source": [ 190 | "# plot and give different colors\n", 191 | "import matplotlib.pyplot as plt\n", 192 | "plt.figure(figsize=(10, 5))\n", 193 | "plt.plot(human_scores, label='human')\n", 194 | "plt.plot(gpt_scores, label='gpt')\n", 195 | "plt.legend()\n", 196 | "plt.show()" 197 | ] 198 | }, 199 | { 200 | "cell_type": "code", 201 | "execution_count": null, 202 | "metadata": {}, 203 | "outputs": [], 204 | "source": [ 205 | "plot_roc_curve(human_scores, gpt_scores)" 206 | ] 207 | }, 208 | { 209 | "cell_type": "code", 210 | "execution_count": null, 211 | "metadata": {}, 212 | "outputs": [], 213 | "source": [ 214 | "###### openai detector ######\n", 215 | "with open( f'results/openai_detect_gpt4.jsonl', 'r') as f:\n", 216 | " openai_detect_gpt = [json.loads(x) for x in f.read().strip().split(\"\\n\")]\n", 217 | "len(openai_detect_gpt)" 218 | ] 219 | }, 220 | { 221 | "cell_type": "code", 222 | "execution_count": null, 223 | "metadata": {}, 224 | "outputs": [], 225 | "source": [ 226 | "# plot and give different colors\n", 227 | "import matplotlib.pyplot as plt\n", 228 | "human_scores2 = [x['gold_gen_prob'] for x in openai_detect_gpt]\n", 229 | "gpt_scores2 = [x['gen_completion_prob'] for x in openai_detect_gpt]\n", 230 | "\n", 231 | "plt.figure(figsize=(10, 5))\n", 232 | "plt.plot(human_scores2, label='human')\n", 233 | "plt.plot(gpt_scores2, label='gpt')\n", 234 | "plt.legend()\n", 235 | "plt.show()" 236 | ] 237 | }, 238 | { 239 | "cell_type": "code", 240 | "execution_count": null, 241 | "metadata": {}, 242 | "outputs": [], 243 | "source": [ 244 | "plot_roc_curve(human_scores2, gpt_scores2)" 245 | ] 246 | }, 247 | { 248 | "cell_type": "code", 249 | "execution_count": null, 250 | "metadata": {}, 251 | "outputs": [], 252 | "source": [ 253 | "###### gptzero detector ######\n", 254 | "with open( f'results/gptzero_gpt4.jsonl', 'r') as f:\n", 255 | " gptzero_detect_gpt = [json.loads(x) for x in f.read().strip().split(\"\\n\")]\n", 256 | "len(gptzero_detect_gpt)" 257 | ] 258 | }, 259 | { 260 | "cell_type": "code", 261 | "execution_count": null, 262 | "metadata": {}, 263 | "outputs": [], 264 | "source": [ 265 | "# plot and give different colors\n", 266 | "import matplotlib.pyplot as plt\n", 267 | "human_scores2 = [x['gold_gen_prob'] for x in gptzero_detect_gpt]\n", 268 | "gpt_scores2 = [x['gen_completion_prob'] for x in gptzero_detect_gpt]\n", 269 | "\n", 270 | "plt.figure(figsize=(10, 5))\n", 271 | "plt.plot(human_scores2, label='human')\n", 272 | "plt.plot(gpt_scores2, label='gpt')\n", 273 | "plt.legend()\n", 274 | "plt.show()" 275 | ] 276 | }, 277 | { 278 | "cell_type": "code", 279 | "execution_count": null, 280 | "metadata": {}, 281 | "outputs": [], 282 | "source": [ 283 | "plot_roc_curve(human_scores2, gpt_scores2)" 284 | ] 285 | } 286 | ], 287 | "metadata": { 288 | "kernelspec": { 289 | "display_name": "vpt", 290 | "language": "python", 291 | "name": "python3" 292 | }, 293 | "language_info": { 294 | "codemirror_mode": { 295 | "name": "ipython", 296 | "version": 3 297 | }, 298 | "file_extension": ".py", 299 | "mimetype": "text/x-python", 300 | "name": "python", 301 | "nbconvert_exporter": "python", 302 | "pygments_lexer": "ipython3", 303 | "version": "3.8.15" 304 | }, 305 | "orig_nbformat": 4, 306 | "vscode": { 307 | "interpreter": { 308 | "hash": "015bfb409bf441c0a66e03b2de1c9b891435fcbf36ed1d1e9d7c8167e73e6b62" 309 | } 310 | } 311 | }, 312 | "nbformat": 4, 313 | "nbformat_minor": 2 314 | } 315 | -------------------------------------------------------------------------------- /DNA-GPT-dist.py: -------------------------------------------------------------------------------- 1 | ## Official release for paper DNA-GPT: https://arxiv.org/abs/2305.17359 2 | ## Authors: Xianjun Yang (UCSB), Wei Cheng (NEC Labs America) 3 | ## MIT License 4 | # Copyright (c) 2023 Xianjun (Nolan) Yang, NEC Labs America 5 | 6 | 7 | ####### install the following packages before using ######### 8 | # %pip install openai 9 | # %pip install spacy 10 | # %pip install nltk 11 | # %pip install gradio 12 | # %pip install rouge_score 13 | # !python3 -m spacy download en_core_web_sm 14 | 15 | 16 | import gradio as gr 17 | import ssl 18 | import nltk 19 | import os 20 | import openai 21 | import re 22 | from gradio import components 23 | 24 | import six 25 | import spacy 26 | from nltk.stem.porter import PorterStemmer 27 | from rouge_score.rouge_scorer import _create_ngrams, _score_ngrams 28 | import six 29 | import numpy as np 30 | 31 | openai_key = "" ### OpenAI AIP key for access to the re-generation service, users are suggested to delete the key after usage to avoid potential leakage of key 32 | openai.api_key = openai_key 33 | temperature = 0.7 ### This parameter controls text quality of chatgpt, by default it was set to 0.7 in the website version of ChatGPT. 34 | max_new_tokens = 300 ### maximum length of generated texts from chatgpt 35 | regen_number = 30 ### for faster response, users can set this value to smaller ones, such as 20 or 10, which will degenerate performance a little bit 36 | question = "" 37 | truncate_ratio = 0.5 38 | threshold = 0.00025 ### for conservative decision, users can set this value to larger values, such as 0.0003 39 | 40 | 41 | 42 | PorterStemmer = PorterStemmer() 43 | nlp = spacy.load('en_core_web_sm') 44 | stopwords = nlp.Defaults.stop_words 45 | 46 | ssl._create_default_https_context = ssl._create_unverified_context 47 | nltk.download('punkt') 48 | 49 | 50 | def tokenize(text, stemmer, stopwords=[]): 51 | """Tokenize input text into a list of tokens. 52 | 53 | This approach aims to replicate the approach taken by Chin-Yew Lin in 54 | the original ROUGE implementation. 55 | 56 | Args: 57 | text: A text blob to tokenize. 58 | stemmer: An optional stemmer. 59 | 60 | Returns: 61 | A list of string tokens extracted from input text. 62 | """ 63 | 64 | # Convert everything to lowercase. 65 | text = text.lower() 66 | # Replace any non-alpha-numeric characters with spaces. 67 | text = re.sub(r"[^a-z0-9]+", " ", six.ensure_str(text)) 68 | 69 | tokens = re.split(r"\s+", text) 70 | if stemmer: 71 | # Only stem words more than 3 characters long. 72 | tokens = [stemmer.stem(x) if len( 73 | x) > 3 else x for x in tokens if x not in stopwords] 74 | 75 | # One final check to drop any empty or invalid tokens. 76 | tokens = [x for x in tokens if re.match(r"^[a-z0-9]+$", six.ensure_str(x))] 77 | 78 | return tokens 79 | 80 | 81 | def get_score_ngrams(target_ngrams, prediction_ngrams): 82 | """calcualte overlap ratio of N-Grams of two documents. 83 | 84 | Args: 85 | target_ngrams: N-Grams set of targe document. 86 | prediction_ngrams: N-Grams set of reference document. 87 | 88 | Returns: 89 | ratio of intersection of N-Grams of two documents, together with dict list of [overlap N-grams: count] 90 | """ 91 | intersection_ngrams_count = 0 92 | ngram_dict = {} 93 | for ngram in six.iterkeys(target_ngrams): 94 | intersection_ngrams_count += min(target_ngrams[ngram], 95 | prediction_ngrams[ngram]) 96 | ngram_dict[ngram] = min(target_ngrams[ngram], prediction_ngrams[ngram]) 97 | target_ngrams_count = sum(target_ngrams.values()) # prediction_ngrams 98 | return intersection_ngrams_count / max(target_ngrams_count, 1), ngram_dict 99 | 100 | 101 | def get_ngram_info(article_tokens, summary_tokens, _ngram): 102 | """calculate N-Gram overlap score of two documents 103 | It use _create_ngrams in rouge_score.rouge_scorer to get N-Grams of two docuemnts, then revoke get_score_ngrams method to calucate overlap score 104 | Args: 105 | article_tokens: tokens of one document. 106 | summary_tokens: tokens of another document. 107 | 108 | Returns: 109 | ratio of intersection of N-Grams of two documents, together with dict list of [overlap N-grams: count], total overlap n-gram count 110 | """ 111 | article_ngram = _create_ngrams( article_tokens , _ngram) 112 | summary_ngram = _create_ngrams( summary_tokens , _ngram) 113 | ngram_score, ngram_dict = get_score_ngrams( article_ngram, summary_ngram) 114 | return ngram_score, ngram_dict, sum( ngram_dict.values() ) 115 | 116 | 117 | def N_gram_detector(ngram_n_ratio): 118 | """calculate N-Gram overlap score from N=3 to N=25 119 | Args: 120 | ngram_n_ratio: a list of ratio of N-Gram overlap scores, N is from 1 to 25. 121 | 122 | Returns: 123 | N-Gram overlap score from N=3 to N=25 with decay weighting n*log(n) 124 | """ 125 | score = 0 126 | non_zero = [] 127 | 128 | for idx, key in enumerate(ngram_n_ratio): 129 | if idx in range(3) and 'score' in key or 'ratio' in key: 130 | score += 0. * ngram_n_ratio[key] 131 | continue 132 | if 'score' in key or 'ratio' in key: 133 | score += (idx+1) * np.log((idx+1)) * ngram_n_ratio[key] 134 | if ngram_n_ratio[key] != 0: 135 | non_zero.append(idx+1) 136 | return score / (sum(non_zero) + 1e-8) 137 | 138 | 139 | def N_gram_detector_ngram(ngram_n_ratio): 140 | """sort the dictionary of N-gram key according to their counts 141 | Args: 142 | ngram_n_ratio: a list of ratio of N-Gram overlap scores, N is from 1 to 25. 143 | 144 | Returns: 145 | sorted dictionary of N-gram [key,value] according to their value counts 146 | """ 147 | ngram = {} 148 | for idx, key in enumerate(ngram_n_ratio): 149 | if idx in range(3) and 'score' in key or 'ratio' in key: 150 | continue 151 | if 'ngramdict' in key: 152 | dict_ngram = ngram_n_ratio[key] 153 | 154 | for key_, value_ in dict_ngram.items(): 155 | ngram[key_] = idx 156 | sorted_dict = dict( 157 | sorted(ngram.items(), key=lambda x: x[1], reverse=True)) 158 | return sorted_dict 159 | 160 | 161 | def tokenize_nltk(text): 162 | """tokenize text using word tokenizer 163 | Args: 164 | text: input text 165 | 166 | Returns: 167 | tokens of words 168 | """ 169 | tokens = nltk.word_tokenize(text) 170 | return tokens 171 | 172 | 173 | def get_ngrams(tokens, n): 174 | ngrams = [] 175 | for i in range(len(tokens) - n + 1): 176 | ngram = ' '.join(tokens[i:i+n]) 177 | ngrams.append(ngram) 178 | return ngrams 179 | 180 | 181 | def getOverlapedTokens(text1, text2): 182 | """get overlap of word tokens of two documents 183 | Args: 184 | text1: input text1 185 | text2: input text2 186 | 187 | Returns: 188 | dict of [token:count] of overlape words in two texts 189 | """ 190 | overlap_dict = {} 191 | tokens1 = tokenize_nltk(text1.lower()) 192 | tokens2 = tokenize_nltk(text2.lower()) 193 | for n in range(3, 25): 194 | ngrams1 = get_ngrams(tokens1, n) 195 | ngrams2 = get_ngrams(tokens2, n) 196 | ngrams_set1 = set(ngrams1) 197 | ngrams_set2 = set(ngrams2) 198 | overlap = ngrams_set1.intersection(ngrams_set2) 199 | for element in overlap: 200 | overlap_dict[element] = n 201 | return overlap_dict 202 | 203 | 204 | def get_html(text, dictionary): 205 | """prepare html format content 206 | Args: 207 | text: input text that is reference 208 | dictionary: overlap dict, overlap is calculated with reference text that is generated by 209 | ChatGPT with prefix input that is the first half of the text. 210 | 211 | Returns: 212 | visualize html to output the results for explain why it is judged as generated by chatgpt 213 | """ 214 | positions = [] 215 | 216 | len_text = len(text) 217 | flag_vec = np.zeros(len_text) 218 | 219 | str_html = "" 220 | for key in dictionary: 221 | start = 0 222 | while True: 223 | index = text.find(key, start) 224 | if index == -1: 225 | break 226 | positions.append((key, index)) 227 | flag_vec[index:index+len(key)] = 1 228 | start = index + len(key) 229 | status = flag_vec[0] 230 | # print(sum(flag_vec)) 231 | for i in range(len_text): 232 | if i == 0: 233 | if status == 0: 234 | str_html = str_html + ""+text[i] 235 | else: 236 | str_html = str_html + ""+text[i] 237 | continue 238 | if flag_vec[i] == status: 239 | str_html = str_html + text[i] 240 | else: 241 | str_html = str_html + "" 242 | if flag_vec[i] == 0: 243 | str_html = str_html + ""+text[i] 244 | else: 245 | str_html = str_html + ""+text[i] 246 | status = flag_vec[i] 247 | if i == len_text - 1: 248 | str_html = str_html + "" 249 | return str_html 250 | 251 | 252 | def truncate_string_by_words(string, max_words): 253 | words = string.split() 254 | if len(words) <= max_words: 255 | return string 256 | else: 257 | truncated_words = words[:max_words] 258 | return ' '.join(truncated_words) 259 | 260 | 261 | def detection(text, option): 262 | """detect if give text is generated by chatgpt 263 | Args: 264 | text: input text 265 | option: target model to be checked: gpt-3.5-turbo or gpt-4-0314. 266 | 267 | Returns: 268 | decision: boolean value that indicate if the text input is generated by chatgpt with version as option 269 | most_matched_generatedtext[0]: most matched re-generated text by chatgpt using half prefix of the input text as the prompt 270 | """ 271 | max_words = 350 #### can be adjusted 272 | if option == "GPT-3.5": 273 | model_name = "gpt-3.5-turbo-instruct" 274 | else: 275 | model_name = "gpt-4-0314" 276 | text = truncate_string_by_words(text, max_words) 277 | ngram_overlap_count =[] 278 | question = "continues the passage from the current text within in total around 300 words:" 279 | input_text = text 280 | human_prefix_prompt = input_text[:int(truncate_ratio*len(input_text))] 281 | # human_gen_text = openai.ChatCompletion.create(model=model_name, 282 | # messages=[{"role": "system", "content": "You are a helpful assistant that continues the passage from the sentences provided."}, 283 | # {"role": "user", 284 | # "content": question}, 285 | # {"role": "assistant", 286 | # "content": human_prefix_prompt}, 287 | # ], 288 | # temperature=temperature, 289 | # max_tokens=max_new_tokens, 290 | # n=regen_number) 291 | from openai import OpenAI 292 | client = OpenAI(api_key=openai.api_key) 293 | completion = client.completions.create( 294 | model="gpt-3.5-turbo-instruct", 295 | prompt=human_prefix_prompt, 296 | max_tokens=max_new_tokens, 297 | temperature=temperature 298 | ) 299 | 300 | input_remaining = input_text[int(truncate_ratio*len(input_text)):] 301 | input_remaining_tokens = tokenize(input_remaining, stemmer=PorterStemmer) 302 | 303 | temp = [] 304 | mx = 0 305 | mx_v = 0 306 | for i in range(regen_number): # len(human_half) 307 | temp1 = {} 308 | gen_text = human_gen_text['choices'][i]['message']['content'] 309 | 310 | ###### optional ####### 311 | gen_text_ = truncate_string_by_words(gen_text, max_words-150) 312 | 313 | gpt_generate_tokens = tokenize( 314 | gen_text_, stemmer=PorterStemmer) 315 | if len(input_remaining_tokens) == 0 or len(gpt_generate_tokens) == 0: 316 | continue 317 | 318 | for _ngram in range(1, 25): 319 | ngram_score, ngram_dict, overlap_count = get_ngram_info( 320 | input_remaining_tokens, gpt_generate_tokens, _ngram) 321 | temp1['human_truncate_ngram_{}_score'.format( 322 | _ngram)] = ngram_score / len(gpt_generate_tokens) 323 | temp1['human_truncate_ngram_{}_ngramdict'.format( 324 | _ngram)] = ngram_dict 325 | temp1['human_truncate_ngram_{}_count'.format( 326 | _ngram)] = overlap_count 327 | 328 | if overlap_count > 0: 329 | if _ngram > mx_v: 330 | mx_v = _ngram 331 | mx = i 332 | 333 | temp.append({'machine': temp1}) 334 | 335 | ngram_overlap_count.append(temp) 336 | gpt_scores = [] 337 | 338 | top_overlap_ngram = [] 339 | max_ind_list = [] 340 | most_matched_generatedtext = [] 341 | for instance in ngram_overlap_count: 342 | human_score = [] 343 | gpt_score = [] 344 | 345 | for i in range(len(instance)): 346 | # human_score.append(N_gram_detector(instance[i]['human'])) 347 | gpt_score.append(N_gram_detector(instance[i]['machine'])) 348 | top_overlap_ngram.append( 349 | N_gram_detector_ngram(instance[i]['machine'])) 350 | 351 | # human_scores.append(sum(human_score)) 352 | gpt_scores.append(np.mean(gpt_score)) 353 | max_value = max(gpt_score) 354 | # print(len(gpt_score)) 355 | max_index = mx # gpt_score.index(max_value) 356 | max_ind_list.append(max_index) 357 | most_matched_generatedtext.append( 358 | human_gen_text['choices'][max_index]['message']['content']) 359 | print(gpt_scores[0]) 360 | 361 | if gpt_scores[0] > threshold: 362 | decision = True 363 | else: 364 | decision = False 365 | return decision, most_matched_generatedtext[0] 366 | 367 | 368 | def generate_html(key, option, text ): 369 | """prepare html format content for decision of DNA-GPT detection tool 370 | Args: 371 | key: key for access openai API, visit https://platform.openai.com/account/api-keys to check keys 372 | option: which version of OpenAI chatgpt to check: chatgpt 3.5 or 4.0 373 | text: input text 374 | 375 | Returns: 376 | visualize html to output the results 377 | """ 378 | openai_key = key 379 | openai.api_key = openai_key 380 | label = "Detection Output: " 381 | res, max_over_lap_generatedtext = detection(text, option) 382 | text = text.lower() 383 | max_over_lap_generatedtext = max_over_lap_generatedtext.lower() 384 | dictionary = getOverlapedTokens(text, max_over_lap_generatedtext) 385 | html_ = get_html(text, dictionary) 386 | html_gen = get_html(max_over_lap_generatedtext, dictionary) 387 | if res: 388 | html_text = f"

{label}:

It is generated by ChatGPT3.5!

Evidence:

Your Input:

" + \ 389 | html_+"





GPT generated text by continue writing uisng half-input prompt:

......."+html_gen 390 | else: 391 | html_text = f"

{label}:

It is generated by human!

" 392 | return html_text 393 | 394 | 395 | def main(): 396 | # input_key = gr.inputs.Textbox( 397 | # label="Enter your key for OpenAI access:", lines=1) 398 | input_key = components.Textbox( 399 | type="text", label="Enter your key for OpenAI access:") 400 | 401 | input_text = components.Textbox( 402 | type="text", label="Enter your text (Ideally between 180 and 300 words)") 403 | 404 | # input_text = gr.inputs.Textbox( 405 | # label="Enter your text (Ideally between 180 and 300 words)", lines=20) 406 | 407 | copyright = "© 2023 NEC Labs America | Author: Xianjun Yang, Wei Cheng" 408 | 409 | # sigle choice option 410 | radio_options = ["GPT-3.5", "GPT-4"] 411 | radio = components.Radio( 412 | radio_options, label="Please choose one version to check:") 413 | 414 | input_group = [input_key, radio, input_text] 415 | 416 | greet_output = gr.outputs.Textbox(label="Detection Output:") 417 | 418 | interface = gr.Interface(fn=generate_html, inputs=input_group, title="DNA-GPT Demo (NEC Labs America)", 419 | outputs=gr.outputs.HTML(), description=copyright) 420 | interface.launch(share=True) 421 | 422 | if __name__ == "__main__": 423 | main() 424 | 425 | --------------------------------------------------------------------------------