\n")
305 | #html_file.write("" + filename + "
\n")
306 | html_file.write("\n")
307 |
308 | # Write the overall summary section
309 | html_file.write("Overall Summary
\n")
310 | overall_summary_file = open(basename + ".overall_summary.txt", "r")
311 | overall_summary_content = overall_summary_file.read()
312 | html_file.write("" + overall_summary_content + "
\n")
313 | overall_summary_file.close()
314 |
315 | # Write the subsection summary section
316 | html_file.write("Subsection Summary
\n")
317 | subsection_summary_files = glob.glob(basename + ".*.summary.txt")
318 | subsection_summary_files.sort()
319 | for subsection_summary_file in subsection_summary_files:
320 | subsection_summary_file_handle = open(subsection_summary_file, "r")
321 | subsection_summary_content = subsection_summary_file_handle.read()
322 | html_file.write("" + subsection_summary_content + "
\n")
323 | subsection_summary_file_handle.close()
324 |
325 | # Write the HTML footer
326 | html_file.write("Original URL\n")
327 | html_file.write("\n")
328 | html_file.write("\n")
329 | html_file.write("\n")
330 |
331 | # Print a message indicating that the HTML file was created
332 | print("Created HTML file: " + basename + ".summary.html")
333 | # Close the HTML file
334 | html_file.close()
335 |
336 | def download_html(url):
337 | # Strip any trailing /'s from the end of the URL
338 | stripped_url = url.rstrip("/")
339 |
340 | # Get the base name of the URL
341 | base_name = stripped_url.split("/")[-1]
342 |
343 | # Download the HTML file
344 | if base_name.endswith(".pdf"):
345 | html_path = "/tmp/" + base_name
346 | else:
347 | html_path = "/tmp/" + base_name + ".html"
348 | print("HTML path: " + html_path)
349 | print("URL: " + url)
350 | os.system("curl -s -o " + html_path + " " + url)
351 |
352 | return html_path
353 |
354 | if __name__ == '__main__':
355 |
356 | model_engine = "text-davinci-003"
357 | max_tokens = 3000
358 | doctype=""
359 | # get the base filename of the first argument without the extension
360 | base_name = os.path.splitext(sys.argv[1])[0]
361 |
362 | # If the command line argument starts with http, use curl to download it to an HTML file
363 | if sys.argv[1].startswith("http"):
364 | # Get the URL from the command line arguments
365 | url = sys.argv[1]
366 | doctype="article"
367 |
368 | # Strip any query parameters from the URL
369 | url = url.split("?")[0]
370 |
371 | # Download the HTML file
372 | html_path = download_html(url)
373 | print(html_path)
374 |
375 | # Strip any trailing /'s from the end of the URL
376 | url = url.rstrip("/")
377 | # Set the base_name to a /tmp file containing the last part of the URL between /'s
378 | base_name = "/tmp/" + url.split("/")[-1]
379 | print(base_name)
380 |
381 | if sys.argv[1].endswith(".pdf"):
382 | text = extract_text_from_pdf(html_path)
383 | else:
384 | # Extract the text from the HTML file
385 | text = extract_text_from_html(html_path)
386 | # If the command line argument references a pdf file
387 | elif sys.argv[1].endswith(".pdf"):
388 | # Get the PDF file path from the command line arguments
389 | pdf_path = sys.argv[1]
390 | doctype="paper"
391 |
392 | # Extract the text from the PDF file
393 | text = extract_text_from_pdf(pdf_path)
394 | elif sys.argv[1].endswith(".html") or sys.argv[1].endswith(".htm"):
395 |
396 | # Get the HTML file path from the command line arguments
397 | html_path = sys.argv[1]
398 | doctype="article"
399 |
400 | # Extract the text from the HTML file
401 | text = extract_text_from_html(html_path)
402 | else:
403 | # Get the text file path from the command line arguments
404 | text_path = sys.argv[1]
405 |
406 | # Read the text file
407 | with open(text_path, "r") as text_file:
408 | text = text_file.read()
409 |
410 | # Checking if output language is set: if not, leave off any language instructions from the prompt
411 | try:
412 | arg = sys.argv[2]
413 | output_language_prompt = " Please use "+sys.argv[2]+" language for the output."
414 | except IndexError:
415 | output_language_prompt = ""
416 |
417 | # Split the text into sections
418 | sections = split_into_sections(text)
419 |
420 | # encode the text as a sequence of tokens
421 | #enc = tiktoken.get_encoding("gpt2")
422 | enc = GPT2TokenizerFast.from_pretrained("gpt2")
423 |
424 | tokens = enc.encode(text)
425 |
426 | # Get the base name of the output file
427 | #base_name, _ = os.path.splitext(pdf_path)
428 |
429 | # Write the extracted text to the output file
430 | with open(base_name + ".full.txt", 'w', encoding='utf-8') as f:
431 | f.write(text)
432 |
433 | print(f"Text extracted from {sys.argv[1]} and written to {base_name}.full.txt")
434 |
435 |
436 | print(f"Total token count: {len(tokens)}")
437 |
438 | # Write each section to a separate text file
439 | for header, content in sections:
440 | print("Header: ", header)
441 | # Split the section into subsections if necessary
442 | subsections = split_section_into_subsections(header, content, enc)
443 |
444 | # Combine adjacent tuples with less than 1000 tokens until they exceed 1000 tokens
445 | combined_subsections = combine_subsections(subsections)
446 |
447 | # Initialize the counter for numbering sequential identical subheaders
448 | subheader_count = 1
449 |
450 | # Process each combined subsection
451 | for subheader, subcontent in combined_subsections:
452 | # Update the subheader if there are multiple sequential identical subheaders
453 | if subheader_count > 1:
454 | subheader += f"-part{subheader_count}"
455 | subheader_count += 1
456 |
457 | # Use tiktoken to encode the subsection content as a sequence of tokens
458 | subcontent_tokens = enc.encode(subcontent)
459 |
460 |
461 | # Get the name of the output file
462 | #print("Subheader: ",subheader)
463 | section_name = re.sub(r'[^a-zA-Z0-9]', '', subheader.replace('/','-'))
464 |
465 | #print("Section name: ",section_name)
466 | output_path = f"{base_name}.{section_name}.full.txt"
467 |
468 | if (len(subcontent) == 0):
469 | subheader_count = subheader_count - 1
470 | else:
471 | # Write the content to the output file
472 | with open(output_path, 'w', encoding='utf-8') as f:
473 | f.write(subcontent)
474 | print(f"{subheader} ({len(subcontent)} characters, {len(subcontent_tokens)} tokens) written to {output_path}")
475 | # Get the name of the summary file
476 | summary_path = f"{base_name}.{section_name}.summary.txt"
477 | # If the summary file does not exist, generate a summary
478 | if os.path.exists(summary_path):
479 | print(f"Summary already exists at {summary_path}")
480 | else:
481 | # Set the prompt for the summary
482 | prompt = f"Please provide a detailed summary of the following section, but if the section content is mostly website context/description, just return 'Section has no content':\n{subcontent}\nPlease provide a detailed summary of the section above. If the section content is mostly website context/description, just return 'Section has no content'.{output_language_prompt}"
483 | # Generate a summary for the subsection
484 | summary = generate_summary(subcontent, prompt, model_engine, max_tokens)
485 | # Write the summary to a summary file
486 | with open(summary_path, 'w', encoding='utf-8') as f:
487 | f.write(summary)
488 | print(f"Summary written to {summary_path}")
489 |
490 | # If there is more than one summary file matching {base_name}.*{section_number}.summary.txt, generate a combined section summary
491 | section_number = section_name.split('.')[0]
492 | if len(glob.glob(f"{base_name}.{section_number}.*.summary.txt")) < 1:
493 | print(f"No summary files found for section {section_number}")
494 | elif len(glob.glob(f"{base_name}.{section_number}.*.summary.txt")) == 1:
495 |
496 | print(f"Only one summary file found for section {section_number}, promoting it to section summary")
497 | # Get the path of the summary file
498 | #print(glob.glob(f"{base_name}.{section_number}.*.summary.txt"))
499 | summary_path = glob.glob(f"{base_name}.{section_number}.*.summary.txt")[0]
500 | # Get the path of the section summary file
501 | section_summary_path = f"{base_name}.{section_name}.section_summary.txt"
502 | # Read the summary file and write it to the section summary file
503 | with open(summary_path, 'r', encoding='utf-8') as f:
504 | summary = f.read()
505 | with open(section_summary_path, 'w', encoding='utf-8') as f:
506 | f.write(summary)
507 |
508 | print(f"Summary promoted to section summary at {section_summary_path}")
509 |
510 | else:
511 | # Read in the section summaries
512 | summaries = []
513 |
514 | summary_pattern = f"{base_name}.*{section_number}.summary.txt"
515 | print(f"Reading summary from {summary_pattern}")
516 | summary_paths = glob.glob(summary_pattern)
517 | summary_paths.sort(key=os.path.getmtime) # sort file names by modification time, oldest first
518 | for summary_path in summary_paths:
519 | print(f"Reading summary from {summary_path}")
520 | with open(summary_path, 'r', encoding='utf-8') as f:
521 | summaries.append(f.read())
522 | # Concatenate the summaries into a single string
523 | subcontent = "\n\n".join(summaries)
524 | # Tokenize the concatenated summaries
525 | subcontent_tokens = enc.encode(subcontent)
526 | print(f"Concatenated {len(summaries)} summaries into a single summary with {len(subcontent)} characters and {len(subcontent_tokens)} tokens")
527 | if len(subcontent_tokens) == 0:
528 | summary_pattern = f"{base_name}.*.summary.txt"
529 | print(f"Reading summary from {summary_pattern}")
530 | summary_paths = glob.glob(summary_pattern)
531 | summary_paths.sort(key=os.path.getmtime) # sort file names by modification time, oldest first
532 |
533 | summaries = []
534 | subcontent_tokens = []
535 | for summary_path in summary_paths:
536 | print(f"Reading summary from {summary_path}")
537 | with open(summary_path, 'r', encoding='utf-8') as f:
538 | summary = f.read()
539 | summary_tokens = enc.encode(summary)
540 | if len(subcontent_tokens) + len(summary_tokens) > max_tokens:
541 | break
542 | summaries.append(summary)
543 | subcontent_tokens += summary_tokens
544 | # Concatenate the summaries into a single string
545 | subcontent = "\n\n".join(summaries)
546 | # Tokenize the concatenated summaries
547 | subcontent_tokens = enc.encode(subcontent)
548 | print(f"Concatenated {len(summaries)} out of {len(summary_paths)} section summaries into a single summary with {len(subcontent)} characters and {len(subcontent_tokens)} tokens")
549 |
550 | # Set the prompt for the overall section summary
551 | prompt = f"Please provide a detailed summary of the following sections:\n{subcontent}\nPlease provide a detailed summary of the sections above.{output_language_prompt}"
552 | # Get the path of the overall section summary file
553 | section_summary_path = f"{base_name}.{section_number}.section_summary.txt"
554 | # If the overall section summary file does not exist, generate a summary
555 | if os.path.exists(section_summary_path):
556 | print(f"Overall section summary already exists at {section_summary_path}")
557 | else:
558 | # Generate the overall section summary
559 | section_summary = generate_summary(content, prompt, model_engine, max_tokens)
560 | # Write the overall section summary to a file
561 | with open(section_summary_path, 'w', encoding='utf-8') as f:
562 | f.write(section_summary)
563 | print(f"Overall section summary written to {section_summary_path}")
564 |
565 |
566 |
567 |
568 | # Check if the overall summary file already exists
569 | overall_summary_path = f"{base_name}.overall_summary.txt"
570 | if os.path.exists(overall_summary_path):
571 | print(f"Overall summary already exists at {overall_summary_path}")
572 | else:
573 | # Read in the abstract, if it exists
574 | try:
575 | abstract_filename=glob.glob(f"{base_name}.Title-Abstract*.full.txt")[0]
576 | with open(f"{abstract_filename}", 'r', encoding='utf-8') as f:
577 | abstract = f.read()
578 | except IndexError:
579 | print(f"No abstract found for {base_name}")
580 | abstract = ""
581 | # Read in the section summaries
582 | summaries = []
583 | summary_pattern = f"{base_name}.*.section_summary.txt"
584 | for summary_path in glob.glob(summary_pattern):
585 | with open(summary_path, 'r', encoding='utf-8') as f:
586 | summaries.append(f.read())
587 | # Concatenate the abstract and summaries into a single string
588 | subcontent = abstract + "\n\n" + "\n\n".join(summaries)
589 | # Tokenize the concatenated abstract and summaries
590 | subcontent_tokens = enc.encode(subcontent)
591 | for summary in summaries:
592 | summary_tokens = enc.encode(summary)
593 | if len(subcontent_tokens) + len(summary_tokens) > max_tokens:
594 | print(f"Exceeded {max_tokens} tokens, stopping concatenation of summaries")
595 | break
596 | subcontent += summary + "\n\n"
597 | subcontent_tokens += summary_tokens
598 | print(f"Concatenated {len(summaries)} summaries into a single summary with {len(subcontent)} characters and {len(subcontent_tokens)} tokens")
599 | if len(subcontent_tokens) < 500:
600 | print(f"Concatenated subsection summaries have less than 500 tokens, reading in all summaries")
601 | summary_pattern = f"{base_name}.*.summary.txt"
602 | for summary_path in glob.glob(summary_pattern):
603 | with open(summary_path, 'r', encoding='utf-8') as f:
604 | summaries.append(f.read())
605 | for summary in summaries:
606 | summary_tokens = enc.encode(summary)
607 | if len(subcontent_tokens) + len(summary_tokens) > max_tokens:
608 | print(f"Exceeded {max_tokens} tokens, stopping concatenation of summaries")
609 | break
610 | subcontent += summary + "\n\n"
611 | subcontent_tokens += summary_tokens
612 |
613 |
614 | # Set the prompt for the overall summary
615 | prompt = f"Please provide a detailed summary of the following {doctype}, based on its abstract and summaries of each section:\n{subcontent}\nPlease provide a detailed summary of the {doctype} described above, based on the provided abstract/introduction and summaries of each section.{output_language_prompt}"
616 | # Generate the overall summary
617 | overall_summary = generate_summary(subcontent, prompt, model_engine, max_tokens)
618 | # Append a newline to the overall summary
619 | overall_summary += "\n"
620 | # Write the overall summary to a file
621 | with open(overall_summary_path, 'w', encoding='utf-8') as f:
622 | f.write(overall_summary)
623 | print(f"Overall summary written to {overall_summary_path}")
624 |
625 | # Call create_html_file() to create an HTML file with the overall summary
626 | create_html_file(base_name, url)
627 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # gpt-summarizer
2 | ### Extract text, summarize each section w/ GPT, and provide a summarized outline of a paper/article
3 |
4 | This script extracts text from a given file or URL and splits it into sections. It then uses OpenAI's tokenizer to encode the text as a sequence of tokens. It writes the extracted text to an output file and writes each section to a separate text file. It also generates a summary for each subsection and writes the summary to a summary file. If there are multiple summary files for a section, it generates a combined section summary.
5 |
6 | The script expects a PDF or HTML file path, or an HTML URL, to be passed as a command line argument. It extracts the text from the PDF file, splits the text into sections, and uses the tiktoken.get_encoding function to encode the text as a sequence of tokens using the "gpt2" encoding. It writes the extracted text to an output file using the base_name of the file and the .txt extension.
7 |
8 | It processes each section by first using the split_section_into_subsections function to split the section into subsections based on HTML section headings or numbered section headings. If necessary, it further splits any subsections into paragraphs and recombine adjacent ones until they exceed 1000 tokens. It processes each resulting section/part with InstructGPT (text-davinci-003) to generate a summary, and writes each section summary to a summary file. If there are multiple summary files for a section, it generates a combined section summary by concatenating the summaries of the individual subsections.
9 |
10 | It then performs one final round of summarization across all the lower-level summaries, to produce an overall summary of the paper/article.
11 |
12 | The intended use is that both the overall summary and the subsection summaries (in order) are worth reading to determine whether to spend the time reading the entire article/paper, or specific sections of it.
13 |
14 | You can also specify an optional second positional argument to have the summaries generated in the specified language: to do so it adds `" Please use "+sys.argv[2]+" language for the output."` to the prompt if the argument is present.
15 |
16 | ## Usage
17 |
18 | tl;dr:
19 | ```
20 | pip install pdfminer
21 | pip install html2text
22 | pip install tiktoken
23 | pip install openai
24 | export OPENAI_API_KEY=