├── final_report.pdf
├── example
    ├── README.md
    ├── run.sh
    └── dont_run_me_run_the_other_script_instead.py
├── .gitignore
├── LICENSE
├── convert_to_readable.py
├── play_with_model.py
├── README.md
├── demo_play_with_model.py
├── error_calculator.py
├── main2.py
├── punctuator.py
├── main.py
├── data.py
├── lecture_punctuator.py
└── models.py


/final_report.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ragymorkos/LecturePunctuator/HEAD/final_report.pdf


--------------------------------------------------------------------------------
/example/README.md:
--------------------------------------------------------------------------------
 1 | This example produces the preprocessed Europarl English corpus that can be then used for training a model.
 2 | 
 3 | Requires nltk
 4 | 
 5 | Usage example:
 6 | `./run.sh`
 7 | 
 8 | `cd ..`
 9 | 
10 | `python data.py ./example/out`
11 | 
12 | `python main.py ep 256 0.02`
13 | 
14 | `python play_with_model.py Model_ep_h256_lr0.02.pcl`
15 | 
16 | The input text to play_with_model.py should be similar to the contents of the preprocessed files in ./example/out (i.e. lowercased, numeric tokens replaced with <NUM>), but should not contain punctuation tokens.
17 | 
18 | Training time on this dataset with a Nvidia Tesla K20 GPU was about 15 hours (~3500 samples per second)


--------------------------------------------------------------------------------
/example/run.sh:
--------------------------------------------------------------------------------
 1 | wget -qO- http://hltshare.fbk.eu/IWSLT2012/training-monolingual-europarl.tgz | tar xvz
 2 | rm -rf ./out
 3 | echo "Step 1/3"
 4 | mkdir ./out
 5 | grep -v " '[^ ]" ./training-monolingual-europarl/europarl-v7.en | \
 6 | grep -v \'\ s\   | \
 7 | grep -v \'\ ll\  | \
 8 | grep -v \'\ ve\  | \
 9 | grep -v \'\ m\   > step1.txt
10 | echo "Step 2/3"
11 | python dont_run_me_run_the_other_script_instead.py step1.txt step2.txt
12 | echo "Step 3/3"
13 | head -n -400000 step2.txt > ./out/ep.train.txt
14 | tail -n 400000 step2.txt > step3.txt
15 | head -n -200000 step3.txt > ./out/ep.dev.txt
16 | tail -n 200000 step3.txt > ./out/ep.test.txt
17 | echo "Cleaning up..."
18 | rm -f step1.txt step2.txt step3.txt
19 | echo "Preprocessing done. Now you can give the produced ./out dir as <data_dir> argument to data.py script for conversion and continue as described in the main README.md"
20 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
 1 | # Byte-compiled / optimized / DLL files
 2 | __pycache__/
 3 | *.py[cod]
 4 | *$py.class
 5 | 
 6 | # C extensions
 7 | *.so
 8 | 
 9 | # Distribution / packaging
10 | .Python
11 | env/
12 | build/
13 | develop-eggs/
14 | dist/
15 | downloads/
16 | eggs/
17 | .eggs/
18 | lib/
19 | lib64/
20 | parts/
21 | sdist/
22 | var/
23 | *.egg-info/
24 | .installed.cfg
25 | *.egg
26 | 
27 | # PyInstaller
28 | #  Usually these files are written by a python script from a template
29 | #  before PyInstaller builds the exe, so as to inject date/other infos into it.
30 | *.manifest
31 | *.spec
32 | 
33 | # Installer logs
34 | pip-log.txt
35 | pip-delete-this-directory.txt
36 | 
37 | # Unit test / coverage reports
38 | htmlcov/
39 | .tox/
40 | .coverage
41 | .coverage.*
42 | .cache
43 | nosetests.xml
44 | coverage.xml
45 | *,cover
46 | .hypothesis/
47 | 
48 | # Translations
49 | *.mo
50 | *.pot
51 | 
52 | # Django stuff:
53 | *.log
54 | 
55 | # Sphinx documentation
56 | docs/_build/
57 | 
58 | # PyBuilder
59 | target/
60 | 
61 | #Ipython Notebook
62 | .ipynb_checkpoints
63 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | The MIT License (MIT)
 2 | 
 3 | Copyright (c) 2018 Ragy Morkos
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/convert_to_readable.py:
--------------------------------------------------------------------------------
 1 | import sys
 2 | import codecs
 3 | from data import EOS_TOKENS, PUNCTUATION_VOCABULARY
 4 | 
 5 | if __name__ == "__main__":
 6 | 
 7 |     if len(sys.argv) > 1:
 8 |         input_file = sys.argv[1]
 9 |     else:
10 |         sys.exit("Input file path argument missing")
11 | 
12 |     if len(sys.argv) > 2:
13 |         output_file = sys.argv[2]
14 |     else:
15 |         sys.exit("Output file path argument missing")
16 | 
17 |     with_newlines = len(sys.argv) > 3 and bool(int(sys.argv[3]))
18 | 
19 |     with codecs.open(input_file, 'r', 'utf-8') as in_f, codecs.open(output_file, 'w', 'utf-8') as out_f:
20 |         last_was_eos = True
21 |         first = True
22 |         for token in in_f.read().split():
23 |             if token in PUNCTUATION_VOCABULARY:
24 |                 out_f.write(token[:1])
25 |             else:
26 |                 out_f.write(('' if first else ' ') + (token.title() if last_was_eos else token))
27 | 
28 |             last_was_eos = token in EOS_TOKENS
29 |             if with_newlines and last_was_eos:
30 |                 out_f.write('\n')
31 |                 first = True
32 |             else:
33 |                 first = False
34 | 


--------------------------------------------------------------------------------
/example/dont_run_me_run_the_other_script_instead.py:
--------------------------------------------------------------------------------
 1 | # coding: utf-8
 2 | 
 3 | from __future__ import division
 4 | from nltk.tokenize import word_tokenize
 5 | 
 6 | import nltk
 7 | import os
 8 | import codecs
 9 | import re
10 | import sys
11 | 
12 | nltk.download('punkt')
13 | 
14 | NUM = '<NUM>'
15 | 
16 | EOS_PUNCTS = {".": ".PERIOD", "?": "?QUESTIONMARK", "!": "!EXCLAMATIONMARK"}
17 | INS_PUNCTS = {",": ",COMMA", ";": ";SEMICOLON", ":": ":COLON", "-": "-DASH"}
18 | 
19 | forbidden_symbols = re.compile(r"[\[\]\(\)\/\\\>\<\=\+\_\*]")
20 | numbers = re.compile(r"\d")
21 | multiple_punct = re.compile(r'([\.\?\!\,\:\;\-])(?:[\.\?\!\,\:\;\-]){1,}')
22 | 
23 | is_number = lambda x: len(numbers.sub("", x)) / len(x) < 0.6
24 | 
25 | def untokenize(line):
26 |     return line.replace(" '", "'").replace(" n't", "n't").replace("can not", "cannot")
27 | 
28 | def skip(line):
29 | 
30 |     if line.strip() == '':
31 |         return True
32 | 
33 |     last_symbol = line[-1]
34 |     if not last_symbol in EOS_PUNCTS:
35 |         return True
36 | 
37 |     if forbidden_symbols.search(line) is not None:
38 |         return True
39 | 
40 |     return False
41 | 
42 | def process_line(line):
43 | 
44 |     tokens = word_tokenize(line)
45 |     output_tokens = []
46 | 
47 |     for token in tokens:
48 | 
49 |         if token in INS_PUNCTS:
50 |             output_tokens.append(INS_PUNCTS[token])
51 |         elif token in EOS_PUNCTS:
52 |             output_tokens.append(EOS_PUNCTS[token])
53 |         elif is_number(token):
54 |             output_tokens.append(NUM)
55 |         else:
56 |             output_tokens.append(token.lower())
57 | 
58 |     return untokenize(" ".join(output_tokens) + " ")
59 | 
60 | skipped = 0
61 | 
62 | with codecs.open(sys.argv[2], 'w', 'utf-8') as out_txt:
63 |     with codecs.open(sys.argv[1], 'r', 'utf-8') as text:
64 | 
65 |         for line in text:
66 | 
67 |             line = line.replace("\"", "").strip()
68 |             line = multiple_punct.sub(r"\g<1>", line)
69 | 
70 |             if skip(line):
71 |                 skipped += 1
72 |                 continue
73 | 
74 |             line = process_line(line)
75 | 
76 |             out_txt.write(line + '\n')
77 | 
78 | print "Skipped %d lines" % skipped
79 | 


--------------------------------------------------------------------------------
/play_with_model.py:
--------------------------------------------------------------------------------
  1 | # coding: utf-8
  2 | 
  3 | from __future__ import division
  4 | 
  5 | import models
  6 | import data
  7 | 
  8 | import theano
  9 | import sys
 10 | import codecs
 11 | 
 12 | import theano.tensor as T
 13 | import numpy as np
 14 | 
 15 | def to_array(arr, dtype=np.int32):
 16 |     # minibatch of 1 sequence as column
 17 |     return np.array([arr], dtype=dtype).T
 18 | 
 19 | def convert_punctuation_to_readable(punct_token):
 20 |     if punct_token == data.SPACE:
 21 |         return " "
 22 |     else:
 23 |         return punct_token[0]
 24 | 
 25 | def punctuate(predict, word_vocabulary, punctuation_vocabulary, reverse_punctuation_vocabulary, reverse_word_vocabulary, text, f_out, show_unk):
 26 | 
 27 |     if len(text) == 0:
 28 |         sys.exit("Input text from stdin missing.")
 29 | 
 30 |     text = [w for w in text.split() if w not in punctuation_vocabulary] + [data.END]
 31 | 
 32 |     i = 0
 33 | 
 34 |     while True:
 35 | 
 36 |         subsequence = text[i:i+data.MAX_SEQUENCE_LEN]
 37 | 
 38 |         if len(subsequence) == 0:
 39 |             break
 40 | 
 41 |         converted_subsequence = [word_vocabulary.get(w, word_vocabulary[data.UNK]) for w in subsequence]
 42 |         if show_unk:
 43 |             subsequence = [reverse_word_vocabulary[w] for w in converted_subsequence]
 44 | 
 45 |         y = predict(to_array(converted_subsequence))
 46 | 
 47 |         f_out.write(subsequence[0])
 48 | 
 49 |         last_eos_idx = 0
 50 |         punctuations = []
 51 |         for y_t in y:
 52 | 
 53 |             p_i = np.argmax(y_t.flatten())
 54 |             punctuation = reverse_punctuation_vocabulary[p_i]
 55 | 
 56 |             punctuations.append(punctuation)
 57 | 
 58 |             if punctuation in data.EOS_TOKENS:
 59 |                 last_eos_idx = len(punctuations) # we intentionally want the index of next element
 60 | 
 61 |         if subsequence[-1] == data.END:
 62 |             step = len(subsequence) - 1
 63 |         elif last_eos_idx != 0:
 64 |             step = last_eos_idx
 65 |         else:
 66 |             step = len(subsequence) - 1
 67 | 
 68 |         for j in range(step):
 69 |             f_out.write(" " + punctuations[j] + " " if punctuations[j] != data.SPACE else " ")
 70 |             if j < step - 1:
 71 |                 f_out.write(subsequence[1+j])
 72 | 
 73 |         if subsequence[-1] == data.END:
 74 |             break
 75 | 
 76 |         i += step
 77 | 
 78 | 
 79 | if __name__ == "__main__":
 80 | 
 81 |     if len(sys.argv) > 1:
 82 |         model_file = sys.argv[1]
 83 |     else:
 84 |         sys.exit("Model file path argument missing")
 85 | 
 86 |     show_unk = False
 87 |     if len(sys.argv) > 2:
 88 |         show_unk = bool(int(sys.argv[2]))
 89 | 
 90 |     x = T.imatrix('x')
 91 | 
 92 |     print "Loading model parameters..."
 93 |     net, _ = models.load(model_file, 1, x)
 94 | 
 95 |     print "Building model..."
 96 |     predict = theano.function(inputs=[x], outputs=net.y)
 97 |     word_vocabulary = net.x_vocabulary
 98 |     punctuation_vocabulary = net.y_vocabulary
 99 |     reverse_word_vocabulary = {v:k for k,v in net.x_vocabulary.items()}
100 |     reverse_punctuation_vocabulary = {v:k for k,v in net.y_vocabulary.items()}
101 | 
102 |     with codecs.getwriter('utf-8')(sys.stdout) as f_out:
103 |         while True:
104 |             text = raw_input("\nTEXT: ").decode('utf-8')
105 |             punctuate(predict, word_vocabulary, punctuation_vocabulary, reverse_punctuation_vocabulary, reverse_word_vocabulary, text, f_out, show_unk)
106 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | Note 1: this repository is not being actively maintained or improved at the moment, but I am quick to respond if you find a bug and open a ticket on GitHub or email me.
 2 | 
 3 | Note 2: Because some videos have manually edited and punctuated subtitles uploaded alongside the video, it can be worth it to check if those are available first. You can check the settings (accessed via the gear icon) for the YouTube video to see if they are available. You can also try to execute `youtube-dl --all-subs --skip-download <YouTube_link>` and you will either get the manually punctuated subtitles (in all available languages) if they exist or an error message saying "WARNING: video doesn't have subtitles".
 4 | 
 5 | # Intro
 6 | 
 7 | This repository contains a tool to provide automatic punctuation of lecture transcripts obtained from YouTube. YouTube's transcripts are automatically generated by Google's automatic speech recognition (ASR) functionality, but are not punctuated. My tool is specially configured to punctuate lecture transcripts, since transcripts can be helpful to learners for a variety of reasons. The tool's current average F-score for correctly punctuating transcripts of lecture videos using commas, periods, question marks and semicolons is 67.3. Note that this F-score will probably be much lower if you try my tool on non-lecture or non-educational videos as the context will be different.
 8 | 
 9 | My work on this tool slightly builds upon that of https://github.com/ottokart/punctuator2, from which my repository is forked. I used the training structure in that repository to train a customized bidirectional recurrent neural network on manually punctuated transcripts scraped from Udacity's YouTube channel (https://www.youtube.com/user/Udacity) and from Princeton's COS 226 class (https://cuvids.io/app/course/2/). The training data that was used can be found in the "training_data" subdirectory inside the root of this repository. For more information and details regarding the background and methodology for training the neural network and regarding the project in general, including possible ways to improve the F-score, please refer to the "final_report.pdf" file in the root of the repository. Please note that some of the things in the report might be outdated or irrelevant to this repository's functionality.
10 | 
11 | Please reach out to me for the training data used for this project.
12 | 
13 | # Requirements (Ubuntu)
14 | 
15 | * Python 2.7+
16 | * pip (if it is not installed on your system, install with `sudo apt install python-pip`)
17 | * Theano (if it is not installed on your system, install with `sudo pip install Theano`)
18 | * youtube-dl (if it is not installed on your system, install with `sudo pip install --upgrade youtube_dl`)
19 | * ffmpeg (if it is not installed on your system, install with `sudo apt install ffmpeg`)
20 | * The neural network file: Please reach out to me if you'd like to use mine.
21 | 
22 | I have not yet tested this repository on a non-Linux OS. Feel free to send me the installation instructions or put a pull request on the readme if you were to make it work on MacOS or Windows.
23 | 
24 | # Usage
25 | 
26 | If you have a link to a YouTube lecture video or playlist, simply execute `python lecture_punctuator.py <YouTube_link>`, where `<YouTube_link>` could be the link of a YouTube video, playlist, or channel. The program will output the transcript(s) in the current working directory to a subdirectory called "transcripts". Please make sure that the path to `lecture_punctuator.py` is configured correctly. The output format for a transcript is `<Video Title>_<YouTube ID>`.
27 | 


--------------------------------------------------------------------------------
/demo_play_with_model.py:
--------------------------------------------------------------------------------
  1 | # coding: utf-8
  2 | 
  3 | from __future__ import division
  4 | 
  5 | from nltk.tokenize import word_tokenize
  6 | 
  7 | import models
  8 | import data
  9 | 
 10 | import theano
 11 | import sys
 12 | import codecs
 13 | import re
 14 | 
 15 | import theano.tensor as T
 16 | import numpy as np
 17 | 
 18 | numbers = re.compile(r'\d')
 19 | is_number = lambda x: len(numbers.sub('', x)) / len(x) < 0.6
 20 | 
 21 | def to_array(arr, dtype=np.int32):
 22 |     # minibatch of 1 sequence as column
 23 |     return np.array([arr], dtype=dtype).T
 24 | 
 25 | def convert_punctuation_to_readable(punct_token):
 26 |     if punct_token == data.SPACE:
 27 |         return ' '
 28 |     elif punct_token.startswith('-'):
 29 |         return ' ' + punct_token[0] + ' '
 30 |     else:
 31 |         return punct_token[0] + ' '
 32 | 
 33 | def punctuate(predict, word_vocabulary, punctuation_vocabulary, reverse_punctuation_vocabulary, reverse_word_vocabulary, words, f_out, show_unk):
 34 | 
 35 |     if len(words) == 0:
 36 |         sys.exit("Input text from stdin missing.")
 37 | 
 38 |     if words[-1] != data.END:
 39 |         words += [data.END]
 40 | 
 41 |     i = 0
 42 | 
 43 |     while True:
 44 | 
 45 |         subsequence = words[i:i+data.MAX_SEQUENCE_LEN]
 46 | 
 47 |         if len(subsequence) == 0:
 48 |             break
 49 | 
 50 |         converted_subsequence = [word_vocabulary.get(
 51 |                 "<NUM>" if is_number(w) else w.lower(),
 52 |                 word_vocabulary[data.UNK])
 53 |             for w in subsequence]
 54 | 
 55 |         if show_unk:
 56 |             subsequence = [reverse_word_vocabulary[w] for w in converted_subsequence]
 57 | 
 58 |         y = predict(to_array(converted_subsequence))
 59 | 
 60 |         f_out.write(subsequence[0].title())
 61 | 
 62 |         last_eos_idx = 0
 63 |         punctuations = []
 64 |         for y_t in y:
 65 | 
 66 |             p_i = np.argmax(y_t.flatten())
 67 |             punctuation = reverse_punctuation_vocabulary[p_i]
 68 | 
 69 |             punctuations.append(punctuation)
 70 | 
 71 |             if punctuation in data.EOS_TOKENS:
 72 |                 last_eos_idx = len(punctuations) # we intentionally want the index of next element
 73 | 
 74 |         if subsequence[-1] == data.END:
 75 |             step = len(subsequence) - 1
 76 |         elif last_eos_idx != 0:
 77 |             step = last_eos_idx
 78 |         else:
 79 |             step = len(subsequence) - 1
 80 | 
 81 |         for j in range(step):
 82 |             current_punctuation = punctuations[j]
 83 |             f_out.write(convert_punctuation_to_readable(current_punctuation))
 84 |             if j < step - 1:
 85 |                 if current_punctuation in data.EOS_TOKENS:
 86 |                     f_out.write(subsequence[1+j].title())
 87 |                 else:
 88 |                     f_out.write(subsequence[1+j])
 89 | 
 90 |         if subsequence[-1] == data.END:
 91 |             break
 92 | 
 93 |         i += step
 94 | 
 95 | 
 96 | if __name__ == "__main__":
 97 | 
 98 |     if len(sys.argv) > 1:
 99 |         model_file = sys.argv[1]
100 |     else:
101 |         sys.exit("Model file path argument missing")
102 | 
103 |     show_unk = False
104 |     if len(sys.argv) > 2:
105 |         show_unk = bool(int(sys.argv[2]))
106 | 
107 |     x = T.imatrix('x')
108 | 
109 |     print "Loading model parameters..."
110 |     net, _ = models.load(model_file, 1, x)
111 | 
112 |     print "Building model..."
113 |     predict = theano.function(inputs=[x], outputs=net.y)
114 |     word_vocabulary = net.x_vocabulary
115 |     punctuation_vocabulary = net.y_vocabulary
116 |     reverse_word_vocabulary = {v:k for k,v in net.x_vocabulary.items()}
117 |     reverse_punctuation_vocabulary = {v:k for k,v in net.y_vocabulary.items()}
118 | 
119 |     human_readable_punctuation_vocabulary = [p[0] for p in punctuation_vocabulary if p != data.SPACE]
120 |     tokenizer = word_tokenize
121 |     untokenizer = lambda text: text.replace(" '", "'").replace(" n't", "n't").replace("can not", "cannot")
122 | 
123 |     with codecs.getwriter('utf-8')(sys.stdout) as f_out:
124 |         while True:
125 |             text = raw_input("\nTEXT: ").decode('utf-8')
126 | 
127 |             words = [w for w in untokenizer(' '.join(tokenizer(text))).split()
128 |                      if w not in punctuation_vocabulary and w not in human_readable_punctuation_vocabulary]
129 | 
130 |             punctuate(predict, word_vocabulary, punctuation_vocabulary, reverse_punctuation_vocabulary, reverse_word_vocabulary, words, f_out, show_unk)
131 | 


--------------------------------------------------------------------------------
/error_calculator.py:
--------------------------------------------------------------------------------
  1 | # coding: utf-8
  2 | 
  3 | """
  4 | Computes and prints the overall classification error and precision, recall, F-score over punctuations.
  5 | """
  6 | 
  7 | from numpy import nan
  8 | import data
  9 | import codecs
 10 | import sys
 11 | 
 12 | MAPPING = {}#{"!EXCLAMATIONMARK": ".PERIOD", "?QUESTIONMARK": ".PERIOD", ":COLON": ".PERIOD", ";SEMICOLON": ".PERIOD"} # Can be used to estimate 2-class performance for example
 13 | 
 14 | def compute_error(target_paths, predicted_paths):
 15 |     counter = 0
 16 |     total_correct = 0
 17 | 
 18 |     correct = 0.
 19 |     substitutions = 0.
 20 |     deletions = 0.
 21 |     insertions = 0.
 22 | 
 23 |     true_positives = {}
 24 |     false_positives = {}
 25 |     false_negatives = {}
 26 | 
 27 |     for target_path, predicted_path in zip(target_paths, predicted_paths):
 28 | 
 29 |         target_punctuation = " "
 30 |         predicted_punctuation = " "
 31 | 
 32 |         t_i = 0
 33 |         p_i = 0
 34 | 
 35 |         with codecs.open(target_path, 'r', 'utf-8') as target, codecs.open(predicted_path, 'r', 'utf-8') as predicted:
 36 | 
 37 |             target_stream = target.read().split()
 38 |             predicted_stream = predicted.read().split()
 39 |             
 40 |             while True:
 41 | 
 42 |                 if data.PUNCTUATION_MAPPING.get(target_stream[t_i], target_stream[t_i]) in data.PUNCTUATION_VOCABULARY:
 43 |                     while data.PUNCTUATION_MAPPING.get(target_stream[t_i], target_stream[t_i]) in data.PUNCTUATION_VOCABULARY: # skip multiple consecutive punctuations
 44 |                         target_punctuation = data.PUNCTUATION_MAPPING.get(target_stream[t_i], target_stream[t_i])
 45 |                         target_punctuation = MAPPING.get(target_punctuation, target_punctuation)
 46 |                         t_i += 1
 47 |                 else:
 48 |                     target_punctuation = " "
 49 | 
 50 |                 if predicted_stream[p_i] in data.PUNCTUATION_VOCABULARY:
 51 |                     predicted_punctuation = MAPPING.get(predicted_stream[p_i], predicted_stream[p_i])
 52 |                     p_i += 1
 53 |                 else:
 54 |                     predicted_punctuation = " "
 55 | 
 56 |                 is_correct = target_punctuation == predicted_punctuation
 57 | 
 58 |                 counter += 1 
 59 |                 total_correct += is_correct
 60 | 
 61 |                 if predicted_punctuation == " " and target_punctuation != " ":
 62 |                     deletions += 1
 63 |                 elif predicted_punctuation != " " and target_punctuation == " ":
 64 |                     insertions += 1
 65 |                 elif predicted_punctuation != " " and target_punctuation != " " and predicted_punctuation == target_punctuation:
 66 |                     correct += 1
 67 |                 elif predicted_punctuation != " " and target_punctuation != " " and predicted_punctuation != target_punctuation:
 68 |                     substitutions += 1
 69 | 
 70 |                 true_positives[target_punctuation] = true_positives.get(target_punctuation, 0.) + float(is_correct)
 71 |                 false_positives[predicted_punctuation] = false_positives.get(predicted_punctuation, 0.) + float(not is_correct)
 72 |                 false_negatives[target_punctuation] = false_negatives.get(target_punctuation, 0.) + float(not is_correct)
 73 | 
 74 |                 assert target_stream[t_i] == predicted_stream[p_i] or predicted_stream[p_i] == "<unk>", \
 75 |                        ("File: %s \n" + \
 76 |                        "Error: %s (%s) != %s (%s) \n" + \
 77 |                        "Target context: %s \n" + \
 78 |                        "Predicted context: %s") % \
 79 |                        (target_path,
 80 |                         target_stream[t_i], t_i, predicted_stream[p_i], p_i,
 81 |                         " ".join(target_stream[t_i-2:t_i+2]),
 82 |                         " ".join(predicted_stream[p_i-2:p_i+2]))
 83 | 
 84 |                 t_i += 1
 85 |                 p_i += 1
 86 | 
 87 |                 if t_i >= len(target_stream)-1 and p_i >= len(predicted_stream)-1:
 88 |                     break
 89 | 
 90 |     overall_tp = 0.0
 91 |     overall_fp = 0.0
 92 |     overall_fn = 0.0
 93 | 
 94 |     print "-"*46
 95 |     print "{:<16} {:<9} {:<9} {:<9}".format('PUNCTUATION','PRECISION','RECALL','F-SCORE')
 96 |     for p in data.PUNCTUATION_VOCABULARY:
 97 | 
 98 |         if p == data.SPACE:
 99 |             continue
100 | 
101 |         overall_tp += true_positives.get(p,0.)
102 |         overall_fp += false_positives.get(p,0.)
103 |         overall_fn += false_negatives.get(p,0.)
104 | 
105 |         punctuation = p
106 |         precision = (true_positives.get(p,0.) / (true_positives.get(p,0.) + false_positives[p])) if p in false_positives else nan
107 |         recall = (true_positives.get(p,0.) / (true_positives.get(p,0.) + false_negatives[p])) if p in false_negatives else nan
108 |         f_score = (2. * precision * recall / (precision + recall)) if (precision + recall) > 0 else nan        
109 |         print u"{:<16} {:<9} {:<9} {:<9}".format(punctuation, round(precision,3)*100, round(recall,3)*100, round(f_score,3)*100).encode('utf-8')
110 |     print "-"*46
111 |     pre = overall_tp/(overall_tp+overall_fp) if overall_fp else nan
112 |     rec = overall_tp/(overall_tp+overall_fn) if overall_fn else nan
113 |     f1 = (2.*pre*rec)/(pre+rec) if (pre + rec) else nan
114 |     print "{:<16} {:<9} {:<9} {:<9}".format("Overall", round(pre,3)*100, round(rec,3)*100, round(f1,3)*100)
115 |     print "Err: %s%%" % round((100.0 - float(total_correct) / float(counter-1) * 100.0), 2)
116 |     print "SER: %s%%" % round((substitutions + deletions + insertions) / (correct + substitutions + deletions) * 100, 1)
117 | 
118 | 
119 | if __name__ == "__main__":
120 | 
121 |     if len(sys.argv) > 1:
122 |         target_path = sys.argv[1]
123 |     else:
124 |         sys.exit("Ground truth file path argument missing")
125 | 
126 |     if len(sys.argv) > 2:
127 |         predicted_path = sys.argv[2]
128 |     else:
129 |         sys.exit("Model predictions file path argument missing")
130 | 
131 |     compute_error([target_path], [predicted_path])    
132 |         
133 | 


--------------------------------------------------------------------------------
/main2.py:
--------------------------------------------------------------------------------
  1 | # coding: utf-8
  2 | from __future__ import division
  3 | 
  4 | from collections import OrderedDict
  5 | from time import time
  6 | 
  7 | import models
  8 | import data
  9 | 
 10 | import theano
 11 | import sys
 12 | import os.path
 13 | 
 14 | import theano.tensor as T
 15 | import numpy as np
 16 | 
 17 | from main import get_minibatch
 18 | 
 19 | MAX_EPOCHS = 50
 20 | MINIBATCH_SIZE = 128
 21 | L2_REG = 0.0
 22 | CLIPPING_THRESHOLD = 2.0
 23 | PATIENCE_EPOCHS = 1
 24 | 
 25 | """
 26 | Second stage training
 27 | """
 28 | 
 29 | if __name__ == "__main__":
 30 | 
 31 |     if len(sys.argv) > 1:
 32 |         model_name = sys.argv[1]
 33 |     else:
 34 |         sys.exit("'Model name' argument missing!")
 35 | 
 36 |     if len(sys.argv) > 2:
 37 |         num_hidden = int(sys.argv[2])
 38 |     else:
 39 |         sys.exit("'Hidden layer size' argument missing!")
 40 | 
 41 |     if len(sys.argv) > 3:
 42 |         learning_rate = float(sys.argv[3])
 43 |     else:
 44 |         sys.exit("'Learning rate' argument missing!")
 45 | 
 46 |     if len(sys.argv) > 4:
 47 |         stage1_model_file_name = sys.argv[4]
 48 |     else:
 49 |         sys.exit("'Stage 1 model path' argument missing!")
 50 | 
 51 |     model_file_name = "Model_stage2_%s_h%d_lr%s.pcl" % (model_name, num_hidden, learning_rate)
 52 | 
 53 |     print num_hidden, learning_rate, model_file_name
 54 | 
 55 |     word_vocabulary = data.read_vocabulary(data.WORD_VOCAB_FILE)
 56 |     punctuation_vocabulary = data.iterable_to_dict(data.PUNCTUATION_VOCABULARY)
 57 | 
 58 |     x = T.imatrix('x')
 59 |     y = T.imatrix('y')
 60 |     p = T.matrix('p')
 61 |     lr = T.scalar('lr')
 62 | 
 63 |     continue_with_previous = False
 64 |     if os.path.isfile(model_file_name):
 65 | 
 66 |         while True:
 67 |             resp = raw_input("Found an existing model with the name %s. Do you want to:\n[c]ontinue training the existing model?\n[r]eplace the existing model and train a new one?\n[e]xit?\n>" % model_file_name)
 68 |             resp = resp.lower().strip()
 69 |             if resp not in ('c', 'r', 'e'):
 70 |                 continue
 71 |             if resp == 'e':
 72 |                 sys.exit()
 73 |             elif resp == 'c':
 74 |                 continue_with_previous = True
 75 |             break
 76 | 
 77 |     if continue_with_previous:
 78 |         net, state = models.load(model_file_name, MINIBATCH_SIZE, x, p)
 79 |         gsums, learning_rate, validation_ppl_history, starting_epoch, rng = state
 80 |         best_ppl = min(validation_ppl_history)
 81 |     else:
 82 |         rng = np.random
 83 |         rng.seed(1)
 84 | 
 85 |         print "Building model..."
 86 |         net = models.GRUstage2(
 87 |             rng=rng,
 88 |             x=x,
 89 |             minibatch_size=MINIBATCH_SIZE,
 90 |             n_hidden=num_hidden,
 91 |             x_vocabulary=word_vocabulary,
 92 |             y_vocabulary=punctuation_vocabulary,
 93 |             stage1_model_file_name=stage1_model_file_name,
 94 |             p=p
 95 |             )
 96 | 
 97 |         starting_epoch = 0
 98 |         best_ppl = np.inf
 99 |         validation_ppl_history = []
100 | 
101 |         gsums = [theano.shared(np.zeros_like(param.get_value(borrow=True))) for param in net.params]
102 | 
103 |     cost = net.cost(y) + L2_REG * net.L2_sqr
104 | 
105 |     gparams = T.grad(cost, net.params)
106 |     updates = OrderedDict()
107 | 
108 |     # Compute norm of gradients
109 |     norm = T.sqrt(T.sum(
110 |                [T.sum(gparam ** 2) for gparam in gparams]
111 |            ))
112 | 
113 | 
114 |     # Adagrad: "Adaptive subgradient methods for online learning and stochastic optimization" (2011)
115 |     for gparam, param, gsum in zip(gparams, net.params, gsums):
116 |         gparam = T.switch(
117 |             T.ge(norm, CLIPPING_THRESHOLD),
118 |             gparam / norm * CLIPPING_THRESHOLD,
119 |             gparam
120 |         ) # Clipping of gradients
121 |         updates[gsum] = gsum + (gparam ** 2)
122 |         updates[param] = param - lr * (gparam / (T.sqrt(updates[gsum] + 1e-6)))
123 | 
124 |     train_model = theano.function(
125 |         inputs=[x, p, y, lr],
126 |         outputs=cost,
127 |         updates=updates
128 |     )
129 | 
130 |     validate_model = theano.function(
131 |         inputs=[x, p, y],
132 |         outputs=net.cost(y)
133 |     )
134 | 
135 |     print "Training..."
136 |     for epoch in range(starting_epoch, MAX_EPOCHS):
137 |         t0 = time()
138 |         total_neg_log_likelihood = 0
139 |         total_num_output_samples = 0
140 |         iteration = 0
141 |         for X, Y, P in get_minibatch(data.TRAIN_FILE2, MINIBATCH_SIZE, shuffle=True, with_pauses=True):
142 |             total_neg_log_likelihood += train_model(X, P, Y, learning_rate)
143 |             total_num_output_samples += np.prod(Y.shape)
144 |             iteration += 1
145 |             if iteration % 100 == 0:
146 |                 sys.stdout.write("PPL: %.4f; Speed: %.2f sps\n" % (np.exp(total_neg_log_likelihood / total_num_output_samples), total_num_output_samples / max(time() - t0, 1e-100)))
147 |                 sys.stdout.flush()
148 |         print "Total number of training labels: %d" % total_num_output_samples
149 | 
150 |         total_neg_log_likelihood = 0
151 |         total_num_output_samples = 0
152 |         for X, Y, P in get_minibatch(data.DEV_FILE2, MINIBATCH_SIZE, shuffle=False, with_pauses=True):
153 |             total_neg_log_likelihood += validate_model(X, P, Y)
154 |             total_num_output_samples += np.prod(Y.shape)
155 |         print "Total number of validation labels: %d" % total_num_output_samples
156 | 
157 |         ppl = np.exp(total_neg_log_likelihood / total_num_output_samples)
158 |         validation_ppl_history.append(ppl)
159 | 
160 |         print "Validation perplexity is %s" % np.round(ppl, 4)
161 | 
162 |         if ppl <= best_ppl:
163 |             best_ppl = ppl
164 |             net.save(model_file_name, gsums=gsums, learning_rate=learning_rate, validation_ppl_history=validation_ppl_history, best_validation_ppl=best_ppl, epoch=epoch, random_state=rng.get_state())
165 |         elif best_ppl not in validation_ppl_history[-PATIENCE_EPOCHS:]:
166 |             print "Finished!"
167 |             break
168 | 


--------------------------------------------------------------------------------
/punctuator.py:
--------------------------------------------------------------------------------
  1 | # coding: utf-8
  2 | from __future__ import division
  3 | 
  4 | import models
  5 | import data
  6 | 
  7 | import theano
  8 | import sys
  9 | import codecs
 10 | 
 11 | import theano.tensor as T
 12 | import numpy as np
 13 | 
 14 | MAX_SUBSEQUENCE_LEN = 200
 15 | 
 16 | def to_array(arr, dtype=np.int32):
 17 |     # minibatch of 1 sequence as column
 18 |     return np.array([arr], dtype=dtype).T
 19 | 
 20 | def convert_punctuation_to_readable(punct_token):
 21 |     if punct_token == data.SPACE:
 22 |         return " "
 23 |     else:
 24 |         return punct_token[0]
 25 | 
 26 | def restore_with_pauses(output_file, text, pauses, word_vocabulary, reverse_punctuation_vocabulary, predict_function):
 27 |     i = 0
 28 |     with codecs.open(output_file, 'w', 'utf-8') as f_out:
 29 |         while True:
 30 | 
 31 |             subsequence = text[i:i+MAX_SUBSEQUENCE_LEN]
 32 |             subsequence_pauses = pauses[i:i+MAX_SUBSEQUENCE_LEN]
 33 | 
 34 |             if len(subsequence) == 0:
 35 |                 break
 36 | 
 37 |             converted_subsequence = [word_vocabulary.get(w, word_vocabulary[data.UNK]) for w in subsequence]
 38 | 
 39 |             y = predict_function(to_array(converted_subsequence), to_array(subsequence_pauses, dtype=theano.config.floatX))
 40 | 
 41 |             f_out.write(subsequence[0])
 42 | 
 43 |             last_eos_idx = 0
 44 |             punctuations = []
 45 |             for y_t in y:
 46 | 
 47 |                 p_i = np.argmax(y_t.flatten())
 48 |                 punctuation = reverse_punctuation_vocabulary[p_i]
 49 | 
 50 |                 punctuations.append(punctuation)
 51 | 
 52 |                 if punctuation in data.EOS_TOKENS:
 53 |                     last_eos_idx = len(punctuations) # we intentionally want the index of next element
 54 | 
 55 |             if subsequence[-1] == data.END:
 56 |                 step = len(subsequence) - 1
 57 |             elif last_eos_idx != 0:
 58 |                 step = last_eos_idx
 59 |             else:
 60 |                 step = len(subsequence) - 1
 61 | 
 62 |             for j in range(step):
 63 |                 f_out.write(" " + punctuations[j] + " " if punctuations[j] != data.SPACE else " ")
 64 |                 if j < step - 1:
 65 |                     f_out.write(subsequence[1+j])
 66 | 
 67 |             if subsequence[-1] == data.END:
 68 |                 break
 69 | 
 70 |             i += step
 71 | 
 72 | def restore(output_file, text, word_vocabulary, reverse_punctuation_vocabulary, predict_function):
 73 |     i = 0
 74 |     with codecs.open(output_file, 'w', 'utf-8') as f_out:
 75 |         while True:
 76 | 
 77 |             subsequence = text[i:i+MAX_SUBSEQUENCE_LEN]
 78 | 
 79 |             if len(subsequence) == 0:
 80 |                 break
 81 | 
 82 |             converted_subsequence = [word_vocabulary.get(w, word_vocabulary[data.UNK]) for w in subsequence]
 83 | 
 84 |             y = predict_function(to_array(converted_subsequence))
 85 | 
 86 |             f_out.write(subsequence[0])
 87 | 
 88 |             last_eos_idx = 0
 89 |             punctuations = []
 90 |             for y_t in y:
 91 | 
 92 |                 p_i = np.argmax(y_t.flatten())
 93 |                 punctuation = reverse_punctuation_vocabulary[p_i]
 94 | 
 95 |                 punctuations.append(punctuation)
 96 | 
 97 |                 if punctuation in data.EOS_TOKENS:
 98 |                     last_eos_idx = len(punctuations) # we intentionally want the index of next element
 99 | 
100 |             if subsequence[-1] == data.END:
101 |                 step = len(subsequence) - 1
102 |             elif last_eos_idx != 0:
103 |                 step = last_eos_idx
104 |             else:
105 |                 step = len(subsequence) - 1
106 | 
107 |             for j in range(step):
108 |                 f_out.write(" " + punctuations[j] + " " if punctuations[j] != data.SPACE else " ")
109 |                 if j < step - 1:
110 |                     f_out.write(subsequence[1+j])
111 | 
112 |             if subsequence[-1] == data.END:
113 |                 break
114 | 
115 |             i += step
116 | 
117 | if __name__ == "__main__":
118 | 
119 |     if len(sys.argv) > 1:
120 |         model_file = sys.argv[1]
121 |     else:
122 |         sys.exit("Model file path argument missing")
123 | 
124 |     if len(sys.argv) > 2:
125 |         output_file = sys.argv[2]
126 |     else:
127 |         sys.exit("Output file path argument missing")
128 | 
129 |     use_pauses = len(sys.argv) > 3 and bool(int(sys.argv[3]))
130 | 
131 |     x = T.imatrix('x')
132 |     
133 |     if use_pauses:
134 |     
135 |         p = T.matrix('p')
136 | 
137 |         print "Loading model parameters..."
138 |         net, _ = models.load(model_file, 1, x, p)
139 | 
140 |         print "Building model..."
141 |         predict = theano.function(
142 |             inputs=[x, p],
143 |             outputs=net.y
144 |         )
145 | 
146 |     else:
147 | 
148 |         print "Loading model parameters..."
149 |         net, _ = models.load(model_file, 1, x)
150 | 
151 |         print "Building model..."
152 |         predict = theano.function(
153 |             inputs=[x],
154 |             outputs=net.y
155 |         )
156 | 
157 |     word_vocabulary = net.x_vocabulary
158 |     punctuation_vocabulary = net.y_vocabulary
159 | 
160 |     reverse_word_vocabulary = {v:k for k,v in word_vocabulary.items()}
161 |     reverse_punctuation_vocabulary = {v:k for k,v in punctuation_vocabulary.items()}
162 | 
163 |     input_text = codecs.getreader('utf-8')(sys.stdin).read()
164 | 
165 |     if len(input_text) == 0:
166 |         sys.exit("Input text from stdin missing.")
167 | 
168 |     text = [w for w in input_text.split() if w not in punctuation_vocabulary and w not in data.PUNCTUATION_MAPPING and not w.startswith(data.PAUSE_PREFIX)] + [data.END]
169 |     pauses = [float(s.replace(data.PAUSE_PREFIX,"").replace(">","")) for s in input_text.split() if s.startswith(data.PAUSE_PREFIX)]
170 | 
171 |     if not use_pauses:
172 |         restore(output_file, text, word_vocabulary, reverse_punctuation_vocabulary, predict)
173 |     else:
174 |         if not pauses:
175 |             pauses = [0.0 for _ in range(len(text)-1)]
176 |         restore_with_pauses(output_file, text, pauses, word_vocabulary, reverse_punctuation_vocabulary, predict)
177 | 


--------------------------------------------------------------------------------
/main.py:
--------------------------------------------------------------------------------
  1 | # coding: utf-8
  2 | from __future__ import division
  3 | 
  4 | from collections import OrderedDict
  5 | from time import time
  6 | 
  7 | import models
  8 | import data
  9 | 
 10 | import theano
 11 | import cPickle
 12 | import sys
 13 | import os.path
 14 | 
 15 | import theano.tensor as T
 16 | import numpy as np
 17 | 
 18 | MAX_EPOCHS = 50
 19 | MINIBATCH_SIZE = 128
 20 | L2_REG = 0.0
 21 | CLIPPING_THRESHOLD = 2.0
 22 | PATIENCE_EPOCHS = 1
 23 | 
 24 | """
 25 | Bi-directional RNN with attention
 26 | For a sequence of N words, the model makes N punctuation decisions (no punctuation before the first word, but there's a decision after the last word or before </S>)
 27 | """
 28 | 
 29 | def get_minibatch(file_name, batch_size, shuffle, with_pauses=False):
 30 | 
 31 |     dataset = data.load(file_name)
 32 | 
 33 |     if shuffle:
 34 |         np.random.shuffle(dataset)
 35 | 
 36 |     X_batch = []
 37 |     Y_batch = []
 38 |     if with_pauses:
 39 |         P_batch = []
 40 | 
 41 |     if len(dataset) < batch_size:
 42 |         print "WARNING: Not enough samples in '%s'. Reduce mini-batch size to %d or use a dataset with at least %d words." % (
 43 |             file_name,
 44 |             len(dataset),
 45 |             MINIBATCH_SIZE * data.MAX_SEQUENCE_LEN)
 46 | 
 47 |     for subsequence in dataset:
 48 | 
 49 |         X_batch.append(subsequence[0])
 50 |         Y_batch.append(subsequence[1])
 51 |         if with_pauses:
 52 |             P_batch.append(subsequence[2])
 53 |         
 54 |         if len(X_batch) == batch_size:
 55 | 
 56 |             # Transpose, because the model assumes the first axis is time
 57 |             X = np.array(X_batch, dtype=np.int32).T
 58 |             Y = np.array(Y_batch, dtype=np.int32).T
 59 |             if with_pauses:
 60 |                 P = np.array(P_batch, dtype=theano.config.floatX).T
 61 |             
 62 |             if with_pauses:
 63 |                 yield X, Y, P
 64 |             else:
 65 |                 yield X, Y
 66 | 
 67 |             X_batch = []
 68 |             Y_batch = []
 69 |             if with_pauses:
 70 |                 P_batch = []
 71 | 
 72 | if __name__ == "__main__":
 73 |     
 74 |     if len(sys.argv) > 1:
 75 |         model_name = sys.argv[1]
 76 |     else:
 77 |         sys.exit("'Model name' argument missing!")
 78 | 
 79 |     if len(sys.argv) > 2:
 80 |         num_hidden = int(sys.argv[2])
 81 |     else:
 82 |         sys.exit("'Hidden layer size' argument missing!")
 83 | 
 84 |     if len(sys.argv) > 3:
 85 |         learning_rate = float(sys.argv[3])
 86 |     else:
 87 |         sys.exit("'Learning rate' argument missing!")
 88 | 
 89 |     model_file_name = "Model_%s_h%d_lr%s.pcl" % (model_name, num_hidden, learning_rate)
 90 | 
 91 |     print num_hidden, learning_rate, model_file_name
 92 | 
 93 |     word_vocabulary = data.read_vocabulary(data.WORD_VOCAB_FILE)
 94 |     punctuation_vocabulary = data.iterable_to_dict(data.PUNCTUATION_VOCABULARY)
 95 | 
 96 |     x = T.imatrix('x')
 97 |     y = T.imatrix('y')
 98 |     lr = T.scalar('lr')
 99 | 
100 |     continue_with_previous = False
101 |     if os.path.isfile(model_file_name):
102 | 
103 |         while True:
104 |             resp = raw_input("Found an existing model with the name %s. Do you want to:\n[c]ontinue training the existing model?\n[r]eplace the existing model and train a new one?\n[e]xit?\n>" % model_file_name)
105 |             resp = resp.lower().strip()
106 |             if resp not in ('c', 'r', 'e'):
107 |                 continue
108 |             if resp == 'e':
109 |                 sys.exit()
110 |             elif resp == 'c':
111 |                 continue_with_previous = True
112 |             break
113 | 
114 |     if continue_with_previous:
115 |         print "Loading previous model state" 
116 | 
117 |         net, state = models.load(model_file_name, MINIBATCH_SIZE, x)
118 |         gsums, learning_rate, validation_ppl_history, starting_epoch, rng = state
119 |         best_ppl = min(validation_ppl_history)
120 | 
121 |     else:
122 |         rng = np.random
123 |         rng.seed(1)
124 | 
125 |         print "Building model..."
126 |         net = models.GRU(
127 |             rng=rng,
128 |             x=x,
129 |             minibatch_size=MINIBATCH_SIZE,
130 |             n_hidden=num_hidden,
131 |             x_vocabulary=word_vocabulary,
132 |             y_vocabulary=punctuation_vocabulary
133 |             )
134 | 
135 |         starting_epoch = 0
136 |         best_ppl = np.inf
137 |         validation_ppl_history = []
138 |         
139 |         gsums = [theano.shared(np.zeros_like(param.get_value(borrow=True))) for param in net.params]
140 | 
141 |     cost = net.cost(y) + L2_REG * net.L2_sqr
142 | 
143 |     gparams = T.grad(cost, net.params)
144 |     updates = OrderedDict()
145 | 
146 |     # Compute norm of gradients
147 |     norm = T.sqrt(T.sum(
148 |                [T.sum(gparam ** 2) for gparam in gparams]
149 |            ))
150 | 
151 |     
152 |     # Adagrad: "Adaptive subgradient methods for online learning and stochastic optimization" (2011)    
153 |     for gparam, param, gsum in zip(gparams, net.params, gsums):
154 |         gparam = T.switch(
155 |             T.ge(norm, CLIPPING_THRESHOLD),
156 |             gparam / norm * CLIPPING_THRESHOLD,
157 |             gparam
158 |         ) # Clipping of gradients
159 |         updates[gsum] = gsum + (gparam ** 2)
160 |         updates[param] = param - lr * (gparam / (T.sqrt(updates[gsum] + 1e-6)))
161 | 
162 |     train_model = theano.function(
163 |         inputs=[x, y, lr],
164 |         outputs=cost,
165 |         updates=updates
166 |     )
167 | 
168 |     validate_model = theano.function(
169 |         inputs=[x, y],
170 |         outputs=net.cost(y)
171 |     )
172 | 
173 |     print "Training..."
174 |     for epoch in range(starting_epoch, MAX_EPOCHS):
175 |         t0 = time()
176 |         total_neg_log_likelihood = 0
177 |         total_num_output_samples = 0
178 |         iteration = 0 
179 |         for X, Y in get_minibatch(data.TRAIN_FILE, MINIBATCH_SIZE, shuffle=True):
180 |             total_neg_log_likelihood += train_model(X, Y, learning_rate)
181 |             total_num_output_samples += np.prod(Y.shape)
182 |             iteration += 1
183 |             if iteration % 100 == 0:
184 |                 sys.stdout.write("PPL: %.4f; Speed: %.2f sps\n" % (np.exp(total_neg_log_likelihood / total_num_output_samples), total_num_output_samples / max(time() - t0, 1e-100)))
185 |                 sys.stdout.flush()
186 |         print "Total number of training labels: %d" % total_num_output_samples
187 | 
188 |         total_neg_log_likelihood = 0
189 |         total_num_output_samples = 0
190 |         for X, Y in get_minibatch(data.DEV_FILE, MINIBATCH_SIZE, shuffle=False):
191 |             total_neg_log_likelihood += validate_model(X, Y)
192 |             total_num_output_samples += np.prod(Y.shape)
193 |         print "Total number of validation labels: %d" % total_num_output_samples
194 |         
195 |         ppl = np.exp(total_neg_log_likelihood / total_num_output_samples)
196 |         validation_ppl_history.append(ppl)
197 | 
198 |         print "Validation perplexity is %s" % np.round(ppl, 4)
199 | 
200 |         if ppl <= best_ppl:
201 |             best_ppl = ppl
202 |             net.save(model_file_name, gsums=gsums, learning_rate=learning_rate, validation_ppl_history=validation_ppl_history, best_validation_ppl=best_ppl, epoch=epoch, random_state=rng.get_state())
203 |         elif best_ppl not in validation_ppl_history[-PATIENCE_EPOCHS:]:
204 |             print "Finished!"
205 |             print "Best validation perplexity was %s" % best_ppl
206 |             break
207 | 


--------------------------------------------------------------------------------
/data.py:
--------------------------------------------------------------------------------
  1 | # coding: utf-8
  2 | from __future__ import division
  3 | 
  4 | import random
  5 | import os
  6 | import sys
  7 | import operator
  8 | import cPickle
  9 | import codecs
 10 | import fnmatch
 11 | import shutil
 12 | 
 13 | DATA_PATH = "../data"
 14 | 
 15 | # path to text file in the format:
 16 | # word1 0.123 0.123 ... 0.123
 17 | # word2 0.123 0.123 ... 0.123 etc...
 18 | # e.g. glove.6B.50d.txt
 19 | PRETRAINED_EMBEDDINGS_PATH = None
 20 | 
 21 | END = "</S>"
 22 | UNK = "<UNK>"
 23 | 
 24 | SPACE = "_SPACE"
 25 | 
 26 | MAX_WORD_VOCABULARY_SIZE = 100000
 27 | MIN_WORD_COUNT_IN_VOCAB = 2
 28 | MAX_SEQUENCE_LEN = 50
 29 | 
 30 | TRAIN_FILE = os.path.join(DATA_PATH, "train")
 31 | DEV_FILE = os.path.join(DATA_PATH, "dev")
 32 | TEST_FILE = os.path.join(DATA_PATH, "test")
 33 | 
 34 | # Stage 2
 35 | TRAIN_FILE2 = os.path.join(DATA_PATH, "train2")
 36 | DEV_FILE2 = os.path.join(DATA_PATH, "dev2")
 37 | TEST_FILE2 = os.path.join(DATA_PATH, "test2")
 38 | 
 39 | WORD_VOCAB_FILE = os.path.join(DATA_PATH, "vocabulary")
 40 | 
 41 | PUNCTUATION_VOCABULARY = [SPACE, ",COMMA", ".PERIOD", "?QUESTIONMARK", "!EXCLAMATIONMARK", ":COLON", ";SEMICOLON", "-DASH"]
 42 | PUNCTUATION_MAPPING = {}
 43 | 
 44 | # Comma, period & question mark only:
 45 | # PUNCTUATION_VOCABULARY = {SPACE, ",COMMA", ".PERIOD", "?QUESTIONMARK"}
 46 | # PUNCTUATION_MAPPING = {"!EXCLAMATIONMARK": ".PERIOD", ":COLON": ",COMMA", ";SEMICOLON": ".PERIOD", "-DASH": ",COMMA"}
 47 | 
 48 | EOS_TOKENS = {".PERIOD", "?QUESTIONMARK", "!EXCLAMATIONMARK"}
 49 | CRAP_TOKENS = {"<doc>", "<doc.>"} # punctuations that are not included in vocabulary nor mapping, must be added to CRAP_TOKENS
 50 | PAUSE_PREFIX = "<sil="
 51 | 
 52 | # replacement for pickling that takes less RAM. Useful for large datasets.
 53 | def dump(d, path):
 54 |     with open(path, 'w') as f:
 55 |         for s in d:
 56 |             f.write("%s\n" % repr(s))
 57 | 
 58 | def load(path):
 59 |     d = []
 60 |     with open(path, 'r') as f:
 61 |         for l in f:
 62 |             d.append(eval(l))
 63 |     return d
 64 | 
 65 | def add_counts(word_counts, line):
 66 |     for w in line.split():
 67 |         if w in CRAP_TOKENS or w in PUNCTUATION_VOCABULARY or w in PUNCTUATION_MAPPING or w.startswith(PAUSE_PREFIX):
 68 |             continue
 69 |         word_counts[w] = word_counts.get(w, 0) + 1
 70 | 
 71 | def build_vocabulary(word_counts):
 72 |     return [wc[0] for wc in reversed(sorted(word_counts.items(), key=operator.itemgetter(1))) if wc[1] >= MIN_WORD_COUNT_IN_VOCAB and wc[0] != UNK][:MAX_WORD_VOCABULARY_SIZE] # Unk will be appended to end
 73 | 
 74 | def write_vocabulary(vocabulary, file_name):
 75 |     if END not in vocabulary:
 76 |         vocabulary.append(END)
 77 |     if UNK not in vocabulary:
 78 |         vocabulary.append(UNK)
 79 | 
 80 |     print "Vocabulary size: %d" % len(vocabulary)
 81 | 
 82 |     with codecs.open(file_name, 'w', 'utf-8') as f:
 83 |         f.write("\n".join(vocabulary))
 84 | 
 85 | def iterable_to_dict(arr):
 86 |     return dict((x.strip(), i) for (i, x) in enumerate(arr))
 87 | 
 88 | def read_vocabulary(file_name):
 89 |     with codecs.open(file_name, 'r', 'utf-8') as f:
 90 |         return iterable_to_dict(f.readlines())
 91 | 
 92 | def write_processed_dataset(input_files, output_file):
 93 |     """
 94 |     data will consist of two sets of aligned subsequences (words and punctuations) of MAX_SEQUENCE_LEN tokens (actually punctuation sequence will be 1 element shorter).
 95 |     If a sentence is cut, then it will be added to next subsequence entirely (words before the cut belong to both sequences)
 96 |     """
 97 | 
 98 |     data = []
 99 | 
100 |     word_vocabulary = read_vocabulary(WORD_VOCAB_FILE)
101 |     punctuation_vocabulary = iterable_to_dict(PUNCTUATION_VOCABULARY)
102 | 
103 |     num_total = 0
104 |     num_unks = 0
105 | 
106 |     current_words = []
107 |     current_punctuations = []
108 |     current_pauses = []
109 | 
110 |     last_eos_idx = 0 # if it's still 0 when MAX_SEQUENCE_LEN is reached, then the sentence is too long and skipped.
111 |     last_token_was_punctuation = True # skipt first token if it's punctuation
112 |     last_pause = 0.0
113 | 
114 |     skip_until_eos = False # if a sentence does not fit into subsequence, then we need to skip tokens until we find a new sentence
115 | 
116 |     for input_file in input_files:
117 | 
118 |         with codecs.open(input_file, 'r', 'utf-8') as text:
119 | 
120 |             for line in text:
121 | 
122 |                 for token in line.split():
123 | 
124 |                     # First map oov punctuations to known punctuations
125 |                     if token in PUNCTUATION_MAPPING:
126 |                         token = PUNCTUATION_MAPPING[token]
127 | 
128 |                     if skip_until_eos:
129 | 
130 |                         if token in EOS_TOKENS:
131 |                             skip_until_eos = False
132 | 
133 |                         continue
134 | 
135 |                     elif token in CRAP_TOKENS:
136 |                         continue
137 | 
138 |                     elif token.startswith(PAUSE_PREFIX):
139 |                         last_pause = float(token.replace(PAUSE_PREFIX,"").replace(">",""))
140 | 
141 |                     elif token in punctuation_vocabulary:
142 | 
143 |                         if last_token_was_punctuation: # if we encounter sequences like: "... !EXLAMATIONMARK .PERIOD ...", then we only use the first punctuation and skip the ones that follow
144 |                             continue
145 | 
146 |                         if token in EOS_TOKENS:
147 |                             last_eos_idx = len(current_punctuations) # no -1, because the token is not added yet
148 | 
149 |                         punctuation = punctuation_vocabulary[token]
150 | 
151 |                         current_punctuations.append(punctuation)
152 |                         last_token_was_punctuation = True
153 | 
154 |                     else:
155 | 
156 |                         if not last_token_was_punctuation:
157 |                             current_punctuations.append(punctuation_vocabulary[SPACE])
158 | 
159 |                         word = word_vocabulary.get(token, word_vocabulary[UNK])
160 | 
161 |                         current_words.append(word)
162 |                         current_pauses.append(last_pause)
163 |                         last_token_was_punctuation = False
164 | 
165 |                         num_total += 1
166 |                         num_unks += int(word == word_vocabulary[UNK])
167 | 
168 |                     if len(current_words) == MAX_SEQUENCE_LEN: # this also means, that last token was a word
169 |                         
170 |                         assert len(current_words) == len(current_punctuations) + 1, "#words: %d; #punctuations: %d" % (len(current_words), len(current_punctuations))
171 |                         assert current_pauses == [] or len(current_words) == len(current_pauses), "#words: %d; #pauses: %d" % (len(current_words), len(current_pauses))
172 | 
173 |                         # Sentence did not fit into subsequence - skip it
174 |                         if last_eos_idx == 0: 
175 |                             skip_until_eos = True
176 | 
177 |                             current_words = []
178 |                             current_punctuations = []
179 |                             current_pauses = []
180 | 
181 |                             last_token_was_punctuation = True # next sequence starts with a new sentence, so is preceded by eos which is punctuation
182 | 
183 |                         else:
184 |                             subsequence = [
185 |                                 current_words[:-1] + [word_vocabulary[END]],
186 |                                 current_punctuations,
187 |                                 current_pauses[1:]
188 |                             ]
189 | 
190 |                             data.append(subsequence)
191 | 
192 |                             # Carry unfinished sentence to next subsequence
193 |                             current_words = current_words[last_eos_idx+1:]
194 |                             current_punctuations = current_punctuations[last_eos_idx+1:]
195 |                             current_pauses = current_pauses[last_eos_idx+1:]
196 | 
197 |                         last_eos_idx = 0 # sequence always starts with a new sentence
198 | 
199 |     print "%.2f%% UNK-s in %s" % (num_unks / num_total * 100, output_file)
200 | 
201 |     dump(data, output_file)
202 | 
203 | def create_dev_test_train_split_and_vocabulary(root_path, create_vocabulary, train_output, dev_output, test_output, pretrained_embeddings_path=None):
204 | 
205 |     train_txt_files = []
206 |     dev_txt_files = []
207 |     test_txt_files = []
208 | 
209 |     if create_vocabulary and not pretrained_embeddings_path:
210 |         word_counts = dict()
211 |     
212 |     for root, dirnames, filenames in os.walk(root_path):
213 |         for filename in fnmatch.filter(filenames, '*.txt'):
214 | 
215 |             path = os.path.join(root, filename)
216 | 
217 |             if filename.endswith(".test.txt"):
218 |                 test_txt_files.append(path)
219 | 
220 |             elif filename.endswith(".dev.txt"):
221 |                 dev_txt_files.append(path)
222 | 
223 |             else:
224 |                 train_txt_files.append(path)
225 | 
226 |                 if create_vocabulary and not pretrained_embeddings_path:
227 |                     with codecs.open(path, 'r', 'utf-8') as text:
228 |                         for line in text:
229 |                             add_counts(word_counts, line)
230 | 
231 |     if create_vocabulary:
232 |         if pretrained_embeddings_path:
233 |             vocabulary = []
234 |             embeddings = []
235 |             with codecs.open(pretrained_embeddings_path, 'r', 'utf-8') as f:
236 |                 for line in f:
237 |                     line = line.split()
238 |                     w = line[0]
239 |                     e = [float(x) for x in line[1:]]
240 |                     vocabulary.append(w)
241 |                     embeddings.append(e)
242 | 
243 |             with open("We.pcl", 'wb') as f:
244 |                 cPickle.dump(embeddings, f, cPickle.HIGHEST_PROTOCOL)
245 |         else:
246 |             vocabulary = build_vocabulary(word_counts)
247 |         write_vocabulary(vocabulary, WORD_VOCAB_FILE)
248 | 
249 |     write_processed_dataset(train_txt_files, train_output)
250 |     write_processed_dataset(dev_txt_files, dev_output)
251 |     write_processed_dataset(test_txt_files, test_output)
252 | 
253 | if __name__ == "__main__":
254 | 
255 |     if len(sys.argv) > 1:
256 |         path = sys.argv[1]
257 |     else:
258 |         sys.exit("The path to stage1 source data directory with txt files is missing")
259 | 
260 |     replace = False
261 |     if os.path.exists(DATA_PATH):
262 | 
263 |         while True:
264 |             resp = raw_input("Data path '%s' already exists. Do you want to:\n[r]eplace the files in existing data path?\n[e]xit?\n>" % DATA_PATH)
265 |             resp = resp.lower().strip()
266 |             if resp not in ('r', 'e'):
267 |                 continue
268 |             if resp == 'e':
269 |                 sys.exit()
270 |             elif resp == 'r':
271 |                 replace = True
272 |             break
273 | 
274 |     if replace and os.path.exists(DATA_PATH):
275 |         shutil.rmtree(DATA_PATH)
276 | 
277 |     os.makedirs(DATA_PATH)
278 |     
279 |     create_dev_test_train_split_and_vocabulary(path, True, TRAIN_FILE, DEV_FILE, TEST_FILE, PRETRAINED_EMBEDDINGS_PATH)
280 | 
281 |     # Stage 2
282 |     if len(sys.argv) > 2:
283 |         path2 = sys.argv[2]
284 |         create_dev_test_train_split_and_vocabulary(path2, False, TRAIN_FILE2, DEV_FILE2, TEST_FILE2)


--------------------------------------------------------------------------------
/lecture_punctuator.py:
--------------------------------------------------------------------------------
  1 | # lecture_punctuator.py
  2 | # Author: Ragy Morkos
  3 | 
  4 | #################################################################################
  5 | 
  6 | import sys
  7 | import os
  8 | import re
  9 | import string
 10 | 
 11 | #################################################################################
 12 | 
 13 | def make_new_directory(new_directory_name):
 14 |     current_directory = os.getcwd()
 15 |     new_directory = current_directory + "/" + new_directory_name
 16 |     while os.path.isdir(new_directory):
 17 |         new_directory += new_directory_name[-1]
 18 |     os.makedirs(new_directory)
 19 |     return new_directory
 20 | 
 21 | #################################################################################
 22 | 
 23 | def convert_to_srt(filename):
 24 |     new_filename = filename[:-3] + "srt"
 25 |     os.system("ffmpeg -i \"" + filename + "\" \"" + new_filename + "\"")
 26 |     os.system("rm \"" + filename + "\"")
 27 |     return new_filename
 28 | 
 29 | #################################################################################
 30 | 
 31 | # Adapted from https://gist.github.com/nimatrueway/4589700f49c691e5413c5b2df4d02f4f
 32 | def fix_srt_overlap(filename):
 33 |     timeFramePattern = re.compile("(\d+):(\d+):(\d+),(\d+) --> (\d+):(\d+):(\d+),(\d+)")
 34 | 
 35 |     class Subtitle:
 36 |         def __init__(self, idx, fromTime, toTime, text):
 37 |             self.idx = idx
 38 |             self.fromTime = fromTime
 39 |             self.toTime = toTime
 40 |             self.text = text
 41 | 
 42 |     def getDuration(parts):
 43 |         hour = int(parts[0])
 44 |         minute = int(parts[1])
 45 |         second = int(parts[2])
 46 |         millisecond = int(parts[3])
 47 |         return (hour * 3600000000000) + (minute * 60000000000) + (second * 1000000000) + (millisecond * 1000000)
 48 | 
 49 |     def printDuration(duration):
 50 |         hour = duration / 3600000000000
 51 |         duration -= hour * 3600000000000
 52 |         minute = duration / 60000000000
 53 |         duration -= minute * 60000000000
 54 |         second = duration / 1000000000
 55 |         duration -= second * 1000000000
 56 |         millisecond = duration / 1000000
 57 |         return "%02d:%02d:%02d,%03d" % (hour, minute, second, millisecond)
 58 | 
 59 |     def readOneSubtitle(i, inputt):
 60 |         if i >= len(inputt):
 61 |             return i, None
 62 | 
 63 |         # read idx
 64 |         idx = inputt[i]
 65 |         i += 1
 66 |         if not idx:
 67 |             return i, None
 68 |         idx = int(idx)
 69 |         
 70 |         # read timing
 71 |         if i >= len(inputt):
 72 |             return None
 73 |         timing = inputt[i]
 74 |         i += 1
 75 |         timing = timeFramePattern.findall(timing)
 76 |         if not timing:
 77 |             return i, None
 78 |         timing = timing[0]
 79 |         fromTime = getDuration(timing[:4])
 80 |         toTime = getDuration(timing[4:])
 81 | 
 82 |         # read content
 83 |         j = i
 84 |         while j < len(inputt):
 85 |             if len(inputt[j]) == 0:
 86 |                 j += 1
 87 |             else:
 88 |                 break
 89 |         
 90 |         if j >= len(inputt):
 91 |             return i, None
 92 |         if inputt[j].isdigit():
 93 |             return readOneSubtitle(j, inputt)
 94 | 
 95 |         content = inputt[i]
 96 |         i += 1
 97 |         if not content:
 98 |             return None
 99 |         content += "\n"
100 |         while i < len(inputt):
101 |             scanned = inputt[i]
102 |             i += 1
103 |             if not scanned:
104 |                 break
105 |             content += scanned + "\n"
106 |         
107 |         return i, Subtitle(idx, fromTime, toTime, content)
108 | 
109 |     def writeOneSubtitle(file, subtitle, idx):
110 |         file.write(str(idx[0]) + "\n" + printDuration(subtitle.fromTime) + " --> " + printDuration(subtitle.toTime) + "\n" + subtitle.text + "\n\n")
111 |         idx[0] += 1
112 | 
113 |     old_name = filename + ".old"
114 |     os.rename(filename, old_name)
115 |     filee = open(old_name, "r")
116 |     s = ""
117 |     
118 |     for linee in filee:
119 |         line = linee.strip()
120 |         if line:
121 |             if line.isdigit():
122 |                 s += "\n"
123 |             s += line + "\n"
124 |     
125 |     if s[0] == "\n":
126 |         s = s[1:]
127 |     
128 |     with open(filename, "w") as filee:
129 |         filee.write(s)
130 |     
131 |     newFilePath = filename + ".fixed"
132 |     with open(filename, "r") as f:
133 |        inputt = f.readlines()
134 |     inputt = [x.strip() for x in inputt]
135 | 
136 |     newFile = open(newFilePath, "w")
137 |     newIdx = [1]
138 |     lastSubtitle = None
139 |     i = 0
140 |     while True:
141 |         i, subtitle = readOneSubtitle(i, inputt)
142 |         if lastSubtitle:
143 |             if subtitle:
144 |                 subtitle.text = subtitle.text.strip()
145 |                 if len(subtitle.text) == 0: # skip over empty subtitles
146 |                     continue
147 |                 
148 |                 # skip over super-short subtitles that basically contain what their previous subtitle contains, and just prolong previous subtitle
149 |                 if (subtitle.toTime - subtitle.fromTime < 1000000 * 150) and (subtitle.text in lastSubtitle.text):
150 |                     lastSubtitle.toTime = subtitle.toTime
151 |                     continue
152 |                 
153 |                 # if first-line of current subtitle is repeating last-line of previous-subtitle remove it
154 |                 currentLines = subtitle.text.split("\n")
155 |                 currentLines = [x for x in currentLines if x != ""]
156 |                 lastLines = lastSubtitle.text.split("\n")
157 |                 lastLines = [x for x in lastLines if x != ""]
158 |                 if currentLines[0] == lastLines[-1]:
159 |                     subtitle.text = "\n".join(currentLines[1:])
160 | 
161 |                 # if first-line of current subtitle is repeating last-line of previous-subtitle remove it
162 |                 if subtitle.fromTime < lastSubtitle.toTime:
163 |                     lastSubtitle.toTime = subtitle.fromTime - 1000000
164 | 
165 |             writeOneSubtitle(newFile, lastSubtitle, newIdx)
166 | 
167 |         if not subtitle:
168 |             break
169 | 
170 |         lastSubtitle = subtitle
171 | 
172 |     os.rename(filename, filename + ".bak")
173 |     os.rename(newFilePath, filename)
174 | 
175 |     filee.close()
176 |     newFile.close()
177 | 
178 |     os.system("rm \"" + filename + ".bak\"")
179 |     os.system("rm \"" + old_name + "\"")
180 | 
181 | #################################################################################
182 | 
183 | # function that takes as input directory with trascripts in srt format and converts them to YouTube format
184 | def convert_to_youtube(filename):
185 |     with open(filename, "r") as f:
186 |         s = ""
187 |         for linee in f:
188 |             line = linee.strip()
189 |             if "-->" in line:
190 |                 s += line[:8] + " "
191 |             elif line and not line.isdigit():
192 |                 s += (line + "\n")
193 |     os.system("rm \"" + filename + "\"")
194 |     with open(filename, "w") as f:
195 |         f.write(s)
196 | 
197 | #################################################################################
198 | 
199 | def remove_timecodes(timecoded_filename, nontimecoded_filename):
200 |     with open(timecoded_filename, "r") as f_original, open(nontimecoded_filename, "w") as f:
201 |         s = ""
202 |         for line in f_original:
203 |             s += " ".join(line.strip().split()[1:]) + " "
204 |         s = s.strip()
205 |         f.write(s)
206 | 
207 | #################################################################################
208 | 
209 | def restore_timings(timecoded_file, punctuated_filee):
210 |     with open(timecoded_file, "r") as original_file, open(punctuated_filee, "r") as punctuated_file:
211 |         punctuated = punctuated_file.read().strip().split()
212 |         result = ""
213 |         counter = 0
214 |         for line in original_file:
215 |             original = line.strip().split()
216 |             result += original[0]
217 |             wasPreviousQuestionMark = False
218 |             for i in range(1, len(original)):
219 |                 if counter < len(punctuated):
220 |                     if punctuated[counter] == "i":
221 |                         punctuated[counter] = "I"
222 |                     
223 |                     if counter + 2 < len(punctuated) and punctuated[counter + 1] == "I.":
224 |                         if punctuated[counter] == "and":
225 |                             punctuated[counter + 1] = "I"
226 |                             punctuated[counter + 2] = punctuated[counter + 2].lower()
227 |                         else:
228 |                             punctuated[counter] += "."
229 |                             punctuated[counter + 1] = "I"
230 |                                 
231 |                     punctuated[counter] = string.replace(punctuated[counter], "'S", "'s")
232 |                     punctuated[counter] = string.replace(punctuated[counter], "'L", "'l")
233 |                     punctuated[counter] = string.replace(punctuated[counter], "'R", "'r'")
234 |                     punctuated[counter] = string.replace(punctuated[counter], "'M", "'m")
235 |                     punctuated[counter] = string.replace(punctuated[counter], "java", "Java")
236 |                     punctuated[counter] = string.replace(punctuated[counter], "alice", "Alice")
237 |                     punctuated[counter] = string.replace(punctuated[counter], "bob", "Bob")
238 |                     punctuated[counter] = string.replace(punctuated[counter], "princeton", "Princeton")
239 |                     if wasPreviousQuestionMark:
240 |                         punctuated[counter] = punctuated[counter].title()
241 |                     if punctuated[counter][-1] == "?":
242 |                         wasPreviousQuestionMark = True
243 |                     result += " " + punctuated[counter]
244 |                     counter += 1
245 |                 else:
246 |                     break
247 |             result += "\n"
248 |         if result[-2] != ".":
249 |             result = result.strip()
250 |             result += ".\n"
251 |     os.system("rm \"" + punctuated_filee + "\"")
252 |     with open(punctuated_filee, "w") as timed_file:
253 |         timed_file.write(result)
254 | 
255 | #################################################################################
256 | 
257 | if __name__ == "__main__":
258 |     if len(sys.argv) < 2:
259 |         raise BaseException("Please input a YouTube link.")
260 |     if len(sys.argv) > 2:
261 |         raise BaseException("Please input only one YouTub link.")
262 | 
263 |     # make two temporary directories for subtitle processing
264 |     temp_directory = make_new_directory("temp")
265 |     temp_directory_2 = make_new_directory("temp_2")
266 | 
267 |     # make final directory for transcripts
268 |     final_directory = make_new_directory("transcripts")
269 | 
270 |     # download subtitle/subtitles to temporary directory
271 |     os.system("youtube-dl --ignore-errors --write-auto-sub --skip-download -o \"" + temp_directory + "/%(title)s_%(id)s.%(ext)s\" " + sys.argv[1])
272 | 
273 |     for filename in os.listdir(temp_directory):
274 |         full_filename = os.path.join(temp_directory, filename)
275 |         
276 |         full_filename = convert_to_srt(full_filename)
277 |         fix_srt_overlap(full_filename)
278 |         convert_to_youtube(full_filename)
279 | 
280 |         nontimecoded_filename = os.path.join(temp_directory_2, filename)
281 |         remove_timecodes(full_filename, nontimecoded_filename)
282 |         
283 |         path = os.path.dirname(os.path.realpath(__file__))
284 |         neural_network = os.path.join(path, "Neural_Network_Trained_on_Udacity_Transcripts.pcl")
285 |         punctuator = os.path.join(path, "punctuator.py")
286 |         convert_to_readable = os.path.join(path, "convert_to_readable.py")
287 | 
288 |         punctuated_filename = os.path.join(final_directory, filename[:-3] + "srt")
289 | 
290 |         os.system("cat \"" + nontimecoded_filename + "\" | python \"" + punctuator + "\" \"" + neural_network + "\" \"" + punctuated_filename + "\"")
291 | 
292 |         os.system("python \"" + convert_to_readable + "\" \"" + punctuated_filename + "\" \"" + punctuated_filename + "1\"")
293 |         os.system("rm \"" + punctuated_filename + "\"")
294 |         os.rename(punctuated_filename + "1", punctuated_filename)
295 |         os.system("rm \"" + nontimecoded_filename + "\"")
296 | 
297 |         restore_timings(full_filename, punctuated_filename)
298 | 
299 |         os.system("rm \"" + full_filename + "\"")
300 | 
301 |     os.system("rm -r \"" + temp_directory + "\"")
302 |     os.system("rm -r \"" + temp_directory_2 + "\"")
303 | 


--------------------------------------------------------------------------------
/models.py:
--------------------------------------------------------------------------------
  1 | # coding: utf-8
  2 | from __future__ import division
  3 | 
  4 | import theano
  5 | import cPickle
  6 | import os
  7 | import theano.tensor as T
  8 | import numpy as np
  9 | 
 10 | def PReLU(a, x):
 11 |     return T.maximum(0.0, x) + a * T.minimum(0.0, x)
 12 | 
 13 | def ReLU(x):
 14 |     return T.maximum(0.0, x)
 15 | 
 16 | def _get_shape(i, o, keepdims):
 17 |     if (i == 1 or o == 1) and not keepdims:
 18 |         return (max(i,o),)
 19 |     else:
 20 |         return (i, o)
 21 | 
 22 | def _slice(tensor, size, i):
 23 |     """Gets slice of columns of the tensor"""
 24 |     if tensor.ndim == 2:
 25 |         return tensor[:, i*size:(i+1)*size]
 26 |     elif tensor.ndim == 1:
 27 |         return tensor[i*size:(i+1)*size]
 28 |     else:
 29 |         raise NotImplementedError("Tensor should be 1 or 2 dimensional")
 30 | 
 31 | def weights_const(i, o, name, const, keepdims=False):
 32 |     W_values = np.ones(_get_shape(i, o, keepdims)).astype(theano.config.floatX) * const
 33 |     return theano.shared(value=W_values, name=name, borrow=True)
 34 | 
 35 | def weights_identity(i, o, name, const, keepdims=False):
 36 |     #"A Simple Way to Initialize Recurrent Networks of Rectified Linear Units" (2015) (http://arxiv.org/abs/1504.00941)
 37 |     W_values = np.eye(*_get_shape(i, o, keepdims)).astype(theano.config.floatX) * const
 38 |     return theano.shared(value=W_values, name=name, borrow=True)
 39 | 
 40 | def weights_Glorot(i, o, name, rng, is_logistic_sigmoid=False, keepdims=False):
 41 |     #http://jmlr.org/proceedings/papers/v9/glorot10a/glorot10a.pdf
 42 |     d = np.sqrt(6. / (i + o))
 43 |     if is_logistic_sigmoid:
 44 |         d *= 4.
 45 |     W_values = rng.uniform(low=-d, high=d, size=_get_shape(i, o, keepdims)).astype(theano.config.floatX)
 46 |     return theano.shared(value=W_values, name=name, borrow=True)
 47 | 
 48 | def load(file_path, minibatch_size, x, p=None):
 49 |     import models
 50 |     import cPickle
 51 |     import theano
 52 |     import numpy as np
 53 | 
 54 |     with open(file_path, 'rb') as f:
 55 |         state = cPickle.load(f)
 56 | 
 57 |     Model = getattr(models, state["type"])
 58 | 
 59 |     rng = np.random
 60 |     rng.set_state(state["random_state"])
 61 | 
 62 |     net = Model(
 63 |         rng=rng,
 64 |         x=x,
 65 |         minibatch_size=minibatch_size,
 66 |         n_hidden=state["n_hidden"],
 67 |         x_vocabulary=state["x_vocabulary"],
 68 |         y_vocabulary=state["y_vocabulary"],
 69 |         stage1_model_file_name=state.get("stage1_model_file_name", None),
 70 |         p=p
 71 |         )
 72 | 
 73 |     for net_param, state_param in zip(net.params, state["params"]):
 74 |         net_param.set_value(state_param, borrow=True)
 75 | 
 76 |     gsums = [theano.shared(gsum) for gsum in state["gsums"]] if state["gsums"] else None
 77 | 
 78 |     return net, (gsums, state["learning_rate"], state["validation_ppl_history"], state["epoch"], rng)
 79 | 
 80 | class GRULayer(object):
 81 | 
 82 |     def __init__(self, rng, n_in, n_out, minibatch_size):
 83 |         super(GRULayer, self).__init__()
 84 |         # Notation from: An Empirical Exploration of Recurrent Network Architectures
 85 | 
 86 |         self.n_in = n_in
 87 |         self.n_out = n_out
 88 | 
 89 |         # Initial hidden state
 90 |         self.h0 = theano.shared(value=np.zeros((minibatch_size, n_out)).astype(theano.config.floatX), name='h0', borrow=True)
 91 | 
 92 |         # Gate parameters:
 93 |         self.W_x = weights_Glorot(n_in, n_out*2, 'W_x', rng)
 94 |         self.W_h = weights_Glorot(n_out, n_out*2, 'W_h', rng)
 95 |         self.b = weights_const(1, n_out*2, 'b', 0)
 96 |         # Input parameters
 97 |         self.W_x_h = weights_Glorot(n_in, n_out, 'W_x_h', rng)
 98 |         self.W_h_h = weights_Glorot(n_out, n_out, 'W_h_h', rng)
 99 |         self.b_h = weights_const(1, n_out, 'b_h', 0)
100 | 
101 |         self.params = [self.W_x, self.W_h, self.b, self.W_x_h, self.W_h_h, self.b_h]
102 | 
103 |     def step(self, x_t, h_tm1):
104 | 
105 |         rz = T.nnet.sigmoid(T.dot(x_t, self.W_x) + T.dot(h_tm1, self.W_h) + self.b)
106 |         r = _slice(rz, self.n_out, 0)
107 |         z = _slice(rz, self.n_out, 1)
108 | 
109 |         h = T.tanh(T.dot(x_t, self.W_x_h) + T.dot(h_tm1 * r, self.W_h_h) + self.b_h)
110 | 
111 |         h_t = z * h_tm1 + (1. - z) * h
112 | 
113 |         return h_t
114 | 
115 | class GRU(object):
116 | 
117 |     def __init__(self, rng, x, minibatch_size, n_hidden, x_vocabulary, y_vocabulary, stage1_model_file_name=None, p=None):
118 | 
119 |         assert not stage1_model_file_name and not p, "Stage 1 model can't have stage 1 model"
120 | 
121 |         x_vocabulary_size = len(x_vocabulary)
122 |         y_vocabulary_size = len(y_vocabulary)
123 | 
124 |         self.n_hidden = n_hidden
125 |         self.x_vocabulary = x_vocabulary
126 |         self.y_vocabulary = y_vocabulary
127 | 
128 |         # input model
129 |         pretrained_embs_path = "We.pcl"
130 |         if os.path.exists(pretrained_embs_path):
131 |             print "Found pretrained embeddings in '%s'. Using them..." % pretrained_embs_path
132 |             with open(pretrained_embs_path, 'rb') as f:
133 |                 We = cPickle.load(f)
134 |             n_emb = len(We[0])
135 |             We.append([0.1]*n_emb) # END
136 |             We.append([0.0]*n_emb) # UNK - both quite arbitrary initializations
137 | 
138 |             We = np.array(We).astype(theano.config.floatX)
139 |             self.We = theano.shared(value=We, name="We", borrow=True)
140 |         else:
141 |             n_emb = n_hidden
142 |             self.We = weights_Glorot(x_vocabulary_size, n_emb, 'We', rng) # Share embeddings between forward and backward model
143 | 
144 |         self.GRU_f = GRULayer(rng=rng, n_in=n_emb, n_out=n_hidden, minibatch_size=minibatch_size)
145 |         self.GRU_b = GRULayer(rng=rng, n_in=n_emb, n_out=n_hidden, minibatch_size=minibatch_size)
146 | 
147 |         # output model
148 |         self.GRU = GRULayer(rng=rng, n_in=n_hidden*2, n_out=n_hidden, minibatch_size=minibatch_size)
149 |         self.Wy = weights_const(n_hidden, y_vocabulary_size, 'Wy', 0)
150 |         self.by = weights_const(1, y_vocabulary_size, 'by', 0)
151 | 
152 |         # attention model
153 |         n_attention = n_hidden * 2 # to match concatenated forward and reverse model states
154 |         self.Wa_h = weights_Glorot(n_hidden, n_attention, 'Wa_h', rng) # output model previous hidden state to attention model weights
155 |         self.Wa_c = weights_Glorot(n_attention, n_attention, 'Wa_c', rng) # contexts to attention model weights
156 |         self.ba = weights_const(1, n_attention, 'ba', 0)
157 |         self.Wa_y = weights_Glorot(n_attention, 1, 'Wa_y', rng) # gives weights to contexts
158 | 
159 |         # Late fusion parameters
160 |         self.Wf_h = weights_const(n_hidden, n_hidden, 'Wf_h', 0)
161 |         self.Wf_c = weights_const(n_attention, n_hidden, 'Wf_c', 0)
162 |         self.Wf_f = weights_const(n_hidden, n_hidden, 'Wf_f', 0)
163 |         self.bf = weights_const(1, n_hidden, 'by', 0)
164 | 
165 |         self.params = [self.We,
166 |                        self.Wy, self.by,
167 |                        self.Wa_h, self.Wa_c, self.ba, self.Wa_y,
168 |                        self.Wf_h, self.Wf_c, self.Wf_f, self.bf]
169 | 
170 |         self.params += self.GRU.params + self.GRU_f.params + self.GRU_b.params
171 | 
172 |         # bi-directional recurrence
173 |         def input_recurrence(x_f_t, x_b_t, h_f_tm1, h_b_tm1):
174 |             h_f_t = self.GRU_f.step(x_t=x_f_t, h_tm1=h_f_tm1)
175 |             h_b_t = self.GRU_b.step(x_t=x_b_t, h_tm1=h_b_tm1)
176 |             return [h_f_t, h_b_t]
177 | 
178 |         def output_recurrence(x_t, h_tm1, Wa_h, Wa_y, Wf_h, Wf_c, Wf_f, bf, Wy, by, context, projected_context):
179 | 
180 |             # Attention model
181 |             h_a = T.tanh(projected_context + T.dot(h_tm1, Wa_h))
182 |             alphas = T.exp(T.dot(h_a, Wa_y))
183 |             alphas = alphas.reshape((alphas.shape[0], alphas.shape[1])) # drop 2-axis (sized 1)
184 |             alphas = alphas / alphas.sum(axis=0, keepdims=True)
185 |             weighted_context = (context * alphas[:,:,None]).sum(axis=0)
186 | 
187 |             h_t = self.GRU.step(x_t=x_t, h_tm1=h_tm1)
188 | 
189 |             # Late fusion
190 |             lfc = T.dot(weighted_context, Wf_c) # late fused context
191 |             fw = T.nnet.sigmoid(T.dot(lfc, Wf_f) + T.dot(h_t, Wf_h) + bf) # fusion weights
192 |             hf_t = lfc * fw + h_t # weighted fused context + hidden state
193 | 
194 |             z = T.dot(hf_t, Wy) + by
195 |             y_t = T.nnet.softmax(z)
196 | 
197 |             return [h_t, hf_t, y_t, alphas]
198 | 
199 |         x_emb = self.We[x.flatten()].reshape((x.shape[0], minibatch_size, n_emb))
200 | 
201 |         [h_f_t, h_b_t], _ = theano.scan(fn=input_recurrence,
202 |             sequences=[x_emb, x_emb[::-1]], # forward and backward sequences
203 |             outputs_info=[self.GRU_f.h0, self.GRU_b.h0])
204 | 
205 |         # 0-axis is time steps, 1-axis is batch size and 2-axis is hidden layer size
206 |         context = T.concatenate([h_f_t, h_b_t[::-1]], axis=2)
207 |         projected_context = T.dot(context, self.Wa_c) + self.ba
208 | 
209 |         [_, self.last_hidden_states, self.y, self.alphas], _ = theano.scan(fn=output_recurrence,
210 |             sequences=[context[1:]], # ignore the 1st word in context, because there's no punctuation before that
211 |             non_sequences=[self.Wa_h, self.Wa_y, self.Wf_h, self.Wf_c, self.Wf_f, self.bf, self.Wy, self.by, context, projected_context],
212 |             outputs_info=[self.GRU.h0, None, None, None])
213 | 
214 |         print "Number of parameters is %d" % sum(np.prod(p.shape.eval()) for p in self.params)
215 | 
216 |         self.L1 = sum(abs(p).sum() for p in self.params)
217 |         self.L2_sqr = sum((p**2).sum() for p in self.params)
218 | 
219 |     def cost(self, y):
220 |         num_outputs = self.y.shape[0]*self.y.shape[1] # time steps * number of parallel sequences in batch
221 |         output = self.y.reshape((num_outputs, self.y.shape[2]))
222 |         return -T.sum(T.log(output[T.arange(num_outputs), y.flatten()]))
223 | 
224 |     def save(self, file_path, gsums=None, learning_rate=None, validation_ppl_history=None, best_validation_ppl=None, epoch=None, random_state=None):
225 |         import cPickle
226 |         state = {
227 |             "type":                     self.__class__.__name__,
228 |             "n_hidden":                 self.n_hidden,
229 |             "x_vocabulary":             self.x_vocabulary,
230 |             "y_vocabulary":             self.y_vocabulary,
231 |             "stage1_model_file_name":   self.stage1_model_file_name if hasattr(self, "stage1_model_file_name") else None,
232 |             "params":                   [p.get_value(borrow=True) for p in self.params],
233 |             "gsums":                    [s.get_value(borrow=True) for s in gsums] if gsums else None,
234 |             "learning_rate":            learning_rate,
235 |             "validation_ppl_history":   validation_ppl_history,
236 |             "epoch":                    epoch,
237 |             "random_state":             random_state
238 |         }
239 | 
240 |         with open(file_path, 'wb') as f:
241 |             cPickle.dump(state, f, protocol=cPickle.HIGHEST_PROTOCOL)
242 | 
243 | 
244 | class GRUstage2(GRU):
245 | 
246 |     def __init__(self, rng, x, minibatch_size, n_hidden, x_vocabulary, y_vocabulary, stage1_model_file_name, p=None):
247 | 
248 |         y_vocabulary_size = len(y_vocabulary)
249 | 
250 |         self.stage1_model_file_name = stage1_model_file_name
251 |         self.stage1, _ = load(stage1_model_file_name, minibatch_size, x)
252 | 
253 |         self.n_hidden = n_hidden
254 |         self.x_vocabulary = x_vocabulary
255 |         self.y_vocabulary = y_vocabulary
256 | 
257 |         # output model
258 |         self.GRU = GRULayer(rng=rng, n_in=self.stage1.n_hidden + 1, n_out=n_hidden, minibatch_size=minibatch_size)
259 |         self.Wy = weights_const(n_hidden, y_vocabulary_size, 'Wy', 0)
260 |         self.by = weights_const(1, y_vocabulary_size, 'by', 0)
261 | 
262 |         self.params = [self.Wy, self.by]
263 |         self.params += self.GRU.params
264 | 
265 |         def recurrence(x_t, p_t, h_tm1, Wy, by):
266 | 
267 |             h_t = self.GRU.step(x_t=T.concatenate((x_t, p_t.dimshuffle((0, 'x'))), axis=1), h_tm1=h_tm1)
268 | 
269 |             z = T.dot(h_t, Wy) + by
270 |             y_t = T.nnet.softmax(z)
271 | 
272 |             return [h_t, y_t]
273 | 
274 |         [_, self.y], _ = theano.scan(fn=recurrence,
275 |             sequences=[self.stage1.last_hidden_states, p],
276 |             non_sequences=[self.Wy, self.by],
277 |             outputs_info=[self.GRU.h0, None])
278 | 
279 |         print "Number of parameters is %d" % sum(np.prod(p.shape.eval()) for p in self.params)
280 |         print "Number of parameters with stage1 params is %d" % sum(np.prod(p.shape.eval()) for p in self.params + self.stage1.params)
281 | 
282 |         self.L1 = sum(abs(p).sum() for p in self.params)
283 |         self.L2_sqr = sum((p**2).sum() for p in self.params)


--------------------------------------------------------------------------------