├── 12BarBluesOmnibook.txt ├── LICENSE ├── README.md ├── datasets.py ├── exponential_families.py ├── markov_continuator.py ├── markov_finite.py ├── markov_steerable.py ├── maxent.py ├── maxent_tests ├── maxent.py ├── minimize.py ├── optimize.py ├── test_spam.py └── tokenizing.py ├── minimize.py └── timidifyit.sh /12BarBluesOmnibook.txt: -------------------------------------------------------------------------------- 1 | AnotherHairdo 2 | C7/C7|C7/C7|C7/C7|C7/C7|F7/F7|F7/F7|C7/C7|C7/A7|Dm/Dm|G7/G7|C7/C7|C7/G7 3 | auPrivave 4 | C7/C7|Dm/D7|C7/C7|Gm/C7|F7/F7|F7/F7|C7/C7|Em/A7|Dm/Dm|G7/G7|C7/A7|Dm/G7 5 | BackHomeBlues 6 | C7/C7|C7/C7|C7/C7|C7/C7|F7/F7|F7/F7|C7/C7|Em/A7|Dm/Dm|G7/G7|C7/C7|Dm/G7 7 | Barbados 8 | C/C|Dm/G7|C7/C7|Gm/C7|F7/F7|F7/F7|Am/D7|Gm/Gm|C7/C7|F7/D7|G7/C7 9 | BilliesBounce 10 | C7/C7|F7/F7|C7/C7|C7/C7|F7/F7|F7/F7|C7/C7|Em/A7|Dm/Dm|G7/G7|C7/A7|D7/G7 11 | Bloomdido 12 | C/C|C7/C7|C7/C7|C7/C7|F7/F7|Fm/Fm|C7/C7|Ebm/Ebm|Dm/Dm|G7/G7|C7/C7|Dm/G7 13 | BlueBird 14 | C7/C7|C7/C7|C7/C7|C7/C7|F7/F7|F7/F7|C7/C7|Em/A7|Dm/Dm|G7/G7|C7/C7|Dm/G7 15 | Bluesfast 16 | C7/C7|C7/C7|C7/C7|C7/C7|F7/F7|F7/F7|C7/C7|C7/A7|Dm/Dm|G7/G7|C7/C7|Dm/G7 17 | bluesForAlice 18 | C/C|BhalfDim/E7|Am/D7|Gm/C7|F7/F7|Fm/Bb7|Em/Em|Ebm/Ab7|Dm7/Dm7|G7/G7|C7/C7|Dm/G7 19 | Buzzy 20 | C7/C7|C7/C7|C7/C7|C7/C7|F7/F7|F7/F7|C7/C7|A7/A7|Dm/Dm|G7/G7|C7/C7|Dm/G7 21 | Cheryl 22 | C7/C7|C7/C7|Gm/Gm|C7/C7|F7/F7|F7/F7|C7/C7|Em/A7|Dm/Dm|G7/G7|C7/C7|Dm/G7 23 | ChiChi 24 | C7/C7|Dm/G7|C7/C7|Gm/C7|F7/F7|Fm/Fm|Em/Em|Ebm/Ebm|Dm/Dm|G7/G7|Em/A7|Dm/G7 25 | CosmicRays 26 | C/C|F7/F7|C7/C7|Gm/C7|F/F|Fm/Fm|Em/Em|Em/A7|Dm/Dm|G7/G7|A/A|Dm/G7 27 | KCBlues 28 | C7/C7|C7/C7|C7/C7|C7/C7|F7/F7|F7/F7|C7/C7|C7/C7|Dm/Dm|G7/G7|C7/C7|G7/G7 29 | LairdBird 30 | C/C|BhalfDim/E7|Am/D7|Gm/C7|F7/F7|Fm/Fm|Em/Em|Ebm/Ebm|Dm/Dm|G7/G7|C/A7|Dm/G7 31 | Mohawk 32 | C7/C7|F7/F7|C/C|Gm/C7|F7/F7|F7/F7|C/C|A7/A7|Dm/Dm|G7/G7|C7/C7|C7/C7 33 | Mohawk2 34 | C7/C7|F7/F7|C7/C7|C7/C7|F7/F7|F7/F7|C7/C7|A7/A7|Dm/Dm|G7/G7|C7/C7|Dm/G7 35 | NowsTheTime 36 | C7/C7|C7/C7|C7/C7|C7/C7|F7/F7|F7/F7|C7/C7|A7/A7|Dm/Dm|G7/G7|C7/C7|G7/G7 37 | ParkersMood 38 | C7/C7|F7/F7|C7/C7|C7/C7|F7/F7|F7/F7|C/C|C7/Ab7|Dm/Dm|Dm/G7|C7/C7|Dm/G7 39 | Perhaps 40 | C7/C7|C7/C7|C7/C7|C7/C7|F7/F7|F7/F7|C7/C7|Em/A7|Dm/Dm|G7/G7|C7/C7|Dm/G7 41 | SiSi 42 | C/C|B halfDim/E7|Am/D7|Gm/C7|F7/F7|F7/F7|C/C|Em/A7|Dm/Dm|G7/G7|C7/C7|Dm/G7 43 | Visa 44 | C7/C7|F7/F7|C7/C7|C7/C7|F7/F7|F7/F7|C7/C7|A7/A7|Dm/Dm|G7/G7|C7/C7|G7/G7 -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2017, Kyle Kastner 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Usage 2 | ===== 3 | 4 | `python markov_continuator.py` 5 | 6 | or 7 | 8 | `python markov_steerable.py` 9 | 10 | or 11 | 12 | `python exponential_families.py` 13 | 14 | 15 | Info 16 | ==== 17 | This repo implements (parts of) several ideas from Sony CSL, led by Francois Pachet. See References for paper citations/links. Hear some example audio [at this link](https://badsamples.tumblr.com/post/164569290687/experiments-in-musical-textures-based-on-work). 18 | 19 | Exponential families is in a semi-modifiable form (see constants at the top of the file) 20 | but takes a while to run the first time. 21 | 22 | Continuator experiment is easier to modify (constants near the top) but is not very exciting. 23 | 24 | Using the constraint experiments (steerable) will generate chord progressions, but needs more know-how to use. Here the modifications should happen at the bottom of the file. 25 | 26 | This is *not* meant to be easy to use on new data, or out-of-the box usable without knowledge of the papers. 27 | The goal was for me to understand a series of interesting works, 28 | and hopefully provide some code that could be read/hacked up by other motivated people to understand the core ideas in these papers. 29 | 30 | That said, feel free to fork and add whatever you want :D 31 | 32 | 33 | A Bit on the Constrained Markov Process 34 | ======================================= 35 | An instance of a CMP object can be used by the .insert, and .branch methods. When creating these objects, one can specify the following: 36 | 37 | order to calculate likelihoods over 38 | 39 | max order, over which there *cannot* be any copying from the dataset (default None disables the check) 40 | 41 | ptype, which is how the probability is calculated for longer history 42 | 43 | constraints, passed as a dictionary 44 | 45 | These constraints are *requirements* that each sequence returned from .branch 46 | must satisfy. "start", "end", and "position" arguments all define that a given place in the sequence 47 | must have one of the elements of the list in that position. The alldiff constraint specifies that every sequence element *must* be different. 48 | 49 | Calling the .insert method with a sequence (list of tokens) will add it to the CMP object. 50 | 51 | Finally when calling branch the two key arguments are the sequence to start from, and the length of the sequence to generate. This is the expensive call, .insert should be fast. 52 | 53 | Constraints pt 2 54 | ================ 55 | For example, this snippet approximates the setup from "Markov Constraints: Steerable Generation of Markov Sequences" 56 | 57 | ``` 58 | order = 1 59 | m = CMP(order, 60 | max_order=None, 61 | ptype="fixed", 62 | named_constraints={"not_contains": ["C7"], 63 | "position": {8: ["F7"]}, 64 | "alldiff": True, 65 | "end": ["G7"]}) 66 | m.insert(chord_seq1) 67 | m.insert(chord_seq2) 68 | m.insert(chord_seq3) 69 | t = m.branch(["C7"], 15) 70 | ``` 71 | 72 | Adding constraints can greatly *decrease* runtime, due to reducing the search space. 73 | ptype "fixed" is faster to evaluate than the other options ("max", "avg") but gives different results. 74 | Changing the branch search type is not recommended, but could be useful in new problems 75 | 76 | 77 | Tips 78 | ==== 79 | Best sounding results come from editing `/etc/timidity/timidity.cfg` 80 | and using `source /etc/timidity/fluidr3_gm.cfg` instead of the default `source /etc/timidity/freepats.cfg` 81 | This may require the fluid-soundfont package, which I installed with `sudo apt-get install fluidsynth` 82 | To play midi files, do `timidity whatever_sample.mid`. 83 | Use the helper function `timidifyit.sh myfile.mid` to convert mid files into wav 84 | 85 | 86 | Requirements 87 | ============ 88 | numpy 89 | 90 | scipy 91 | 92 | music21 93 | 94 | pretty\_midi 95 | 96 | python 2.7 (3 may work, not tested) 97 | 98 | timidity (optional) 99 | 100 | 101 | References 102 | ========== 103 | The Continuator: Musical Interaction with Style 104 | F. Pachet 105 | https://www.csl.sony.fr/downloads/papers/uploads/pachet-02f.pdf 106 | 107 | Finite-Length Markov Processes With Constraints 108 | F. Pachet, P. Roy, G. Barbieri 109 | https://www.csl.sony.fr/downloads/papers/2011/pachet-11b.pdf 110 | 111 | Markov Constraints: Steerable Generation of Markov Sequences 112 | F. Pachet, P. Roy 113 | https://www.csl.sony.fr/downloads/papers/2011/pachet-09c.pdf 114 | 115 | Avoiding Plagiarism in Markov Sequence Generation 116 | A. Papadopolous, P. Roy, F. Pachet 117 | https://www.csl.sony.fr/downloads/papers/2014/papadopoulos-14a.pdf 118 | 119 | Enforcing Meter in Finite-Length Markov Sequences 120 | P. Roy, F. Pachet 121 | https://www.csl.sony.fr/downloads/papers/2013/roy-13a.pdf 122 | 123 | Non-Conformant Harmonization: The Real Book in the Style of Take 6 124 | F. Pachet, P. Roy 125 | https://www.csl.sony.fr/downloads/papers/2014/pachet-14a.pdf 126 | 127 | Style Imitation and Chord Invention in Polyphonic Music with Exponential Families 128 | Gaetan Hadjeres, Jason Sakellariou, Francois Pachet 129 | https://arxiv.org/abs/1609.05152 130 | 131 | The omnibook data came from here 132 | http://francoispachet.fr/texts/12BarBluesOmnibook.txt 133 | 134 | All Bach data is pulled from Music21 (https://github.com/cuthbertLab/music21) 135 | -------------------------------------------------------------------------------- /datasets.py: -------------------------------------------------------------------------------- 1 | # Author: Kyle Kastner 2 | # License: BSD 3-clause 3 | # pulled from pthbldr 4 | # Ideas from Junyoung Chung and Kyunghyun Cho 5 | # See https://github.com/jych/cle for a library in this style 6 | from music21 import converter, interval, pitch, harmony, analysis, spanner, midi, meter 7 | import numpy as np 8 | from collections import Counter 9 | from scipy.io import loadmat, wavfile 10 | from scipy.linalg import svd 11 | from functools import reduce 12 | import shutil 13 | import string 14 | import tarfile 15 | import fnmatch 16 | import zipfile 17 | import gzip 18 | import os 19 | import json 20 | import re 21 | import csv 22 | import time 23 | import signal 24 | import multiprocessing 25 | try: 26 | import cPickle as pickle 27 | except ImportError: 28 | import pickle 29 | 30 | floatX = "float32" 31 | 32 | def get_dataset_dir(dataset_name, data_dir=None, folder=None, create_dir=True): 33 | pth = os.getcwd() + os.sep + dataset_name 34 | if not os.path.exists(pth): 35 | os.mkdir(pth) 36 | return pth 37 | 38 | 39 | def download(url, server_fname, local_fname=None, progress_update_percentage=5): 40 | """ 41 | An internet download utility modified from 42 | http://stackoverflow.com/questions/22676/ 43 | how-do-i-download-a-file-over-http-using-python/22776#22776 44 | """ 45 | try: 46 | import urllib 47 | urllib.urlretrieve('http://google.com') 48 | except AttributeError: 49 | import urllib.request as urllib 50 | u = urllib.urlopen(url) 51 | if local_fname is None: 52 | local_fname = server_fname 53 | full_path = local_fname 54 | meta = u.info() 55 | with open(full_path, 'wb') as f: 56 | try: 57 | file_size = int(meta.get("Content-Length")) 58 | except TypeError: 59 | print("WARNING: Cannot get file size, displaying bytes instead!") 60 | file_size = 100 61 | print("Downloading: %s Bytes: %s" % (server_fname, file_size)) 62 | file_size_dl = 0 63 | block_sz = int(1E7) 64 | p = 0 65 | while True: 66 | buffer = u.read(block_sz) 67 | if not buffer: 68 | break 69 | file_size_dl += len(buffer) 70 | f.write(buffer) 71 | if (file_size_dl * 100. / file_size) > p: 72 | status = r"%10d [%3.2f%%]" % (file_size_dl, file_size_dl * 73 | 100. / file_size) 74 | print(status) 75 | p += progress_update_percentage 76 | 77 | 78 | def music21_to_chord_duration(p): 79 | """ 80 | Takes in a Music21 score, and outputs two lists 81 | List for chords (by string name) 82 | List for durations 83 | """ 84 | p_chords = p.chordify() 85 | p_chords_o = p_chords.flat.getElementsByClass('Chord') 86 | chord_list = [] 87 | duration_list = [] 88 | for ch in p_chords_o: 89 | chord_list.append(ch.primeFormString) 90 | #chord_list.append(ch.pitchedCommonName) 91 | duration_list.append(ch.duration.quarterLength) 92 | return chord_list, duration_list 93 | 94 | 95 | def music21_to_pitch_duration(p): 96 | """ 97 | Takes in a Music21 score, outputs 3 list of list 98 | One for pitch 99 | One for duration 100 | list for part times of each voice 101 | """ 102 | parts = [] 103 | parts_times = [] 104 | parts_delta_times = [] 105 | for i, pi in enumerate(p.parts): 106 | part = [] 107 | part_time = [] 108 | part_delta_time = [] 109 | total_time = 0 110 | for n in pi.stream().flat.notesAndRests: 111 | if n.isRest: 112 | part.append(0) 113 | else: 114 | try: 115 | part.append(n.midi) 116 | except AttributeError: 117 | continue 118 | part_time.append(total_time + n.duration.quarterLength) 119 | total_time = part_time[-1] 120 | part_delta_time.append(n.duration.quarterLength) 121 | parts.append(part) 122 | parts_times.append(part_time) 123 | parts_delta_times.append(part_delta_time) 124 | return parts, parts_times, parts_delta_times 125 | 126 | 127 | # http://stackoverflow.com/questions/2281850/timeout-function-if-it-takes-too-long-to-finish 128 | # only works on Unix platforms though 129 | class timeout: 130 | def __init__(self, seconds=1, error_message='Timeout'): 131 | self.seconds = seconds 132 | self.error_message = error_message 133 | def handle_timeout(self, signum, frame): 134 | raise ValueError(self.error_message) 135 | def __enter__(self): 136 | signal.signal(signal.SIGALRM, self.handle_timeout) 137 | signal.alarm(self.seconds) 138 | def __exit__(self, type, value, traceback): 139 | signal.alarm(0) 140 | 141 | 142 | def _single_extract_music21(files, data_path, skip_chords, verbose, force_time_sig_denom, n): 143 | if verbose: 144 | print("Starting file {} of {}".format(n, len(files))) 145 | f = files[n] 146 | file_path = os.path.join(data_path, f) 147 | 148 | start_time = time.time() 149 | 150 | try: 151 | p = converter.parse(file_path) 152 | k = p.analyze("key") 153 | parse_time = time.time() 154 | if verbose: 155 | r = parse_time - start_time 156 | print("Parse time {}:{}".format(f, r)) 157 | except (AttributeError, IndexError, UnicodeDecodeError, 158 | UnicodeEncodeError, harmony.ChordStepModificationException, 159 | ZeroDivisionError, 160 | ValueError, 161 | midi.MidiException, 162 | analysis.discrete.DiscreteAnalysisException, 163 | pitch.PitchException, 164 | spanner.SpannerException) as err: 165 | print("Parse failed for {}".format(f)) 166 | return ("null",) 167 | 168 | p.keySignature = k 169 | 170 | # none if there is no data aug 171 | an = "B" if "major" in k.name else "D" 172 | 173 | try: 174 | time_sigs = [str(ts).split(" ")[-1].split(">")[0] for ts in p.recurse().getElementsByClass(meter.TimeSignature)] 175 | nums = [int(ts.split("/")[0]) for ts in time_sigs] 176 | num_check = all([n == nums[0] for n in nums]) 177 | denoms = [int(ts.split("/")[1]) for ts in time_sigs] 178 | denom_check = all([d == denoms[0] for d in denoms]) 179 | if force_time_sig_denom is not None: 180 | quarter_check = denoms[0] == force_time_sig_denom 181 | else: 182 | quarter_check = True 183 | if not num_check or not denom_check or not quarter_check: 184 | raise TypeError("Invalid") 185 | 186 | pc = pitch.Pitch(an) 187 | i = interval.Interval(k.tonic, pc) 188 | p = p.transpose(i) 189 | k = p.analyze("key") 190 | transpose_time = time.time() 191 | if verbose: 192 | r = transpose_time - start_time 193 | print("Transpose time {}:{}".format(f, r)) 194 | 195 | if skip_chords: 196 | chords = ["null"] 197 | chord_durations = ["null"] 198 | else: 199 | chords, chord_durations = music21_to_chord_duration(p) 200 | pitches, parts_times, parts_delta_times = music21_to_pitch_duration(p) 201 | pitch_duration_time = time.time() 202 | if verbose: 203 | r = pitch_duration_time - start_time 204 | print("music21 to pitch_duration time {}:{}".format(f, r)) 205 | except TypeError: 206 | #raise ValueError("Non-transpose not yet supported") 207 | return ("null",) 208 | """ 209 | pc = pitch.Pitch(an) 210 | i = interval.Interval(k.tonic, pc) 211 | # FIXME: In this case chords are unnormed? 212 | if skip_chords: 213 | chords = ["null"] 214 | chord_durations = ["null"] 215 | else: 216 | chords, chord_durations = music21_to_chord_duration(p) 217 | pitches, durations = music21_to_pitch_duration(p) 218 | 219 | kt = k.tonic.pitchClass 220 | pct = pc.pitchClass 221 | assert kt >= 0 222 | if kt <= 6: 223 | pitches -= kt 224 | else: 225 | pitches -= 12 226 | pitches += (12 - kt) 227 | # now centered at C 228 | 229 | if "minor" in k.name: 230 | # C -> B -> B flat -> A 231 | pitches -= 3 232 | 233 | if pct <= 6: 234 | pitches += pct 235 | else: 236 | pitches -= 12 237 | pitches += pct 238 | """ 239 | 240 | str_key = "{} minor".format(an) if "minor" in k.name else "{} major".format(an) 241 | 242 | ttime = time.time() 243 | if verbose: 244 | r = ttime - start_time 245 | print("Overall file time {}:{}".format(f, r)) 246 | str_time_sig = time_sigs[0] 247 | return (pitches, parts_times, parts_delta_times, str_key, str_time_sig, f, p.quarterLength, chords, chord_durations) 248 | 249 | 250 | # http://stackoverflow.com/questions/29494001/how-can-i-abort-a-task-in-a-multiprocessing-pool-after-a-timeout 251 | def abortable_worker(func, *args, **kwargs): 252 | # returns ("null",) if timeout 253 | timeout = kwargs.get('timeout', None) 254 | p = multiprocessing.dummy.Pool(1) 255 | res = p.apply_async(func, args=args) 256 | try: 257 | out = res.get(timeout) # Wait timeout seconds for func to complete. 258 | return out 259 | except multiprocessing.TimeoutError: 260 | return ("null",) 261 | 262 | 263 | def count_unique(keys): 264 | uniq_keys = np.unique(keys) 265 | bins = uniq_keys.searchsorted(keys) 266 | return uniq_keys, np.bincount(bins) 267 | 268 | 269 | def _music_extract(data_path, pickle_path, ext=".xml", 270 | pitch_augmentation=False, 271 | skip_chords=True, 272 | skip_drums=True, 273 | lower_voice_limit=None, 274 | upper_voice_limit=None, 275 | equal_voice_count=4, 276 | force_denom=None, 277 | parse_timeout=100, 278 | multiprocess_count=4, 279 | verbose=False): 280 | 281 | if not os.path.exists(pickle_path): 282 | print("Pickled file %s not found, creating. This may take a few minutes..." % pickle_path) 283 | itime = time.time() 284 | 285 | all_transposed_pitch = [] 286 | all_transposed_parts_times = [] 287 | all_transposed_parts_delta_times = [] 288 | all_transposed_keys = [] 289 | all_time_sigs = [] 290 | all_file_names = [] 291 | all_transposed_chord = [] 292 | all_transposed_chord_duration = [] 293 | all_quarter_length = [] 294 | 295 | if 'basestring' not in globals(): 296 | basestring = str 297 | 298 | if isinstance(data_path, basestring): 299 | files = sorted([fi for fi in os.listdir(data_path) if fi.endswith(ext)]) 300 | else: 301 | files = sorted([ap for ap in data_path if ap.endswith(ext)]) 302 | 303 | #import pretty_midi 304 | print("Processing {} files".format(len(files))) 305 | force_denom = 4 306 | if multiprocess_count is not None: 307 | from multiprocessing import Pool 308 | import functools 309 | pool = Pool(4) 310 | 311 | ex = functools.partial(_single_extract_music21, 312 | files, data_path, 313 | skip_chords, verbose, force_denom) 314 | abortable_ex = functools.partial(abortable_worker, ex, timeout=parse_timeout) 315 | result = pool.map(abortable_ex, range(len(files))) 316 | pool.close() 317 | pool.join() 318 | else: 319 | result = [] 320 | for n in range(len(files)): 321 | r = _single_extract_music21(files, data_path, skip_chords, 322 | verbose, force_denom, n) 323 | result.append(r) 324 | 325 | for n, r in enumerate(result): 326 | if r[0] != "null": 327 | (pitches, parts_times, parts_delta_times, 328 | key, time_signature, fname, quarter_length, 329 | chords, chord_durations) = r 330 | 331 | all_transposed_chord.append(chords) 332 | all_transposed_chord_duration.append(chord_durations) 333 | all_transposed_pitch.append(pitches) 334 | all_transposed_parts_times.append(parts_times) 335 | all_transposed_parts_delta_times.append(parts_delta_times) 336 | all_transposed_keys.append(key) 337 | all_time_sigs.append(time_signature) 338 | all_file_names.append(fname) 339 | all_quarter_length.append(quarter_length) 340 | else: 341 | print("Result {} timed out".format(n)) 342 | gtime = time.time() 343 | if verbose: 344 | r = gtime - itime 345 | print("Overall time {}".format(r)) 346 | d = {"data_pitch": all_transposed_pitch, 347 | "data_parts_times": all_transposed_parts_times, 348 | "data_parts_delta_times": all_transposed_parts_delta_times, 349 | "data_key": all_transposed_keys, 350 | "data_time_sig": all_time_sigs, 351 | "data_chord": all_transposed_chord, 352 | "data_chord_duration": all_transposed_chord_duration, 353 | "data_quarter_length": all_quarter_length, 354 | "file_names": all_file_names} 355 | with open(pickle_path, "wb") as f: 356 | print("Saving pickle file %s" % pickle_path) 357 | pickle.dump(d, f) 358 | print("Pickle file %s saved" % pickle_path) 359 | else: 360 | print("Loading cached data from {}".format(pickle_path)) 361 | with open(pickle_path, "rb") as f: 362 | d = pickle.load(f) 363 | 364 | major_pitch = [] 365 | minor_pitch = [] 366 | 367 | major_time_sigs = [] 368 | minor_time_sigs = [] 369 | 370 | major_part_times = [] 371 | minor_part_times = [] 372 | 373 | major_part_delta_times = [] 374 | minor_part_delta_times = [] 375 | 376 | major_chord = [] 377 | minor_chord = [] 378 | 379 | major_chord_duration = [] 380 | minor_chord_duration = [] 381 | 382 | major_filename = [] 383 | minor_filename = [] 384 | 385 | major_quarter_length = [] 386 | minor_quarter_length = [] 387 | 388 | major_part_times = [] 389 | minor_part_times = [] 390 | 391 | major_time_sigs = [] 392 | minor_time_sigs = [] 393 | 394 | keys = [] 395 | for i in range(len(d["data_key"])): 396 | k = d["data_key"][i] 397 | ts = d["data_time_sig"][i] 398 | ddp = d["data_pitch"][i] 399 | ddt = d["data_parts_times"][i] 400 | ddtd = d["data_parts_delta_times"][i] 401 | nm = d["file_names"][i] 402 | ql = d["data_quarter_length"][i] 403 | try: 404 | ch = d["data_chord"][i] 405 | chd = d["data_chord_duration"][i] 406 | except IndexError: 407 | ch = "null" 408 | chd = -1 409 | 410 | if "major" in k: 411 | major_pitch.append(ddp) 412 | major_time_sigs.append(ts) 413 | major_part_times.append(ddt) 414 | major_part_delta_times.append(ddtd) 415 | major_filename.append(nm) 416 | major_chord.append(ch) 417 | major_chord_duration.append(chd) 418 | major_quarter_length.append(ql) 419 | keys.append(k) 420 | elif "minor" in k: 421 | minor_pitch.append(ddp) 422 | minor_time_sigs.append(ts) 423 | minor_part_times.append(ddt) 424 | minor_part_delta_times.append(ddtd) 425 | minor_filename.append(nm) 426 | minor_chord.append(ch) 427 | minor_chord_duration.append(chd) 428 | minor_quarter_length.append(ql) 429 | keys.append(k) 430 | else: 431 | raise ValueError("Unknown key %s" % k) 432 | 433 | all_pitches = major_pitch + minor_pitch 434 | all_time_sigs = major_time_sigs + minor_time_sigs 435 | all_part_times = major_part_times + minor_part_times 436 | all_part_delta_times = major_part_delta_times + minor_part_delta_times 437 | all_filenames = major_filename + minor_filename 438 | all_chord = major_chord + minor_chord 439 | all_chord_duration = major_chord_duration + minor_chord_duration 440 | all_quarter_length = major_quarter_length + minor_quarter_length 441 | 442 | all_notes = np.unique([ni for p in all_pitches for pi in p for ni in pi]) 443 | n_notes = len(all_notes) 444 | 445 | final_chord_set = [] 446 | final_chord_duration_set = [] 447 | for n in range(len(all_chord)): 448 | final_chord_set.extend(all_chord[n]) 449 | final_chord_duration_set.extend(all_chord_duration[n]) 450 | 451 | final_chord_set = sorted(set(final_chord_set)) 452 | final_chord_lookup = {k: v for k, v in zip(final_chord_set, range(len(final_chord_set)))} 453 | final_chord_duration_set = sorted(set(final_chord_duration_set)) 454 | final_chord_duration_lookup = {k: v for k, v in zip(final_chord_duration_set, range(len(final_chord_duration_set)))} 455 | 456 | final_chord = [] 457 | final_chord_duration = [] 458 | for n in range(len(all_chord)): 459 | final_chord.append(np.array([final_chord_lookup[ch] for ch in all_chord[n]]).astype(floatX)) 460 | final_chord_duration.append(np.array([final_chord_duration_lookup[chd] for chd in all_chord_duration[n]]).astype(floatX)) 461 | 462 | final_pitches = [] 463 | final_time_sigs = [] 464 | final_durations = [] 465 | final_part_times = [] 466 | final_part_delta_times = [] 467 | final_filenames = [] 468 | final_keys = [] 469 | final_quarter_length = [] 470 | 471 | invalid_idx = [] 472 | for i in range(len(all_pitches)): 473 | n = len(all_pitches[i]) 474 | if lower_voice_limit is None and upper_voice_limit is None: 475 | cond = True 476 | else: 477 | raise ValueError("Voice limiting not yet implemented...") 478 | 479 | #if cond: 480 | if n == equal_voice_count: 481 | final_pitches.append(all_pitches[i]) 482 | final_time_sigs.append(all_time_sigs[i]) 483 | final_part_times.append(all_part_times[i]) 484 | final_part_delta_times.append(all_part_delta_times[i]) 485 | final_filenames.append(all_filenames[i]) 486 | final_keys.append(keys[i]) 487 | final_quarter_length.append(all_quarter_length[i]) 488 | else: 489 | invalid_idx.append(i) 490 | if verbose: 491 | print("Skipping file {}: {} had invalid note count {}, {} required".format( 492 | i, all_filenames[i], n, equal_voice_count)) 493 | 494 | # drop and align 495 | final_chord = [fc for n, fc in enumerate(final_chord) 496 | if n not in invalid_idx] 497 | final_chord_duration = [fcd for n, fcd in enumerate(final_chord_duration) 498 | if n not in invalid_idx] 499 | 500 | all_chord = final_chord 501 | all_chord_duration = final_chord_duration 502 | all_time_sigs = final_time_sigs 503 | 504 | all_pitches = final_pitches 505 | all_part_times = final_part_times 506 | all_part_delta_times = final_part_delta_times 507 | all_filenames = final_filenames 508 | all_keys = final_keys 509 | all_quarter_length = final_quarter_length 510 | 511 | pitch_list = list(np.unique([ni for p in all_pitches for pi in p for ni in pi])) 512 | part_delta_times_list = list(np.unique([ni for pdt in all_part_delta_times for pdti in pdt for ni in pdti])) 513 | 514 | basic_durs = [.125, .25, .33, .5, .66, .75, 1., 1.5, 2., 2.5, 3, 3.5, 4., 5., 6., 8.] 515 | if len(part_delta_times_list) > len(basic_durs): 516 | from scipy.cluster.vq import kmeans2, vq 517 | raise ValueError("Duration clustering nyi") 518 | 519 | #cent, lbl = kmeans2(np.array(duration_list), 200) 520 | 521 | # relative to quarter length 522 | 523 | ul = np.percentile(duration_list, 90) 524 | duration_list = [dl if dl < ul else ul for dl in duration_list] 525 | counts, tt = np.histogram(duration_list, 30) 526 | cent = tt[:-1] + (tt[1:] - tt[:-1]) * .5 527 | cent = cent[cent > basic_durs[-1]] 528 | cent = sorted(basic_durs + list(cent)) 529 | 530 | all_durations_new = [] 531 | for adi in all_durations: 532 | shp = adi.shape 533 | fixed = vq(adi.flatten(), cent)[0] 534 | fixed = fixed.astype(floatX) 535 | 536 | code_where = [] 537 | for n, ci in enumerate(cent): 538 | code_where.append(np.where(fixed == n)) 539 | 540 | for n, cw in enumerate(code_where): 541 | fixed[cw] = cent[n] 542 | 543 | fixed = fixed.reshape(shp) 544 | all_durations_new.append(fixed) 545 | all_durations = all_durations_new 546 | duration_list = list(np.unique(np.concatenate([np.unique(adi) for adi in all_durations]))) 547 | 548 | pitch_lu = {k: v for v, k in enumerate(pitch_list)} 549 | duration_lu = {k: v for v, k in enumerate(part_delta_times_list)} 550 | 551 | quarter_length_list = sorted([float(ql) for ql in list(set(all_quarter_length))]) 552 | all_quarter_length = [float(ql) for ql in all_quarter_length] 553 | 554 | r = {"list_of_data_pitch": all_pitches, 555 | "list_of_data_time": all_part_times, 556 | "list_of_data_time_delta": all_part_delta_times, 557 | "list_of_data_key": all_keys, 558 | "list_of_data_time_sig": all_time_sigs, 559 | "list_of_data_chord": all_chord, 560 | "list_of_data_chord_duration": all_chord_duration, 561 | "list_of_data_quarter_length": all_quarter_length, 562 | "chord_list": final_chord_set, 563 | "chord_duration_list": final_chord_duration_set, 564 | "pitch_list": pitch_list, 565 | "part_delta_times_list": part_delta_times_list, 566 | "quarter_length_list": quarter_length_list, 567 | "filename_list": all_filenames} 568 | return r 569 | 570 | 571 | def check_fetch_bach_chorales_music21(): 572 | """ Move files into pthbldr dir, in case python is on nfs. """ 573 | from music21 import corpus 574 | all_bach_paths = corpus.getComposer("bach") 575 | partial_path = get_dataset_dir("bach_chorales_music21") 576 | for path in all_bach_paths: 577 | if "riemenschneider" in path: 578 | continue 579 | filename = os.path.split(path)[-1] 580 | local_path = os.path.join(partial_path, filename) 581 | if not os.path.exists(local_path): 582 | shutil.copy2(path, local_path) 583 | return partial_path 584 | 585 | 586 | def fetch_bach_chorales_music21(keys=["B major", "D minor"], 587 | truncate_length=100, 588 | equal_voice_count=4, 589 | force_denom=4, 590 | compress_pitch=False, 591 | compress_duration=False, 592 | verbose=True): 593 | """ 594 | Bach chorales, transposed to C major or A minor (depending on original key). 595 | Only contains chorales with 4 voices populated. 596 | Requires music21. 597 | 598 | n_timesteps : 34270 599 | n_features : 4 600 | n_classes : 12 (duration), 54 (pitch) 601 | 602 | Returns 603 | ------- 604 | summary : dict 605 | A dictionary cantaining data and image statistics. 606 | 607 | summary["list_of_data_pitch"] : list of array 608 | Pitches for each piece 609 | summary["list_of_data_duration"] : list of array 610 | Durations for each piece 611 | summary["list_of_data_key"] : list of str 612 | String key for each piece 613 | summary["list_of_data_chord"] : list of str 614 | String chords for each piece 615 | summary["list_of_data_chord_duration"] : list of str 616 | String chords for each piece 617 | summary["pitch_list"] : list 618 | summary["duration_list"] : list 619 | 620 | pitch_list and duration_list give the mapping back from array value to 621 | actual data value. 622 | """ 623 | 624 | data_path = check_fetch_bach_chorales_music21() 625 | pickle_path = os.path.join(data_path, "__processed_bach.pkl") 626 | mu = _music_extract(data_path, pickle_path, ext=".mxl", 627 | skip_chords=False, equal_voice_count=equal_voice_count, 628 | force_denom=force_denom, 629 | verbose=verbose) 630 | 631 | lp = mu["list_of_data_pitch"] 632 | lt = mu["list_of_data_time"] 633 | ltd = mu["list_of_data_time_delta"] 634 | lql = mu["list_of_data_quarter_length"] 635 | 636 | del mu["list_of_data_chord"] 637 | del mu["list_of_data_chord_duration"] 638 | del mu["chord_list"] 639 | del mu["chord_duration_list"] 640 | 641 | def _len_prune(l): 642 | return [[lii[:truncate_length] for lii in li] for li in l] 643 | 644 | lp2 = _len_prune(lp) 645 | lt2 = _len_prune(lt) 646 | ltd2 = _len_prune(ltd) 647 | 648 | def _key_prune(l): 649 | k = mu["list_of_data_key"] 650 | assert len(l) == len(k) 651 | return [li for li, ki in zip(l, k) if ki in keys] 652 | 653 | lp2 = _key_prune(lp2) 654 | lt2 = _key_prune(lt2) 655 | ltd2 = _key_prune(ltd2) 656 | lql2 = _key_prune(lql) 657 | 658 | lp = lp2 659 | lt = lt2 660 | ltd = ltd2 661 | lql = lql2 662 | 663 | mu["list_of_data_pitch"] = lp 664 | mu["list_of_data_time"] = lt 665 | mu["list_of_data_time_delta"] = ltd 666 | mu["list_of_data_quarter_length"] = lql 667 | return mu 668 | 669 | 670 | def quantized_to_pretty_midi(quantized, 671 | quantized_bin_size, 672 | save_dir="samples", 673 | name_tag="sample_{}.mid", 674 | add_to_name=0, 675 | lower_pitch_limit=12, 676 | list_of_quarter_length=None, 677 | max_hold_bars=1, 678 | default_quarter_length=47, 679 | voice_params="woodwinds"): 680 | """ 681 | takes in list of list of list, or list of array with axis 0 time, axis 1 voice_number (S,A,T,B) 682 | outer list is over samples, middle list is over voice, inner list is over time 683 | """ 684 | 685 | is_seq_of_seq = False 686 | try: 687 | quantized[0][0] 688 | if not hasattr(quantized[0], "flatten"): 689 | is_seq_of_seq = True 690 | except: 691 | try: 692 | quantized[0].shape 693 | except AttributeError: 694 | raise ValueError("quantized must be a sequence of sequence (such as list of array, or list of list) or numpy array") 695 | 696 | # list of list or mb? 697 | n_samples = len(quantized) 698 | all_pitches = [] 699 | all_durations = [] 700 | 701 | max_hold = int(max_hold_bars / quantized_bin_size) 702 | if max_hold < max_hold_bars: 703 | max_hold = max_hold_bars 704 | 705 | for ss in range(n_samples): 706 | pitches = [] 707 | durations = [] 708 | if is_seq_of_seq: 709 | voices = len(quantized[ss]) 710 | qq = quantized[ss] 711 | else: 712 | voices = quantized[ss].shape[1] 713 | qq = quantized[ss].T 714 | for i in range(voices): 715 | q = qq[i] 716 | pitch_i = [0] 717 | dur_i = [] 718 | cur = None 719 | count = 0 720 | for qi in q: 721 | if qi != cur:# or count > max_hold: 722 | if cur is None: 723 | cur = qi 724 | count += 1 725 | continue 726 | pitch_i.append(qi) 727 | quarter_count = quantized_bin_size * (count + 1) 728 | dur_i.append(quarter_count) 729 | cur = qi 730 | count = 0 731 | else: 732 | count += 1 733 | quarter_count = quantized_bin_size * (count + 1) 734 | dur_i.append(quarter_count) 735 | pitches.append(pitch_i) 736 | durations.append(dur_i) 737 | all_pitches.append(pitches) 738 | all_durations.append(durations) 739 | pitches_and_durations_to_pretty_midi(all_pitches, all_durations, 740 | save_dir=save_dir, 741 | name_tag=name_tag, 742 | add_to_name=add_to_name, 743 | lower_pitch_limit=lower_pitch_limit, 744 | list_of_quarter_length=list_of_quarter_length, 745 | default_quarter_length=default_quarter_length, 746 | voice_params=voice_params) 747 | 748 | 749 | def pitches_and_durations_to_pretty_midi(pitches, durations, 750 | save_dir="samples", 751 | name_tag="sample_{}.mid", 752 | add_to_name=0, 753 | lower_pitch_limit=12, 754 | list_of_quarter_length=None, 755 | default_quarter_length=47, 756 | voice_params="woodwinds"): 757 | # allow list of list of list 758 | """ 759 | takes in list of list of list, or list of array with axis 0 time, axis 1 voice_number (S,A,T,B) 760 | outer list is over samples, middle list is over voice, inner list is over time 761 | durations assumed to be scaled to quarter lengths e.g. 1 is 1 quarter note 762 | 2 is a half note, etc 763 | """ 764 | is_seq_of_seq = False 765 | try: 766 | pitches[0][0] 767 | durations[0][0] 768 | if not hasattr(pitches, "flatten") and not hasattr(durations, "flatten"): 769 | is_seq_of_seq = True 770 | except: 771 | raise ValueError("pitches and durations must be a list of array, or list of list of list (time, voice, pitch/duration)") 772 | 773 | if is_seq_of_seq: 774 | if hasattr(pitches[0], "flatten"): 775 | # it's a list of array, convert to list of list of list 776 | pitches = [[[pitches[i][j, k] for j in range(pitches[i].shape[0])] for k in range(pitches[i].shape[1])] for i in range(len(pitches))] 777 | durations = [[[durations[i][j, k] for j in range(durations[i].shape[0])] for k in range(durations[i].shape[1])] for i in range(len(durations))] 778 | 779 | 780 | import pretty_midi 781 | # BTAS mapping 782 | def weird(): 783 | voice_mappings = ["Sitar", "Orchestral Harp", "Acoustic Guitar (nylon)", 784 | "Pan Flute"] 785 | voice_velocity = [20, 80, 80, 40] 786 | voice_offset = [0, 0, 0, 0] 787 | voice_decay = [1., 1., 1., .95] 788 | return voice_mappings, voice_velocity, voice_offset, voice_decay 789 | 790 | if voice_params == "weird": 791 | voice_mappings, voice_velocity, voice_offset, voice_decay = weird() 792 | elif voice_params == "weird_r": 793 | voice_mappings, voice_velocity, voice_offset, voice_decay = weird() 794 | voice_mappings = voice_mappings[::-1] 795 | voice_velocity = voice_velocity[::-1] 796 | voice_offset = voice_offset[::-1] 797 | elif voice_params == "nylon": 798 | voice_mappings = ["Acoustic Guitar (nylon)"] * 4 799 | voice_velocity = [20, 16, 25, 10] 800 | voice_offset = [0, 0, 0, -12] 801 | voice_decay = [1., 1., 1., 1.] 802 | voice_decay = voice_decay[::-1] 803 | elif voice_params == "legend": 804 | # LoZ 805 | voice_mappings = ["Acoustic Guitar (nylon)"] * 3 + ["Pan Flute"] 806 | voice_velocity = [20, 16, 25, 5] 807 | voice_offset = [0, 0, 0, -12] 808 | voice_decay = [1., 1., 1., .95] 809 | elif voice_params == "organ": 810 | voice_mappings = ["Church Organ"] * 4 811 | voice_velocity = [40, 30, 30, 60] 812 | voice_offset = [0, 0, 0, 0] 813 | voice_decay = [.98, .98, .98, .98] 814 | elif voice_params == "piano": 815 | voice_mappings = ["Acoustic Grand Piano"] * 4 816 | voice_velocity = [40, 30, 30, 60] 817 | voice_offset = [0, 0, 0, 0] 818 | voice_decay = [1., 1., 1., 1.] 819 | elif voice_params == "electric_piano": 820 | voice_mappings = ["Electric Piano 1"] * 4 821 | voice_velocity = [40, 30, 30, 60] 822 | voice_offset = [0, 0, 0, 0] 823 | voice_decay = [1., 1., 1., 1.] 824 | elif voice_params == "harpsichord": 825 | voice_mappings = ["Harpsichord"] * 4 826 | voice_velocity = [40, 30, 30, 60] 827 | voice_offset = [0, 0, 0, 0] 828 | voice_decay = [1., 1., 1., 1.] 829 | elif voice_params == "woodwinds": 830 | voice_mappings = ["Bassoon", "Clarinet", "English Horn", "Oboe"] 831 | voice_velocity = [50, 30, 30, 40] 832 | voice_offset = [0, 0, 0, 0] 833 | voice_decay = [1., 1., 1., 1.] 834 | else: 835 | # eventually add and define dictionary support here 836 | raise ValueError("Unknown voice mapping specified") 837 | 838 | # normalize 839 | mm = float(max(voice_velocity)) 840 | mi = float(min(voice_velocity)) 841 | dynamic_range = min(80, (mm - mi)) 842 | # keep same scale just make it louder? 843 | voice_velocity = [int((80 - dynamic_range) + int(v - mi)) for v in voice_velocity] 844 | 845 | if not is_seq_of_seq: 846 | order = durations.shape[-1] 847 | else: 848 | try: 849 | # TODO: reorganize so list of array and list of list of list work 850 | order = durations[0].shape[-1] 851 | except: 852 | order = len(durations[0]) 853 | voice_mappings = voice_mappings[-order:] 854 | voice_velocity = voice_velocity[-order:] 855 | voice_offset = voice_offset[-order:] 856 | voice_decay = voice_decay[-order:] 857 | if not is_seq_of_seq: 858 | pitches = [pitches[:, i, :] for i in range(pitches.shape[1])] 859 | durations = [durations[:, i, :] for i in range(durations.shape[1])] 860 | 861 | n_samples = len(durations) 862 | for ss in range(n_samples): 863 | durations_ss = durations[ss] 864 | pitches_ss = pitches[ss] 865 | # same number of voices 866 | assert len(durations_ss) == len(pitches_ss) 867 | # time length match 868 | assert all([len(durations_ss[i]) == len(pitches_ss[i]) for i in range(len(pitches_ss))]) 869 | pm_obj = pretty_midi.PrettyMIDI() 870 | # Create an Instrument instance for a cello instrument 871 | def mkpm(name): 872 | return pretty_midi.instrument_name_to_program(name) 873 | 874 | def mki(p): 875 | return pretty_midi.Instrument(program=p) 876 | 877 | pm_programs = [mkpm(n) for n in voice_mappings] 878 | pm_instruments = [mki(p) for p in pm_programs] 879 | 880 | if list_of_quarter_length is None: 881 | # qpm to s per quarter = 60 s per min / quarters per min 882 | time_scale = 60. / default_quarter_length 883 | else: 884 | time_scale = 60. / list_of_quarter_length[ss] 885 | 886 | time_offset = np.zeros((order,)) 887 | 888 | # swap so that SATB order becomes BTAS for voice matching 889 | pitches_ss = pitches_ss[::-1] 890 | durations_ss = durations_ss[::-1] 891 | 892 | # time 893 | for ii in range(len(durations_ss[0])): 894 | # voice 895 | for jj in range(order): 896 | try: 897 | pitches_isj = pitches_ss[jj][ii] 898 | durations_isj = durations_ss[jj][ii] 899 | except IndexError: 900 | # voices may stop short 901 | continue 902 | p = int(pitches_isj) 903 | d = durations_isj 904 | if d < 0: 905 | continue 906 | if p < 0: 907 | continue 908 | # hack out the whole last octave? 909 | s = time_scale * time_offset[jj] 910 | e = time_scale * (time_offset[jj] + voice_decay[jj] * d) 911 | time_offset[jj] += d 912 | if p < lower_pitch_limit: 913 | continue 914 | note = pretty_midi.Note(velocity=voice_velocity[jj], 915 | pitch=p + voice_offset[jj], 916 | start=s, end=e) 917 | # Add it to our instrument 918 | pm_instruments[jj].notes.append(note) 919 | # Add the instrument to the PrettyMIDI object 920 | for pm_instrument in pm_instruments: 921 | pm_obj.instruments.append(pm_instrument) 922 | # Write out the MIDI data 923 | 924 | sv = save_dir + os.sep + name_tag.format(ss + add_to_name) 925 | try: 926 | pm_obj.write(sv) 927 | except ValueError: 928 | print("Unable to write file {} due to mido error".format(sv)) 929 | -------------------------------------------------------------------------------- /exponential_families.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import numpy as np 3 | from scipy.cluster.vq import vq 4 | import os 5 | import cPickle as pickle 6 | import copy 7 | import collections 8 | from collections import defaultdict, Counter 9 | from maxent import SparseMaxEnt, log_sum_exp 10 | import cPickle as pickle 11 | import time 12 | 13 | from datasets import fetch_bach_chorales_music21 14 | from datasets import quantized_to_pretty_midi 15 | 16 | # default tempo of the saved midi 17 | default_quarter_length = 70 18 | # options include "nylon", "harpsichord", "woodwinds", "piano", "electric_piano", "organ", "legend", "weird" 19 | voice_type = "piano" 20 | # use data from a given key, can be "major" or "minor" 21 | # be sure to remove all the tmp_*.pkl files in the directory if you change this! 22 | key = "major" 23 | # l1 weight for training 24 | l1 = 3E-5 25 | # number of iterations to do resampling 26 | total_itr = 15 27 | # number of candidate notes per voice for random part of proposal distribution 28 | num_cands = 4 29 | # length of generation in quarter notes 30 | song_len = 500 31 | # song index into the dataset - random initial notes come from this song 32 | song_ind = 9 33 | # proportion of propsals that come from the model versus random 34 | model_proportion = 0.99 35 | # temperature of softmax for sampling 36 | temperature = 0.01 37 | # random seeds for shuffling data, training the model, and sampling 38 | shuffle_seed = 1999 39 | model_seed = 2100 40 | randomness_seed = 2147 41 | # directory to save in 42 | save_dir = "samples" 43 | if not os.path.exists(save_dir): 44 | os.mkdir(save_dir) 45 | 46 | def pitch_and_duration_to_piano_roll(list_of_pitch_voices, list_of_duration_voices, min_dur): 47 | def expand(pitch, dur, min_dur): 48 | assert len(pitch) == len(dur) 49 | expanded = [int(d // min_dur) for d in dur] 50 | check = [d / min_dur for d in dur] 51 | assert all([e == c for e, c in zip(expanded, check)]) 52 | stretch = [[p] * e for p, e in zip(pitch, expanded)] 53 | # flatten out to 1 voice 54 | return [pi for p in stretch for pi in p] 55 | 56 | res = [] 57 | for lpv, ldv in zip(list_of_pitch_voices, list_of_duration_voices): 58 | qi = expand(lpv, ldv, min_dur) 59 | res.append(qi) 60 | 61 | min_len = min([len(ri) for ri in res]) 62 | res = [ri[:min_len] for ri in res] 63 | piano_roll = np.array(res).transpose() 64 | return piano_roll 65 | 66 | 67 | def get_data(offset=88, shuffle=True): 68 | if os.path.exists("tmp_data.pkl"): 69 | print("Found existing data storage {}, loading...".format("tmp_data.pkl")) 70 | with open("tmp_data.pkl", "r") as f: 71 | r = pickle.load(f) 72 | return r 73 | 74 | mu = fetch_bach_chorales_music21() 75 | 76 | order = len(mu["list_of_data_pitch"][0]) 77 | 78 | random_state = np.random.RandomState(shuffle_seed) 79 | 80 | lp = mu["list_of_data_pitch"] 81 | lt = mu["list_of_data_time"] 82 | ltd = mu["list_of_data_time_delta"] 83 | lql = mu["list_of_data_quarter_length"] 84 | fnames = mu["filename_list"] 85 | 86 | if key != None: 87 | keep_lp = [] 88 | keep_lt = [] 89 | keep_ltd = [] 90 | keep_lql = [] 91 | keep_fnames = [] 92 | lk = mu["list_of_data_key"] 93 | for n in range(len(lp)): 94 | if key in lk[n]: 95 | keep_lp.append(lp[n]) 96 | keep_lt.append(lt[n]) 97 | keep_ltd.append(ltd[n]) 98 | keep_lql.append(lql[n]) 99 | keep_fnames.append(fnames[n]) 100 | lp = copy.deepcopy(keep_lp) 101 | lt = copy.deepcopy(keep_lt) 102 | ltd = copy.deepcopy(keep_ltd) 103 | lql = copy.deepcopy(keep_lql) 104 | fnames = copy.deepcopy(keep_fnames) 105 | 106 | all_pr = [] 107 | all_len = [] 108 | for ii in range(len(lp)): 109 | # 16th note piano roll 110 | pr = pitch_and_duration_to_piano_roll(lp[ii], ltd[ii], .0625) 111 | # only want things that are on the beats! 112 | # 16th notes into quarters is a subdivision of 4 113 | pr = pr[:len(pr) - len(pr) % 4] 114 | pr = pr[::4] 115 | # also avoid bars with silences in a voice 116 | nonsil = np.where(pr != 0)[0] 117 | pr = pr[nonsil] 118 | all_len.append(len(pr)) 119 | all_pr.append(pr) 120 | 121 | note_set = set() 122 | for n, pr in enumerate(all_pr): 123 | uq = set(tuple(np.unique(pr))) 124 | note_set = note_set | uq 125 | note_set = sorted(list(set(note_set))) 126 | 127 | """ 128 | name_tag = "actual_{}.mid" 129 | save_dir = "samples/samples" 130 | 131 | quantized_to_pretty_midi(all_pr[:10], .125, 132 | save_dir=save_dir, 133 | name_tag=name_tag, 134 | default_quarter_length=80, 135 | voice_params="nylon") 136 | """ 137 | 138 | lut = {} 139 | rlut = {} 140 | i = 0 141 | for n in sorted(list(note_set)): 142 | lut[n] = i 143 | rlut[i] = n 144 | i += 1 145 | all_start = np.cumsum(all_len) 146 | all_start = np.append(0, all_start) 147 | return all_pr, lut, rlut, all_start 148 | 149 | offset = 88 150 | h_context = 3 151 | all_pieces, lut, rlut, song_start_idx = get_data(offset) 152 | dataset = np.concatenate(all_pieces, axis=0) 153 | 154 | n_classes = len(lut.keys()) 155 | n_classes = offset 156 | n_features_per = offset 157 | 158 | # -h_context to h_context, ignore self = 2 * 3 159 | # 3 for vertical minus self 160 | # 3 for prev diag 161 | # 3 for future diag 162 | n_features = n_features_per * (2 * 3 + 3 + 3 + 3) 163 | dataset = np.ascontiguousarray(dataset) 164 | 165 | def feature_fn(X, i): 166 | which_voice = wv = i 167 | features = [] 168 | notes = X 169 | for ii in range(h_context, len(notes) - h_context): 170 | tot = 0 171 | # hard coded for 4 voices 172 | nv = [n for n in [0, 1, 2, 3] if n != which_voice] 173 | 174 | h_span = list(range(ii - h_context, ii + h_context + 1)) 175 | h_span = [h for h in h_span if h != ii] 176 | h_n = [] 177 | for hi in h_span: 178 | h_ni = lut[int(notes[hi, wv].ravel())] + tot * offset 179 | tot += 1 180 | h_n.append(h_ni) 181 | h_n = np.array(h_n).ravel() 182 | 183 | vm1_n = notes[ii - 1, nv].ravel() 184 | for nn in range(len(vm1_n)): 185 | vm1_n[nn] = lut[int(vm1_n[nn])] + tot * offset 186 | tot += 1 187 | 188 | v_n = notes[ii, nv].ravel() 189 | for nn in range(len(v_n)): 190 | v_n[nn] = lut[int(v_n[nn])] + tot * offset 191 | tot += 1 192 | 193 | vp1_n = notes[ii + 1, nv].ravel() 194 | for nn in range(len(v_n)): 195 | vp1_n[nn] = lut[int(vp1_n[nn])] + tot * offset 196 | tot += 1 197 | features_i = np.concatenate((h_n, vm1_n, v_n, vp1_n)) 198 | features.append(features_i) 199 | return [None] * h_context + features + [None] * h_context 200 | 201 | def feature_fn0(X): 202 | return feature_fn(X, 0) 203 | 204 | def feature_fn1(X): 205 | return feature_fn(X, 1) 206 | 207 | def feature_fn2(X): 208 | return feature_fn(X, 2) 209 | 210 | def feature_fn3(X): 211 | return feature_fn(X, 3) 212 | 213 | feature_fns = [feature_fn0, feature_fn1, feature_fn2, feature_fn3] 214 | 215 | labels = {} 216 | for which_voice in [0, 1, 2, 3]: 217 | labels[which_voice] = [lut[d] for d in dataset[:, which_voice]] 218 | 219 | def get_models(dataset, labels): 220 | models = [] 221 | random_state = np.random.RandomState(model_seed) 222 | for which_voice in [0, 1, 2, 3]: 223 | if not os.path.exists("saved_sme_{}.pkl".format(which_voice)): 224 | model = SparseMaxEnt(feature_fns[which_voice], n_features=n_features, n_classes=n_classes, 225 | random_state=random_state) 226 | start_time = time.time() 227 | model.fit(dataset, labels[which_voice], l1) 228 | stop_time = time.time() 229 | print("Total training time {}".format(stop_time - start_time)) 230 | with open("saved_sme_{}.pkl".format(which_voice), "w") as f: 231 | pickle.dump(model, f) 232 | else: 233 | print("Found saved model saved_sme_{}.pkl, loading...".format(which_voice)) 234 | with open("saved_sme_{}.pkl".format(which_voice), "r") as f: 235 | model = pickle.load(f) 236 | models.append(model) 237 | return models 238 | 239 | models = get_models(dataset, labels) 240 | 241 | random_state = np.random.RandomState(randomness_seed) 242 | song_start = song_start_idx[song_ind] 243 | song_stop = song_start_idx[song_ind + 1] 244 | generated = copy.copy(dataset[song_start:song_stop]) 245 | 246 | new_generated = [] 247 | for ii in range(generated.shape[1]): 248 | new_g = copy.copy(generated[:, ii]) 249 | c = Counter(new_g) 250 | cands = [v for v, count in c.most_common(num_cands)] 251 | rand_g = random_state.choice(cands, size=song_len, replace=True) 252 | new_generated.append(rand_g) 253 | generated = np.array(new_generated).T 254 | 255 | def save_midi(generated, itr): 256 | print("Saving, iteration {}".format(itr)) 257 | name_tag = "generated_{}".format(itr) + "_{}.mid" 258 | 259 | quantized_to_pretty_midi([generated[2 * h_context:-2 * h_context]], .25, 260 | save_dir=save_dir, 261 | name_tag=name_tag, 262 | default_quarter_length=default_quarter_length, 263 | voice_params=voice_type) 264 | 265 | # sampling loop 266 | for n in range(total_itr): 267 | print("Iteration {}".format(n)) 268 | if n % 1 == 0 or n == (total_itr - 1): 269 | save_midi(generated, n) 270 | 271 | # all voices, over the comb range 272 | # metropolized gibbs comb? 273 | # Idea of gibbs comb from OrMachine 274 | moves = list(range(generated.shape[1])) * (2 * h_context + 1) 275 | random_state.shuffle(moves) 276 | 277 | comb_offset = random_state.randint(10000) % (2 * h_context + 1) 278 | all_changed = [] 279 | for m in moves: 280 | j = m 281 | poss = list(sorted(set([g for g in generated[h_context:-h_context, j]]))) 282 | rvv = random_state.choice(poss, len(generated[h_context:-h_context]), replace=True) 283 | 284 | l = models[j].predict_proba(generated) 285 | valid_sub = l[h_context:-h_context].argmax(axis=1) 286 | argmax_cvv = np.array([rlut[vs] for vs in valid_sub]) 287 | 288 | valid_sub = l[h_context:-h_context] 289 | def np_softmax(v, t): 290 | v = v / float(t) 291 | e_X = np.exp(v - v.max(axis=-1, keepdims=True)) 292 | out = e_X / e_X.sum(axis=-1, keepdims=True) 293 | return out 294 | 295 | t = temperature 296 | if t > 0: 297 | valid_sub = np_softmax(valid_sub, t) 298 | valid_draw = np.array([random_state.multinomial(1, v).argmax() for v in valid_sub]) 299 | else: 300 | valid_draw = l[h_context:-h_context].argmax(axis=1) 301 | 302 | cvv = [] 303 | for n, vs in enumerate(valid_draw): 304 | # hacking around issues 305 | if vs >= 58: 306 | cvv.append(argmax_cvv[n]) 307 | assert argmax_cvv[n] < 88 308 | else: 309 | cvv.append(rlut[vs]) 310 | cvv = np.array(cvv) 311 | 312 | # flip coin to choose between random and copy 313 | choose = np.array(random_state.rand(len(cvv)) > model_proportion).astype("int16") 314 | vv = choose * rvv + (1 - choose) * cvv 315 | 316 | nlls = [models[t].loglikelihoods(generated, generated[:, t]) for t in range(len(models))] 317 | nlls_j = nlls[j] 318 | 319 | new_generated = copy.copy(generated) 320 | new_generated[h_context:-h_context, j] = vv 321 | new_nlls = [models[t].loglikelihoods(new_generated, new_generated[:, t]) for t in range(len(models))] 322 | new_nlls_j = new_nlls[j] 323 | 324 | accept_ind = np.array([1 if (viv % (2 * h_context + 1)) == comb_offset else 0 for viv in range(len(vv))]) 325 | accept_pos = np.where(accept_ind)[0] 326 | score_ind = np.array([1. if (viv % (2 * h_context + 1)) == comb_offset else 0. for viv in range(len(vv))]) 327 | 328 | accept_winds = (np.where(accept_ind)[0][None] + np.arange(-h_context, h_context + 1)[:, None]).T 329 | 330 | not_j = np.array([t for t in range(len(models)) if t != j]) 331 | 332 | for ii in range(len(accept_pos)): 333 | pos = accept_pos[ii] 334 | hidx = pos + np.arange(-h_context, h_context + 1) 335 | hidx = hidx[hidx > h_context] 336 | hidx = hidx[hidx < (len(nlls_j) - h_context)] 337 | if (pos < h_context + 1) or (pos > len(nlls_j) - h_context - 1): 338 | continue 339 | if len(hidx) == 0: 340 | continue 341 | htop = log_sum_exp(new_nlls_j[hidx]) 342 | hbot = log_sum_exp(nlls_j[hidx]) 343 | 344 | vtop = log_sum_exp(np.array([nlls[nj][pos] for nj in not_j])) 345 | vbot = log_sum_exp(np.array([new_nlls[nj][pos] for nj in not_j])) 346 | 347 | dm1top = log_sum_exp(np.array([nlls[nj][pos - 1] for nj in not_j])) 348 | dm1bot = log_sum_exp(np.array([new_nlls[nj][pos - 1] for nj in not_j])) 349 | 350 | dp1top = log_sum_exp(np.array([nlls[nj][pos + 1] for nj in not_j])) 351 | dp1bot = log_sum_exp(np.array([new_nlls[nj][pos + 1] for nj in not_j])) 352 | 353 | dtop = dm1top + dp1top 354 | dbot = dm1bot + dp1bot 355 | 356 | top = np.exp(htop + vtop + dtop) 357 | bot = np.exp(hbot + vbot + dbot) 358 | score_ind[np.where(accept_ind)[0][ii]] *= (top / float(bot)) 359 | 360 | accept_roll = random_state.rand(len(vv)) 361 | accept = np.array((accept_ind * accept_roll) < (score_ind)).astype("int32") 362 | comb_offset += 1 363 | if comb_offset >= 2 * h_context + 1: 364 | comb_offset = comb_offset % (2 * h_context + 1) 365 | old_generated = copy.copy(generated) 366 | generated[h_context:-h_context, j] = accept * vv + (1 - accept) * generated[h_context:-h_context, j] 367 | changed = np.sum(generated[:, j] != old_generated[:, j]) / float(len(generated[:, j])) 368 | all_changed.append(changed) 369 | print("Average change ratio: {}".format(np.mean(all_changed))) 370 | save_midi(generated, total_itr) 371 | -------------------------------------------------------------------------------- /markov_continuator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import numpy as np 3 | from scipy.cluster.vq import vq 4 | import os 5 | import cPickle as pickle 6 | import copy 7 | import collections 8 | from collections import defaultdict 9 | 10 | from datasets import pitches_and_durations_to_pretty_midi 11 | from datasets import quantized_to_pretty_midi 12 | from datasets import fetch_bach_chorales_music21 13 | 14 | # key to use - changing to major may need different history lengths 15 | key = "minor" 16 | # tempo of output 17 | default_quarter_length = 70 18 | # what voice to synthesize with 19 | voice_type = "woodwinds" 20 | # history to consider 21 | split = 2 22 | # how long to generate 23 | clip_gen = 20 24 | 25 | # 0 Soprano, 1 Alto, 2 Tenor, 3 Bass 26 | which_voice = 0 27 | random_seed = 1999 28 | 29 | mu = fetch_bach_chorales_music21() 30 | order = len(mu["list_of_data_pitch"][0]) 31 | random_state = np.random.RandomState(random_seed) 32 | 33 | lp = mu["list_of_data_pitch"] 34 | lt = mu["list_of_data_time"] 35 | ltd = mu["list_of_data_time_delta"] 36 | lql = mu["list_of_data_quarter_length"] 37 | 38 | if key != None: 39 | keep_lp = [] 40 | keep_lt = [] 41 | keep_ltd = [] 42 | keep_lql = [] 43 | lk = mu["list_of_data_key"] 44 | for n in range(len(lp)): 45 | if key in lk[n]: 46 | keep_lp.append(lp[n]) 47 | keep_lt.append(lt[n]) 48 | keep_ltd.append(ltd[n]) 49 | keep_lql.append(lql[n]) 50 | lp = copy.deepcopy(keep_lp) 51 | lt = copy.deepcopy(keep_lt) 52 | ltd = copy.deepcopy(keep_ltd) 53 | lql = copy.deepcopy(keep_lql) 54 | 55 | 56 | # https://csl.sony.fr/downloads/papers/uploads/pachet-02f.pdf 57 | # https://stackoverflow.com/questions/11015320/how-to-create-a-trie-in-python 58 | class Continuator: 59 | def __init__(self, random_state): 60 | self.root = dict() 61 | self.index = dict() 62 | # 0 indexed 63 | self.continuation_offset = 0 64 | self.random_state = random_state 65 | # use this to reduce the complexity of queries 66 | self.max_seq_len_seen = 0 67 | 68 | 69 | def insert(self, list_of_symbol, continuation_offset=None): 70 | if isinstance(list_of_symbol, (str, unicode)): 71 | raise AttributeError("list of symbol must not be string") 72 | word = list_of_symbol 73 | if continuation_offset is None: 74 | for n, wi in enumerate(word): 75 | # 1 indexed to match the paper 76 | self.index[n + self.continuation_offset + 1] = wi 77 | self.continuation_offset += len(word) 78 | continuation_offset = self.continuation_offset 79 | self.max_seq_len_seen = max(len(word), self.max_seq_len_seen) 80 | co = continuation_offset 81 | root = self.root 82 | current = root 83 | word_slice = word[:-1] 84 | for letter in word_slice[::-1]: 85 | if letter not in current: 86 | current[letter] = [co, {}] 87 | else: 88 | current[letter].insert(len(current[letter]) - 1, co) 89 | current = current[letter][-1] 90 | current["_end"] = None 91 | if len(word) > 1: 92 | self.insert(word[:-1], co - 1) 93 | 94 | 95 | def _prefix_search(self, prefix): 96 | root = self.root 97 | current = root 98 | subword = prefix[::-1] 99 | continuations = [] 100 | for letter in subword: 101 | if letter in current and "_end" in current[letter][-1].keys(): 102 | continuations += current[letter][:-1] 103 | return continuations 104 | elif letter not in current: 105 | # node not found 106 | return [] 107 | current = current[letter][-1] 108 | # short sequence traversed to partial point of tree ("BC case from paper") 109 | continuations = [] 110 | for k in current.keys(): 111 | continuations += current[k][:-1] 112 | return continuations 113 | 114 | 115 | def _index_lookup(self, indices): 116 | return [self.index[i] for i in indices] 117 | 118 | 119 | def _next(self, prefix): 120 | ci = self._prefix_search(prefix) 121 | if len(ci) > 0: 122 | possibles = self._index_lookup(ci) 123 | else: 124 | sub_prefix = prefix[-self.max_seq_len_seen + 1:] 125 | possibles = None 126 | for i in range(len(sub_prefix)): 127 | ci = self._prefix_search(sub_prefix[i:]) 128 | if len(ci) > 0: 129 | possibles = self._index_lookup(ci) 130 | break 131 | if possibles is not None: 132 | # choose one of possibles 133 | irange = np.arange(len(possibles)) 134 | i = self.random_state.choice(irange) 135 | p = possibles[i] 136 | else: 137 | p = "" 138 | return [p] 139 | 140 | def continuate(self, seq, max_steps=-1): 141 | if isinstance(seq, (str, unicode)): 142 | raise AttributeError("prefix must list of symbols, not string") 143 | res = None 144 | i = 0 145 | new_seq = [] 146 | while res != [""]: 147 | if max_steps > 0 and i > max_steps: 148 | break 149 | if res is not None: 150 | new_seq = new_seq + res 151 | res = t._next(seq) 152 | i += 1 153 | return new_seq 154 | 155 | ''' 156 | # tests from 157 | # https://csl.sony.fr/downloads/papers/uploads/pachet-02f.pdf 158 | random_state = np.random.RandomState(1999) 159 | t = Continuator(random_state) 160 | t.insert(["A", "B", "C", "D"]) 161 | t.insert(["A", "B", "B", "C"]) 162 | ret = t.continuate(["A", "B"]) 163 | # should be ["B", "B", "C", "D"] 164 | 165 | # Test the duration / tuple case 166 | random_state = np.random.RandomState(1999) 167 | t = Continuator(random_state) 168 | t.insert([("A", 1), ("B", 1), ("C", 1), ("D", 1)]) 169 | t.insert([("A", 1), ("B", 1), ("B", 1), ("C", 1)]) 170 | ret = t.continuate([("A", 1), ("B", 1)]) 171 | # should be [("B", 1), ("B", 1), ("C", 1), ("D", 1)] 172 | ''' 173 | 174 | random_state = np.random.RandomState(random_seed) 175 | t = Continuator(random_state) 176 | 177 | inds = range(len(lp)) 178 | for ii in inds: 179 | pii = lp[ii][which_voice] 180 | tdii = ltd[ii][which_voice] 181 | if len(pii) % split != 0: 182 | offset = split * (len(pii) // split) 183 | pii = pii[:offset] 184 | tdii = tdii[:offset] 185 | 186 | if len(tdii) < split or len(pii) < split: 187 | continue 188 | 189 | tdr = np.array(tdii).reshape(len(tdii) // split, -1) 190 | pr = np.array(pii).reshape(len(pii) // split, -1) 191 | 192 | for i in range(len(tdr)): 193 | tdri = tdr[i] 194 | pri = pr[i] 195 | comb = [(pi, tdi) for pi, tdi in zip(pri, tdri)] 196 | t.insert(comb) 197 | 198 | tri = tdr[which_voice] 199 | pri = pr[which_voice] 200 | comb = [(pi, tdi) for pi, tdi in zip(pri, tdri)] 201 | ret = t.continuate(comb, clip_gen) 202 | 203 | pitches = [[[r[0] for r in ret]]] 204 | durations = [[[r[1] for r in ret]]] 205 | name_tag = "continuated_{}.mid" 206 | pitches_and_durations_to_pretty_midi(pitches, durations, 207 | save_dir="samples/", 208 | name_tag=name_tag, 209 | default_quarter_length=default_quarter_length, 210 | voice_params=voice_type) 211 | 212 | pii = [[lp[0][which_voice]]] 213 | tdii = [[ltd[0][which_voice]]] 214 | name_tag = "original_{}.mid" 215 | pitches_and_durations_to_pretty_midi(pii, tdii, 216 | save_dir="samples/", 217 | name_tag=name_tag, 218 | default_quarter_length=default_quarter_length, 219 | voice_params=voice_type) 220 | -------------------------------------------------------------------------------- /markov_finite.py: -------------------------------------------------------------------------------- 1 | # Author: Kyle Kastner 2 | # License: BSD 3-Clause 3 | # built out of code from Gabriele Barbieri 4 | # https://github.com/gabrielebarbieri/markovchain 5 | # All mistakes my own 6 | from collections import defaultdict 7 | import copy 8 | import nltk 9 | import pandas as pd 10 | import os 11 | import numpy as np 12 | from functools import partial 13 | import string 14 | 15 | NLTK_PACKAGES = ['punkt', 'word2vec_sample', 'cmudict'] 16 | START_SYMBOL = '' 17 | END_SYMBOL = '' 18 | 19 | _WORD2VEC = None 20 | _CMU_DICT = None 21 | 22 | BLACKLIST = ['"', '``', "''"] 23 | 24 | DYLAN_DATA = 'data/Dylan' 25 | DYLAN_POPULARITY = 'data/dylan_popularity.csv' 26 | 27 | def get_dylan_most_popular_songs(n=5): 28 | df = pd.read_csv(DYLAN_POPULARITY) 29 | return [os.path.join(DYLAN_DATA, f) for f in df.file.head(n)] 30 | 31 | 32 | def get_rhymes(word): 33 | try: 34 | d = get_cmu_dict() 35 | return set(get_rhyme_from_pronunciation(p) for p in d[word] if p) 36 | except KeyError: 37 | return [] 38 | 39 | 40 | def get_rhyme_from_pronunciation(pronunciation): 41 | 42 | stresses = [] 43 | for p in pronunciation: 44 | try: 45 | stresses.append(int(p[-1])) 46 | except ValueError: 47 | pass 48 | stress = str(max(stresses)) 49 | 50 | # the reversed is needed to deal with the "because" case 51 | for i, e in enumerate(reversed(pronunciation)): 52 | if e.endswith(stress): 53 | return ','.join(pronunciation[len(pronunciation) - 1 - i:]) 54 | 55 | 56 | def process_word(word, replace_dict=None): 57 | processed_word = word.lower() 58 | if replace_dict is not None: 59 | for k, v in replace_dict.items(): 60 | if k in processed_word: 61 | processed_word = processed_word.replace(k, v) 62 | return processed_word 63 | 64 | 65 | def tokenize(string, replace_dict=None): 66 | words = [process_word(token, replace_dict) for token in nltk.word_tokenize(string) if token not in BLACKLIST] 67 | # strip nonrhyming words, which includes stuff like punctuation 68 | return [START_SYMBOL] + words + [END_SYMBOL] 69 | 70 | 71 | def tokenize_corpus(sources, replace_dict=None): 72 | sentences = [] 73 | for file_name in sources: 74 | try: 75 | with open(file_name) as f: 76 | sentences += [tokenize(line, replace_dict) for line in f if line.strip()] 77 | except IOError: 78 | pass 79 | return sentences 80 | 81 | 82 | def get_coeffs(transitions): 83 | return {prefix: sum(probabilities.values()) for prefix, probabilities in transitions.items()} 84 | 85 | 86 | def normalize_it(values, coeff): 87 | return {suffix: value / float(coeff) for suffix, value in values.items()} 88 | 89 | 90 | def normalize(transitions, coeffs=None): 91 | # normalization coeffs are optional 92 | if coeffs is None: 93 | coeffs = get_coeffs(transitions) 94 | res = empty_transitions() 95 | for prefix, probabilities in transitions.items(): 96 | res[prefix] = normalize_it(probabilities, coeffs[prefix]) 97 | return res 98 | 99 | 100 | def empty_transitions(): 101 | m = defaultdict(lambda: defaultdict(float)) 102 | return m 103 | 104 | 105 | def markov_transitions(sequences, order): 106 | # order None returns the dict structure directly 107 | m = empty_transitions() 108 | for seq in sequences: 109 | for n_gram in zip(*(seq[i:] for i in range(order + 1))): 110 | prefix = n_gram[:-1] 111 | suffix = n_gram[-1] 112 | m[prefix][suffix] += 1. 113 | return normalize(m) 114 | 115 | 116 | def make_markov_corpus(sequences, order_upper): 117 | return {order: markov_transitions(sequences, order) for order in range(order_upper + 1)} 118 | 119 | 120 | def filter_(transitions, values): 121 | if values is None: 122 | return transitions 123 | if hasattr(values, "keys"): 124 | res = copy.deepcopy(transitions) 125 | res_keys = res.keys() 126 | # make the partial keys up front... annoying 127 | r = [[(rk, rk[-i:]) for idx, rk in enumerate(res_keys)] for i in range(1, len(res_keys[0]) + 1)] 128 | res_k = [[rii[0] for rii in ri] for ri in r] 129 | res_m = [[rii[1] for rii in ri] for ri in r] 130 | for prefix, suffix in values.items(): 131 | i = len(prefix) - 1 132 | p_m = res_m[i] 133 | p_k = res_k[i] 134 | if prefix in p_m: 135 | # find indices 136 | ii = [n for n, _ in enumerate(p_m) if prefix == _] 137 | # delete full matches 138 | for di in ii: 139 | # looks ugly due to checks 140 | if p_k[di] in res: 141 | for su in suffix: 142 | if su in res[p_k[di]]: 143 | del res[p_k[di]][su] 144 | return res 145 | else: 146 | res = {} 147 | for prefix, probs in transitions.items(): 148 | filtered = {suffix: probs[suffix] for suffix in probs.keys() if suffix in values} 149 | if filtered: 150 | res[prefix] = filtered 151 | return res 152 | 153 | 154 | def propagate_(constrained_markov, coeffs): 155 | if coeffs is None: 156 | return constrained_markov 157 | 158 | res = {} 159 | for prefix, probs in constrained_markov.items(): 160 | transitions = {} 161 | for suffix, value in probs.items(): 162 | index = prefix[1:] + (suffix,) 163 | if index in coeffs: 164 | transitions[suffix] = value * coeffs[index] 165 | else: 166 | # for lower order ones 167 | index = prefix + (suffix,) 168 | if index in coeffs: 169 | transitions[suffix] = value * coeffs[index] 170 | # if it's not in the coeffs, just pass 171 | if transitions: 172 | res[prefix] = transitions 173 | return res 174 | 175 | 176 | def make_constrained_markov(markovs, constraints): 177 | # Section 4.3 of https://www.ijcai.org/Proceedings/11/Papers/113.pdf 178 | # Finite-Length Markov Processes with Constraints 179 | # Pachet, Roy, Barbieri 180 | # constraints are hard requirements for the sequence 181 | # dict rules are transitions which should be disallowed 182 | coeffs = None 183 | orders = markovs.keys() 184 | max_order = max(orders) 185 | markov_process = [] 186 | for index, values in reversed(list(enumerate(constraints))): 187 | transitions = markovs[min(index, max_order)] 188 | filtered = filter_(transitions, values) 189 | filtered = propagate_(filtered, coeffs) 190 | if not filtered: 191 | raise RuntimeError('The constraints satisfaction problem has no solution. ' 192 | 'Try to relax your constraints') 193 | coeffs = get_coeffs(filtered) 194 | # prepend because of reverse 195 | markov_process.insert(0, normalize(filtered, coeffs)) 196 | return markov_process 197 | 198 | 199 | def generate_from_constrained_markov_process(constrained_markov_process, random_state): 200 | max_order = len(constrained_markov_process[-1].keys()[0]) 201 | sequence = [] 202 | for index, markov in enumerate(constrained_markov_process): 203 | prefix = tuple(sequence[-min(index, max_order):]) 204 | probs = markov[prefix] 205 | value = random_state.choice(probs.keys(), p=probs.values()) 206 | sequence.append(value) 207 | return sequence 208 | 209 | 210 | class Trie(object): 211 | def __init__(self): 212 | self.root = defaultdict() 213 | self._end = "_end" 214 | self.orders = [] 215 | 216 | def insert(self, list_of_items): 217 | current = self.root 218 | for item in list_of_items: 219 | current = current.setdefault(item, {}) 220 | current.setdefault(self._end) 221 | self.orders = sorted(list(set(self.orders + [len(list_of_items)]))) 222 | 223 | def order_insert(self, order, list_of_items): 224 | s = 0 225 | e = order 226 | while e < len(list_of_items): 227 | # + 1 due to numpy slicing 228 | e = s + order + 1 229 | self.insert(list_of_items[s:e]) 230 | s += 1 231 | 232 | def search(self, list_of_items): 233 | # items of the list should be hashable 234 | # returns True if item in Trie, else False 235 | if len(list_of_items) not in self.orders: 236 | raise ValueError("item {} has invalid length {} for search, only {} supported".format(list_of_items, len(list_of_items), self.orders)) 237 | current = self.root 238 | for item in list_of_items: 239 | if item not in current: 240 | return False 241 | current = current[item] 242 | if self._end in current: 243 | return True 244 | return False 245 | 246 | def order_search(self, order, list_of_items): 247 | # returns true if subsequence at offset is found 248 | s = 0 249 | e = order 250 | searches = [] 251 | while e < len(list_of_items): 252 | # + 1 due to numpy slicing 253 | e = s + order + 1 254 | searches.append(self.search(list_of_items[s:e])) 255 | s += 1 256 | return searches 257 | 258 | def test_markov_process(): 259 | # constraints are listed PER STEP 260 | # rule constraints are list of list, e.g. [[prefix1, prefix2],[suffix]] 261 | # this one will match tutorial 262 | order = 1 263 | # mc should exactly match end of 4.4 264 | c = [None, None, None, ["D"]] 265 | # this one checks hard unary constraints that *SHOULD* happen 266 | # c = [["E"], ["C"], ["C"], ["D"]] 267 | # can have multiple unary constraints - output should be in this set 268 | #c = [["E", "C"], ["E", "C"], ["E", "C"], ["D"]] 269 | # this one checks pairwise transitions that shouldn't happen 270 | #c = [None, None, {("E",): ["D","C"], ("C",): ["D"]}, ["D"]] 271 | 272 | # can also do higher order 273 | #order = 2 274 | #c = [None, None, ["E"]] 275 | # binary constraints up to markov order 276 | #c = [None, None, {("C", "D"): ["E"]}] 277 | # can accept constraints that are shorter, for partial match 278 | #c = [None, None, {"E": ["E"]}] 279 | corpus = [["E", "C", "D", "E", "C", "C"], 280 | ["C", "C", "E", "E", "D", "C"]] 281 | # turn it into words 282 | ms = make_markov_corpus(corpus, order) 283 | mc = make_constrained_markov(ms, c) 284 | random_state = np.random.RandomState(100) 285 | for i in range(5): 286 | print(generate_from_constrained_markov_process(mc, random_state)) 287 | # can also seed the generation, thus bypassing the prior 288 | #print(generate_from_constrained_markov_process(mc, random_state, starting_seed=["C"])) 289 | # also for higher order, seed with order length 290 | #print(generate_from_constrained_markov_process(mc, random_state, starting_seed=["E", "C"])) 291 | 292 | def test_dylan(): 293 | sources = get_dylan_most_popular_songs(40) 294 | order = 2 295 | corpus = tokenize_corpus(sources) 296 | ms = make_markov_corpus(corpus, order) 297 | # add not constraints 298 | # like !D, !C etc 299 | # can we iteratively create the max-order constraints? 300 | # using suffix tree, writing to c and recompiling mc? 301 | length = 10 302 | c = [[""]] + [None] * (length + 1) + [[""]] 303 | # remove you 304 | #c[1] = {("",): ["you"]} 305 | #c[2] = {("", "you"): ["'ve"]} 306 | #c[5] = {("been", "through"): ["all"]} 307 | # check partial match 308 | #c[5] = {("through",): ["all"]} 309 | mc = make_constrained_markov(ms, c) 310 | random_state = np.random.RandomState(100) 311 | for i in range(10): 312 | #print(generate_from_constrained_markov_process(mc, random_state, starting_seed=[""])) 313 | print(generate_from_constrained_markov_process(mc, random_state)) 314 | 315 | def test_dylan_maxorder(): 316 | sources = get_dylan_most_popular_songs(40) 317 | corpus = tokenize_corpus(sources) 318 | corpus = [[ci for ci in c if ci not in [".", "'", ","]] 319 | for c in corpus] 320 | checker = Trie() 321 | max_order = 5 322 | [checker.order_insert(max_order, c) for c in corpus] 323 | 324 | order = 2 325 | ms = make_markov_corpus(corpus, order) 326 | length = 10 327 | c = [[""]] + [None] * (length + 1) + [[""]] 328 | mc = make_constrained_markov(ms, c) 329 | random_state = np.random.RandomState(100) 330 | generations = [] 331 | for i in range(1000): 332 | generations.append(generate_from_constrained_markov_process(mc, random_state)) 333 | 334 | passes = [checker.order_search(max_order, g) for g in generations] 335 | passing_generations = [g for n, g in enumerate(generations) if not any(passes[n])] 336 | print(passing_generations) 337 | 338 | if __name__ == "__main__": 339 | #test_markov_process() 340 | #test_dylan() 341 | #test_dylan_maxorder() 342 | -------------------------------------------------------------------------------- /markov_steerable.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import numpy as np 3 | from scipy.cluster.vq import vq 4 | import os 5 | import cPickle as pickle 6 | import copy 7 | import collections 8 | from collections import defaultdict, Counter, namedtuple 9 | import heapq 10 | import music21 11 | from datasets import pitches_and_durations_to_pretty_midi 12 | 13 | from functools import partial 14 | 15 | 16 | class cls_memoize(object): 17 | """cache the return value of a method 18 | 19 | This class is meant to be used as a decorator of methods. The return value 20 | from a given method invocation will be cached on the instance whose method 21 | was invoked. All arguments passed to a method decorated with memoize must 22 | be hashable. 23 | 24 | If a memoized method is invoked directly on its class the result will not 25 | be cached. Instead the method will be invoked like a static method: 26 | class Obj(object): 27 | @memoize 28 | def add_to(self, arg): 29 | return self + arg 30 | Obj.add_to(1) # not enough arguments 31 | Obj.add_to(1, 2) # returns 3, result is not cached 32 | """ 33 | def __init__(self, func): 34 | self.func = func 35 | def __get__(self, obj, objtype=None): 36 | if obj is None: 37 | return self.func 38 | return partial(self, obj) 39 | def __call__(self, *args, **kw): 40 | obj = args[0] 41 | try: 42 | cache = obj.__cache 43 | except AttributeError: 44 | cache = obj.__cache = {} 45 | key = (self.func, args[1:], frozenset(kw.items())) 46 | try: 47 | res = cache[key] 48 | except KeyError: 49 | res = cache[key] = self.func(*args, **kw) 50 | return res 51 | 52 | 53 | class Trie(object): 54 | def __init__(self): 55 | self.root = collections.defaultdict() 56 | self._end = "_end" 57 | self.orders = [] 58 | 59 | def insert(self, list_of_items): 60 | current = self.root 61 | for item in list_of_items: 62 | current = current.setdefault(item, {}) 63 | current.setdefault(self._end) 64 | self.orders = sorted(list(set(self.orders + [len(list_of_items)]))) 65 | 66 | def order_insert(self, order, list_of_items): 67 | s = 0 68 | e = order 69 | while e < len(list_of_items): 70 | # + 1 due to numpy slicing 71 | e = s + order + 1 72 | self.insert(list_of_items[s:e]) 73 | s += 1 74 | 75 | def search(self, list_of_items): 76 | # items of the list should be hashable 77 | # returns True if item in Trie, else False 78 | if len(list_of_items) not in self.orders: 79 | raise ValueError("item {} has invalid length {} for search, only {} supported".format(list_of_items, len(list_of_items), self.orders)) 80 | current = self.root 81 | for item in list_of_items: 82 | if item not in current: 83 | return False 84 | current = current[item] 85 | if self._end in current: 86 | return True 87 | return False 88 | 89 | @cls_memoize 90 | def partial(self, prefix_tuple): 91 | prefix = prefix_tuple 92 | # items of the list should be hashable 93 | # Returns valid keys for continuation 94 | if len(prefix) + 1 not in self.orders: 95 | raise ValueError("item {} has invalid length {} for partial search, only {} supported".format(prefix, len(prefix), [o - 1 for o in self.orders])) 96 | current = self.root 97 | for p in prefix: 98 | if p not in current: 99 | return [] 100 | current = current[p] 101 | return [c for c in current.keys() if c != self._end] 102 | 103 | 104 | Node = namedtuple("Node", 105 | ["level", "proposed_note", "log_prob", "previous_notes"], 106 | verbose=False, rename=False) 107 | 108 | 109 | class CMP(object): 110 | """ Constrained Markov Process 111 | 112 | Implements tools/ideas from the following papers: 113 | 114 | The Continuator: Musical Interaction with Style 115 | F. Pachet 116 | https://www.csl.sony.fr/downloads/papers/uploads/pachet-02f.pdf 117 | 118 | Finite-Length Markov Processes With Constraints 119 | F. Pachet, P. Roy, G. Barbieri 120 | https://www.csl.sony.fr/downloads/papers/2011/pachet-11b.pdf 121 | 122 | Markov Constraints: Steerable Generation of Markov Sequences 123 | F. Pachet, P. Roy 124 | https://www.csl.sony.fr/downloads/papers/2011/pachet-09c.pdf 125 | 126 | Avoiding Plagiarism in Markov Sequence Generation 127 | A. Papadopolous, P. Roy, F. Pachet 128 | https://www.csl.sony.fr/downloads/papers/2014/papadopoulos-14a.pdf 129 | 130 | Enforcing Meter in Finite-Length Markov Sequences 131 | P. Roy, F. Pachet 132 | https://www.csl.sony.fr/downloads/papers/2013/roy-13a.pdf 133 | 134 | Non-Conformant Harmonization: The Real Book in the Style of Take 6 135 | F. Pachet, P. Roy 136 | https://www.csl.sony.fr/downloads/papers/2014/pachet-14a.pdf 137 | """ 138 | def __init__(self, order, max_order=None, ptype="max", named_constraints={}): 139 | 140 | self.order = order 141 | self.goods = [Trie() for i in range(0, self.order)] 142 | self.max_order = max_order 143 | constraint_types = ["end", "start", "position", "alldiff", "contains", "not_contains"] 144 | # need to flesh out API 145 | # position is dict of dict of list 146 | # alldiff key indicates window size 147 | assert all([k in constraint_types for k in named_constraints.keys()]) 148 | self.named_constraints = named_constraints 149 | self.bad = Trie() 150 | self.ptype = ptype 151 | assert ptype in ["fixed", "max", "avg"] 152 | 153 | def insert(self, list_of_items): 154 | if self.max_order is not None: 155 | self.bad.order_insert(self.max_order, list_of_items) 156 | for i in list(range(0, self.order)): 157 | self.goods[i].order_insert(i + 1, list_of_items) 158 | 159 | def partial(self, prefix_tuple): 160 | prefix = prefix_tuple 161 | if self.max_order is not None: 162 | prefix = prefix[-self.max_order:] 163 | else: 164 | prefix = prefix[-self.order:] 165 | return self._partial(prefix) 166 | 167 | @cls_memoize 168 | def _partial(self, prefix_tuple): 169 | # subclass to memoize more values 170 | # returns dict of key: prob 171 | prefix = prefix_tuple 172 | all_p = [] 173 | all_gp = [] 174 | for i in list(range(0, self.order))[::-1]: 175 | gp = self.goods[i].partial(prefix[-(i + 1):]) 176 | # already checked for self.max_order 177 | if self.max_order is not None: 178 | bp = self.bad.partial(prefix[-self.max_order:]) 179 | else: 180 | bp = [] 181 | p = list(set(gp) - set(bp)) 182 | if self.ptype == "fixed": 183 | all_p += p 184 | all_gp += gp 185 | break 186 | else: 187 | if len(p) > 0: 188 | all_p += p 189 | all_gp += gp 190 | if self.ptype == "max": 191 | break 192 | 193 | """ 194 | d = {k: 1. / len(ps) for k in ps} 195 | return d 196 | """ 197 | 198 | sums = Counter(all_gp) 199 | tot = sum(sums.values()) 200 | d = {k: float(v) / tot for k, v in sums.items()} 201 | return d 202 | 203 | def check_constraint(self, node, sequence, depth_index, max_length): 204 | generated = sequence[-(depth_index + 1):] 205 | if "alldiff" in self.named_constraints: 206 | # windowed alldiff? 207 | if len(set(generated)) != len(generated): 208 | return False 209 | 210 | if "start" in self.named_constraints: 211 | valid_start = self.named_constraints["start"] 212 | if generated[0] not in valid_start: 213 | return False 214 | 215 | if "end" in self.named_constraints: 216 | valid_end = self.named_constraints["end"] 217 | if depth_index == (max_length - 1) and generated[-1] not in valid_end: 218 | return False 219 | 220 | if "position" in self.named_constraints: 221 | position_checks = self.named_constraints["position"] 222 | for k, v in position_checks.items(): 223 | if len(generated) > k and generated[k] not in v: 224 | return False 225 | 226 | if "contains" in self.named_constraints: 227 | contained_elems = self.named_constraints["contains"] 228 | if depth_index == (max_length - 1): 229 | for c in contained_elems: 230 | if c not in generated: 231 | return False 232 | 233 | if "not_contains" in self.named_constraints: 234 | not_contained_elems = self.named_constraints["not_contains"] 235 | for nc in not_contained_elems: 236 | if nc in generated: 237 | return False 238 | return True 239 | 240 | def branch(self, seed_list, length, search="depth", return_on=-1): 241 | # seach options 242 | # depth 243 | # best 244 | # breadth 245 | # dtob depth-to-best, depth til 1 solution found, then best 246 | res = tuple(seed_list) 247 | 248 | options = self.partial(res) 249 | 250 | el = [] 251 | def dpush(i, p=None): 252 | el.append((-p, i)) 253 | 254 | def dpop(): 255 | return el.pop()[1] 256 | 257 | def brpush(i, p=None): 258 | el.append((-p, i)) 259 | 260 | def brpop(): 261 | return el.pop(0)[1] 262 | 263 | def bpush(i, p=None): 264 | el.append((-p, i)) 265 | 266 | def bpop(): 267 | heapq.heapify(el) 268 | return heapq.heappop(el)[1] 269 | 270 | if search == "dtb" or search == "depth": 271 | push = dpush 272 | pop = dpop 273 | elif search == "breadth": 274 | push = brpush 275 | pop = brpop 276 | elif search == "best": 277 | push = bpush 278 | pop = bpop 279 | else: 280 | raise ValueError("Unknown value for 'search', got {}".format(search)) 281 | 282 | 283 | best_log_prob = -float("inf") 284 | for k, v in options.items(): 285 | log_prob = np.log(v) 286 | n = Node(0, k, log_prob, tuple(res)) 287 | push(n, log_prob) 288 | 289 | soln = {} 290 | break_while = False 291 | while len(el) > 0 and break_while is False: 292 | current = pop() 293 | index = current[0] 294 | cur_note = current[1] 295 | cur_log_prob = current[2] 296 | # always adding a number between 0 and -inf, stopping immediately 297 | # would be the upper bound on the sequence probability 298 | if cur_log_prob < best_log_prob: 299 | continue 300 | cur_seq = current[3] 301 | new_seq = cur_seq + (cur_note,) 302 | if index >= length: 303 | if cur_seq not in soln: 304 | # soln: log_prob 305 | soln[cur_seq] = cur_log_prob 306 | if cur_log_prob > best_log_prob: 307 | best_log_prob = cur_log_prob 308 | if search == "dtb": 309 | heapq.heapify(el) 310 | push = bpush 311 | pop = bpop 312 | 313 | if return_on > 0: 314 | if len(soln.keys()) >= return_on: 315 | break_while = True 316 | else: 317 | if self.check_constraint(current, new_seq, index, length): 318 | options = self.partial(new_seq) 319 | for k, v in options.items(): 320 | new_log_prob = cur_log_prob + np.log(v) 321 | if new_log_prob >= best_log_prob: 322 | n = Node(index + 1, k, new_log_prob, new_seq) 323 | push(n, new_log_prob) 324 | 325 | res = sorted([(v, k[len(seed_list):]) for k, v in soln.items()])[::-1] 326 | return res 327 | 328 | 329 | def realize_chord(chordstring, numofpitch=3, baseoctave=4, direction="ascending"): 330 | """ 331 | given a chordstring like Am7, return a list of numofpitch pitches, starting in octave baseoctave, and ascending 332 | if direction == "descending", reverse the list of pitches before returning them 333 | """ 334 | # https://github.com/shimpe/canon-generator 335 | # http://web.mit.edu/music21/doc/moduleReference/moduleHarmony.html 336 | try: 337 | pitches = music21.harmony.ChordSymbol(chordstring).pitches 338 | except ValueError: 339 | # enharmonic equivalents 340 | orig_chordstring = chordstring 341 | if "halfDim" in chordstring: 342 | chordstring = chordstring.replace("halfDim", "/o7") 343 | if chordstring[:2] == "Eb": 344 | chordstring = "D#" + chordstring[2:] 345 | elif chordstring[:2] == "Ab": 346 | chordstring = "G#" + chordstring[2:] 347 | elif chordstring[:2] == "Bb": 348 | chordstring = "A#" + chordstring[2:] 349 | try: 350 | pitches = music21.harmony.ChordSymbol(chordstring).pitches 351 | except ValueError: 352 | from IPython import embed; embed(); raise ValueError() 353 | 354 | num_iter = numofpitch / len(pitches) + 1 355 | octave_correction = baseoctave - pitches[0].octave 356 | result = [] 357 | actual_pitches = 0 358 | for i in range(num_iter): 359 | for p in pitches: 360 | if actual_pitches < numofpitch: 361 | newp = copy.deepcopy(p) 362 | newp.octave = newp.octave + octave_correction 363 | result.append(newp) 364 | actual_pitches += 1 365 | else: 366 | if direction == "ascending": 367 | return result 368 | else: 369 | result.reverse() 370 | return result 371 | octave_correction += 1 372 | 373 | if direction == "ascending": 374 | return result 375 | else: 376 | result.reverse() 377 | return result 378 | 379 | 380 | def render_chords(list_of_chord_lists, name_tag, dur=2, tempo=110, voices=4, 381 | voice_type="piano", save_dir="samples/"): 382 | r = list_of_chord_lists 383 | midi_p = [] 384 | for ri in r: 385 | rch = [realize_chord(rii, voices) for rii in ri] 386 | rt = [] 387 | for rchi in rch: 388 | rt.append([rchi[idx].midi for idx in range(len(rchi))]) 389 | midi_p.append(rt) 390 | 391 | midi_d = [[[dur for midi_ppii in midi_ppi] for midi_ppi in midi_pi] for midi_pi in midi_p] 392 | 393 | # BTAS to SATB 394 | midi_p = [np.array(midi_pi) for midi_pi in midi_p] 395 | midi_d = [np.array(midi_di) for midi_di in midi_d] 396 | 397 | midi_pp = [] 398 | midi_dd = [] 399 | for p, d in zip(midi_p, midi_d): 400 | # hack to avoid strange chords 401 | w = np.where((p[:, 3] - p[:, 2]) > 12)[0] 402 | p[w, 3] = 0. 403 | midi_pp.append(p) 404 | midi_dd.append(d) 405 | 406 | # BTAS to SATB 407 | midi_pp = [midi_pi[:, ::-1] for midi_pi in midi_pp] 408 | midi_dd = [midi_di[:, ::-1] for midi_di in midi_dd] 409 | 410 | name_stub = name_tag.split(".")[0] 411 | text_tag = save_dir + "/" + name_stub + ".txt" 412 | for i in range(len(midi_pp)): 413 | with open(text_tag.format(i), "w") as f: 414 | r = " | ".join(list_of_chord_lists[i]) 415 | f.writelines([r]) 416 | 417 | pitches_and_durations_to_pretty_midi(midi_pp, midi_dd, 418 | save_dir=save_dir, 419 | name_tag=name_tag, 420 | default_quarter_length=tempo, 421 | voice_params=voice_type) 422 | 423 | def transpose(chord_seq): 424 | roots = ["C", "C#", "D", "Eb", "E", "F", "F#", "G", "Ab", "A", "Bb", "B"] 425 | roots2map = {k: v for v, k in enumerate(roots)} 426 | # 2 octaves for easier transpose 427 | oct_roots = roots + roots 428 | map2roots = {k: v for k, v in enumerate(oct_roots)} 429 | 430 | prototype = [] 431 | for c in chord_seq: 432 | if c[:-1] in roots2map: 433 | prototype.append(roots2map[c[:-1]]) 434 | elif c[:2] in roots2map: 435 | prototype.append(roots2map[c[:2]]) 436 | elif c[0] in roots2map: 437 | prototype.append(roots2map[c[0]]) 438 | else: 439 | print(c) 440 | from IPython import embed; embed(); raise ValueError() 441 | 442 | chord_types = ["m", "7", "halfDim"] 443 | chord_function = [] 444 | for c in chord_seq: 445 | if "halfDim" in c: 446 | chord_function.append("halfDim") 447 | continue 448 | elif c[-1] not in ["m", "7"]: 449 | chord_function.append("") 450 | continue 451 | chord_function.append(c[-1]) 452 | 453 | assert len(chord_function) == len(prototype) 454 | all_t = [] 455 | for i in range(len(roots)): 456 | t = [map2roots[p + i] + cf for p, cf in zip(prototype, chord_function)] 457 | all_t.append(t) 458 | return all_t 459 | 460 | 461 | # hardcode the data for now 462 | with open("12BarBluesOmnibook.txt", "r") as f: 463 | r = f.readlines() 464 | names = r[::2] 465 | bars = r[1::2] 466 | names = [n.strip() for n in names] 467 | bars = [b.strip() for b in bars] 468 | 469 | pairs = zip(names, bars) 470 | new_bars = [] 471 | for n, b in pairs: 472 | bb = [bi.split("/") for bi in b.split("|")] 473 | bb = [bbii for bbi in bb for bbii in bbi] 474 | new_bars.append(bb) 475 | pairs = zip(names, new_bars) 476 | 477 | final_pairs = [] 478 | for p in pairs: 479 | t_p = transpose(p[1]) 480 | final_pairs += [(p[0], ti_p) for ti_p in t_p] 481 | pairs = final_pairs 482 | 483 | # chord length 484 | dur = 2 485 | # synthesis tempo 486 | tempo = 110 487 | # number of examples considered, be careful as big numbers cause much larger runtime 488 | dataset_size = 12 489 | # history considered for likelihood scores 490 | order = 1 491 | 492 | m = CMP(order, 493 | max_order=None, 494 | ptype="fixed", 495 | named_constraints={"not_contains": ["C7"], 496 | "position": {8: ["F7"]}, 497 | "alldiff": True, 498 | "end": ["G7"]}, 499 | verbose=True) 500 | 501 | # too many songs and bad things happen... 502 | for n, p in enumerate(pairs): 503 | m.insert(p[1]) 504 | if n > 12: 505 | break 506 | 507 | t = m.branch(["C7"], 15) 508 | if len(t) == 0: 509 | raise ValueError("No solution found!") 510 | 511 | res = t[0][1] 512 | res = ("C7",) + res 513 | # repeat it 2x 514 | render_chords([res + res], "sample_branch_{}.mid", dur=dur, tempo=tempo) 515 | import sys 516 | sys.exit() 517 | -------------------------------------------------------------------------------- /maxent.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from collections import Counter, defaultdict 3 | from minimize import minimize 4 | import scipy as sp 5 | import copy 6 | import hashlib 7 | 8 | class memoize(object): 9 | def __init__(self, func): 10 | self.func = func 11 | self.lu = {} 12 | 13 | def __call__(self, *args): 14 | try: 15 | ha = hash(args) 16 | return self.lu[args] 17 | # numpy array in there 18 | except TypeError: 19 | new_args = [] 20 | for a in args: 21 | try: 22 | hash(a) 23 | new_args.append(a) 24 | except TypeError: 25 | b = a.view(np.uint8) 26 | b_as_str = hashlib.sha1(b).hexdigest() 27 | new_args.append(b_as_str) 28 | ha = hash(tuple(new_args)) 29 | if ha in self.lu: 30 | return self.lu[ha] 31 | else: 32 | r = self.func(*args) 33 | self.lu[ha] = r 34 | return r 35 | 36 | 37 | def log_sum_exp(x, axis=-1): 38 | """Compute log(sum(exp(x))) in a numerically stable way. 39 | 40 | Use second argument to specify along which dimensions the logsumexp 41 | shall be computed. If -1 (which is also the default), logsumexp is 42 | computed along the last dimension. 43 | 44 | From R. Memisevic 45 | """ 46 | if len(x.shape) < 2: #only one possible dimension to sum over? 47 | x_max = x.max() 48 | return x_max + np.log(np.sum(np.exp(x - x_max))) 49 | else: 50 | if axis != -1: 51 | x = x.transpose(range(axis) + range(axis + 1, len(x.shape)) + [axis]) 52 | last = len(x.shape) - 1 53 | x_max = x.max(last) 54 | return x_max + np.log(np.sum(np.exp(x - x_max[..., None]), last)) 55 | 56 | 57 | def softmax(x): 58 | if x.ndim == 1: 59 | x = x.reshape((1, -1)) 60 | max_x = np.max(x, axis=1).reshape((-1, 1)) 61 | exp_x = np.exp(x - max_x) 62 | return exp_x / np.sum(exp_x, axis=1).reshape((-1, 1)) 63 | 64 | 65 | class SparseMaxEnt(object): 66 | """ Also called a log-linear model, or logistic regression. 67 | Implementation using sparsity for discrete features""" 68 | def __init__(self, feature_function, n_features, n_classes, 69 | random_state=None, shuffle=True, optimizer="lbfgs", 70 | verbose=True): 71 | # feature function returns list of indices 72 | # features are only indicator 73 | # assume sparse setup 74 | self.n_features = n_features 75 | self.n_classes = n_classes 76 | self.random_state = random_state 77 | self.shuffle = shuffle 78 | self.optimizer = optimizer 79 | if random_state == None: 80 | raise ValueError("Random state must not be None!") 81 | self.params = 0.02 * random_state.randn(self.n_classes * self.n_features + self.n_classes) 82 | #self.params = np.zeros((self.n_classes * self.n_features + self.n_classes,)) 83 | self.weights = self.params[:self.n_classes * self.n_features].reshape(self.n_features, self.n_classes) 84 | self.biases = self.params[-self.n_classes:] 85 | # memoize it 86 | self.feature_function = feature_function 87 | self.mem_feature_function = memoize(feature_function) 88 | self.verbose = verbose 89 | 90 | def fit(self, data, labels, l1_weight_cost=0., l2_weight_cost=0.): 91 | if self.optimizer == "lbfgs": 92 | from scipy.optimize import minimize 93 | res = minimize(self.f_and_g, self.params.copy(), 94 | (data, labels, l1_weight_cost, l2_weight_cost), method="L-BFGS-B", jac=True, 95 | options={"ftol": 1E-4}) 96 | p = res.x 97 | elif self.optimizer == "minimize_cg": 98 | max_n_line_search = np.inf 99 | p, g, n_line_searches = minimize(self.params.copy(), 100 | (data, labels, l1_weight_cost, l2_weight_cost), 101 | self.f_and_g, 102 | True, 103 | maxnumlinesearch=max_n_line_search, 104 | verbose=self.verbose) 105 | else: 106 | raise ValueError("Unknown optimizer setting {}".format(self.optimizer)) 107 | 108 | if self.verbose: 109 | print("Training complete!") 110 | self.update_params(p) 111 | 112 | def _oh(self, x, max_classes=None): 113 | if max_classes == None: 114 | n_classes = self.n_classes 115 | else: 116 | n_classes = max_classes 117 | #list of list == lol 118 | # need to normalize... 119 | try: 120 | max_len = max([len(xi) for xi in x]) 121 | empty = np.zeros((len(x), max_len)) - 1 122 | for n, xi in enumerate(x): 123 | empty[n, :len(xi)] = xi 124 | except TypeError: 125 | max_len = 1 126 | empty = np.zeros((len(x), max_len)) - 1 127 | for n, xi in enumerate(x): 128 | empty[n] = xi 129 | 130 | result = np.zeros([len(x)] + [n_classes], dtype="int") 131 | z = np.zeros(len(x)).astype("int64") 132 | for c in range(n_classes): 133 | z *= 0 134 | z[np.where(empty == c)[0]] = 1 135 | result[..., c] += z 136 | return result 137 | 138 | def _uh(self, oh_x): 139 | return oh_x.argmax(len(oh_x.shape)-1) 140 | 141 | def loglikelihoods(self, data, pseudolabels): 142 | # trim means return regardless of matching original data length 143 | active_idxs = self.feature_function(data) 144 | inds = [n for n in range(len(active_idxs)) if hasattr(active_idxs[n], "flatten") or active_idxs[n] != None] 145 | not_inds = [n for n in range(len(active_idxs)) if not hasattr(active_idxs[n], "flatten") and active_idxs[n] == None] 146 | 147 | active_idxs = [active_idxs[ii] for ii in inds] 148 | 149 | label_scores = np.zeros((len(active_idxs), self.n_classes)) 150 | for n in range(len(active_idxs)): 151 | active_idx = active_idxs[n] 152 | active_weights = self.weights[active_idx, :] 153 | active_biases = self.biases 154 | sscores = active_weights.sum(axis=0) + active_biases 155 | label_scores[n] = sscores 156 | sprobs = softmax(label_scores) 157 | 158 | final_probs = [] 159 | si = 0 160 | for ii in range(len(data)): 161 | if ii in inds: 162 | new = sprobs[si] 163 | final_probs.append(new) 164 | si += 1 165 | elif ii in not_inds: 166 | new = 0. * sprobs[0] - 1. 167 | final_probs.append(new) 168 | else: 169 | raise ValueError("This shouldnt happen") 170 | sprobs = np.array(final_probs) 171 | sub_idx = [l for l in list(range(len(data))) if l not in not_inds] 172 | lls = np.zeros_like(sprobs[:, 0]) - 1E8 173 | lls[sub_idx] = np.log(sprobs[list(range(len(data))), pseudolabels][sub_idx]) 174 | return lls 175 | 176 | def predict_proba(self, data): 177 | # trim means return regardless of matching original data length 178 | active_idxs = self.feature_function(data) 179 | inds = [n for n in range(len(active_idxs)) if hasattr(active_idxs[n], "flatten") or active_idxs[n] != None] 180 | not_inds = [n for n in range(len(active_idxs)) if not hasattr(active_idxs[n], "flatten") and active_idxs[n] == None] 181 | 182 | active_idxs = [active_idxs[ii] for ii in inds] 183 | 184 | label_scores = np.zeros((len(active_idxs), self.n_classes)) 185 | for n in range(len(active_idxs)): 186 | active_idx = active_idxs[n] 187 | active_weights = self.weights[active_idx, :] 188 | active_biases = self.biases 189 | sscores = active_weights.sum(axis=0) + active_biases 190 | label_scores[n] = sscores 191 | sprobs = softmax(label_scores) 192 | 193 | final_probs = [] 194 | si = 0 195 | for ii in range(len(data)): 196 | if ii in inds: 197 | new = sprobs[si] 198 | final_probs.append(new) 199 | si += 1 200 | elif ii in not_inds: 201 | new = 0. * sprobs[0] - 1. 202 | final_probs.append(new) 203 | else: 204 | raise ValueError("This shouldnt happen") 205 | return np.array(final_probs) 206 | 207 | def _cost_and_grads(self, data, labels, l1_weight_cost, l2_weight_cost): 208 | assert len(data) == len(labels) 209 | 210 | # switch to block transform... 211 | # preparation for block transform 212 | active_idxs = self.mem_feature_function(data) 213 | if len(active_idxs) != len(labels): 214 | raise ValueError("feature_function should return same number of datapoints! Return None for entries to ignore in training") 215 | 216 | # short circuit OR to avoid issues with array compare 217 | inds = [n for n in range(len(active_idxs)) if hasattr(active_idxs[n], "flatten") or active_idxs[n] != None] 218 | 219 | if self.shuffle: 220 | self.random_state.shuffle(inds) 221 | 222 | active_idxs = [active_idxs[ii] for ii in inds] 223 | labels = [labels[ii] for ii in inds] 224 | 225 | label_scores = np.zeros((len(labels), self.n_classes)) 226 | for n in range(len(active_idxs)): 227 | active_idx = active_idxs[n] 228 | active_weights = self.weights[active_idx, :] 229 | active_biases = self.biases 230 | sscores = active_weights.sum(axis=0) + active_biases 231 | label_scores[n] = sscores 232 | 233 | sprobs = softmax(label_scores) 234 | # https://stats.stackexchange.com/questions/45643/why-l1-norm-for-sparse-models 235 | nll = -np.sum(np.log(sprobs)[list(range(len(labels))), labels]) 236 | nll = nll / float(len(labels)) + l1_weight_cost * np.sum(np.abs(self.weights)).sum() + l2_weight_cost * np.sum(self.weights ** 2).sum() 237 | if self.verbose: 238 | print("nll {}".format(nll)) 239 | 240 | # see non-sparse derivation http://cs231n.github.io/neural-networks-case-study/#loss 241 | dsprobs = sprobs 242 | dsprobs[list(range(len(labels))), labels] -= 1 243 | dsprobs /= float(len(labels)) 244 | 245 | sgrad_w = np.zeros((self.n_features, self.n_classes)) 246 | sgrad_b = np.zeros((self.n_classes,)) 247 | # use cached active_idxs 248 | #for n, (x, y) in enumerate(zip(data, labels)): 249 | # active_idx = sorted(list(set(self.feature_function(x)))) 250 | # if len(active_idx) == 0: 251 | # continue 252 | for n in range(len(active_idxs)): 253 | active_idx = active_idxs[n] 254 | sgrad_w[active_idx] += dsprobs[n] 255 | sgrad_b += dsprobs[n] 256 | sgrad_w += l1_weight_cost * np.sign(self.weights) 257 | sgrad_w += l2_weight_cost * self.weights 258 | grads = np.hstack((sgrad_w.flatten(), sgrad_b)) 259 | if self.verbose: 260 | print("grads_norm {}".format(np.sqrt((grads ** 2).sum()))) 261 | return nll, grads 262 | 263 | def f_and_g(self, x, features, labels, l1_weight_cost, l2_weight_cost): 264 | xold = self.params.copy() 265 | self.update_params(x.copy()) 266 | result = self._cost_and_grads(features, labels, l1_weight_cost, l2_weight_cost) 267 | self.update_params(xold.copy()) 268 | return result 269 | 270 | def update_params(self, new_params): 271 | """ Update model parameters.""" 272 | self.params[:] = new_params.copy() 273 | -------------------------------------------------------------------------------- /maxent_tests/maxent.py: -------------------------------------------------------------------------------- 1 | ../maxent.py -------------------------------------------------------------------------------- /maxent_tests/minimize.py: -------------------------------------------------------------------------------- 1 | ../minimize.py -------------------------------------------------------------------------------- /maxent_tests/optimize.py: -------------------------------------------------------------------------------- 1 | # Note: this was modified by Luke Vilnis for UMass CS585 on 11/21/2013 2 | # to use numpy instead of defunct Numeric/MLab packages 3 | 4 | # ******NOTICE*************** 5 | # optimize.py module by Travis E. Oliphant 6 | # 7 | # You may copy and use this module as you see fit with no 8 | # guarantee implied provided you keep this notice in all copies. 9 | # *****END NOTICE************ 10 | 11 | # A collection of optimization algorithms. Version 0.3.1 12 | 13 | # Minimization routines 14 | """optimize.py 15 | 16 | A collection of general-purpose optimization routines using Numeric 17 | 18 | fmin --- Nelder-Mead Simplex algorithm (uses only function calls) 19 | fminBFGS --- Quasi-Newton method (uses function and gradient) 20 | fminNCG --- Line-search Newton Conjugate Gradient (uses function, gradient 21 | and hessian (if it's provided)) 22 | 23 | """ 24 | import numpy 25 | Num = numpy 26 | MLab = numpy 27 | max = MLab.max 28 | min = MLab.min 29 | abs = Num.absolute 30 | __version__="0.3.1" 31 | 32 | def rosen(x): # The Rosenbrock function 33 | return MLab.sum(100.0*(x[1:]-x[:-1]**2.0)**2.0 + (1-x[:-1])**2.0) 34 | 35 | def rosen_der(x): 36 | xm = x[1:-1] 37 | xm_m1 = x[:-2] 38 | xm_p1 = x[2:] 39 | der = MLab.zeros(x.shape,x.typecode()) 40 | der[1:-1] = 200*(xm-xm_m1**2) - 400*(xm_p1 - xm**2)*xm - 2*(1-xm) 41 | der[0] = -400*x[0]*(x[1]-x[0]**2) - 2*(1-x[0]) 42 | der[-1] = 200*(x[-1]-x[-2]**2) 43 | return der 44 | 45 | def rosen3_hess_p(x,p): 46 | assert(len(x)==3) 47 | assert(len(p)==3) 48 | hessp = Num.zeros((3,),x.typecode()) 49 | hessp[0] = (2 + 800*x[0]**2 - 400*(-x[0]**2 + x[1])) * p[0] \ 50 | - 400*x[0]*p[1] \ 51 | + 0 52 | hessp[1] = - 400*x[0]*p[0] \ 53 | + (202 + 800*x[1]**2 - 400*(-x[1]**2 + x[2]))*p[1] \ 54 | - 400*x[1] * p[2] 55 | hessp[2] = 0 \ 56 | - 400*x[1] * p[1] \ 57 | + 200 * p[2] 58 | 59 | return hessp 60 | 61 | def rosen3_hess(x): 62 | assert(len(x)==3) 63 | hessp = Num.zeros((3,3),x.typecode()) 64 | hessp[0,:] = [2 + 800*x[0]**2 -400*(-x[0]**2 + x[1]), -400*x[0], 0] 65 | hessp[1,:] = [-400*x[0], 202+800*x[1]**2 -400*(-x[1]**2 + x[2]), -400*x[1]] 66 | hessp[2,:] = [0,-400*x[1], 200] 67 | return hessp 68 | 69 | 70 | def fmin(func, x0, args=(), xtol=1e-4, ftol=1e-4, maxiter=None, maxfun=None, fulloutput=0, printmessg=1): 71 | """xopt,{fval,warnflag} = fmin(function, x0, args=(), xtol=1e-4, ftol=1e-4, 72 | maxiter=200*len(x0), maxfun=200*len(x0), fulloutput=0, printmessg=0) 73 | 74 | Uses a Nelder-Mead Simplex algorithm to find the minimum of function 75 | of one or more variables. 76 | """ 77 | x0 = Num.asarray(x0) 78 | assert (len(x0.shape)==1) 79 | N = len(x0) 80 | if maxiter is None: 81 | maxiter = N * 200 82 | if maxfun is None: 83 | maxfun = N * 200 84 | 85 | rho = 1; chi = 2; psi = 0.5; sigma = 0.5; 86 | one2np1 = range(1,N+1) 87 | 88 | sim = Num.zeros((N+1,N),x0.typecode()) 89 | fsim = Num.zeros((N+1,),'d') 90 | sim[0] = x0 91 | fsim[0] = apply(func,(x0,)+args) 92 | nonzdelt = 0.05 93 | zdelt = 0.00025 94 | for k in range(0,N): 95 | y = Num.array(x0,copy=1) 96 | if y[k] != 0: 97 | y[k] = (1+nonzdelt)*y[k] 98 | else: 99 | y[k] = zdelt 100 | 101 | sim[k+1] = y 102 | f = apply(func,(y,)+args) 103 | fsim[k+1] = f 104 | 105 | ind = Num.argsort(fsim) 106 | fsim = Num.take(fsim,ind) # sort so sim[0,:] has the lowest function value 107 | sim = Num.take(sim,ind,0) 108 | 109 | iterations = 1 110 | funcalls = N+1 111 | 112 | while (funcalls < maxfun and iterations < maxiter): 113 | if (max(Num.ravel(abs(sim[1:]-sim[0]))) <= xtol \ 114 | and max(abs(fsim[0]-fsim[1:])) <= ftol): 115 | break 116 | 117 | xbar = Num.add.reduce(sim[:-1],0) / N 118 | xr = (1+rho)*xbar - rho*sim[-1] 119 | fxr = apply(func,(xr,)+args) 120 | funcalls = funcalls + 1 121 | doshrink = 0 122 | 123 | if fxr < fsim[0]: 124 | xe = (1+rho*chi)*xbar - rho*chi*sim[-1] 125 | fxe = apply(func,(xe,)+args) 126 | funcalls = funcalls + 1 127 | 128 | if fxe < fxr: 129 | sim[-1] = xe 130 | fsim[-1] = fxe 131 | else: 132 | sim[-1] = xr 133 | fsim[-1] = fxr 134 | else: # fsim[0] <= fxr 135 | if fxr < fsim[-2]: 136 | sim[-1] = xr 137 | fsim[-1] = fxr 138 | else: # fxr >= fsim[-2] 139 | # Perform contraction 140 | if fxr < fsim[-1]: 141 | xc = (1+psi*rho)*xbar - psi*rho*sim[-1] 142 | fxc = apply(func,(xc,)+args) 143 | funcalls = funcalls + 1 144 | 145 | if fxc <= fxr: 146 | sim[-1] = xc 147 | fsim[-1] = fxc 148 | else: 149 | doshrink=1 150 | else: 151 | # Perform an inside contraction 152 | xcc = (1-psi)*xbar + psi*sim[-1] 153 | fxcc = apply(func,(xcc,)+args) 154 | funcalls = funcalls + 1 155 | 156 | if fxcc < fsim[-1]: 157 | sim[-1] = xcc 158 | fsim[-1] = fxcc 159 | else: 160 | doshrink = 1 161 | 162 | if doshrink: 163 | for j in one2np1: 164 | sim[j] = sim[0] + sigma*(sim[j] - sim[0]) 165 | fsim[j] = apply(func,(sim[j],)+args) 166 | funcalls = funcalls + N 167 | 168 | ind = Num.argsort(fsim) 169 | sim = Num.take(sim,ind,0) 170 | fsim = Num.take(fsim,ind) 171 | iterations = iterations + 1 172 | 173 | x = sim[0] 174 | fval = min(fsim) 175 | warnflag = 0 176 | 177 | if funcalls >= maxfun: 178 | warnflag = 1 179 | if printmessg: 180 | print "Warning: Maximum number of function evaluations has been exceeded." 181 | elif iterations >= maxiter: 182 | warnflag = 2 183 | if printmessg: 184 | print "Warning: Maximum number of iterations has been exceeded" 185 | else: 186 | if printmessg: 187 | print "Optimization terminated successfully." 188 | print " Current function value: %f" % fval 189 | print " Iterations: %d" % iterations 190 | print " Function evaluations: %d" % funcalls 191 | 192 | if fulloutput: 193 | return x, fval, warnflag 194 | else: 195 | return x 196 | 197 | 198 | def zoom(a_lo, a_hi): 199 | pass 200 | 201 | 202 | 203 | def line_search(f, fprime, xk, pk, gfk, args=(), c1=1e-4, c2=0.9, amax=50): 204 | """alpha, fc, gc = line_search(f, xk, pk, gfk, 205 | args=(), c1=1e-4, c2=0.9, amax=1) 206 | 207 | minimize the function f(xk+alpha pk) using the line search algorithm of 208 | Wright and Nocedal in 'Numerical Optimization', 1999, pg. 59-60 209 | """ 210 | 211 | fc = 0 212 | gc = 0 213 | alpha0 = 1.0 214 | phi0 = apply(f,(xk,)+args) 215 | phi_a0 = apply(f,(xk+alpha0*pk,)+args) 216 | fc = fc + 2 217 | derphi0 = Num.dot(gfk,pk) 218 | derphi_a0 = Num.dot(apply(fprime,(xk+alpha0*pk,)+args),pk) 219 | gc = gc + 1 220 | 221 | # check to see if alpha0 = 1 satisfies Strong Wolfe conditions. 222 | if (phi_a0 <= phi0 + c1*alpha0*derphi0) \ 223 | and (abs(derphi_a0) <= c2*abs(derphi0)): 224 | return alpha0, fc, gc 225 | 226 | alpha0 = 0 227 | alpha1 = 1 228 | phi_a1 = phi_a0 229 | phi_a0 = phi0 230 | 231 | i = 1 232 | while 1: 233 | if (phi_a1 > phi0 + c1*alpha1*derphi0) or \ 234 | ((phi_a1 >= phi_a0) and (i > 1)): 235 | return zoom(alpha0, alpha1) 236 | 237 | derphi_a1 = Num.dot(apply(fprime,(xk+alpha1*pk,)+args),pk) 238 | gc = gc + 1 239 | if (abs(derphi_a1) <= -c2*derphi0): 240 | return alpha1 241 | 242 | if (derphi_a1 >= 0): 243 | return zoom(alpha1, alpha0) 244 | 245 | alpha2 = (amax-alpha1)*0.25 + alpha1 246 | i = i + 1 247 | alpha0 = alpha1 248 | alpha1 = alpha2 249 | phi_a0 = phi_a1 250 | phi_a1 = apply(f,(xk+alpha1*pk,)+args) 251 | 252 | 253 | 254 | def line_search_BFGS(f, xk, pk, gfk, args=(), c1=1e-4, alpha0=1): 255 | """alpha, fc, gc = line_search(f, xk, pk, gfk, 256 | args=(), c1=1e-4, alpha0=1) 257 | 258 | minimize over alpha, the function f(xk+alpha pk) using the interpolation 259 | algorithm (Armiijo backtracking) as suggested by 260 | Wright and Nocedal in 'Numerical Optimization', 1999, pg. 56-57 261 | """ 262 | 263 | fc = 0 264 | phi0 = apply(f,(xk,)+args) # compute f(xk) 265 | phi_a0 = apply(f,(xk+alpha0*pk,)+args) # compute f 266 | fc = fc + 2 267 | derphi0 = Num.dot(gfk,pk) 268 | 269 | if (phi_a0 <= phi0 + c1*alpha0*derphi0): 270 | return alpha0, fc, 0 271 | 272 | # Otherwise compute the minimizer of a quadratic interpolant: 273 | 274 | alpha1 = -(derphi0) * alpha0**2 / 2.0 / (phi_a0 - phi0 - derphi0 * alpha0) 275 | phi_a1 = apply(f,(xk+alpha1*pk,)+args) 276 | fc = fc + 1 277 | 278 | if (phi_a1 <= phi0 + c1*alpha1*derphi0): 279 | return alpha1, fc, 0 280 | 281 | # Otherwise loop with cubic interpolation until we find an alpha which satifies 282 | # the first Wolfe condition (since we are backtracking, we will assume that 283 | # the value of alpha is not too small and satisfies the second condition. 284 | 285 | while 1: # we are assuming pk is a descent direction 286 | factor = alpha0**2 * alpha1**2 * (alpha1-alpha0) 287 | a = alpha0**2 * (phi_a1 - phi0 - derphi0*alpha1) - \ 288 | alpha1**2 * (phi_a0 - phi0 - derphi0*alpha0) 289 | a = a / factor 290 | b = -alpha0**3 * (phi_a1 - phi0 - derphi0*alpha1) + \ 291 | alpha1**3 * (phi_a0 - phi0 - derphi0*alpha0) 292 | b = b / factor 293 | 294 | alpha2 = (-b + Num.sqrt(abs(b**2 - 3 * a * derphi0))) / (3.0*a) 295 | phi_a2 = apply(f,(xk+alpha2*pk,)+args) 296 | fc = fc + 1 297 | 298 | if (phi_a2 <= phi0 + c1*alpha2*derphi0): 299 | return alpha2, fc, 0 300 | 301 | if (alpha1 - alpha2) > alpha1 / 2.0 or (1 - alpha2/alpha1) < 0.96: 302 | alpha2 = alpha1 / 2.0 303 | 304 | alpha0 = alpha1 305 | alpha1 = alpha2 306 | phi_a0 = phi_a1 307 | phi_a1 = phi_a2 308 | 309 | epsilon = 1e-8 310 | 311 | def approx_fprime(xk,f,*args): 312 | f0 = apply(f,(xk,)+args) 313 | grad = Num.zeros((len(xk),),'d') 314 | ei = Num.zeros((len(xk),),'d') 315 | for k in range(len(xk)): 316 | ei[k] = 1.0 317 | grad[k] = (apply(f,(xk+epsilon*ei,)+args) - f0)/epsilon 318 | ei[k] = 0.0 319 | return grad 320 | 321 | def approx_fhess_p(x0,p,fprime,*args): 322 | f2 = apply(fprime,(x0+epsilon*p,)+args) 323 | f1 = apply(fprime,(x0,)+args) 324 | return (f2 - f1)/epsilon 325 | 326 | 327 | def fminBFGS(f, x0, fprime=None, args=(), avegtol=1e-5, maxiter=None, fulloutput=0, printmessg=1): 328 | """xopt = fminBFGS(f, x0, fprime=None, args=(), avegtol=1e-5, 329 | maxiter=None, fulloutput=0, printmessg=1) 330 | 331 | Optimize the function, f, whose gradient is given by fprime using the 332 | quasi-Newton method of Broyden, Fletcher, Goldfarb, and Shanno (BFGS) 333 | See Wright, and Nocedal 'Numerical Optimization', 1999, pg. 198. 334 | """ 335 | 336 | app_fprime = 0 337 | if fprime is None: 338 | app_fprime = 1 339 | 340 | x0 = Num.asarray(x0) 341 | if maxiter is None: 342 | maxiter = len(x0)*200 343 | func_calls = 0 344 | grad_calls = 0 345 | k = 0 346 | N = len(x0) 347 | gtol = N*avegtol 348 | I = MLab.eye(N) 349 | Hk = I 350 | 351 | if app_fprime: 352 | gfk = apply(approx_fprime,(x0,f)+args) 353 | func_calls = func_calls + len(x0) + 1 354 | else: 355 | gfk = apply(fprime,(x0,)+args) 356 | grad_calls = grad_calls + 1 357 | xk = x0 358 | sk = [2*gtol] 359 | while (Num.add.reduce(abs(gfk)) > gtol) and (k < maxiter): 360 | pk = -Num.dot(Hk,gfk) 361 | alpha_k, fc, gc = line_search_BFGS(f,xk,pk,gfk,args) 362 | func_calls = func_calls + fc 363 | xkp1 = xk + alpha_k * pk 364 | sk = xkp1 - xk 365 | xk = xkp1 366 | if app_fprime: 367 | gfkp1 = apply(approx_fprime,(xkp1,f)+args) 368 | func_calls = func_calls + gc + len(x0) + 1 369 | else: 370 | gfkp1 = apply(fprime,(xkp1,)+args) 371 | grad_calls = grad_calls + gc + 1 372 | 373 | yk = gfkp1 - gfk 374 | k = k + 1 375 | 376 | rhok = 1 / Num.dot(yk,sk) 377 | A1 = I - sk[:,Num.newaxis] * yk[Num.newaxis,:] * rhok 378 | A2 = I - yk[:,Num.newaxis] * sk[Num.newaxis,:] * rhok 379 | Hk = Num.dot(A1,Num.dot(Hk,A2)) + rhok * sk[:,Num.newaxis] * sk[Num.newaxis,:] 380 | gfk = gfkp1 381 | 382 | 383 | if printmessg or fulloutput: 384 | fval = apply(f,(xk,)+args) 385 | if k >= maxiter: 386 | warnflag = 1 387 | if printmessg: 388 | print "Warning: Maximum number of iterations has been exceeded" 389 | print " Current function value: %f" % fval 390 | print " Iterations: %d" % k 391 | print " Function evaluations: %d" % func_calls 392 | print " Gradient evaluations: %d" % grad_calls 393 | else: 394 | warnflag = 0 395 | if printmessg: 396 | print "Optimization terminated successfully." 397 | print " Current function value: %f" % fval 398 | print " Iterations: %d" % k 399 | print " Function evaluations: %d" % func_calls 400 | print " Gradient evaluations: %d" % grad_calls 401 | 402 | if fulloutput: 403 | return xk, fval, func_calls, grad_calls, warnflag 404 | else: 405 | return xk 406 | 407 | 408 | def fminNCG(f, x0, fprime, fhess_p=None, fhess=None, args=(), avextol=1e-5, maxiter=None, fulloutput=0, printmessg=1): 409 | """xopt = fminNCG(f, x0, fprime, fhess_p=None, fhess=None, args=(), avextol=1e-5, 410 | maxiter=None, fulloutput=0, printmessg=1) 411 | 412 | Optimize the function, f, whose gradient is given by fprime using the 413 | Newton-CG method. fhess_p must compute the hessian times an arbitrary 414 | vector. If it is not given, finite-differences on fprime are used to 415 | compute it. See Wright, and Nocedal 'Numerical Optimization', 1999, 416 | pg. 140. 417 | """ 418 | 419 | x0 = Num.asarray(x0) 420 | fcalls = 0 421 | gcalls = 0 422 | hcalls = 0 423 | approx_hessp = 0 424 | if fhess_p is None and fhess is None: # Define hessian product 425 | approx_hessp = 1 426 | 427 | xtol = len(x0)*avextol 428 | update = [2*xtol] 429 | xk = x0 430 | k = 0 431 | while (Num.add.reduce(abs(update)) > xtol) and (k < maxiter): 432 | # Compute a search direction pk by applying the CG method to 433 | # del2 f(xk) p = - grad f(xk) starting from 0. 434 | b = -apply(fprime,(xk,)+args) 435 | gcalls = gcalls + 1 436 | maggrad = Num.add.reduce(abs(b)) 437 | eta = min([0.5,Num.sqrt(maggrad)]) 438 | termcond = eta * maggrad 439 | xsupi = 0 440 | ri = -b 441 | psupi = -ri 442 | i = 0 443 | dri0 = Num.dot(ri,ri) 444 | 445 | if fhess is not None: # you want to compute hessian once. 446 | A = apply(fhess,(xk,)+args) 447 | hcalls = hcalls + 1 448 | 449 | while Num.add.reduce(abs(ri)) > termcond: 450 | if fhess is None: 451 | if approx_hessp: 452 | Ap = apply(approx_fhess_p,(xk,psupi,fprime)+args) 453 | gcalls = gcalls + 2 454 | else: 455 | Ap = apply(fhess_p,(xk,psupi)+args) 456 | hcalls = hcalls + 1 457 | else: 458 | Ap = Num.dot(A,psupi) 459 | # check curvature 460 | curv = Num.dot(psupi,Ap) 461 | if (curv <= 0): 462 | if (i > 0): 463 | break 464 | else: 465 | xsupi = xsupi + dri0/curv * psupi 466 | break 467 | alphai = dri0 / curv 468 | xsupi = xsupi + alphai * psupi 469 | ri = ri + alphai * Ap 470 | dri1 = Num.dot(ri,ri) 471 | betai = dri1 / dri0 472 | psupi = -ri + betai * psupi 473 | i = i + 1 474 | dri0 = dri1 # update Num.dot(ri,ri) for next time. 475 | 476 | pk = xsupi # search direction is solution to system. 477 | gfk = -b # gradient at xk 478 | alphak, fc, gc = line_search_BFGS(f,xk,pk,gfk,args) 479 | fcalls = fcalls + fc 480 | gcalls = gcalls + gc 481 | 482 | update = alphak * pk 483 | xk = xk + update 484 | k = k + 1 485 | 486 | if printmessg or fulloutput: 487 | fval = apply(f,(xk,)+args) 488 | if k >= maxiter: 489 | warnflag = 1 490 | if printmessg: 491 | print "Warning: Maximum number of iterations has been exceeded" 492 | print " Current function value: %f" % fval 493 | print " Iterations: %d" % k 494 | print " Function evaluations: %d" % fcalls 495 | print " Gradient evaluations: %d" % gcalls 496 | print " Hessian evaluations: %d" % hcalls 497 | else: 498 | warnflag = 0 499 | if printmessg: 500 | print "Optimization terminated successfully." 501 | print " Current function value: %f" % fval 502 | print " Iterations: %d" % k 503 | print " Function evaluations: %d" % fcalls 504 | print " Gradient evaluations: %d" % gcalls 505 | print " Hessian evaluations: %d" % hcalls 506 | 507 | if fulloutput: 508 | return xk, fval, fcalls, gcalls, hcalls, warnflag 509 | else: 510 | return xk 511 | 512 | 513 | 514 | if __name__ == "__main__": 515 | import string 516 | import time 517 | 518 | 519 | times = [] 520 | algor = [] 521 | x0 = [0.8,1.2,0.7] 522 | start = time.time() 523 | x = fmin(rosen,x0) 524 | print x 525 | times.append(time.time() - start) 526 | algor.append('Nelder-Mead Simplex\t') 527 | 528 | start = time.time() 529 | x = fminBFGS(rosen, x0, fprime=rosen_der, maxiter=80) 530 | print x 531 | times.append(time.time() - start) 532 | algor.append('BFGS Quasi-Newton\t') 533 | 534 | start = time.time() 535 | x = fminBFGS(rosen, x0, avegtol=1e-4, maxiter=100) 536 | print x 537 | times.append(time.time() - start) 538 | algor.append('BFGS without gradient\t') 539 | 540 | 541 | start = time.time() 542 | x = fminNCG(rosen, x0, rosen_der, fhess_p=rosen3_hess_p, maxiter=80) 543 | print x 544 | times.append(time.time() - start) 545 | algor.append('Newton-CG with hessian product') 546 | 547 | 548 | start = time.time() 549 | x = fminNCG(rosen, x0, rosen_der, fhess=rosen3_hess, maxiter=80) 550 | print x 551 | times.append(time.time() - start) 552 | algor.append('Newton-CG with full hessian') 553 | 554 | print "\nMinimizing the Rosenbrock function of order 3\n" 555 | print " Algorithm \t\t\t Seconds" 556 | print "===========\t\t\t =========" 557 | for k in range(len(algor)): 558 | print algor[k], "\t -- ", times[k] 559 | 560 | 561 | 562 | 563 | 564 | 565 | 566 | 567 | 568 | 569 | 570 | 571 | 572 | 573 | 574 | -------------------------------------------------------------------------------- /maxent_tests/test_spam.py: -------------------------------------------------------------------------------- 1 | import zipfile 2 | from tokenizing import Tokenizer 3 | from collections import Counter, defaultdict 4 | from maxent import SparseMaxEnt 5 | import numpy as np 6 | from sklearn.feature_extraction.text import TfidfVectorizer 7 | 8 | # uses dataset from 9 | # https://archive.ics.uci.edu/ml/datasets/sms+spam+collection 10 | # based on https://stackoverflow.com/questions/23455728/scikit-learn-balanced-subsampling 11 | def balanced_sample_maker(X, y, sample_size="max", random_state=None): 12 | y_o = y 13 | y = np.array(y) 14 | uniq_levels = np.unique(y) 15 | uniq_counts = {level: sum(y == level) for level in uniq_levels} 16 | if sample_size == "max": 17 | sample_size = max([v for v in uniq_counts.values()]) 18 | if random_state is None: 19 | raise ValueError("Must provide random state") 20 | 21 | # find observation index of each class levels 22 | groupby_levels = {} 23 | for ii, level in enumerate(uniq_levels): 24 | obs_idx = [idx for idx, val in enumerate(y) if val == level] 25 | groupby_levels[level] = obs_idx 26 | 27 | # oversampling on observations of each label 28 | balanced_copy_idx = [] 29 | for gb_level, gb_idx in groupby_levels.iteritems(): 30 | if len(gb_idx) >= sample_size: 31 | over_sample_idx = [] 32 | over_sample_idx += gb_idx 33 | else: 34 | over_sample_idx = [] 35 | # every minority datapoint at least once 36 | over_sample_idx += gb_idx 37 | ex_over_sample_idx = np.random.choice(gb_idx, size=sample_size - len(over_sample_idx), replace=True).tolist() 38 | over_sample_idx += ex_over_sample_idx 39 | balanced_copy_idx += over_sample_idx 40 | np.random.shuffle(balanced_copy_idx) 41 | X_bal = [X[idx] for idx in balanced_copy_idx] 42 | y_bal = [y_o[idx] for idx in balanced_copy_idx] 43 | return (X_bal, y_bal, balanced_copy_idx) 44 | 45 | 46 | fname = "smsspamcollection.zip" 47 | archive = zipfile.ZipFile(fname, 'r') 48 | raw = archive.open(archive.infolist()[0]).readlines() 49 | random_state = np.random.RandomState(1999) 50 | random_state.shuffle(raw) 51 | cats = [l.split("\t")[0] for l in raw] 52 | labels = [0 if l == "ham" else 1 for l in cats] 53 | data = [l.split("\t")[1].rstrip() for l in raw] 54 | 55 | tr = int(.8 * float(len(data))) 56 | train_data = data[:tr] 57 | train_labels = labels[:tr] 58 | #train_data, train_labels, _ = balanced_sample_maker(train_data, train_labels, random_state=random_state) 59 | 60 | test_data = data[tr:] 61 | test_labels = labels[tr:] 62 | #test_data, test_labels, _ = balanced_sample_maker(test_data, test_labels, random_state=random_state) 63 | 64 | tfidf = TfidfVectorizer() 65 | tfidf.fit(train_data) 66 | vocab_size = len(tfidf.get_feature_names()) 67 | n_classes = 2 68 | 69 | def feature_fn(x): 70 | transformed = tfidf.transform([x]) 71 | return transformed.nonzero()[1] 72 | 73 | model = SparseMaxEnt(feature_fn, n_features=vocab_size, n_classes=n_classes, 74 | random_state=random_state) 75 | 76 | model.fit(train_data, train_labels, 0.) 77 | def do_eval(d): 78 | probs = model.predict_proba(d) 79 | preds = np.argmax(probs, axis=-1) 80 | hams = [dd for n, dd in enumerate(d) if n in np.where(preds == 0)[0]] 81 | spams = [dd for n, dd in enumerate(d) if n in np.where(preds == 1)[0]] 82 | return preds, probs, hams, spams 83 | 84 | train_preds, train_probs, train_hams, train_spams = do_eval(train_data) 85 | train_score = np.mean([tl == tp for tl, tp in zip(train_labels, train_preds)]) 86 | print("Train accuracy {}".format(train_score)) 87 | test_preds, test_probs, test_hams, test_spams = do_eval(test_data) 88 | test_score = np.mean([tl == tp for tl, tp in zip(test_labels, test_preds)]) 89 | print("Test accuracy {}".format(test_score)) 90 | from IPython import embed; embed(); raise ValueError() 91 | -------------------------------------------------------------------------------- /maxent_tests/tokenizing.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | http://sentiment.christopherpotts.net/code-data/happyfuntokenizing.py 6 | 7 | This code implements a basic, Twitter-aware tokenizer. 8 | 9 | A tokenizer is a function that splits a string of text into words. In 10 | Python terms, we map string and unicode objects into lists of unicode 11 | objects. 12 | 13 | There is not a single right way to do tokenizing. The best method 14 | depends on the application. This tokenizer is designed to be flexible 15 | and this easy to adapt to new domains and tasks. The basic logic is 16 | this: 17 | 18 | 1. The tuple regex_strings defines a list of regular expression 19 | strings. 20 | 21 | 2. The regex_strings strings are put, in order, into a compiled 22 | regular expression object called word_re. 23 | 24 | 3. The tokenization is done by word_re.findall(s), where s is the 25 | user-supplied string, inside the tokenize() method of the class 26 | Tokenizer. 27 | 28 | 4. When instantiating Tokenizer objects, there is a single option: 29 | preserve_case. By default, it is set to True. If it is set to 30 | False, then the tokenizer will downcase everything except for 31 | emoticons. 32 | 33 | The __main__ method illustrates by tokenizing a few examples. 34 | 35 | I've also included a Tokenizer method tokenize_random_tweet(). If the 36 | twitter library is installed (http://code.google.com/p/python-twitter/) 37 | and Twitter is cooperating, then it should tokenize a random 38 | English-language tweet. 39 | """ 40 | 41 | __author__ = "Christopher Potts" 42 | __copyright__ = "Copyright 2011, Christopher Potts" 43 | __credits__ = [] 44 | __license__ = "Creative Commons Attribution-NonCommercial-ShareAlike 3.0 Unported License: http://creativecommons.org/licenses/by-nc-sa/3.0/" 45 | __version__ = "1.0" 46 | __maintainer__ = "Christopher Potts" 47 | __email__ = "See the author's website" 48 | 49 | ###################################################################### 50 | 51 | import re 52 | import htmlentitydefs 53 | 54 | ###################################################################### 55 | # The following strings are components in the regular expression 56 | # that is used for tokenizing. It's important that phone_number 57 | # appears first in the final regex (since it can contain whitespace). 58 | # It also could matter that tags comes after emoticons, due to the 59 | # possibility of having text like 60 | # 61 | # <:| and some text >:) 62 | # 63 | # Most imporatantly, the final element should always be last, since it 64 | # does a last ditch whitespace-based tokenization of whatever is left. 65 | 66 | # This particular element is used in a couple ways, so we define it 67 | # with a name: 68 | emoticon_string = r""" 69 | (?: 70 | [<>]? 71 | [:;=8] # eyes 72 | [\-o\*\']? # optional nose 73 | [\)\]\(\[dDpP/\:\}\{@\|\\] # mouth 74 | | 75 | [\)\]\(\[dDpP/\:\}\{@\|\\] # mouth 76 | [\-o\*\']? # optional nose 77 | [:;=8] # eyes 78 | [<>]? 79 | )""" 80 | 81 | # The components of the tokenizer: 82 | regex_strings = ( 83 | # Phone numbers: 84 | r""" 85 | (?: 86 | (?: # (international) 87 | \+?[01] 88 | [\-\s.]* 89 | )? 90 | (?: # (area code) 91 | [\(]? 92 | \d{3} 93 | [\-\s.\)]* 94 | )? 95 | \d{3} # exchange 96 | [\-\s.]* 97 | \d{4} # base 98 | )""" 99 | , 100 | # Emoticons: 101 | emoticon_string 102 | , 103 | # HTML tags: 104 | r"""<[^>]+>""" 105 | , 106 | # Twitter username: 107 | r"""(?:@[\w_]+)""" 108 | , 109 | # Twitter hashtags: 110 | r"""(?:\#+[\w_]+[\w\'_\-]*[\w_]+)""" 111 | , 112 | # Remaining word types: 113 | r""" 114 | (?:[a-z][a-z'\-_]+[a-z]) # Words with apostrophes or dashes. 115 | | 116 | (?:[+\-]?\d+[,/.:-]\d+[+\-]?) # Numbers, including fractions, decimals. 117 | | 118 | (?:[\w_]+) # Words without apostrophes or dashes. 119 | | 120 | (?:\.(?:\s*\.){1,}) # Ellipsis dots. 121 | | 122 | (?:\S) # Everything else that isn't whitespace. 123 | """ 124 | ) 125 | 126 | ###################################################################### 127 | # This is the core tokenizing regex: 128 | 129 | word_re = re.compile(r"""(%s)""" % "|".join(regex_strings), re.VERBOSE | re.I | re.UNICODE) 130 | 131 | # The emoticon string gets its own regex so that we can preserve case for them as needed: 132 | emoticon_re = re.compile(regex_strings[1], re.VERBOSE | re.I | re.UNICODE) 133 | 134 | # These are for regularizing HTML entities to Unicode: 135 | html_entity_digit_re = re.compile(r"&#\d+;") 136 | html_entity_alpha_re = re.compile(r"&\w+;") 137 | amp = "&" 138 | 139 | ###################################################################### 140 | 141 | class Tokenizer: 142 | def __init__(self, preserve_case=False): 143 | self.preserve_case = preserve_case 144 | 145 | def tokenize(self, s): 146 | """ 147 | Argument: s -- any string or unicode object 148 | Value: a tokenize list of strings; conatenating this list returns the original string if preserve_case=False 149 | """ 150 | # Try to ensure unicode: 151 | try: 152 | s = unicode(s) 153 | except UnicodeDecodeError: 154 | s = str(s).encode('string_escape') 155 | s = unicode(s) 156 | # Fix HTML character entitites: 157 | s = self.__html2unicode(s) 158 | # Tokenize: 159 | words = word_re.findall(s) 160 | # Possible alter the case, but avoid changing emoticons like :D into :d: 161 | if not self.preserve_case: 162 | words = map((lambda x : x if emoticon_re.search(x) else x.lower()), words) 163 | return words 164 | 165 | def unk_tokenize(self, s, vocabulary_lookup): 166 | pass 167 | 168 | def __html2unicode(self, s): 169 | """ 170 | Internal metod that seeks to replace all the HTML entities in 171 | s with their corresponding unicode characters. 172 | """ 173 | # First the digits: 174 | ents = set(html_entity_digit_re.findall(s)) 175 | if len(ents) > 0: 176 | for ent in ents: 177 | entnum = ent[2:-1] 178 | try: 179 | entnum = int(entnum) 180 | s = s.replace(ent, unichr(entnum)) 181 | except: 182 | pass 183 | # Now the alpha versions: 184 | ents = set(html_entity_alpha_re.findall(s)) 185 | ents = filter((lambda x : x != amp), ents) 186 | for ent in ents: 187 | entname = ent[1:-1] 188 | try: 189 | s = s.replace(ent, unichr(htmlentitydefs.name2codepoint[entname])) 190 | except: 191 | pass 192 | s = s.replace(amp, " and ") 193 | return s 194 | 195 | ############################################################################### 196 | 197 | if __name__ == '__main__': 198 | tok = Tokenizer(preserve_case=False) 199 | samples = ( 200 | u"RT @ #happyfuncoding: this is a typical Twitter tweet :-)", 201 | u"HTML entities & other Web oddities can be an ácute pain >:(", 202 | u"It's perhaps noteworthy that phone numbers like +1 (800) 123-4567, (800) 123-4567, and 123-4567 are treated as words despite their whitespace." 203 | ) 204 | 205 | for s in samples: 206 | print "======================================================================" 207 | print s 208 | tokenized = tok.tokenize(s) 209 | print "\n".join(tokenized) 210 | -------------------------------------------------------------------------------- /minimize.py: -------------------------------------------------------------------------------- 1 | #This program is distributed WITHOUT ANY WARRANTY; without even the implied 2 | #warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 3 | # 4 | # 5 | #This file contains a Python version of Carl Rasmussen's Matlab-function 6 | #minimize.m 7 | # 8 | #minimize.m is copyright (C) 1999 - 2006, Carl Edward Rasmussen. 9 | #Python adaptation by Roland Memisevic 2008. 10 | # 11 | # 12 | #The following is the original copyright notice that comes with the 13 | #function minimize.m 14 | #(from http://www.kyb.tuebingen.mpg.de/bs/people/carl/code/minimize/Copyright): 15 | # 16 | # 17 | #"(C) Copyright 1999 - 2006, Carl Edward Rasmussen 18 | # 19 | #Permission is granted for anyone to copy, use, or modify these 20 | #programs and accompanying documents for purposes of research or 21 | #education, provided this copyright notice is retained, and note is 22 | #made of any changes that have been made. 23 | # 24 | #These programs and documents are distributed without any warranty, 25 | #express or implied. As the programs were written for research 26 | #purposes only, they have not been tested to the degree that would be 27 | #advisable in any important application. All use of these programs is 28 | #entirely at the user's own risk." 29 | 30 | 31 | """minimize.py 32 | 33 | This module contains a function 'minimize' that performs unconstrained 34 | gradient based optimization using nonlinear conjugate gradients. 35 | 36 | The function is a straightforward Python-translation of Carl Rasmussen's 37 | Matlab-function minimize.m 38 | 39 | """ 40 | 41 | 42 | from numpy import dot, isinf, isnan, any, sqrt, isreal, real, nan, inf 43 | 44 | def minimize(X, args, f, grad=None, maxnumlinesearch=None, maxnumfuneval=None, red=1.0, verbose=True): 45 | INT = 0.1;# don't reevaluate within 0.1 of the limit of the current bracket 46 | EXT = 3.0; # extrapolate maximum 3 times the current step-size 47 | MAX = 20; # max 20 function evaluations per line search 48 | RATIO = 10; # maximum allowed slope ratio 49 | SIG = 0.1;RHO = SIG/2;# SIG and RHO are the constants controlling the Wolfe- 50 | #Powell conditions. SIG is the maximum allowed absolute ratio between 51 | #previous and new slopes (derivatives in the search direction), thus setting 52 | #SIG to low (positive) values forces higher precision in the line-searches. 53 | #RHO is the minimum allowed fraction of the expected (from the slope at the 54 | #initial point in the linesearch). Constants must satisfy 0 < RHO < SIG < 1. 55 | #Tuning of SIG (depending on the nature of the function to be optimized) may 56 | #speed up the minimization; it is probably not worth playing much with RHO. 57 | 58 | SMALL = 10.**-16 #minimize.m uses matlab's realmin 59 | 60 | if maxnumlinesearch == None: 61 | if maxnumfuneval == None: 62 | raise "Specify maxnumlinesearch or maxnumfuneval" 63 | else: 64 | S = 'Function evaluation' 65 | length = maxnumfuneval 66 | else: 67 | if maxnumfuneval != None: 68 | raise "Specify either maxnumlinesearch or maxnumfuneval (not both)" 69 | else: 70 | S = 'Linesearch' 71 | length = maxnumlinesearch 72 | 73 | i = 0 # zero the run length counter 74 | ls_failed = 0 # no previous line search has failed 75 | if grad is None: 76 | raise ValueError("Grad must be either True (if f returns grad) or a function") 77 | 78 | if grad == True: 79 | f0, df0 = f(X, *args) 80 | else: 81 | f0 = f(X, *args) # get function value and gradient 82 | df0 = grad(X, *args) 83 | fX = [f0] 84 | i = i + (length<0) # count epochs?! 85 | s = -df0; d0 = -dot(s,s) # initial search direction (steepest) and slope 86 | x3 = red/(1.0-d0) # initial step is red/(|s|+1) 87 | 88 | while i < abs(length): # while not finished 89 | i = i + (length>0) # count iterations?! 90 | 91 | X0 = X; F0 = f0; dF0 = df0 # make a copy of current values 92 | if length>0: 93 | M = MAX 94 | else: 95 | M = min(MAX, -length-i) 96 | while 1: # keep extrapolating as long as necessary 97 | x2 = 0; f2 = f0; d2 = d0; f3 = f0; df3 = df0 98 | success = 0 99 | while (not success) and (M > 0): 100 | try: 101 | M = M - 1; i = i + (length<0) # count epochs?! 102 | if grad == True: 103 | f3, df3 = f(X+x3*s, *args) 104 | else: 105 | f3 = f(X+x3*s, *args) 106 | df3 = grad(X+x3*s, *args) 107 | if isnan(f3) or isinf(f3) or any(isnan(df3)+isinf(df3)): 108 | print "error" 109 | return 110 | success = 1 111 | except: # catch any error which occured in f 112 | x3 = (x2+x3)/2 # bisect and try again 113 | if f3 < F0: 114 | X0 = X+x3*s; F0 = f3; dF0 = df3 # keep best values 115 | d3 = dot(df3,s) # new slope 116 | if d3 > SIG*d0 or f3 > f0+x3*RHO*d0 or M == 0: 117 | # are we done extrapolating? 118 | break 119 | x1 = x2; f1 = f2; d1 = d2 # move point 2 to point 1 120 | x2 = x3; f2 = f3; d2 = d3 # move point 3 to point 2 121 | A = 6*(f1-f2)+3*(d2+d1)*(x2-x1) # make cubic extrapolation 122 | B = 3*(f2-f1)-(2*d1+d2)*(x2-x1) 123 | Z = B+sqrt(complex(B*B-A*d1*(x2-x1))) 124 | if Z != 0.0: 125 | x3 = x1-d1*(x2-x1)**2/Z # num. error possible, ok! 126 | else: 127 | x3 = inf 128 | if (not isreal(x3)) or isnan(x3) or isinf(x3) or (x3 < 0): 129 | # num prob | wrong sign? 130 | x3 = x2*EXT # extrapolate maximum amount 131 | elif x3 > x2*EXT: # new point beyond extrapolation limit? 132 | x3 = x2*EXT # extrapolate maximum amount 133 | elif x3 < x2+INT*(x2-x1): # new point too close to previous point? 134 | x3 = x2+INT*(x2-x1) 135 | x3 = real(x3) 136 | 137 | while (abs(d3) > -SIG*d0 or f3 > f0+x3*RHO*d0) and M > 0: 138 | # keep interpolating 139 | if (d3 > 0) or (f3 > f0+x3*RHO*d0): # choose subinterval 140 | x4 = x3; f4 = f3; d4 = d3 # move point 3 to point 4 141 | else: 142 | x2 = x3; f2 = f3; d2 = d3 # move point 3 to point 2 143 | if f4 > f0: 144 | x3 = x2-(0.5*d2*(x4-x2)**2)/(f4-f2-d2*(x4-x2)) 145 | # quadratic interpolation 146 | else: 147 | A = 6*(f2-f4)/(x4-x2)+3*(d4+d2) # cubic interpolation 148 | B = 3*(f4-f2)-(2*d2+d4)*(x4-x2) 149 | if A != 0: 150 | x3=x2+(sqrt(B*B-A*d2*(x4-x2)**2)-B)/A 151 | # num. error possible, ok! 152 | else: 153 | x3 = inf 154 | if isnan(x3) or isinf(x3): 155 | x3 = (x2+x4)/2 # if we had a numerical problem then bisect 156 | x3 = max(min(x3, x4-INT*(x4-x2)),x2+INT*(x4-x2)) 157 | # don't accept too close 158 | if grad == True: 159 | f3, df3 = f(X+x3*s, *args) 160 | else: 161 | f3 = f(X+x3*s, *args) 162 | df3 = grad(X+x3*s, *args) 163 | if f3 < F0: 164 | X0 = X+x3*s; F0 = f3; dF0 = df3 # keep best values 165 | M = M - 1; i = i + (length<0) # count epochs?! 166 | d3 = dot(df3,s) # new slope 167 | 168 | if abs(d3) < -SIG*d0 and f3 < f0+x3*RHO*d0: # if line search succeeded 169 | X = X+x3*s; f0 = f3; fX.append(f0) # update variables 170 | if verbose: print '%s %6i; Value %4.6e\r' % (S, i, f0) 171 | s = (dot(df3,df3)-dot(df0,df3))/dot(df0,df0)*s - df3 172 | # Polack-Ribiere CG direction 173 | df0 = df3 # swap derivatives 174 | d3 = d0; d0 = dot(df0,s) 175 | if d0 > 0: # new slope must be negative 176 | s = -df0; d0 = -dot(s,s) # otherwise use steepest direction 177 | x3 = x3 * min(RATIO, d3/(d0-SMALL)) # slope ratio but max RATIO 178 | ls_failed = 0 # this line search did not fail 179 | else: 180 | X = X0; f0 = F0; df0 = dF0 # restore best point so far 181 | if ls_failed or (i>abs(length)):# line search failed twice in a row 182 | break # or we ran out of time, so we give up 183 | s = -df0; d0 = -dot(s,s) # try steepest 184 | x3 = 1/(1-d0) 185 | ls_failed = 1 # this line search failed 186 | if verbose: print "\n" 187 | return X, fX, i 188 | -------------------------------------------------------------------------------- /timidifyit.sh: -------------------------------------------------------------------------------- 1 | file=$1 2 | timidity --output-24bit -Ow $file 3 | bn=${file%.mid} 4 | ffmpeg -i $bn.wav -acodec pcm_s16le -ar 44100 $bn_1.wav 5 | mv $bn_1.wav $bn.wav 6 | --------------------------------------------------------------------------------