├── .gitignore ├── tidal-sound-explorer.gif ├── requirements.txt ├── config.py ├── utils.py ├── segmentor.py ├── README.md ├── add_to_BootTidal.hs ├── player.py ├── plotter.py └── analyze_samples.py /.gitignore: -------------------------------------------------------------------------------- 1 | /__pycache__/ 2 | /.idea/ 3 | -------------------------------------------------------------------------------- /tidal-sound-explorer.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShaiRosenblit/tidal-sound-explorer/HEAD/tidal-sound-explorer.gif -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pandas 2 | numpy 3 | python-osc 4 | scipy 5 | matplotlib 6 | path 7 | pydub 8 | librosa 9 | scikit-learn 10 | tqdm 11 | joblib 12 | -------------------------------------------------------------------------------- /config.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | sample_folders = [ 4 | #'/Users/shai/Library/Application Support/SuperCollider/downloaded-quarks/Dirt-Samples', 5 | #'/Users/shai/Documents/tidal/sounds/samples-yt', 6 | #'/Users/shai/Documents/tidal/sounds/samples-extra', 7 | '~/.local/share/SuperCollider/downloaded-quarks/Dirt-Samples' 8 | ] 9 | 10 | segment_csv = '~/Documents/tidal/segments.csv' 11 | 12 | -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import numpy as np 3 | 4 | import config 5 | import os 6 | 7 | def segment_csv_path(): 8 | result = os.path.expanduser(config.segment_csv) 9 | try: 10 | os.makedirs(os.path.dirname(result)) 11 | except FileExistsError: 12 | # directory already exists 13 | pass 14 | return result 15 | 16 | def load_df(): 17 | df = pd.read_csv(segment_csv_path()) 18 | df = df.rename(columns={'Unnamed: 0': 'seg_number_in_sample'}) 19 | newcols = dict() 20 | newcols['s'] = df.seg_sound.str.split(':').str[0] 21 | newcols['n'] = df.seg_sound.str.split(':').str[1].astype(int) 22 | for c in df.columns: 23 | if pd.api.types.is_numeric_dtype(df[c]): 24 | newcols[f"{c}_norm"] = scale_col(df[c]) 25 | newcols[f"{c}_percentile"] = df[c].argsort().argsort() / len(df) 26 | newcols['col_x'] = df['pca_0'] 27 | newcols['col_y'] = df['pca_1'] 28 | newcols['col_x'] = scale_col(newcols['col_x']) 29 | newcols['col_y'] = scale_col(newcols['col_y']) 30 | newcols['color'] = df['cluster'] 31 | newcols['point_size'] = 1 32 | df = pd.concat([df, pd.DataFrame(newcols, index=df.index)], axis=1) 33 | return df 34 | 35 | 36 | def scale_col(col: pd.Series): 37 | col = col - col.min() 38 | col = col / col.max() 39 | return col 40 | -------------------------------------------------------------------------------- /segmentor.py: -------------------------------------------------------------------------------- 1 | import youtube_dl 2 | import argparse 3 | from path import Path 4 | from pydub import AudioSegment 5 | 6 | import os 7 | import sys 8 | 9 | import config 10 | 11 | from find_beats import find_beats_and_bpm, segment_and_analyze_sample 12 | 13 | 14 | def download_youtube_audio(youtube_id: str, sample_name: str, target_folder: Path) -> Path: 15 | # TODO: lower download quality to accelerate stuff 16 | url = "https://www.youtube.com/watch?v=" + youtube_id 17 | file = target_folder.joinpath(f"{sample_name}/{youtube_id}.wav") 18 | 19 | ydl_opts = { 20 | 'format': 'worstaudio/worst', 21 | 'postprocessors': [{ 22 | 'key': 'FFmpegExtractAudio', 23 | 'preferredcodec': 'wav', 24 | 'preferredquality': '192', 25 | }], 26 | 'outtmpl': str(file), 27 | # 'cachedir': str(target_folder) 28 | } 29 | with youtube_dl.YoutubeDL(ydl_opts) as ydl: 30 | ydl.download([url]) 31 | track = AudioSegment.from_file(file) 32 | track.export(file, format='wav') 33 | print(f"File path: {file}\ntrack duration: {track.duration_seconds}") 34 | return file 35 | 36 | 37 | if __name__ == '__main__': 38 | samples_path = Path('/Users/shai/Documents/tidal/sounds/samples-yt/') 39 | 40 | # track = AudioSegment.from_file('~/Documents/tidal/sounds/samples-extra/yt/sophie1.wav') 41 | # track = AudioSegment.from_file("/Users/shai/Documents/tidal/sounds/samples-extra/yt/sophie1.wav") 42 | my_parser = argparse.ArgumentParser(description="Dowload file from youtube and analyze it's audio") 43 | 44 | my_parser.add_argument('youtube_id', 45 | type=str, 46 | help='The youtube id') 47 | my_parser.add_argument('sample_name', 48 | type=str, 49 | help='Sample name') 50 | 51 | # Execute the parse_args() method 52 | args = my_parser.parse_args() 53 | yt_id = args.youtube_id 54 | samp_name = args.sample_name 55 | audio_file_path = download_youtube_audio(yt_id, samp_name, samples_path) 56 | beats_df, bpm, beats = find_beats_and_bpm(audio_file_path) 57 | # segments_df = find_onsets(audio_file_path) 58 | print(f"bpm: {bpm}") 59 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # tidal-sound-explorer 2 | tidal-sound-explorer is a tool for analyzing a libarary of audio samples and end explore them on a 2D plane using TidalCycles live coding languge. 3 | The idea is heavily inspired by the Flucoma project. Currently non of the Flucoma tools is used here, I prefred to implemet it all in python by myself (it's easier and more fun for me :) ) 4 | 5 | The workflow is as flows: 6 | 1. Change the local paths in config.py 7 | 2. Run the script `analyze_samples.py`. This script will: 8 | 1. Read all of the samples from your local tidal cycles sounds directory. 9 | 2. Segment long samples into small chunck using onset detector. 10 | 3. Extract audio features for each segment. 11 | 4. The segments information (including path, strat and end time, and audio features) will be stored in a csv file. (The segments audio is not copied into new files) 12 | 3. Add to your `BootTidal.hs` file the code from `add_to_BootTidal.hs` 13 | 4. Run the scripts `player.py`. This script will take care of listening to tidal, selecting the samples based on the tidal message and sending it to Supercollider over OSC. 14 | 5. Run the scripts `plotter.py`. This script will plot the 2D embedding of the samples and will interactively in real time will plot the played samples. (It is not necessary to run this script to make sounds) 15 | 6. Run tidalcycles (and Supercollider and Superdirt of course). Try executing the following code: 16 | ``` 17 | d1 18 | $ struct "t*16" 19 | $ x (range "0.6" "0.7" $ fast "1" sine) 20 | # y (range "0.2" "0.3" $ fast "1" cosine) 21 | # c "w" -- color 22 | # room 0.5 23 | ``` 24 | If everything went well you should see the some points moving and hear some (hopefully cool!) sounds: 25 | ![cool gif](tidal-sound-explorer.gif) 26 | 27 | (notice that you can pass any superdirt effect with the message, like `# room 0.5` in the example) 28 | 29 | If "x" or "y" are specified (as in the example above) the nearest point in the 2D plane will be played. 30 | If they are not specified a random segment will be selected. 31 | 32 | It is possible to filter the segments by any column in the data frame. Here is an example 33 | ``` 34 | d1 35 | $ struct "t(5,8)" 36 | $ pF "keep_only_below_spectral_centroid" 500 37 | # pF "keep_only_above_spectral_centroid" 200 38 | ``` 39 | 40 | This example will play only segments with `spectral_centroid` between 200 and 500 (i.e relatively low pitch sounds). 41 | Since "x" and "y" are not specified, a random segment will be selected. 42 | The format of the filtering message is `"keep_only_{above/below/equal}_{column name}"`. 43 | 44 | And here is another example that demostrates filtering by sample name (it should work if you have [extra-samples](https://github.com/yaxu/spicule/tree/master/extra-samples)): 45 | ``` 46 | d1 47 | $ stack [ 48 | id 49 | $ struct "t(3,8)" 50 | $ pF "keep_only_below_spectral_centroid" 300 51 | # pF "keep_only_above_spectral_centroid" 200 52 | # pS "keep_only_equal_s" "kick" 53 | # release 0.2 54 | # gain 1.5 55 | , id 56 | $ fast "{1!4 2}%7" 57 | $ fast "{1!4 2}%8" 58 | $ struct "t(8,8)" 59 | $ pS "keep_only_equal_s" "foley" 60 | # pF "keep_only_below_spectral_centroid" 6000 61 | # pF "keep_only_above_spectral_centroid" 5000 62 | # release 0.1 63 | # gain 1.2 64 | # room 0.4 65 | # c "r" 66 | , id 67 | $ pS "keep_only_equal_s" "snare(2,4,1)" 68 | # gain 2 69 | # c "g" 70 | ] 71 | ``` 72 | 73 | 74 | There are many more things to explore using this tool (better segmentation, better features, more 2d embeddings, and more...). Feel free to contact me if you are interested :) 75 | -------------------------------------------------------------------------------- /add_to_BootTidal.hs: -------------------------------------------------------------------------------- 1 | :{ 2 | let 3 | target = 4 | Target {oName = "plotter", -- A friendly name for the target (only used in error messages) 5 | oAddress = "127.0.0.1", -- The target's network address, normally "localhost" 6 | oPort = 57121, -- The network port the target is listening on 7 | oLatency = 0.1, -- Additional delay, to smooth out network jitter/get things in sync 8 | oSchedule = Live, -- The scheduling method - see below 9 | oWindow = Nothing, -- Not yet used 10 | oHandshake = False, -- SuperDirt specific 11 | oBusPort = Nothing -- Also SuperDirt specific 12 | } 13 | oscplay_named_params = OSC "/play" Named {requiredArgs = []} 14 | oscmap = [(target {oLatency = 0.1}, [oscplay_named_params]), 15 | (superdirtTarget {oLatency = 0.1}, [superdirtShape]) 16 | ] 17 | :} 18 | 19 | tidal <- startStream defaultConfig oscmap 20 | 21 | :{ 22 | let only = (hush >>) 23 | p = streamReplace tidal 24 | hush = streamHush tidal 25 | panic = do hush 26 | once $ sound "superpanic" 27 | list = streamList tidal 28 | mute = streamMute tidal 29 | unmute = streamUnmute tidal 30 | unmuteAll = streamUnmuteAll tidal 31 | unsoloAll = streamUnsoloAll tidal 32 | solo = streamSolo tidal 33 | unsolo = streamUnsolo tidal 34 | once = streamOnce tidal 35 | first = streamFirst tidal 36 | asap = once 37 | nudgeAll = streamNudgeAll tidal 38 | all = streamAll tidal 39 | resetCycles = streamResetCycles tidal 40 | setcps = asap . cps 41 | getcps = streamGetcps tidal 42 | getnow = streamGetnow tidal 43 | xfade i = transition tidal True (Sound.Tidal.Transition.xfadeIn 4) i 44 | xfadeIn i t = transition tidal True (Sound.Tidal.Transition.xfadeIn t) i 45 | histpan i t = transition tidal True (Sound.Tidal.Transition.histpan t) i 46 | wait i t = transition tidal True (Sound.Tidal.Transition.wait t) i 47 | waitT i f t = transition tidal True (Sound.Tidal.Transition.waitT f t) i 48 | jump i = transition tidal True (Sound.Tidal.Transition.jump) i 49 | jumpIn i t = transition tidal True (Sound.Tidal.Transition.jumpIn t) i 50 | jumpIn' i t = transition tidal True (Sound.Tidal.Transition.jumpIn' t) i 51 | jumpMod i t = transition tidal True (Sound.Tidal.Transition.jumpMod t) i 52 | jumpMod' i t p = transition tidal True (Sound.Tidal.Transition.jumpMod' t p) i 53 | mortal i lifespan release = transition tidal True (Sound.Tidal.Transition.mortal lifespan release) i 54 | interpolate i = transition tidal True (Sound.Tidal.Transition.interpolate) i 55 | interpolateIn i t = transition tidal True (Sound.Tidal.Transition.interpolateIn t) i 56 | clutch i = transition tidal True (Sound.Tidal.Transition.clutch) i 57 | clutchIn i t = transition tidal True (Sound.Tidal.Transition.clutchIn t) i 58 | anticipate i = transition tidal True (Sound.Tidal.Transition.anticipate) i 59 | anticipateIn i t = transition tidal True (Sound.Tidal.Transition.anticipateIn t) i 60 | forId i t = transition tidal False (Sound.Tidal.Transition.mortalOverlay t) i 61 | d1 = p 1 . (|< orbit 0) 62 | d2 = p 2 . (|< orbit 1) 63 | d3 = p 3 . (|< orbit 2) 64 | d4 = p 4 . (|< orbit 3) 65 | d5 = p 5 . (|< orbit 4) 66 | d6 = p 6 . (|< orbit 5) 67 | d7 = p 7 . (|< orbit 6) 68 | d8 = p 8 . (|< orbit 7) 69 | d9 = p 9 . (|< orbit 8) 70 | d10 = p 10 . (|< orbit 9) 71 | d11 = p 11 . (|< orbit 10) 72 | d12 = p 12 . (|< orbit 11) 73 | d13 = p 13 74 | d14 = p 14 75 | d15 = p 15 76 | d16 = p 16 77 | x = pF "query_col_x" 78 | y = pF "query_col_y" 79 | c = pS "c" 80 | xy x_ y_ = (x x_) |>| (y y_) 81 | yx y_ x_ = (y y_) |>| (x x_) 82 | circle x_ y_ rad rate = xy (range (x_+2*rad) (x_) $ fast rate sine) 83 | (range (y_+2*rad) (y_) $ fast rate cosine) 84 | :} 85 | -------------------------------------------------------------------------------- /player.py: -------------------------------------------------------------------------------- 1 | from pythonosc import udp_client 2 | from pythonosc.osc_server import AsyncIOOSCUDPServer 3 | from pythonosc.dispatcher import Dispatcher 4 | import asyncio 5 | import pandas as pd 6 | from scipy.spatial import KDTree 7 | import socket 8 | import json 9 | import numpy as np 10 | import functools 11 | import config 12 | 13 | from utils import load_df 14 | 15 | ip = "127.0.0.1" 16 | udp_port = 5005 17 | UDP_PORT_plotter2player = 5006 18 | server_port = 57121 19 | client_port = 57120 20 | 21 | 22 | def send_message_to_tidal(message_dict): 23 | filters = tuple((k, v) for k, v in message_dict.items() if k.startswith('keep_only')) 24 | filt_idx = filter_df(filters) 25 | if len(filt_idx) == 0: 26 | return 27 | query_cols = [] 28 | query_vals = [] 29 | for k, v in message_dict.items(): 30 | if k.startswith('query_'): 31 | query_cols.append(k[6:]) 32 | query_vals.append(v) 33 | 34 | filt_sub_space = df.loc[filt_idx, query_cols].values 35 | if len(query_cols) > 0: 36 | if 'nth_nearest' in message_dict: 37 | nth_nearest = int(message_dict['nth_nearest']) 38 | else: 39 | nth_nearest = 0 40 | dists = ((filt_sub_space - query_vals)**2).sum(axis=1) 41 | nth_nearest = min(nth_nearest, (len(dists) - 1)) 42 | sub_df_idx = np.argpartition(dists, nth_nearest)[nth_nearest] 43 | idx = filt_idx[sub_df_idx] 44 | else: 45 | idx = np.random.choice(filt_idx) 46 | 47 | print(f'Playing smaple {df.iloc[idx]["seg_sound"]}, ' 48 | f'{df.iloc[idx]["seg_number_in_sample"]}, ' 49 | f'cycle - {message_dict["cycle"]}') 50 | message_list = [ 51 | "s", df.iloc[idx]['s'], 52 | "n", int(df.iloc[idx]['n']), 53 | "begin", df.iloc[idx]['seg_start'], 54 | "end", df.iloc[idx]['seg_end'], 55 | "sustain", 10, 56 | ] 57 | for k, v in message_dict.items(): 58 | if v is None: 59 | v = 0 60 | message_list.extend([k, v]) 61 | client.send_message("/dirt/play", message_list) 62 | return idx 63 | 64 | def empty_socket(sock): 65 | while True: 66 | try: 67 | sock.recvfrom(65535) 68 | except socket.error: 69 | break 70 | 71 | 72 | def update_func(*args): 73 | 74 | if not hasattr(update_func, 'x'): 75 | update_func.x = 0 76 | update_func.y = 0 77 | if not hasattr(update_func, 'data_from_plotter'): 78 | update_func.data_from_plotter = {} 79 | 80 | message_dict = dict(zip(args[1::2], args[2::2])) 81 | if "s" in message_dict or (len(message_dict)==1 and (list(message_dict.keys())[0] is not str)): 82 | return 83 | if ('x' not in message_dict) or ('y' not in message_dict): 84 | message_dict['x'] = None 85 | message_dict['y'] = None 86 | 87 | update_func.x = message_dict['x'] 88 | update_func.y = message_dict['y'] 89 | try: 90 | data_from_plotter, _ = sock_plotter2player.recvfrom(65535) 91 | empty_socket(sock_plotter2player) 92 | data_from_plotter = json.loads(data_from_plotter) 93 | update_func.data_from_plotter = data_from_plotter 94 | print('*'*40) 95 | print(f'Got data from plotter: {update_func.data_from_plotter}') 96 | print('~'*40) 97 | except socket.error: 98 | pass 99 | 100 | if 'offset_xy_key' in message_dict: 101 | if message_dict['offset_xy_key'] in update_func.data_from_plotter: 102 | message_dict['query_col_x'] += update_func.data_from_plotter[message_dict['offset_xy_key']][0] 103 | message_dict['query_col_y'] += update_func.data_from_plotter[message_dict['offset_xy_key']][1] 104 | 105 | idx = send_message_to_tidal(message_dict) 106 | if idx is not None: 107 | message_dict['idx'] = int(idx) 108 | if 'gain' not in message_dict: 109 | message_dict['gain'] = 1 110 | # print(message_dict) 111 | sock.sendto(json.dumps(message_dict, indent=2).encode('utf-8'), (ip, udp_port)) 112 | 113 | dispatcher = Dispatcher() 114 | dispatcher.set_default_handler(update_func) 115 | 116 | 117 | async def loop(): 118 | print('Here we go!!!') 119 | while True: 120 | await asyncio.sleep(0) 121 | 122 | 123 | async def main(): 124 | 125 | server = AsyncIOOSCUDPServer((ip, server_port), dispatcher, asyncio.get_event_loop()) 126 | transport, protocol = await server.create_serve_endpoint() # Create datagram endpoint and start serving 127 | # await print_loop() 128 | # await loop() # Enter main loop of program 129 | await asyncio.gather(loop()) 130 | 131 | transport.close() # Clean up serve endpoint 132 | 133 | 134 | @functools.lru_cache(maxsize=None) 135 | def filter_df(filters): 136 | print(f'filters - {filters}') 137 | if len(filters) == 0: 138 | return df.index 139 | 140 | filt_df = df 141 | for filt, val in filters: 142 | if filt.startswith('keep_only_above_'): 143 | filt_df = filt_df[filt_df[filt[16:]] > val] 144 | elif filt.startswith('keep_only_below_'): 145 | filt_df = filt_df[filt_df[filt[16:]] < val] 146 | elif filt.startswith('keep_only_equal_'): 147 | filt_df = filt_df[filt_df[filt[16:]] == val] 148 | elif filt.startswith('keep_only_isin_'): 149 | words = val.split('.') 150 | filt_df = filt_df[filt_df[filt[15:]].isin(words)] 151 | elif filt.startswith('keep_only_start_'): 152 | filt_df = filt_df[filt_df[filt[16:]].str.startswith(val)] 153 | else: 154 | raise ValueError(f"unsupprted filter {filt}") 155 | return filt_df.index 156 | 157 | 158 | if __name__ == "__main__": 159 | global df 160 | df = load_df() 161 | kdt = KDTree(df[['col_x', 'col_y']]) 162 | client = udp_client.SimpleUDPClient("127.0.0.1", client_port) 163 | sock = socket.socket(socket.AF_INET, # Internet 164 | socket.SOCK_DGRAM) # UDP 165 | sock_plotter2player = socket.socket(socket.AF_INET, # Internet 166 | socket.SOCK_DGRAM) # UDP 167 | 168 | sock_plotter2player.settimeout(0) 169 | sock_plotter2player.bind(("127.0.0.1", UDP_PORT_plotter2player)) 170 | 171 | asyncio.run(main()) 172 | -------------------------------------------------------------------------------- /plotter.py: -------------------------------------------------------------------------------- 1 | import json 2 | import socket 3 | import sys 4 | 5 | import numpy as np 6 | import pandas as pd 7 | import matplotlib.pyplot as plt 8 | import matplotlib.animation as animation 9 | from matplotlib import cm 10 | 11 | from utils import load_df 12 | 13 | 14 | UDP_IP = "127.0.0.1" 15 | UDP_PORT = 5005 16 | UDP_PORT_plotter2player = 5006 17 | MAX_POINTS_IN_QUEUE = 30 # adjust this number if the event rate is too high and the plot gets out of sync 18 | 19 | sock = socket.socket(socket.AF_INET, 20 | socket.SOCK_DGRAM) 21 | sock.bind((UDP_IP, UDP_PORT)) 22 | 23 | sock_plotter2player = socket.socket(socket.AF_INET, 24 | socket.SOCK_DGRAM) 25 | 26 | plt.style.use('dark_background') 27 | 28 | 29 | class Scope: 30 | def __init__(self, fig, ax, plot_df, dt, init_random_keys=True): 31 | print("Get ready for some points!!!") 32 | plt.rcParams['keymap.xscale'].remove('k') 33 | plt.rcParams['keymap.save'].remove('s') 34 | self.fig = fig 35 | self.ax = ax 36 | self.dt = dt 37 | self.df = plot_df 38 | self.points = np.array([[0, 0, 0, -1]]) # x, y, ,size, last_t 39 | self.points_colors = np.array(['r']) 40 | self.t = 0 41 | self.color_vec = np.array(['b'] * len(plot_df)) 42 | self.size_vec = np.array([1] * len(plot_df)) 43 | self.sc = ax.scatter(plot_df['col_x'], plot_df['col_y'], 44 | alpha=0.5, picker=10, c=plot_df.color, 45 | s=plot_df.point_size, cmap=cm.get_cmap('tab20')) 46 | self.default_point_dur = 0.05 47 | self.default_size_factor = 100 48 | self.default_color = 'w' 49 | self.p = self.ax.scatter([0], [0], visible=False) 50 | self.num_text_elements = 20 51 | self.texts = [self.ax.text(0, 0, '', color='w') for i in range(self.num_text_elements)] 52 | self.poiter_clicked = False 53 | self.fig.canvas.mpl_connect('button_press_event', self.on_click) 54 | self.fig.canvas.mpl_connect('button_release_event', self.on_release) 55 | self.fig.canvas.mpl_connect('motion_notify_event', self.on_motion) 56 | self.fig.canvas.mpl_connect('key_press_event', self.on_press) 57 | self.selected_key = '1' 58 | self.clicked_key_vals = {} 59 | if init_random_keys: 60 | for key in '0123456789': 61 | self.clicked_key_vals[key] = 0.5+np.random.randn()*0.2, 0.5+np.random.randn()*0.2 62 | sock_plotter2player.sendto(json.dumps(self.clicked_key_vals).encode('utf-8'), (UDP_IP, UDP_PORT_plotter2player)) 63 | 64 | 65 | def update(self, data_dict): 66 | self.t += self.dt 67 | keep = self.points[:, -1] > self.t 68 | self.points_colors = self.points_colors[keep] 69 | self.points = self.points[keep, :] # delete old points 70 | if data_dict is not None: 71 | # print(data_dict) 72 | if 'dur' in data_dict: 73 | point_dur = data_dict['dur'] 74 | else: 75 | point_dur = self.default_point_dur 76 | if 'size_factor' in data_dict: 77 | size_factor = data_dict['size_factor'] 78 | else: 79 | size_factor = self.default_size_factor 80 | point_size = data_dict['gain'] * size_factor 81 | new_point = [self.df.iloc[data_dict['idx']]['col_x'], 82 | self.df.iloc[data_dict['idx']]['col_y'], 83 | point_size, 84 | self.t + point_dur 85 | ] 86 | self.points = np.vstack([self.points, new_point]) # add new point 87 | if 'c' in data_dict: 88 | color = data_dict['c'] 89 | else: 90 | color = self.default_color 91 | self.points_colors = np.append(self.points_colors, color) 92 | print(f'{len(self.points)}', data_dict) 93 | 94 | self.p.set_visible(True) 95 | if len(self.points) > 0: 96 | self.p.set_offsets(self.points[:, :2]) 97 | self.p.set_color(self.points_colors) 98 | self.p.set_sizes(self.points[:, 2]) 99 | self.points[:, 2] *= 0.9 100 | else: 101 | self.p.set_visible(False) 102 | for i, (k, v) in enumerate(self.clicked_key_vals.items()): 103 | self.texts[i].set_text(k) 104 | self.texts[i].set_position(v) 105 | self.texts[i].set_color('w') 106 | self.texts[i].set_visible(True) 107 | for i in range(len(self.clicked_key_vals), len(self.texts)): 108 | self.texts[i].set_visible(False) 109 | return self.p, *self.texts 110 | 111 | def on_motion(self, event): 112 | if not self.poiter_clicked: 113 | return 114 | if len(self.clicked_key_vals) > self.num_text_elements: 115 | return 116 | # print('%s click: button=%d, x=%d, y=%d, xdata=%f, ydata=%f' % 117 | # ('double' if event.dblclick else 'single', event.button, 118 | # event.x, event.y, event.xdata, event.ydata)) 119 | self.clicked_key_vals[self.selected_key] = event.xdata, event.ydata 120 | sock_plotter2player.sendto(json.dumps(self.clicked_key_vals).encode('utf-8'), (UDP_IP, UDP_PORT_plotter2player)) 121 | # print('*'*40) 122 | # print(self.clicked_key_vals) 123 | # print('~'*40) 124 | 125 | def on_click(self, event): 126 | if event.button == 1: 127 | self.poiter_clicked = True 128 | self.clicked_key_vals[self.selected_key] = event.xdata, event.ydata 129 | sock_plotter2player.sendto(json.dumps(self.clicked_key_vals).encode('utf-8'), (UDP_IP, UDP_PORT_plotter2player)) 130 | else: 131 | self.poiter_clicked = False 132 | 133 | def on_release(self, event): 134 | if event.button == 1: 135 | self.poiter_clicked = False 136 | 137 | def on_press(self, event): 138 | if event.key == 'tab': 139 | self.clicked_key_vals = {} 140 | for t in self.texts: 141 | t.set_visible(False) 142 | return 143 | if event.key == 'backspace': 144 | if self.selected_key in self.clicked_key_vals: 145 | del self.clicked_key_vals[self.selected_key] 146 | return 147 | print('press', event.key) 148 | self.selected_key = event.key 149 | sys.stdout.flush() 150 | 151 | 152 | 153 | def get_updated_val(update_val=None): 154 | if not hasattr(get_updated_val, 'val'): 155 | get_updated_val.val = 0 156 | if update_val is not None: 157 | get_updated_val.val = update_val 158 | return get_updated_val.val 159 | 160 | 161 | def emitter(): 162 | while True: 163 | try: 164 | data, addr = sock.recvfrom(1024) 165 | while len(scope.points) > MAX_POINTS_IN_QUEUE: 166 | # empty the queue if needed 167 | # (this points will not be plotted but at least the plot will keep sync) 168 | data, addr = sock.recvfrom(1024) 169 | data_dict = json.loads(data) 170 | 171 | except socket.error: 172 | data_dict = None 173 | 174 | yield data_dict 175 | 176 | 177 | df = load_df() 178 | 179 | interval = 1 180 | dt_ = interval / 1000 181 | sock.settimeout(0) 182 | sock_plotter2player.settimeout(0) 183 | 184 | fig_, ax_ = plt.subplots(figsize=(10, 10)) 185 | 186 | scope = Scope(fig_, ax_, df, dt_) 187 | ani = animation.FuncAnimation(fig_, scope.update, emitter, interval=interval, 188 | blit=True) 189 | plt.show() 190 | -------------------------------------------------------------------------------- /analyze_samples.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | import pandas as pd 3 | import os.path 4 | import pydub 5 | from path import Path 6 | from tqdm import tqdm 7 | import numpy as np 8 | import librosa 9 | from librosa.feature.spectral import mfcc, spectral_bandwidth, spectral_centroid, \ 10 | spectral_contrast, spectral_flatness, spectral_rolloff 11 | # import umap 12 | from sklearn.decomposition import PCA 13 | from sklearn.preprocessing import StandardScaler 14 | from joblib import Parallel, delayed 15 | import sklearn.cluster as cluster 16 | import warnings 17 | 18 | import config 19 | import utils 20 | 21 | def analyze_seg(samples, fs): 22 | samples = samples - np.mean(samples) 23 | 24 | feats = dict() 25 | feats['rms'] = np.mean(samples ** 2) ** 0.5 26 | mfcc_mat = mfcc(y=samples, sr=fs, n_mfcc=20) 27 | for i, v in enumerate(mfcc_mat.mean(axis=1)): 28 | feats[f'mfcc_{i}'] = v 29 | 30 | with warnings.catch_warnings(): # supress librosa :UserWarning: Trying to estimate tuning from empty frequency set. 31 | warnings.simplefilter("ignore") 32 | chroma_mat = librosa.feature.chroma_stft(y=samples, sr=fs) 33 | for i, v in enumerate(chroma_mat.mean(axis=1)): 34 | feats[f'chroma_{i}'] = v 35 | feats['note'] = np.argmax(np.mean(chroma_mat, axis=1)) 36 | feats['note_quality'] = np.max(np.mean(chroma_mat, axis=1))-np.median(np.mean(chroma_mat, axis=1)) 37 | # librosa.piptrack(y=samples, sr=fs) 38 | # feats['pitch'] 39 | feats['spectral_bandwidth'] = np.mean(spectral_bandwidth(y=samples, sr=fs)) 40 | feats['spectral_centroid'] = np.mean(spectral_centroid(y=samples, sr=fs)) 41 | feats['spectral_contrast'] = np.mean(spectral_contrast(y=samples, sr=fs)) 42 | feats['spectral_flatness'] = np.mean(spectral_flatness(y=samples)) 43 | feats['spectral_rolloff'] = np.mean(spectral_rolloff(y=samples, sr=fs)) 44 | feats['spectral_bandwidth_std'] = np.std(spectral_bandwidth(y=samples, sr=fs)) 45 | feats['spectral_centroid_std'] = np.std(spectral_centroid(y=samples, sr=fs)) 46 | feats['spectral_contrast_std'] = np.std(spectral_contrast(y=samples, sr=fs)) 47 | feats['spectral_flatness_std'] = np.std(spectral_flatness(y=samples)) 48 | feats['spectral_rolloff_std'] = np.std(spectral_rolloff(y=samples, sr=fs)) 49 | s = pd.Series(samples) 50 | window = int(fs / 50) 51 | amp_env = (s 52 | .rolling(window).max() 53 | .shift(-window) 54 | ) 55 | max_amp = amp_env.max() 56 | attack_thresh = max_amp * 0.9 57 | is_above_attack_thresh = np.where(samples > attack_thresh)[0] 58 | if len(is_above_attack_thresh) > 0: 59 | attack_idx = np.where(samples > attack_thresh)[0][0] 60 | else: 61 | attack_idx = 0 62 | attack_time = attack_idx / fs 63 | decay_val = max_amp * 0.10 64 | decay_idx = np.argmin(np.abs(amp_env[attack_idx:] - decay_val)) + attack_idx 65 | decay_time = decay_idx / fs 66 | feats['decay_dur'] = decay_time - attack_time 67 | feats['attack_dur'] = attack_time 68 | zero_crossing_idx = ((s < 0) & (s.shift(-1) > 0)) | ((s > 0) & (s.shift(-1) < 0)) 69 | feats['zcr'] = zero_crossing_idx.mean() * fs 70 | feats['zcr_attack'] = zero_crossing_idx[:attack_idx].mean() * fs 71 | feats['zcr_decay'] = zero_crossing_idx[attack_idx:decay_idx].mean() * fs 72 | feats['rms_attack'] = (s[:attack_idx] ** 2).mean() ** 0.5 73 | feats['rms_decay'] = (s[attack_idx:decay_idx] ** 2).mean() ** 0.5 74 | return feats 75 | 76 | 77 | def segment_and_analyze_sample(audio_file: Path, sound_name, min_seg_num_samples=2048): 78 | samples, fs = librosa.load(audio_file) 79 | track_dur = len(samples) / fs 80 | len_samples = len(samples) 81 | if (track_dur > 1): 82 | onsets_idx = librosa.onset.onset_detect(y=samples, sr=fs, backtrack=True, units='samples') 83 | else: 84 | onsets_idx = [0, len_samples] 85 | segments_df = pd.DataFrame({'seg_start_idx': onsets_idx[:-1], 'seg_end_idx': onsets_idx[1:]}) 86 | segments_df['seg_num_samples'] = segments_df['seg_end_idx'] - segments_df['seg_start_idx'] 87 | segments_df = segments_df[segments_df.seg_num_samples > min_seg_num_samples].reset_index() 88 | segments_df['seg_start'] = segments_df['seg_start_idx'] / len_samples 89 | segments_df['seg_end'] = segments_df['seg_end_idx'] / len_samples 90 | 91 | segments_df['seg_start_sec'] = segments_df['seg_start'] * track_dur 92 | segments_df['seg_end_sec'] = segments_df['seg_end'] * track_dur 93 | segments_df['seg_dur_sec'] = segments_df['seg_end_sec'] - segments_df['seg_start_sec'] 94 | segments_df['full_sample_dur_sec'] = track_dur 95 | feats_list = [] 96 | for i, r in segments_df.iterrows(): 97 | try: 98 | feats = analyze_seg(samples[int(r.seg_start_idx):int(r.seg_end_idx)], fs) 99 | except Exception as e: 100 | print(f"failed to extrcat features to {sound_name}, segment {i}, {r['seg_dur_sec']}") 101 | raise e 102 | feats_list.append(feats) 103 | feats_df = pd.DataFrame(feats_list) 104 | segments_df['path'] = audio_file 105 | segments_df = pd.concat([segments_df, feats_df], axis=1) 106 | segments_df["seg_sound"] = sound_name 107 | if sum(segments_df.seg_dur_sec == 0) > 0: 108 | print('oh no') 109 | segments_df = segments_df[segments_df.seg_start_sec < 10] # avoid giving too much weight to long samples 110 | return segments_df 111 | 112 | 113 | def gen_haskell_code(segments_df, haskell_file_path): 114 | print(segments_df) 115 | haskell_code_str_ln = ["let"] 116 | seg_sound_str = '", "'.join(segments_df.seg_sound.values) 117 | haskell_code_str_ln.append(f'\t\tseg_sound = ["{seg_sound_str}"]') 118 | seg_start_str = ', '.join(segments_df.seg_start.astype(str).values) 119 | haskell_code_str_ln.append(f'\t\tseg_start = [{seg_start_str}]') 120 | seg_end_str = ', '.join(segments_df.seg_end.astype(str).values) 121 | haskell_code_str_ln.append(f'\t\tseg_end = [{seg_end_str}]') 122 | haskell_code_str_ln.append( 123 | f'\t\tselseg n = (|>| begin (fit 0 seg_start n)) . (|>| end (fit 0 seg_end n)) . (|>| s (fit 0 seg_sound n))') 124 | 125 | haskell_code_str = "\n".join(haskell_code_str_ln) 126 | 127 | print(haskell_code_str) 128 | with open(haskell_file_path, 'w') as f: 129 | f.write(haskell_code_str) 130 | 131 | 132 | def convert_file_to_wav(file): 133 | wav_file = Path(file).with_suffix('.wav').replace(' ', '_') 134 | if Path(wav_file).exists(): 135 | return 136 | command = f'ffmpeg -i "{file}" -ab 160k -ac 2 -ar 44100 -vn "{wav_file}"' 137 | subprocess.call(command, shell=True) 138 | 139 | 140 | def gen_samples_dict(sounds_dir, convert_to_wav=False): 141 | # expand leading ~ 142 | sounds_dir = os.path.expanduser(sounds_dir) 143 | print(sounds_dir) 144 | dirs = Path(sounds_dir).dirs() 145 | # print(dirs) 146 | samples_dict = {} 147 | for directory in dirs: 148 | files = directory.files() 149 | files.sort() 150 | # print('*'*20) 151 | if convert_to_wav: 152 | print('yo') 153 | files_for_conversion = [f for f in files if f.ext.lower() in ['.m4a']] 154 | print(files_for_conversion) 155 | for f in files_for_conversion: 156 | convert_file_to_wav(f) 157 | files = [f for f in files if f.ext.lower() == '.wav'] 158 | print(files) 159 | for i, file in enumerate(files): 160 | # print(file.split('/')[-1]) 161 | samples_dict[f"{directory.split('/')[-1]}:{i}"] = file 162 | return samples_dict 163 | 164 | 165 | def gen_samples_dict_multi(sounds_dirs_multi, convert_to_wav=False): 166 | samples_dict_milti = {} 167 | for sounds_dir in sounds_dirs_multi: 168 | samples_dict_milti.update(gen_samples_dict(sounds_dir, convert_to_wav=convert_to_wav)) 169 | return samples_dict_milti 170 | 171 | 172 | def gen_seg_df(samples_dict): 173 | seg_df_list = \ 174 | Parallel(n_jobs=-1)( 175 | delayed(segment_and_analyze_sample)(sample_path, sound_name) 176 | for sound_name, sample_path in tqdm(samples_dict.items()) 177 | ) 178 | 179 | # for sound_name, sample_path in tqdm(samples_dict.items()): 180 | # cur_df = segment_and_analyze_sample(sample_path) 181 | # cur_df["seg_sound"] = sound_name 182 | # seg_df_list.append(cur_df) 183 | seg_df = pd.concat(seg_df_list) 184 | seg_df = seg_df.fillna(0) 185 | seg_df = seg_df[seg_df.seg_dur_sec > 0] 186 | seg_df = seg_df[seg_df.rms > 0.01] 187 | return seg_df 188 | 189 | 190 | def add_embeddings(df, run_umap=False): 191 | feats_cols = [ 192 | 'seg_dur_sec', 193 | 'rms', 194 | 'mfcc_0', 'mfcc_1', 'mfcc_2', 195 | 'mfcc_3', 'mfcc_4', 'mfcc_5', 'mfcc_6', 'mfcc_7', 'mfcc_8', 'mfcc_9', 196 | 'mfcc_10', 'mfcc_11', 'mfcc_12', 'mfcc_13', 'mfcc_14', 'mfcc_15', 197 | 'mfcc_16', 'mfcc_17', 'mfcc_18', 'mfcc_19', 198 | 'spectral_bandwidth', 'spectral_centroid', 'spectral_contrast', 'spectral_flatness', 'spectral_rolloff', 199 | 'spectral_bandwidth_std', 'spectral_centroid_std', 'spectral_contrast_std', 'spectral_flatness_std', 200 | 'spectral_rolloff_std', 201 | 'decay_dur', 'attack_dur', 202 | 'zcr', 203 | 'zcr_attack', 204 | 'zcr_decay', 205 | 'rms_attack', 206 | 'rms_decay', 207 | ] 208 | if run_umap: 209 | reducer = umap.UMAP( 210 | random_state=42, 211 | n_neighbors=30, 212 | min_dist=0.0, 213 | n_components=2, 214 | ) 215 | sscaler = StandardScaler() 216 | reducer.fit(df[feats_cols]) 217 | umap_embedding = reducer.transform(sscaler.fit_transform(df[feats_cols])) 218 | df[[f'umap_{i}' for i in range(umap_embedding.shape[1])]] = umap_embedding 219 | 220 | pca = PCA() 221 | sscaler = StandardScaler() 222 | pca_embedding = pca.fit_transform(sscaler.fit_transform(df[feats_cols])) 223 | df[[f'pca_{i}' for i in range(pca_embedding.shape[1])]] = pca_embedding 224 | df['cluster'] = cluster.KMeans(n_clusters=8).fit_predict( 225 | np.hstack([pca_embedding[:, :5]])) # , umap_embedding[:, :0]])) 226 | 227 | return df 228 | 229 | 230 | def main(): 231 | samples_dict = gen_samples_dict_multi(config.sample_folders, convert_to_wav=False) 232 | 233 | df = gen_seg_df(samples_dict) 234 | df = add_embeddings(df, run_umap=False) 235 | df.to_csv(utils.segment_csv_path()) 236 | # gen_haskell_code(df, '/Users/shai/Documents/tidal/segments.hs') 237 | 238 | 239 | if __name__ == '__main__': 240 | main() 241 | --------------------------------------------------------------------------------