├── doc ├── inspired.png ├── report.pdf ├── images │ ├── generate.png │ ├── initial.png │ ├── preview.png │ ├── play_pipeline.png │ └── save_pipeline.png └── output │ ├── Charming Domination.webm │ └── The Positive and Negative.webm ├── .gitignore ├── midi ├── charming-domination.mid └── at-the-end-of-the-spring.mid ├── requirements.txt ├── src ├── logger.py ├── parser.py ├── pipeline.py ├── main.py ├── video.py └── ui.glade ├── Makefile ├── LICENSE └── README.md /doc/inspired.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/redbug312/midi-visualizer/HEAD/doc/inspired.png -------------------------------------------------------------------------------- /doc/report.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/redbug312/midi-visualizer/HEAD/doc/report.pdf -------------------------------------------------------------------------------- /doc/images/generate.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/redbug312/midi-visualizer/HEAD/doc/images/generate.png -------------------------------------------------------------------------------- /doc/images/initial.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/redbug312/midi-visualizer/HEAD/doc/images/initial.png -------------------------------------------------------------------------------- /doc/images/preview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/redbug312/midi-visualizer/HEAD/doc/images/preview.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | soundfont/ 3 | venv/ 4 | 5 | tmp.mp4 6 | tmp.mp4~ 7 | output.mp4 8 | pipeline.png 9 | -------------------------------------------------------------------------------- /doc/images/play_pipeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/redbug312/midi-visualizer/HEAD/doc/images/play_pipeline.png -------------------------------------------------------------------------------- /doc/images/save_pipeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/redbug312/midi-visualizer/HEAD/doc/images/save_pipeline.png -------------------------------------------------------------------------------- /midi/charming-domination.mid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/redbug312/midi-visualizer/HEAD/midi/charming-domination.mid -------------------------------------------------------------------------------- /midi/at-the-end-of-the-spring.mid: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/redbug312/midi-visualizer/HEAD/midi/at-the-end-of-the-spring.mid -------------------------------------------------------------------------------- /doc/output/Charming Domination.webm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/redbug312/midi-visualizer/HEAD/doc/output/Charming Domination.webm -------------------------------------------------------------------------------- /doc/output/The Positive and Negative.webm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/redbug312/midi-visualizer/HEAD/doc/output/The Positive and Negative.webm -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | cairocffi==1.3.0 2 | certifi==2022.6.15 3 | cffi==1.15.1 4 | charset-normalizer==2.1.0 5 | decorator==4.4.2 6 | gizeh==0.1.11 7 | idna==3.3 8 | imageio==2.21.1 9 | imageio-ffmpeg==0.4.7 10 | intervaltree==3.1.0 11 | mido==1.2.10 12 | more-itertools==8.14.0 13 | moviepy==1.0.3 14 | numpy==1.23.1 15 | Pillow==9.2.0 16 | proglog==0.1.10 17 | pycparser==2.21 18 | requests==2.28.1 19 | ruamel.yaml==0.17.21 20 | ruamel.yaml.clib==0.2.6 21 | six==1.16.0 22 | sortedcontainers==2.4.0 23 | tqdm==4.64.0 24 | urllib3==1.26.11 25 | vext==0.7.6 26 | vext.gi==0.7.4 27 | xcffib==0.11.1 28 | -------------------------------------------------------------------------------- /src/logger.py: -------------------------------------------------------------------------------- 1 | import gi 2 | gi.require_version('Gtk', '3.0') 3 | from gi.repository import Gtk 4 | from proglog import ProgressBarLogger 5 | 6 | 7 | class Logger(ProgressBarLogger): 8 | def __init__(self, gtk_bar, init_state=None): 9 | ProgressBarLogger.__init__(self, init_state) 10 | self.gtk_bar = gtk_bar 11 | 12 | def bars_callback(self, bar, attr, value, old_value=None): 13 | percentage = value / self.bars[bar]['total'] 14 | self.gtk_bar.set_fraction(percentage) 15 | while Gtk.events_pending(): 16 | Gtk.main_iteration() 17 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PYTHON3 ?= python3 2 | ENV ?= . $(shell pwd)/venv/bin/activate; \ 3 | PYTHONPATH=$(shell pwd) 4 | 5 | .PHONY: start 6 | start: venv soundfont/touhou.sf2 7 | $(ENV) python3 src/main.py 8 | 9 | .PHONY: pipeline.png 10 | pipeline.png: venv 11 | # See https://gstreamer.freedesktop.org/documentation/tutorials/basic/debugging-tools.html?gi-language=c#getting-pipeline-graphs 12 | $(ENV) GST_DEBUG_DUMP_DOT_DIR=/tmp python3 src/pipeline.py 13 | dot -Tpng /tmp/pipeline.dot > pipeline.png 14 | xdg-open pipeline.png 15 | 16 | venv: requirements.txt 17 | $(PYTHON3) -m venv $@ 18 | $(ENV) $(PYTHON3) -m pip install wheel -r $< 19 | touch $@ # update timestamp 20 | 21 | soundfont/touhou.sf2: 22 | mkdir -p soundfont 23 | wget musical-artifacts.com/artifacts/433/Touhou.sf2 -O $@ 24 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 redbug312 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MIDI Visualizer 2 | 3 | Gtk application to visualize MIDI file as piano tutorial videos. 4 | 5 | A project for the course Multimedium Computing Environment (National Taiwan 6 | University, 2018 Spring). 7 | 8 | ![Here's preview of MIDI visualizer](doc/images/preview.png) 9 | 10 | ## Build Environment 11 | 12 | ### Ubuntu 13 | 14 | ```bash 15 | $ sudo apt install python3-venv python3-pip gstreamer1.0-plugins-bad ffmpeg libffi-dev 16 | $ make start 17 | ``` 18 | 19 | The Makefile script downloads a soundfont (249 MB) and creates virtualenv 20 | environment. 21 | 22 | ### Windows 23 | 24 | 1. Install [Python 3.4](https://www.python.org/downloads/release/python-340/) 25 | - Noted that PyGObject for Windows do not support Python 3.5 or above 26 | 2. Install [PyGObject for Windows](https://sourceforge.net/projects/pygobjectwin32/) 27 | 1. Choose these items in GNOME libraries: 28 | - Base packages 29 | - Gst-plugins 30 | - Gst-plugins-extra 31 | - Gst-plugins-more 32 | - Gstreamer 33 | - GTK+ 34 | - JSON-glib 35 | 2. Choose none in non-GNOME libraries 36 | 3. Choose none in development packages 37 | 3. Open the `cmd.exe` to prepare for installing dependencies 38 | ```batch 39 | > python -m pip install --upgrade pip 40 | > pip install requests pycparser 41 | ``` 42 | 4. Download wheel packages from [Unofficial Windows Binaries for Python Extension Packages](https://www.lfd.uci.edu/~gohlke/pythonlibs) 43 | - `cffi‑1.11.5‑cp34‑cp34m‑win_amd64.whl` 44 | - `moviepy‑0.2.3.4‑py2.py3‑none‑any.whl` 45 | 5. Open the `cmd.exe` again to install dependencies 46 | ```batch 47 | > pip install cffi‑1.11.5‑cp34‑cp34m‑win_amd64.whl 48 | > pip install moviepy‑0.2.3.4‑py2.py3‑none‑any.whl 49 | > pip install gizeh mido intervaltree 50 | ``` 51 | 6. Open `C:\Python34\Lib\site-packages\cairocffi\__init__.py` 52 | - Change line 41 and save 53 | ```diff 54 | - cairo = dlopen(ffi, 'cairo', 'cairo-2') 55 | + cairo = dlopen(ffi, 'cairo', 'cairo-2', 'cairo-gobject-2') 56 | ``` 57 | 7. Download soundfont from [musical-artifacts.com](https://musical-artifacts.com/artifacts/433), save it to `soundfont/touhou.sf2` 58 | 8. Execute `python3 src/main.py` 59 | 60 | ## Details Explanation 61 | 62 | ### Pipeline for Playing Video 63 | ![pipeline diagram when playing](doc/images/play_pipeline.png) 64 | 65 | ### Pipeline for Saving Video 66 | ![pipeline diagram when saving](doc/images/save_pipeline.png) 67 | 68 | ## Credits 69 | 1. Gtk framework 70 | - [Gtk+](https://www.gtk.org/): a multi-platform toolkit for creating graphical user interfaces 71 | - [Gstreamer](https://gstreamer.freedesktop.org/): a library for constructing graphs of media-handling components 72 | 2. Dependent packages 73 | - [gizeh](https://github.com/Zulko/gizeh): a Python library for vector graphics 74 | - [moviepy](https://github.com/Zulko/moviepy): a Python library for video editing 75 | - [mido](https://github.com/olemb/mido/): a library for working with MIDI messages and ports 76 | - [intervaltree](https://github.com/chaimleib/intervaltree): a mutable, self-balancing interval tree 77 | 3. Resources 78 | - `midi/charming-domination.mid` from [Chris33711](https://youtu.be/psOjoZmGLnA) 79 | - `midi/at-the-end-of-the-spring.mid` from [Chris33711](https://youtu.be/I3TRDQYr8xI) 80 | - `soundfont/Touhou.sf2` from [Musical Artifacts](https://musical-artifacts.com/artifacts/433) 81 | -------------------------------------------------------------------------------- /src/parser.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import mido 3 | import intervaltree 4 | 5 | 6 | class Note: 7 | def __init__(self, begin, end, index): 8 | self.begin = begin 9 | self.end = end 10 | self.index = index 11 | 12 | def __lt__(self, other): 13 | return self.end < other.end 14 | 15 | def __repr__(self): 16 | return 'Note(%s, %d..%d)' % (self.index, self.begin, self.end) 17 | 18 | 19 | class Midi(): 20 | def __init__(self, file=None): 21 | self.midi = None 22 | self.notes = list() 23 | self.metas = intervaltree.IntervalTree() # indexed by second intervals 24 | self.timeline = intervaltree.IntervalTree() # indexed by tick intervals 25 | self.pending_notes = dict() 26 | if file: 27 | self.parse(file) 28 | 29 | def parse(self, file): 30 | self.midi = mido.MidiFile(file) 31 | 32 | tick = 0 33 | count_notes = 0 34 | current_meta = {'tempo': 500000} 35 | meta_messages = [(tick, dict(current_meta))] 36 | 37 | for track_num, track in enumerate(self.midi.tracks): 38 | for message in track: 39 | tick += message.time 40 | if message.type == 'note_on': 41 | self.notes.append({'note' : message.note, 42 | 'track': track_num}) 43 | self.pending_notes[message.note] = (count_notes, tick) # index, begin_tick 44 | count_notes += 1 45 | elif message.type == 'note_off': 46 | try: 47 | assert message.note in self.pending_notes, 'a note_off before note_on' 48 | except AssertionError: 49 | continue 50 | index, begin = self.pending_notes[message.note] 51 | if tick > begin: 52 | self.timeline[begin:tick] = index 53 | del self.pending_notes[message.note] 54 | elif message.type == 'set_tempo': 55 | current_meta['tempo'] = message.tempo 56 | meta_messages.append((tick, dict(current_meta))) 57 | elif message.type == 'end_of_track': 58 | try: 59 | assert not self.pending_notes, 'no note_off after note_on' 60 | except AssertionError: 61 | self.pending_notes = dict() 62 | meta_messages.append((tick, dict(current_meta))) 63 | tick = 0 64 | 65 | lasttime_sec = 0.0 66 | 67 | for prev_meta, curr_meta in zip(meta_messages, meta_messages[1:]): 68 | interval_sec = mido.tick2second(curr_meta[0] - prev_meta[0], 69 | self.midi.ticks_per_beat, 70 | prev_meta[1]['tempo']) 71 | try: 72 | self.metas[lasttime_sec : lasttime_sec + interval_sec] = \ 73 | dict(prev_meta[1], ticks=(prev_meta[0], curr_meta[0])) 74 | except ValueError: # interval_sec is not a positive number 75 | continue 76 | lasttime_sec += interval_sec 77 | 78 | try: 79 | assert abs(lasttime_sec - self.midi.length) < 1, 'wrong mapping from seconds to ticks' 80 | except AssertionError: 81 | pass 82 | 83 | def second2tick(self, time): 84 | # TODO move to Visualizer and avoid intervaltree lookup 85 | # or maintain Note time unit in sec 86 | index = min(time, self.metas.end() - 1) 87 | meta = next(iter(self.metas[index])) 88 | secs, ticks = (meta[0], meta[1]), meta[2]['ticks'] 89 | return ticks[0] + (ticks[1] - ticks[0]) * (time - secs[0]) / (secs[1] - secs[0]) 90 | -------------------------------------------------------------------------------- /src/pipeline.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import gi 3 | gi.require_version('Gst', '1.0') 4 | gi.require_version('Gtk', '3.0') 5 | from gi.repository import Gst, Gtk 6 | 7 | 8 | class Player: 9 | def __init__(self): 10 | self.pipeline = Gst.Pipeline.new('player') 11 | self.elements = dict() 12 | 13 | self.load_pipe, self.elements['load'] = make_load_pipeline() 14 | self.play_pipe, self.elements['play'] = make_play_pipeline() 15 | self.save_pipe, self.elements['save'] = make_save_pipeline() 16 | 17 | self.pipeline.add(self.load_pipe) 18 | self.pipeline.add(self.play_pipe) 19 | self.load_pipe.link_pads('video_sink', self.play_pipe, 'video_src') 20 | self.load_pipe.link_pads('audio_sink', self.play_pipe, 'audio_src') 21 | 22 | def load(self, mpeg, midi): 23 | mpegsrc = self.elements['load'][0] 24 | midisrc = self.elements['load'][3] 25 | mpegsrc.set_property('location', mpeg) 26 | midisrc.set_property('location', midi) 27 | 28 | def save(self, file): 29 | filesink = self.elements['save'][1] 30 | filesink.set_property('location', file) 31 | 32 | self.pipeline.set_state(Gst.State.NULL) 33 | self.pipeline.remove(self.play_pipe) 34 | self.pipeline.add(self.save_pipe) 35 | self.load_pipe.link_pads('video_sink', self.save_pipe, 'video_src') 36 | self.load_pipe.link_pads('audio_sink', self.save_pipe, 'audio_src') 37 | 38 | self.pipeline.set_state(Gst.State.PLAYING) 39 | bus = self.pipeline.get_bus() 40 | # Refresh slider bar while waiting for EOS 41 | self.refresh_interval = 30 # ms 42 | while bus.timed_pop_filtered(self.refresh_interval * 1e6, Gst.MessageType.EOS) is None: 43 | while Gtk.events_pending(): 44 | Gtk.main_iteration() 45 | 46 | self.pipeline.set_state(Gst.State.NULL) 47 | self.pipeline.remove(self.save_pipe) 48 | self.pipeline.add(self.play_pipe) 49 | self.load_pipe.link_pads('video_sink', self.play_pipe, 'video_src') 50 | self.load_pipe.link_pads('audio_sink', self.play_pipe, 'audio_src') 51 | 52 | def widget(self): 53 | gtksink = self.elements['play'][2] 54 | return gtksink.props.widget 55 | 56 | def draw_pipeline(self): 57 | Gst.debug_bin_to_dot_file(self.pipeline, Gst.DebugGraphDetails.ALL, 'pipeline') 58 | 59 | 60 | def extend_pipe(pipeline, names): 61 | elements = [Gst.ElementFactory.make(name) for name in names] 62 | for element in elements: 63 | pipeline.add(element) 64 | for pred, succ in zip(elements, elements[1:]): 65 | assert pred.link(succ), 'failed to link %s to %s' % (pred, succ) 66 | return elements 67 | 68 | 69 | def make_load_pipeline(): 70 | pipeline = Gst.Pipeline.new('load') 71 | elements = [] 72 | elements += extend_pipe(pipeline, ['filesrc', 'qtdemux']) 73 | elements += extend_pipe(pipeline, ['queue']) 74 | elements += extend_pipe(pipeline, ['filesrc', 'midiparse', 'fluiddec']) 75 | 76 | def on_demux_pad_added(_, pad): 77 | caps = pad.query_caps(None) 78 | if caps.to_string().startswith('video'): 79 | pad.link(elements[2].get_static_pad('sink')) 80 | 81 | elements[1].connect('pad_added', on_demux_pad_added) 82 | elements[5].set_property('soundfont', 'soundfont/touhou.sf2') 83 | 84 | video_sink = Gst.GhostPad.new('video_sink', elements[2].get_static_pad('src')) 85 | audio_sink = Gst.GhostPad.new('audio_sink', elements[5].get_static_pad('src')) 86 | pipeline.add_pad(video_sink) 87 | pipeline.add_pad(audio_sink) 88 | return pipeline, elements 89 | 90 | 91 | def make_play_pipeline(): 92 | # Connected load-play pipeline can be seen as: 93 | # 94 | # env LANG=C gst-launch-1.0 \ 95 | # filesrc location=/tmp/test.mp4 ! qtdemux ! queue ! avdec_h264 ! videoconvert ! autovideosink \ 96 | # filesrc location=/tmp/spring.mid ! midiparse ! fluiddec soundfont=/tmp/th.sf2 ! audioconvert ! autoaudiosink 97 | pipeline = Gst.Pipeline.new('play') 98 | elements = [] 99 | elements += extend_pipe(pipeline, ['avdec_h264', 'videoconvert', 'gtksink']) 100 | elements += extend_pipe(pipeline, ['audioconvert', 'autoaudiosink']) 101 | 102 | video_src = Gst.GhostPad.new('video_src', elements[0].get_static_pad('sink')) 103 | audio_src = Gst.GhostPad.new('audio_src', elements[3].get_static_pad('sink')) 104 | pipeline.add_pad(video_src) 105 | pipeline.add_pad(audio_src) 106 | return pipeline, elements 107 | 108 | 109 | def make_save_pipeline(): 110 | # Connected load-save pipeline can be seen as: 111 | # 112 | # env LANG=C gst-launch-1.0 qtmux name=mux ! filesink location=output.mp4 \ 113 | # filesrc location=/tmp/test.mp4 ! qtdemux ! queue ! mux. \ 114 | # filesrc location=/tmp/spring.mid ! midiparse ! fluiddec soundfont=/tmp/th.sf2 ! audioconvert ! lamemp3enc ! queue ! mux. 115 | # 116 | # Notice that the removal of lamemp3enc would cause buzzing noise 117 | pipeline = Gst.Pipeline.new('save') 118 | elements = [] 119 | elements += extend_pipe(pipeline, ['qtmux', 'filesink']) 120 | elements += extend_pipe(pipeline, ['audioconvert', 'lamemp3enc', 'queue']) 121 | 122 | elements[4].link(elements[0]) # use queue before a mux 123 | video_src = Gst.GhostPad.new('video_src', elements[0].get_request_pad('video_0')) 124 | audio_src = Gst.GhostPad.new('audio_src', elements[2].get_static_pad('sink')) 125 | pipeline.add_pad(video_src) 126 | pipeline.add_pad(audio_src) 127 | return pipeline, elements 128 | 129 | 130 | if __name__ == '__main__': 131 | Gst.init(None) 132 | Gtk.init(None) 133 | player = Player() 134 | player.draw_pipeline() 135 | # player.load('/tmp/test.mp4', 'midi/at-the-end-of-the-spring.mid') 136 | # player.pipeline.set_state(Gst.State.PLAYING) 137 | -------------------------------------------------------------------------------- /src/main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import os 3 | import gi 4 | gi.require_version('Gst', '1.0') 5 | gi.require_version('Gtk', '3.0') 6 | from gi.repository import Gst, Gtk, GLib 7 | 8 | from logger import Logger 9 | from parser import Midi 10 | from pipeline import Player 11 | import video 12 | 13 | 14 | class App: 15 | def __init__(self): 16 | Gtk.init(None) 17 | Gst.init(None) 18 | 19 | self.refresh_interval = 30 # in milliseconds 20 | self.destination = None 21 | self.duration = Gst.CLOCK_TIME_NONE 22 | self.player = Player() 23 | self.builder = self.build_ui() 24 | 25 | widget = self.player.widget() 26 | self.builder.get_object('video_container').pack_start(widget, True, True, 0) 27 | # slider connected not through Glade, for getting handler_id 28 | self.slider_update_signal_id = \ 29 | self.builder.get_object('time_slider').connect('value_changed', self.on_slider_changed) 30 | 31 | bus = self.player.pipeline.get_bus() 32 | bus.add_signal_watch() 33 | bus.connect('message', self.on_message) 34 | 35 | def start(self): 36 | GLib.timeout_add(self.refresh_interval, self.refresh_ui) 37 | Gtk.main() 38 | 39 | def cleanup(self): 40 | try: 41 | os.remove('tmp.mp4~') 42 | except OSError: 43 | pass 44 | if self.player: 45 | self.player.pipeline.set_state(Gst.State.NULL) 46 | 47 | def build_ui(self): 48 | builder = Gtk.Builder() 49 | builder.add_from_file('src/ui.glade') 50 | builder.connect_signals(self) 51 | builder.get_object('main_window').show() 52 | return builder 53 | 54 | def refresh_ui(self): 55 | state = self.player.pipeline.get_state(timeout=self.refresh_interval)[1] 56 | button = self.builder.get_object('play_pause_button') 57 | 58 | if state == Gst.State.PLAYING: 59 | button.get_image().set_from_icon_name(Gtk.STOCK_MEDIA_PAUSE, Gtk.IconSize.BUTTON) 60 | button.set_label('暫停') 61 | else: 62 | button.get_image().set_from_icon_name(Gtk.STOCK_MEDIA_PLAY, Gtk.IconSize.BUTTON) 63 | button.set_label('播放') 64 | return True 65 | 66 | slider = self.builder.get_object('time_slider') 67 | if self.duration == Gst.CLOCK_TIME_NONE: 68 | ret, self.duration = self.player.pipeline.query_duration(Gst.Format.TIME) 69 | slider.set_range(0, self.duration / Gst.SECOND) 70 | slider.set_fill_level(self.duration / Gst.SECOND) 71 | 72 | ret, current = self.player.pipeline.query_position(Gst.Format.TIME) 73 | if ret: 74 | slider.handler_block(self.slider_update_signal_id) 75 | slider.set_value(current / Gst.SECOND) 76 | slider.handler_unblock(self.slider_update_signal_id) 77 | return True 78 | 79 | # Gtk utilizing functions 80 | 81 | def set_window_sensitive(self, sensitive): 82 | self.player.pipeline.set_state(Gst.State.READY if sensitive else Gst.State.NULL) 83 | for gtkobject in ['play_pause_button', 'stop_button', 'time_slider', 84 | 'gtk_open', 'gtk_save', 'gtk_save_as', 'gtk_quit']: 85 | self.builder.get_object(gtkobject).set_sensitive(sensitive) 86 | 87 | # Gtk events: Control bar 88 | 89 | def on_play_pause(self, button): 90 | state = self.player.pipeline.get_state(timeout=10)[1] 91 | state = Gst.State.PAUSED if state == Gst.State.PLAYING else Gst.State.PLAYING 92 | self.player.pipeline.set_state(state) 93 | 94 | def on_stop(self, button): 95 | self.duration = 0 96 | self.player.pipeline.seek_simple(Gst.Format.TIME, Gst.SeekFlags.FLUSH, 0) 97 | self.player.pipeline.set_state(Gst.State.READY) 98 | 99 | def on_slider_changed(self, slider): 100 | value = slider.get_value() 101 | self.player.pipeline.seek_simple(Gst.Format.TIME, Gst.SeekFlags.FLUSH, value * Gst.SECOND) 102 | 103 | # Gtk events: Menu bar 104 | 105 | def on_file_open_activate(self, menuitem): 106 | 107 | open_dialog = self.builder.get_object('open_dialog') 108 | progress_bar = self.builder.get_object('progressing_bar') 109 | hint_label = self.builder.get_object('hint_label') 110 | 111 | response = open_dialog.run() 112 | open_dialog.hide() 113 | if response == Gtk.ResponseType.OK: 114 | self.duration = Gst.CLOCK_TIME_NONE 115 | source = open_dialog.get_filename() 116 | progress_bar.set_fraction(0) 117 | hint_label.set_text('正在解析 MIDI 檔案為影片...') 118 | 119 | self.set_window_sensitive(False) 120 | 121 | midi = Midi(source) 122 | clip = video.midi_videoclip(midi) 123 | logger = Logger(progress_bar) 124 | clip.write_videofile('tmp.mp4', fps=30, audio=False, logger=logger) 125 | 126 | os.rename('tmp.mp4', 'tmp.mp4~') # MoviePy disallows illegal file extension 127 | self.player.load('tmp.mp4~', source) 128 | self.set_window_sensitive(True) 129 | 130 | progress_bar.set_fraction(1) 131 | hint_label.set_visible(False) 132 | self.player.widget().show() 133 | elif response == Gtk.ResponseType.CANCEL: 134 | return 135 | 136 | def on_file_save_activate(self, menuitem, save_as=False): 137 | if not self.destination or save_as: 138 | save_dialog = self.builder.get_object('save_dialog') 139 | response = save_dialog.run() 140 | save_dialog.hide() 141 | if response == Gtk.ResponseType.OK: 142 | self.destination = save_dialog.get_filename() 143 | elif response == Gtk.ResponseType.CANCEL: 144 | return 145 | 146 | if self.destination: 147 | self.set_window_sensitive(False) 148 | self.player.save(self.destination) 149 | self.set_window_sensitive(True) 150 | 151 | def on_file_save_as_activate(self, menuitem): 152 | self.on_file_save_activate(menuitem, save_as=True) 153 | 154 | def on_delete_event(self, widget, event=None): 155 | self.on_stop(None) 156 | Gtk.main_quit() 157 | 158 | def on_help_about_activate(self, menuitem): 159 | about_dialog = self.builder.get_object('about_dialog') 160 | about_dialog.run() 161 | about_dialog.hide() 162 | 163 | # Gst events 164 | 165 | def on_message(self, bus, message): 166 | if message.type == Gst.MessageType.ERROR: 167 | err, debug = message.parse_error() 168 | print('ERROR: {}, {}'.format(message.src.get_name(), err.message)) 169 | self.cleanup() 170 | elif message.type == Gst.MessageType.STATE_CHANGED: 171 | old, new, pending = message.parse_state_changed() 172 | if message.src == self.player: 173 | self.refresh_ui() 174 | elif message.type == Gst.MessageType.EOS: 175 | self.player.pipeline.set_state(Gst.State.READY) 176 | 177 | 178 | if __name__ == '__main__': 179 | app = App() 180 | app.start() 181 | app.cleanup() 182 | -------------------------------------------------------------------------------- /src/video.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import gizeh 3 | import moviepy.editor as mpy 4 | from heapq import heappush, heappop 5 | from more_itertools import peekable, first 6 | from parser import Note 7 | from itertools import count 8 | import numpy as np 9 | 10 | IS_IVORY_KEYS = [x not in [1, 3, 6, 8, 10] for x in range(12)] 11 | 12 | is_ebony = lambda pitch: not IS_IVORY_KEYS[pitch % 12] 13 | is_ivory = lambda pitch: IS_IVORY_KEYS[pitch % 12] 14 | 15 | NEAR_EBONY_KEYS = [[] for _ in range(12)] 16 | for ebony in [1, 3, 6, 8, 10]: 17 | NEAR_EBONY_KEYS[ebony + 1].append(-1) 18 | NEAR_EBONY_KEYS[ebony - 1].append(1) 19 | 20 | OFFSET = [0.0] * 110 21 | 22 | ivory = filter(is_ivory, range(21, 109)) 23 | ivory_offsets = count(start=0.5) 24 | ebony = filter(is_ebony, range(21, 109)) 25 | ebony_offsets = filter(lambda x: x % 7 not in [2, 5], count(start=1)) 26 | 27 | for pitch, off in zip(ivory, ivory_offsets): 28 | OFFSET[pitch] = off / 52 29 | 30 | for pitch, off in zip(ebony, ebony_offsets): 31 | OFFSET[pitch] = off / 52 32 | 33 | PALETTE = { 34 | 'ivory': [ 35 | (0.93, 0.77, 0.45), #F0C674 36 | (0.53, 0.74, 0.71), #8ABEB7 37 | (0.69, 0.57, 0.73), #B294BB 38 | (0.50, 0.63, 0.74), #81A2BE 39 | (0.79, 0.80, 0.79), #CBCFCC, for idle keys 40 | ], 41 | 'ebony': [ 42 | (0.86, 0.57, 0.37), #DE935F 43 | (0.36, 0.55, 0.52), #5E8D87 44 | (0.51, 0.40, 0.55), #85678F 45 | (0.37, 0.50, 0.61), #5F819D 46 | (0.22, 0.24, 0.25), #3A3E42, for idle keys 47 | ], 48 | } 49 | 50 | 51 | class ForeseePart: 52 | def __init__(self, midi, size): 53 | self.midi = midi 54 | self.size = size 55 | self.notes = list() 56 | all_notes = [Note(i[0], i[1], i[2]) for i in midi.timeline.items()] 57 | all_notes = sorted(all_notes, key=lambda n: n.begin) 58 | self.waits = peekable(all_notes) 59 | self.foresee = 2 # sec 60 | 61 | def make_frame(self, time): 62 | now = self.midi.second2tick(time) 63 | future = self.midi.second2tick(time + self.foresee) 64 | NONE = Note(float('inf'), float('inf'), 0) 65 | 66 | while first(self.notes, NONE).end < now: 67 | heappop(self.notes) 68 | while self.waits.peek(NONE).begin <= future: 69 | note = next(self.waits) 70 | heappush(self.notes, note) 71 | 72 | surface = gizeh.Surface(*self.size) 73 | for note in self.notes: 74 | rect = self.spawn_rectangle(note, now, future) 75 | rect.draw(surface) 76 | return surface.get_npimage() 77 | 78 | def spawn_rectangle(self, note, now, future): 79 | w, h = self.size 80 | begin, end = max(note.begin, now), min(note.end, future) 81 | pitch = self.midi.notes[note.index]['note'] 82 | track = self.midi.notes[note.index]['track'] 83 | material = 'ivory' if is_ivory(pitch) else 'ebony' 84 | color = PALETTE[material][track % 4] 85 | 86 | lx = w / 52 if is_ivory(pitch) else w / 52 * 0.7 87 | ly = h * (end - begin) / (future - now) - 5 88 | xy = (w * OFFSET[pitch], 89 | h * (future - end / 2 - begin / 2) / (future - now)) 90 | fill = color 91 | 92 | return gizeh.rectangle(lx=lx, ly=ly, xy=xy, fill=fill) 93 | 94 | 95 | class PianoPart: 96 | def __init__(self, midi, size): 97 | self.midi = midi 98 | self.size = size 99 | self.notes = list() 100 | all_notes = [Note(i[0], i[1], i[2]) for i in midi.timeline.items()] 101 | all_notes = sorted(all_notes, key=lambda n: n.begin) 102 | self.waits = peekable(all_notes) 103 | self.idle_piano = self.init_idle_piano() 104 | 105 | def make_frame(self, time): 106 | now = self.midi.second2tick(time) 107 | NONE = Note(float('inf'), float('inf'), 0) 108 | 109 | while first(self.notes, NONE).end < now: 110 | heappop(self.notes) 111 | while self.waits.peek(NONE).begin <= now: 112 | note = next(self.waits) 113 | heappush(self.notes, note) 114 | 115 | redraw_ivory = {} 116 | redraw_ebony = {} 117 | for note in self.notes: 118 | pitch = self.midi.notes[note.index]['note'] 119 | if is_ivory(pitch): 120 | redraw_ivory[pitch] = note 121 | for neighbor in NEAR_EBONY_KEYS[pitch % 12]: 122 | if pitch + neighbor not in redraw_ebony: 123 | redraw_ebony[pitch + neighbor] = None 124 | else: 125 | redraw_ebony[pitch] = note 126 | 127 | surface = gizeh.Surface(*self.size) 128 | arr = np.frombuffer(surface._cairo_surface.get_data(), np.uint8) 129 | arr += self.idle_piano 130 | surface._cairo_surface.mark_dirty() 131 | 132 | for pitch, note in redraw_ivory.items(): 133 | rect = self.spawn_ivory_key(pitch, note) 134 | rect.draw(surface) 135 | for pitch, note in redraw_ebony.items(): 136 | rect = self.spawn_ebony_key(pitch, note) 137 | rect.draw(surface) 138 | 139 | return surface.get_npimage() 140 | 141 | def init_idle_piano(self): 142 | surface = gizeh.Surface(*self.size) 143 | for pitch in filter(is_ivory, range(21, 109)): 144 | rect = self.spawn_ivory_key(pitch, None) 145 | rect.draw(surface) 146 | for pitch in filter(is_ebony, range(21, 109)): 147 | rect = self.spawn_ebony_key(pitch, None) 148 | rect.draw(surface) 149 | 150 | w, h = self.size 151 | image = surface.get_npimage() 152 | image = image[:, :, [2, 1, 0]] 153 | image = np.dstack([image, 255 * np.ones((h, w), dtype=np.uint8)]) 154 | image = image.flatten() 155 | return image 156 | 157 | def spawn_ivory_key(self, pitch, note=None): 158 | w, h = self.size 159 | color = PALETTE['ivory'][-1] 160 | if note: 161 | pitch = self.midi.notes[note.index]['note'] 162 | track = self.midi.notes[note.index]['track'] 163 | material = 'ivory' if is_ivory(pitch) else 'ebony' 164 | color = PALETTE[material][track % 4] 165 | 166 | lx = w / 52 167 | ly = h 168 | xy = (w * OFFSET[pitch], h / 2) 169 | fill = color 170 | stroke = PALETTE['ebony'][-1] 171 | stroke_width = 1 172 | return gizeh.rectangle(lx=lx, ly=ly, xy=xy, fill=fill, stroke=stroke, 173 | stroke_width=stroke_width) 174 | 175 | def spawn_ebony_key(self, pitch, note=None): 176 | w, h = self.size 177 | color = PALETTE['ebony'][-1] 178 | if note: 179 | pitch = self.midi.notes[note.index]['note'] 180 | track = self.midi.notes[note.index]['track'] 181 | material = 'ivory' if is_ivory(pitch) else 'ebony' 182 | color = PALETTE[material][track % 4] 183 | 184 | lx = w / 52 * 0.7 185 | ly = h * 2 / 3 186 | xy = (w * OFFSET[pitch], h / 3) 187 | fill = color 188 | return gizeh.rectangle(lx=lx, ly=ly, xy=xy, fill=fill) 189 | 190 | 191 | def midi_videoclip(midi, size=(640, 360)): 192 | lower_size = (size[0], int(size[0] / 52 * 6)) 193 | upper_size = (size[0], size[1] - lower_size[1]) 194 | lower_part = PianoPart(midi, lower_size) 195 | upper_part = ForeseePart(midi, upper_size) 196 | 197 | duration = midi.midi.length # expensive call 198 | upper_clip = mpy.VideoClip(upper_part.make_frame, duration=duration) 199 | lower_clip = mpy.VideoClip(lower_part.make_frame, duration=duration) 200 | final_clip = mpy.clips_array([[upper_clip], [lower_clip]]) 201 | 202 | return final_clip 203 | 204 | 205 | if __name__ == '__main__': 206 | from parser import Midi 207 | midi = Midi('midi/at-the-end-of-the-spring.mid') 208 | clip = midi_videoclip(midi) 209 | clip.write_videofile('/tmp/test.mp4', fps=30, audio=False) 210 | -------------------------------------------------------------------------------- /src/ui.glade: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 100 7 | 1 8 | 10 9 | 10 | 11 | 12 | audio/midi 13 | audio/x-midi 14 | 15 | 16 | 17 | True 18 | False 19 | gtk-media-pause 20 | 21 | 22 | True 23 | False 24 | gtk-media-stop 25 | 26 | 27 | False 28 | MIDI Visualizer 29 | 640 30 | 480 31 | 32 | 33 | 34 | True 35 | False 36 | vertical 37 | 38 | 39 | True 40 | False 41 | 42 | 43 | True 44 | False 45 | 檔案(_F) 46 | True 47 | 48 | 49 | True 50 | False 51 | 52 | 53 | gtk-open 54 | True 55 | False 56 | True 57 | True 58 | 59 | 60 | 61 | 62 | 63 | gtk-save 64 | True 65 | False 66 | False 67 | True 68 | True 69 | 70 | 71 | 72 | 73 | 74 | gtk-save-as 75 | True 76 | False 77 | False 78 | True 79 | True 80 | 81 | 82 | 83 | 84 | 85 | True 86 | False 87 | 88 | 89 | 90 | 91 | gtk-quit 92 | True 93 | False 94 | True 95 | True 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | True 106 | False 107 | 求助(_H) 108 | True 109 | 110 | 111 | True 112 | False 113 | 114 | 115 | gtk-about 116 | True 117 | False 118 | True 119 | True 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | False 133 | True 134 | 0 135 | 136 | 137 | 138 | 139 | 640 140 | 360 141 | True 142 | False 143 | vertical 144 | 145 | 146 | True 147 | False 148 | 請開啟 MIDI 檔案 149 | 150 | 151 | True 152 | True 153 | 0 154 | 155 | 156 | 157 | 158 | True 159 | True 160 | 1 161 | 162 | 163 | 164 | 165 | True 166 | False 167 | 168 | 169 | False 170 | True 171 | 2 172 | 173 | 174 | 175 | 176 | True 177 | False 178 | 3 179 | 4 180 | 3 181 | 3 182 | 3 183 | 184 | 185 | 暫停 186 | True 187 | False 188 | True 189 | True 190 | play_pause_icon 191 | 192 | 193 | 194 | False 195 | True 196 | 0 197 | 198 | 199 | 200 | 201 | 停止 202 | True 203 | False 204 | True 205 | True 206 | stop_icon 207 | 208 | 209 | 210 | False 211 | True 212 | 1 213 | 214 | 215 | 216 | 217 | True 218 | False 219 | True 220 | adjustment1 221 | 100 222 | 1 223 | False 224 | 225 | 226 | True 227 | True 228 | 2 229 | 230 | 231 | 232 | 233 | False 234 | True 235 | 3 236 | 237 | 238 | 239 | 240 | 241 | 242 | False 243 | dialog 244 | main_window 245 | MIDI Visualizer 246 | 1.0.0 247 | 使用 Gtk3 開發，多媒體資訊系統作業二。 248 | 將 MIDI 檔案轉換為鋼琴演奏教學影片。 249 | 洪崇凱 250 | applications-multimedia 251 | gpl-3-0 252 | 253 | 254 | False 255 | vertical 256 | 2 257 | 258 | 259 | False 260 | end 261 | 262 | 263 | False 264 | False 265 | 0 266 | 267 | 268 | 269 | 270 | 271 | 272 | 273 | 274 | 275 | 276 | 277 | 278 | 279 | 280 | 281 | False 282 | dialog 283 | main_window 284 | True 285 | midi_filter 286 | 287 | 288 | False 289 | vertical 290 | 2 291 | 292 | 293 | False 294 | end 295 | 296 | 297 | 取消(_C) 298 | True 299 | True 300 | True 301 | True 302 | 303 | 304 | True 305 | True 306 | 0 307 | 308 | 309 | 310 | 311 | 開啟(_O) 312 | True 313 | True 314 | True 315 | True 316 | True 317 | 318 | 319 | True 320 | True 321 | 1 322 | 323 | 324 | 325 | 326 | False 327 | False 328 | 0 329 | 330 | 331 | 332 | 333 | 334 | 335 | 336 | 337 | open_cancel 338 | open_ok 339 | 340 | 341 | 342 | False 343 | dialog 344 | main_window 345 | save 346 | True 347 | 348 | 349 | False 350 | vertical 351 | 2 352 | 353 | 354 | False 355 | end 356 | 357 | 358 | 取消(_C) 359 | True 360 | True 361 | True 362 | True 363 | 364 | 365 | True 366 | True 367 | 0 368 | 369 | 370 | 371 | 372 | 儲存(_S) 373 | True 374 | True 375 | True 376 | True 377 | True 378 | 379 | 380 | True 381 | True 382 | 1 383 | 384 | 385 | 386 | 387 | False 388 | False 389 | 0 390 | 391 | 392 | 393 | 394 | 395 | save_cancel 396 | save_ok 397 | 398 | 399 | 400 | 401 | video/mp4 402 | 403 | 404 | 405 | --------------------------------------------------------------------------------