6 |
5 |
6 |
14 |
15 |
16 | Check out an example:
17 |
18 | https://github.com/okio-ai/nendo-example-apps/assets/17432850/061a07a2-64ac-4647-872a-8b6f1370fab8
19 |
20 |
21 | ## Features
22 |
23 | - Generate music videos of remixed audio from youtube links
24 | - Smells Like Teen Spirit (Dub Version)? Shake It Off (Trash Metal Remix)? Yes please!
25 | - Endless creativity, inspiration and memes
26 |
27 | ## Requirements
28 |
29 | As this app uses the [Nendo musicgen plugin](https://github.com/okio-ai/nendo_plugin_musicgen), you have to install Pytorch 2.0.0 or higher first:
30 |
31 | `pip install "torch>=2.0"`
32 |
33 | > Note: On Mac OSX, the instructions for installing pytorch differ. Please refer to the [pytorch installation instructions](https://pytorch.org/get-started/locally/).
34 |
35 | ## Installation
36 |
37 | Install nendo and the required plugins via `requirements.txt`:
38 |
39 | ```bash
40 | git clone https://github.com/okio-ai/nendo-example-apps.git
41 | cd nendo-example-apps/video-remixer
42 | pip install -r requirements.txt
43 | ```
44 |
45 | ## Usage
46 |
47 | Create a video remix from a youtube link:
48 |
49 | ```bash
50 | python main.py -l https://www.youtube.com/watch\?v\=OPf0YbXqDm0 -p "jazz bebop" -o "uptown_jazz.mp4"
51 | ```
52 |
53 | Create an audio-only remix from a file:
54 |
55 | ```bash
56 | python main.py -f /path/to/file.mp3 -p "jazz bebop" -oa "jazz_remix.mp3"
57 | ```
58 |
59 | ## Nendo Functionality
60 |
61 | ### Plugins in use:
62 |
63 | - [nendo-plugin-classify-core](https://github.com/okio-ai/nendo_plugin_classify_core)
64 | - [nendo-plugin-stemify-demucs](https://github.com/okio-ai/nendo_plugin_stemify_demucs)
65 | - [nendo-plugin-musicgen](https://github.com/okio-ai/nendo_plugin_musicgen)
66 | - [nendo-plugin-fx-core](https://github.com/okio-ai/nendo_plugin_fx_core)
67 |
68 |
69 | The full audio remixing code in this example takes up about 30 lines of code,
70 | to do MIR, stemification, audio generation and effects processing.
71 | Here is the full example from [main.py](main.py):
72 |
73 | ```python
74 | def run_nendo_plugin_chain(
75 | path_to_audio: str, prompt: str, vocal_gain: float, model: str,
76 | ) -> str:
77 | nd = Nendo(
78 | config=NendoConfig(
79 | plugins=[
80 | "nendo_plugin_stemify_demucs",
81 | "nendo_plugin_musicgen",
82 | "nendo_plugin_classify_core",
83 | "nendo_plugin_fx_core",
84 | ],
85 | )
86 | )
87 |
88 | track = nd.library.add_track(file_path=path_to_audio)
89 |
90 | track = nd.plugins.classify_core(track=track)
91 | bpm = int(float(track.get_plugin_data("nendo_plugin_classify_core", "tempo")))
92 | key = track.get_plugin_data("nendo_plugin_classify_core", "key")
93 | scale = track.get_plugin_data("nendo_plugin_classify_core", "scale")
94 |
95 | stems = nd.plugins.stemify_demucs(
96 | track=track, model="mdx_extra", stem_types=["vocals", "no_vocals"]
97 | )
98 | vocals, background = stems[0], stems[1]
99 |
100 | remixed_bg = nd.plugins.musicgen(
101 | track=background,
102 | prompt=prompt,
103 | bpm=bpm,
104 | key=key,
105 | scale=scale,
106 | conditioning_length=2,
107 | n_samples=1,
108 | model=model,
109 | )[0]
110 |
111 | vocals = nd.plugins.fx_core.highpass(track=vocals, cutoff_frequency_hz=100)
112 | vocals = nd.plugins.fx_core.reverb(track=vocals, wet_level=0.2, dry_level=0.8)
113 |
114 | remix = remixed_bg.overlay(vocals, gain_db=vocal_gain)
115 | return remix.resource.src
116 | ```
117 |
118 |
--------------------------------------------------------------------------------
/video-remixer/main.py:
--------------------------------------------------------------------------------
1 | import yt_dlp
2 | import argparse
3 | import subprocess
4 | from moviepy.editor import VideoFileClip, AudioFileClip
5 | import os
6 | from nendo import NendoConfig, Nendo
7 |
8 |
9 | def yt_download(
10 | link: str,
11 | output_path: str,
12 | start_time: str,
13 | end_time: str,
14 | ) -> str:
15 | start_time = f"00:{start_time}.00"
16 | end_time = f"00:{end_time}.00"
17 | ydl_opts = {
18 | "format": "bestvideo+bestaudio/best",
19 | "outtmpl": output_path,
20 | "nocheckcertificate": True,
21 | "ignoreerrors": True,
22 | "no_warnings": True,
23 | "quiet": True,
24 | "postprocessor_args": [
25 | "-ss",
26 | start_time,
27 | "-to",
28 | end_time,
29 | ],
30 | }
31 | with yt_dlp.YoutubeDL(ydl_opts) as ydl:
32 | ydl.download([link])
33 |
34 | return f"{output_path}.webm"
35 |
36 |
37 | def remix_video(video_path: str, audio_path: str, output_path: str) -> str:
38 | video = VideoFileClip(video_path)
39 | audio = AudioFileClip(audio_path)
40 |
41 | video_with_audio = video.set_audio(audio)
42 | video_with_audio.write_videofile(
43 | output_path,
44 | codec="libx264",
45 | audio_codec="aac",
46 | temp_audiofile="temp-audio.m4a",
47 | remove_temp=True,
48 | )
49 | return output_path
50 |
51 |
52 | def extract_audio(video_path: str, output_path: str) -> str:
53 | cmd = [
54 | "ffmpeg",
55 | "-y",
56 | "-i",
57 | video_path, # Input video file
58 | "-q:a",
59 | "0", # Best audio quality
60 | "-map",
61 | "a", # Map to audio stream
62 | "-vn", # No video
63 | output_path, # Output audio file
64 | ]
65 | subprocess.check_call(cmd)
66 | return output_path
67 |
68 |
69 | def run_nendo_plugin_chain(
70 | path_to_audio: str,
71 | prompt: str,
72 | vocal_gain: float,
73 | model: str,
74 | conditioning_length: float,
75 | prompt_strength: float,
76 | output_audio_path: str,
77 | ) -> str:
78 | nd = Nendo(
79 | config=NendoConfig(
80 | log_level="warning",
81 | plugins=[
82 | "nendo_plugin_stemify_demucs",
83 | "nendo_plugin_musicgen",
84 | "nendo_plugin_classify_core",
85 | "nendo_plugin_fx_core",
86 | ],
87 | )
88 | )
89 |
90 | track = nd.library.add_track(file_path=path_to_audio)
91 |
92 | track = nd.plugins.classify_core(track=track)
93 | bpm = int(float(track.get_plugin_value("tempo")))
94 | key = track.get_plugin_value("key")
95 | scale = track.get_plugin_value("scale")
96 |
97 | stems = nd.plugins.stemify_demucs(
98 | track=track, model="mdx_extra", stem_types=["vocals", "no_vocals"],
99 | )
100 | vocals, background = stems[0], stems[1]
101 |
102 | remixed_bg = nd.plugins.musicgen(
103 | track=background,
104 | prompt=prompt,
105 | bpm=bpm,
106 | key=key,
107 | scale=scale,
108 | conditioning_length=conditioning_length,
109 | cfg_coef=prompt_strength,
110 | n_samples=1,
111 | model=model,
112 | )[0]
113 |
114 | vocals = nd.plugins.fx_core.highpass(track=vocals, cutoff_frequency_hz=100)
115 | vocals = nd.plugins.fx_core.reverb(track=vocals, wet_level=0.2, dry_level=0.8)
116 |
117 | remix = remixed_bg.overlay(vocals, gain_db=vocal_gain)
118 | remix.export(output_audio_path)
119 | return output_audio_path
120 |
121 |
122 | def parse_args() -> argparse.Namespace:
123 | parser = argparse.ArgumentParser()
124 | parser.add_argument(
125 | "-l",
126 | "--link",
127 | type=str,
128 | help="Youtube link to download or audio file path for the remix.",
129 | )
130 | parser.add_argument(
131 | "-p", "--prompt", type=str, required=True, help="Prompt to use for the remix."
132 | )
133 | parser.add_argument("-s", "--start-time", type=str, default="00:00")
134 | parser.add_argument("-e", "--end-time", type=str, default="00:30")
135 | parser.add_argument("-v", "--vocal-gain", type=float, default=0.5)
136 | parser.add_argument(
137 | "-m", "--model", type=str, default="facebook/musicgen-stereo-medium"
138 | )
139 | parser.add_argument("-c", "--conditioning-length", type=float, default=5)
140 | parser.add_argument("-ps", "--prompt-strength", type=float, default=3.5)
141 | parser.add_argument(
142 | "-o",
143 | "--output-video-path",
144 | type=str,
145 | default="output.mp4",
146 | help="Output path for the remix.",
147 | )
148 | parser.add_argument(
149 | "-oa",
150 | "--output-audio-path",
151 | type=str,
152 | default="remixed_audio.mp3",
153 | help="Output path for the audio of the remix.",
154 | )
155 | return parser.parse_args()
156 |
157 |
158 | def main(
159 | link_or_path: str,
160 | start_time: str,
161 | end_time: str,
162 | prompt: str,
163 | output_video_path: str,
164 | output_audio_path: str,
165 | vocal_gain: float,
166 | conditioning_length: float,
167 | prompt_strength: float,
168 | model: str,
169 | ):
170 | is_link = link_or_path.startswith("https://")
171 | if is_link:
172 | print("Downloading video...")
173 | video_path = yt_download(
174 | link=link_or_path, start_time=start_time, end_time=end_time, output_path="video"
175 | )
176 | audio_path = extract_audio(video_path, output_path="audio.mp3")
177 | else:
178 | print("Using local audio file...")
179 | audio_path = link_or_path
180 |
181 | print("Running nendo plugin chain...")
182 | run_nendo_plugin_chain(
183 | audio_path, prompt,
184 | vocal_gain=vocal_gain,
185 | model=model,
186 | conditioning_length=conditioning_length,
187 | prompt_strength=prompt_strength,
188 | output_audio_path=output_audio_path
189 | )
190 |
191 | if is_link:
192 | print("Remixing audio and video...")
193 | remix_video(video_path, output_audio_path, output_path=output_video_path)
194 | os.remove(video_path)
195 | os.remove(audio_path)
196 |
197 | print("Done!")
198 |
199 |
200 | if __name__ == "__main__":
201 | args = parse_args()
202 | main(
203 | link_or_path=args.link,
204 | prompt=args.prompt,
205 | start_time=args.start_time,
206 | end_time=args.end_time,
207 | output_video_path=args.output_video_path,
208 | output_audio_path=args.output_audio_path,
209 | conditioning_length=args.conditioning_length,
210 | prompt_strength=args.prompt_strength,
211 | vocal_gain=args.vocal_gain,
212 | model=args.model,
213 | )
214 |
--------------------------------------------------------------------------------