├── custom_nodes ├── README.md ├── node_text_to_speech.py ├── save_audio_to_davinci.py ├── save_image_to_davinci.py └── nodes_phi_3_contitioning.py └── README.md /custom_nodes/README.md: -------------------------------------------------------------------------------- 1 | In order to use DaVinci node you must have DaVinci Resolve Studio connected to the API. 2 | For more information check the help seciton in DaVinci Resolve Studio HELP>DOCUMENTATION>DEVELOPER. 3 | It will open a folder, search for scripting and the for README.txt file, the API documentation. 4 | -------------------------------------------------------------------------------- /custom_nodes/node_text_to_speech.py: -------------------------------------------------------------------------------- 1 | import folder_paths 2 | import os 3 | from openai import OpenAI 4 | 5 | class TextToSpeech: 6 | def __init__(self): 7 | self.output_dir = folder_paths.get_output_directory() 8 | 9 | @classmethod 10 | def INPUT_TYPES(s): 11 | return { 12 | "required": { 13 | "text_prompt": ("STRING", { 14 | "multiline": True, 15 | "default": "Enter your text here"}), 16 | "voice": (['nova', 'shimmer', 'echo', 'onyx', 'fable', 'alloy'],), 17 | "filename": ("STRING", {"default": "output"}), 18 | }, 19 | } 20 | 21 | RETURN_TYPES = ("AUDIO", ) 22 | FUNCTION = "generate_tts" 23 | OUTPUT_NODE = True 24 | CATEGORY = "audio" 25 | 26 | def generate_tts(self, text_prompt, voice, filename): 27 | try: 28 | os.environ["OPENAI_API_KEY"] = "Your_OpenAI_Key" 29 | client = OpenAI() 30 | if text_prompt and filename: 31 | speech_file_path = os.path.join(self.output_dir, f"{filename}.mp3") 32 | response = client.audio.speech.create( 33 | model="tts-1", 34 | voice=voice, 35 | input=text_prompt 36 | ) 37 | response.stream_to_file(speech_file_path) 38 | print(f'Audio generated and saved to {speech_file_path}') 39 | return (speech_file_path,) 40 | else: 41 | print("Text prompt or filename is missing.") 42 | return None 43 | except Exception as e: 44 | print(f"An error occurred while generating TTS: {e}") 45 | return None 46 | 47 | NODE_CLASS_MAPPINGS = { 48 | "TextToSpeech": TextToSpeech 49 | } 50 | 51 | NODE_DISPLAY_NAME_MAPPINGS = { 52 | "TextToSpeech": "Text to Speech" 53 | } 54 | -------------------------------------------------------------------------------- /custom_nodes/save_audio_to_davinci.py: -------------------------------------------------------------------------------- 1 | import folder_paths 2 | import os 3 | import sys 4 | 5 | os.environ["RESOLVE_SCRIPT_API"] = "/Library/Application Support/Blackmagic Design/DaVinci Resolve/Developer/Scripting" 6 | os.environ["RESOLVE_SCRIPT_LIB"] = "/Applications/DaVinci Resolve/DaVinci Resolve.app/Contents/Libraries/Fusion/fusionscript.so" 7 | sys.path.append(os.path.join(os.environ["RESOLVE_SCRIPT_API"], "Modules")) 8 | import DaVinciResolveScript as dvr_script 9 | 10 | class SaveAudioToDaVinci: 11 | def __init__(self): 12 | self.output_dir = folder_paths.get_output_directory() 13 | 14 | @classmethod 15 | def INPUT_TYPES(s): 16 | return { 17 | "required": { 18 | "audio_path": ("AUDIO", ), 19 | "filename_prefix": ("STRING", {"default": "ComfyUI"}), 20 | }, 21 | } 22 | RETURN_TYPES = () 23 | FUNCTION = "save_audio" 24 | 25 | OUTPUT_NODE = True 26 | 27 | CATEGORY = "audio" 28 | 29 | def save_audio(self, audio_path, filename_prefix): 30 | try: 31 | if not audio_path or not os.path.exists(audio_path): 32 | print(f"Invalid audio path: {audio_path}") 33 | return {} 34 | 35 | full_output_folder, filename, counter, subfolder, filename_prefix = folder_paths.get_save_image_path(filename_prefix, self.output_dir, 0, 0) 36 | audio_output_path = os.path.join(full_output_folder, os.path.basename(audio_path)) 37 | 38 | os.makedirs(full_output_folder, exist_ok=True) 39 | os.rename(audio_path, audio_output_path) 40 | 41 | resolve = dvr_script.scriptapp("Resolve") 42 | project = resolve.GetProjectManager().GetCurrentProject() 43 | media_pool = project.GetMediaPool() 44 | 45 | if media_pool.ImportMedia([audio_output_path]): 46 | print(f"Audio file imported successfully: {audio_output_path}") 47 | else: 48 | print(f"Failed to import audio file: {audio_output_path}") 49 | 50 | return {"audio_path": audio_output_path} 51 | except Exception as e: 52 | print(f"An error occurred while importing audio to DaVinci: {e}") 53 | return {"error": str(e)} 54 | 55 | NODE_CLASS_MAPPINGS = { 56 | "SaveAudioToDaVinci": SaveAudioToDaVinci 57 | } 58 | 59 | NODE_DISPLAY_NAME_MAPPINGS = { 60 | "SaveAudioToDaVinci": "Save Audio to DaVinci" 61 | } 62 | -------------------------------------------------------------------------------- /custom_nodes/save_image_to_davinci.py: -------------------------------------------------------------------------------- 1 | import folder_paths 2 | import os 3 | import json 4 | import numpy as np 5 | from PIL import Image 6 | from PIL.PngImagePlugin import PngInfo 7 | from comfy.cli_args import args 8 | import DaVinciResolveScript as dvr_script 9 | 10 | class SaveImageToDaVinci: 11 | def __init__(self): 12 | self.output_dir = folder_paths.get_output_directory() 13 | self.type = "output" 14 | self.prefix_append = "" 15 | self.compress_level = 4 16 | 17 | @classmethod 18 | def INPUT_TYPES(s): 19 | return {"required": 20 | {"images": ("IMAGE", ), 21 | "filename_prefix": ("STRING", {"default": "ComfyUI"})}, 22 | "hidden": {"prompt": "PROMPT", "extra_pnginfo": "EXTRA_PNGINFO"}, 23 | } 24 | 25 | RETURN_TYPES = () 26 | FUNCTION = "save_images" 27 | 28 | OUTPUT_NODE = True 29 | 30 | CATEGORY = "image" 31 | 32 | def save_images(self, images, filename_prefix="ComfyUI_To_DaVinci", prompt=None, extra_pnginfo=None): 33 | filename_prefix += self.prefix_append 34 | full_output_folder, filename, counter, subfolder, filename_prefix = folder_paths.get_save_image_path(filename_prefix, self.output_dir, images[0].shape[1], images[0].shape[0]) 35 | results = list() 36 | for (batch_number, image) in enumerate(images): 37 | i = 255. * image.cpu().numpy() 38 | img = Image.fromarray(np.clip(i, 0, 255).astype(np.uint8)) 39 | metadata = None 40 | if not args.disable_metadata: 41 | metadata = PngInfo() 42 | if prompt is not None: 43 | metadata.add_text("prompt", json.dumps(prompt)) 44 | if extra_pnginfo is not None: 45 | for x in extra_pnginfo: 46 | metadata.add_text(x, json.dumps(extra_pnginfo[x])) 47 | 48 | filename_with_batch_num = filename.replace("%batch_num%", str(batch_number)) 49 | file = f"{filename_with_batch_num}_{counter:05}_.png" 50 | 51 | img.save(os.path.join(full_output_folder, file), pnginfo=metadata, compress_level=self.compress_level) 52 | # Save the image to the output folder 53 | file_to_davinci = f'{full_output_folder}{file}' 54 | resolve = dvr_script.scriptapp("Resolve") 55 | project = resolve.GetProjectManager().GetCurrentProject() 56 | media_pool = project.GetMediaPool() 57 | filename = filename.replace('\\', '/') 58 | media_pool.ImportMedia([file_to_davinci]) 59 | 60 | results.append({ 61 | "filename": file, 62 | "subfolder": subfolder, 63 | "type": self.type 64 | }) 65 | counter += 1 66 | 67 | return { "ui": { "images": results } } 68 | 69 | NODE_CLASS_MAPPINGS = { 70 | "SaveImageToDaVinci": SaveImageToDaVinci 71 | } 72 | 73 | # A dictionary that contains the friendly/humanly readable titles for the nodes 74 | NODE_DISPLAY_NAME_MAPPINGS = { 75 | "SaveImageToDaVinci": "Save Image To DaVinci" 76 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ### Comprehensive Guide for Using the Text-to-Speech Node and Saving Audio to DaVinci Resolve Studio 2 | ## These nodes don't work with the free version of DaVinci Resolve. 3 | 4 | How to use the Text-to-Speech (TTS) node and save the generated audio to DaVinci Resolve Studio using ComfyUI. This guide will walk you through each step, ensuring you can leverage this powerful integration effectively. 5 | 6 | #### Prerequisites 7 | 1. **ComfyUI** installed on your machine. 8 | 2. **DaVinci Resolve Studio** installed and running. 9 | 3. Basic understanding of how to use nodes in ComfyUI. 10 | 4. Access to OpenAI API for voice generation (optional, if not using default voices). 11 | 5. Install OpenAI API via pip 12 | 13 | --- 14 | 15 | ### Step-by-Step Guide 16 | 17 | #### 1. Setting Up the Text-to-Speech Node 18 | 19 | 1. **Open DaVinci Resolve Studio**: 20 | Launch DaVinci Resolve Studio on your machine and ensure it is running before proceeding with ComfyUI. 21 | 22 | 2. **Open ComfyUI**: 23 | Launch ComfyUI on your machine. 24 | 25 | 3. **Create a New Node**: 26 | Navigate to the node creation interface in ComfyUI. 27 | 28 | 4. **Add the Text-to-Speech Node**: 29 | - Locate the TTS node in the node library. 30 | - Drag and drop the TTS node into your workspace. 31 | 32 | 5. **Configure the TTS Node**: 33 | - **Text Input**: Enter the text you want to convert to speech. 34 | - **Voice Selection**: Choose a voice from the available options. If using OpenAI API, ensure you have your API key set up and select a voice from the OpenAI API list. 35 | - **File Name**: Specify the name for the output audio file. 36 | 37 | 6. **Output Directory**: 38 | - The files will be saved in the default output directory of ComfyUI. 39 | 40 | #### 2. Generating the Audio File 41 | 42 | 1. **Queue the Node**: 43 | - Connect the TTS node to Save Audio To DaVinci 44 | - Click the ‘Queue’ button to start the process. 45 | 46 | 2. **Monitor Progress**: 47 | - Wait for the node to process the text and generate the audio file. 48 | - Once completed, the audio file will be imported in the active folder in DaVinci. 49 | 50 | #### 3. Additional Tips 51 | 52 | - **Batch Processing**: You can create multiple TTS nodes if you need to generate and import multiple audio files. 53 | - **Voice Customization**: Explore different voices and settings in the TTS node to find the best fit for your project. 54 | --- 55 | 56 | ### Example Workflow 57 | 58 | Here’s an example workflow to illustrate the process: 59 | 60 | 1. **Setup**: 61 | - Open DaVinci Resolve Studio. 62 | - Add a TTS node in ComfyUI. 63 | - Enter text: “Hello, this is a demo speech for DaVinci Resolve Studio.” 64 | - Choose a voice: OpenAI's ‘en-US-Wavenet-D’. 65 | - Set file name: `demo_speech`. 66 | 67 | 2. **Queue and Generate**: 68 | - Queue the TTS node. 69 | - Wait for the audio file to be generated and imported. 70 | --- 71 | 72 | ### Troubleshooting 73 | 74 | - **Audio Not Generated**: Ensure all node settings are correctly configured and the output directory is writable. 75 | - **API Issues**: If using OpenAI API, ensure your API key is valid and you have sufficient credits. 76 | 77 | --- 78 | 79 | Happy editing! 80 | 81 | Contributing 82 | 83 | -------------------------------------------------------------------------------- /custom_nodes/nodes_phi_3_contitioning.py: -------------------------------------------------------------------------------- 1 | class phi_3_conditioning: 2 | 3 | @classmethod 4 | # nodes in 5 | def INPUT_TYPES(s): 6 | return { 7 | "required": { 8 | "clip": ("CLIP",), 9 | "text": ("STRING", { 10 | "multiline": True, 11 | "default": "Describe a photo of a sunset over a lake." 12 | }), 13 | "Enable_Phy_3_Prompt": (["enable", "disable"],), 14 | } 15 | } 16 | 17 | # nodes out 18 | RETURN_TYPES = ("CONDITIONING",) 19 | 20 | # OUTPUT_NODE = False 21 | FUNCTION = "encode" 22 | 23 | CATEGORY = "Example" 24 | 25 | def encode(self, clip, text, Enable_Phy_3_Prompt): 26 | if Enable_Phy_3_Prompt == "enable": 27 | tokens = clip.tokenize(text) 28 | print(type(tokens)) 29 | import torch 30 | from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline 31 | torch.random.manual_seed(0) 32 | model = AutoModelForCausalLM.from_pretrained( 33 | "microsoft/Phi-3-mini-128k-instruct", 34 | device_map="cuda", 35 | torch_dtype="auto", 36 | trust_remote_code=True, 37 | ) 38 | tokenizer = AutoTokenizer.from_pretrained("microsoft/Phi-3-mini-128k-instruct") 39 | messages = [ 40 | {"role": "user", "content": "You are a helpful digital assistant expert in prompt engeneering for stable diffusion."}, 41 | {"role": "user", "content": "Picture of a woman"}, 42 | {"role": "assistant", "content": "a serene and elegant woman, standing confidently in a softly lit room. Her posture is poised and graceful, exuding a sense of calm and stability. She is dressed in a flowing, pastel-colored dress that gently sways with her every movement, reflecting the tranquility of her demeanor. The lighting is warm and inviting, casting a soft glow on her features, highlighting her gentle smile and kind eyes. Her hair is styled in a simple yet sophisticated manner, adding to her overall composed and balanced appearance. This image, rendered at a high resolution of 8k, captures the essence of stability and grace, making her presence both comforting and inspiring."}, 43 | {"role": "user", "content": text}, 44 | ] 45 | pipe = pipeline( 46 | "text-generation", 47 | model=model, 48 | tokenizer=tokenizer, 49 | ) 50 | generation_args = { 51 | "max_new_tokens": 500, 52 | "return_full_text": False, 53 | "temperature": 0.0, 54 | "do_sample": False, 55 | } 56 | text_output = pipe(messages, **generation_args) 57 | print(text) 58 | generated_text = text_output[0]['generated_text'] 59 | print(generated_text) 60 | tokens = clip.tokenize(generated_text) 61 | else: 62 | tokens = clip.tokenize(text) 63 | 64 | cond, pooled = clip.encode_from_tokens(tokens, return_pooled=True) 65 | return ([[cond, {"pooled_output": pooled}]], ) 66 | 67 | 68 | 69 | NODE_CLASS_MAPPINGS = { 70 | "phi_3_conditioning": phi_3_conditioning 71 | } 72 | 73 | # A dictionary that contains the friendly/humanly readable titles for the nodes 74 | NODE_DISPLAY_NAME_MAPPINGS = { 75 | "phi_3_conditioning": "phi_3_conditioning" 76 | } 77 | --------------------------------------------------------------------------------