└── addons └── ThreadedResourceSaveLoadPlugin ├── ThreadedResourceLoader.gd ├── ThreadedResourceLoader.gd.uid ├── ThreadedResourceSaveLoadPlugin.gd ├── ThreadedResourceSaveLoadPlugin.gd.uid ├── ThreadedResourceSaver.gd ├── ThreadedResourceSaver.gd.uid └── plugin.cfg /addons/ThreadedResourceSaveLoadPlugin/ThreadedResourceLoader.gd: -------------------------------------------------------------------------------- 1 | extends RefCounted 2 | class_name ThreadedResourceLoader 3 | 4 | signal loadStarted(totalResources: int) 5 | signal loadProgress(completedCount: int, totalResources: int) 6 | signal loadCompleted(loadedFiles: Array[Resource]) 7 | signal loadError(path: String) 8 | 9 | static var ignoreWarnings: bool = false 10 | 11 | var MAX_THREADS: int 12 | var _semaphore: Semaphore 13 | var _mutex: Mutex 14 | var _loadThreads: Array[Thread] = [] 15 | var _loadQueue: Array[Array] = [] 16 | var _totalResourcesAmount: int = 0 17 | var _completedResourcesAmount: int = 0 18 | var _failedResourcesAmount: int = 0 19 | var _loadedFiles: Array[Resource] = [] 20 | var _isStopping: bool = false 21 | var _loadingHasStarted: bool = false 22 | var _selfRefToKeepAlive: ThreadedResourceLoader 23 | 24 | 25 | func _init(threadsAmount: int = OS.get_processor_count() - 1) -> void: 26 | _selfRefToKeepAlive = self 27 | _semaphore = Semaphore.new() 28 | _mutex = Mutex.new() 29 | MAX_THREADS = threadsAmount 30 | 31 | _initThreadPool() 32 | 33 | 34 | func _initThreadPool() -> void: 35 | var thread: Thread 36 | for i in range(MAX_THREADS): 37 | thread = Thread.new() 38 | _loadThreads.append(thread) 39 | thread.start(_loadThreadWorker) 40 | 41 | 42 | func add(resources: Array[Array]) -> ThreadedResourceLoader: 43 | _mutex.lock() 44 | if _loadingHasStarted: 45 | _mutex.unlock() 46 | push_error("loading has already started, current call ignored") 47 | return self 48 | 49 | for params in resources: 50 | if params.size() == 0: 51 | push_error("empty params array will be ignored") 52 | continue 53 | elif typeof(params[0]) != TYPE_STRING or params[0].strip_edges() == "": 54 | push_error("invalid param value: \"{0}\", it should be a non empty string, will be ignored".format([params[0]])) 55 | continue 56 | 57 | _loadQueue.append(params) 58 | 59 | _totalResourcesAmount = _loadQueue.size() 60 | _mutex.unlock() 61 | 62 | return self 63 | 64 | 65 | func start() -> ThreadedResourceLoader: 66 | _mutex.lock() 67 | if _loadingHasStarted: 68 | _mutex.unlock() 69 | push_error("loading has already started, current call ignored") 70 | return self 71 | 72 | _loadingHasStarted = true 73 | 74 | call_deferred("emit_signal", "loadStarted", _totalResourcesAmount) 75 | 76 | if _totalResourcesAmount == 0: 77 | if not ThreadedResourceSaver.ignoreWarnings: 78 | push_warning("load queue is empty, immediate finish loading signal emission") 79 | call_deferred("emit_signal", "loadCompleted", _loadedFiles) 80 | _mutex.unlock() 81 | return self 82 | 83 | for _i in range(min(MAX_THREADS, _totalResourcesAmount)): 84 | _semaphore.post.call_deferred() 85 | 86 | _mutex.unlock() 87 | 88 | return self 89 | 90 | 91 | func _loadThreadWorker() -> void: 92 | while true: 93 | _semaphore.wait() 94 | _mutex.lock() 95 | 96 | if _isStopping: 97 | _mutex.unlock() 98 | break 99 | 100 | if _loadQueue.is_empty(): 101 | _mutex.unlock() 102 | continue 103 | 104 | var loadItem: Array = _loadQueue.pop_back() 105 | var isQueueEmpty: bool = _loadQueue.is_empty() 106 | _mutex.unlock() 107 | 108 | var resource: Resource = ResourceLoader.load.callv(loadItem) 109 | 110 | _mutex.lock() 111 | if resource: 112 | _completedResourcesAmount += 1 113 | _loadedFiles.append(resource) 114 | call_deferred("emit_signal", "loadProgress", _completedResourcesAmount, _totalResourcesAmount) 115 | else: 116 | _failedResourcesAmount += 1 117 | call_deferred("emit_signal", "loadError", loadItem[0]) 118 | 119 | var isLoadComplete: bool = _completedResourcesAmount + _failedResourcesAmount >= _totalResourcesAmount 120 | 121 | if isLoadComplete: 122 | call_deferred("emit_signal", "loadCompleted", _loadedFiles) 123 | _mutex.unlock() 124 | _stopLoadThreads.call_deferred() 125 | _clearSelfRef.call_deferred() 126 | else: 127 | _mutex.unlock() 128 | 129 | if not isQueueEmpty: 130 | _semaphore.post() 131 | 132 | 133 | func _stopLoadThreads() -> void: 134 | _mutex.lock() 135 | if _isStopping: 136 | _mutex.unlock() 137 | return 138 | _isStopping = true 139 | _mutex.unlock() 140 | 141 | for _i in range(MAX_THREADS): 142 | _semaphore.post() 143 | 144 | for thread in _loadThreads: 145 | if thread.is_alive(): 146 | thread.wait_to_finish() 147 | 148 | 149 | func _clearSelfRef() -> void: 150 | _selfRefToKeepAlive = null 151 | 152 | 153 | # force threads cleanup on instance freed 154 | # (preventing thread leaks if freed instance before it finished the job) 155 | func _notification(what: int) -> void: 156 | if what == NOTIFICATION_PREDELETE: 157 | # Force immediate thread cleanup when being deleted 158 | _mutex.lock() 159 | _isStopping = true 160 | _mutex.unlock() 161 | 162 | # don't use separate func coz ref will be invalid 163 | for _i in range(MAX_THREADS): 164 | _semaphore.post() 165 | 166 | for thread in _loadThreads: 167 | if thread.is_started(): 168 | thread.wait_to_finish() 169 | -------------------------------------------------------------------------------- /addons/ThreadedResourceSaveLoadPlugin/ThreadedResourceLoader.gd.uid: -------------------------------------------------------------------------------- 1 | uid://dyuw7pn5akupt 2 | -------------------------------------------------------------------------------- /addons/ThreadedResourceSaveLoadPlugin/ThreadedResourceSaveLoadPlugin.gd: -------------------------------------------------------------------------------- 1 | @tool 2 | extends EditorPlugin 3 | 4 | 5 | func _enter_tree(): 6 | pass 7 | 8 | 9 | func _exit_tree(): 10 | pass 11 | -------------------------------------------------------------------------------- /addons/ThreadedResourceSaveLoadPlugin/ThreadedResourceSaveLoadPlugin.gd.uid: -------------------------------------------------------------------------------- 1 | uid://bh52fhqa2ho6i 2 | -------------------------------------------------------------------------------- /addons/ThreadedResourceSaveLoadPlugin/ThreadedResourceSaver.gd: -------------------------------------------------------------------------------- 1 | extends RefCounted 2 | class_name ThreadedResourceSaver 3 | 4 | signal saveStarted(totalResources: int) 5 | signal saveProgress(completedCount: int, totalResources: int) 6 | signal saveCompleted(savedPaths: Array[String]) 7 | signal saveError(path: String, errorCode: Error) 8 | 9 | static var ignoreWarnings: bool = false 10 | 11 | var MAX_THREADS: int 12 | var _semaphore: Semaphore 13 | var _mutex: Mutex 14 | var _saveThreads: Array[Thread] = [] 15 | var _saveQueue: Array[Array] = [] 16 | var _totalResourcesAmount: int = 0 17 | var _completedResourcesAmount: int = 0 18 | var _failedResourcesAmount: int = 0 19 | var _savedPaths: Array[String] = [] 20 | var _verifyFilesAccess : bool = true 21 | var _isStopping: bool = false 22 | var _savingHasStarted: bool = false 23 | var _selfRefToKeepAlive: ThreadedResourceSaver 24 | 25 | 26 | func _init(verifyFilesAccess: bool = false, threadsAmount: int = OS.get_processor_count() - 1) -> void: 27 | _selfRefToKeepAlive = self 28 | _semaphore = Semaphore.new() 29 | _mutex = Mutex.new() 30 | _verifyFilesAccess = verifyFilesAccess 31 | MAX_THREADS = threadsAmount 32 | 33 | _initThreadPool() 34 | 35 | 36 | func _initThreadPool() -> void: 37 | var thread: Thread 38 | for i in range(MAX_THREADS): 39 | thread = Thread.new() 40 | _saveThreads.append(thread) 41 | thread.start(_saveThreadWorker) 42 | 43 | 44 | # typing 45 | # resources: Array[{ resource: Resource, path: String }] 46 | func add(resources: Array[Array]) -> ThreadedResourceSaver: 47 | _mutex.lock() 48 | if _savingHasStarted: 49 | _mutex.unlock() 50 | push_error("saving has already started, current call ignored") 51 | return self 52 | 53 | for params in resources: 54 | if not (params[0] is Resource): 55 | push_error("invalid param value: \"{0}\", it should be a Resource, will be ignored".format([params[0]])) 56 | continue 57 | 58 | if params.size() == 0: 59 | push_error("empty params array will be ignored") 60 | continue 61 | else: 62 | var resourcePathIsEmpty: bool = params[0].resource_path.strip_edges() == "" 63 | 64 | if params.size() == 1: 65 | if resourcePathIsEmpty: 66 | push_error("resource_path is empty and no save path param been provided, resource will be ignored") 67 | continue 68 | else: 69 | if not ThreadedResourceSaver.ignoreWarnings: 70 | push_warning("save path param is empty, resource_path will be used instead: \"{0}\"".format([params[0].resource_path])) 71 | params.append(params[0].resource_path) 72 | # params amount > 1 73 | else: 74 | if typeof(params[1]) != TYPE_STRING: 75 | push_error("invalid save path param value: \"{0}\", it should be a string, resource will be ignored".format([params[1]])) 76 | continue 77 | 78 | var savePathParamIsEmpty: bool = params[1].strip_edges() == "" 79 | 80 | if savePathParamIsEmpty: 81 | if resourcePathIsEmpty: 82 | push_error("resource_path and save path param are both empty, resource will be ignored") 83 | continue 84 | else: 85 | if not ThreadedResourceSaver.ignoreWarnings: 86 | push_warning("save path param is empty, resource_path will be used instead: \"{0}\"".format([params[0].resource_path])) 87 | params[1] = params[0].resource_path 88 | 89 | _saveQueue.append(params) 90 | 91 | _totalResourcesAmount = _saveQueue.size() 92 | _mutex.unlock() 93 | 94 | return self 95 | 96 | 97 | func start() -> ThreadedResourceSaver: 98 | _mutex.lock() 99 | if _savingHasStarted: 100 | _mutex.unlock() 101 | push_error("saving has already started, current call ignored") 102 | return self 103 | 104 | _savingHasStarted = true 105 | 106 | call_deferred("emit_signal", "saveStarted", _totalResourcesAmount) 107 | 108 | if _totalResourcesAmount == 0: 109 | if not ThreadedResourceSaver.ignoreWarnings: 110 | push_warning("save queue is empty, immediate finish saving signal emission") 111 | call_deferred("emit_signal", "saveCompleted", _savedPaths) 112 | _mutex.unlock() 113 | return self 114 | 115 | for _i in range(min(MAX_THREADS, _totalResourcesAmount)): 116 | _semaphore.post.call_deferred() 117 | 118 | _mutex.unlock() 119 | 120 | return self 121 | 122 | 123 | func _saveThreadWorker() -> void: 124 | while true: 125 | _semaphore.wait() 126 | _mutex.lock() 127 | 128 | if _isStopping: 129 | _mutex.unlock() 130 | break 131 | 132 | if _saveQueue.is_empty(): 133 | _mutex.unlock() 134 | continue 135 | 136 | var saveParams: Array = _saveQueue.pop_back() 137 | var isQueueEmpty: bool = _saveQueue.is_empty() 138 | 139 | _mutex.unlock() 140 | 141 | var error: Error = ResourceSaver.save.callv(saveParams) 142 | 143 | _mutex.lock() 144 | if error == OK: 145 | _completedResourcesAmount += 1 146 | _savedPaths.append(saveParams[1]) 147 | call_deferred("emit_signal", "saveProgress", _completedResourcesAmount, _totalResourcesAmount) 148 | else: 149 | _failedResourcesAmount += 1 150 | call_deferred("emit_signal", "saveError", saveParams[1], error) 151 | 152 | var isSaveComplete: bool = _completedResourcesAmount + _failedResourcesAmount >= _totalResourcesAmount 153 | 154 | if isSaveComplete: 155 | _mutex.unlock() 156 | _verifyFileReadinessAccess.call_deferred() 157 | else: 158 | _mutex.unlock() 159 | 160 | if not isQueueEmpty: 161 | _semaphore.post() 162 | 163 | 164 | func _verifyFileReadinessAccess() -> void: 165 | _mutex.lock() 166 | var savedPathsCopy: Array[String] = _savedPaths.duplicate() 167 | _mutex.unlock() 168 | 169 | if not _verifyFilesAccess: 170 | call_deferred("emit_signal", "saveCompleted", savedPathsCopy) 171 | _stopSaveThreads.call_deferred() 172 | _clearSelfRef.call_deferred() 173 | return 174 | 175 | var file: FileAccess 176 | for path in savedPathsCopy: 177 | file = FileAccess.open(path, FileAccess.READ) 178 | if file: 179 | file.close() 180 | else: 181 | call_deferred("emit_signal", "saveError", path, ERR_FILE_CANT_READ) 182 | _stopSaveThreads.call_deferred() 183 | _clearSelfRef.call_deferred() 184 | return 185 | 186 | call_deferred("emit_signal", "saveCompleted", savedPathsCopy) 187 | _stopSaveThreads.call_deferred() 188 | _clearSelfRef.call_deferred() 189 | 190 | 191 | func _stopSaveThreads() -> void: 192 | _mutex.lock() 193 | if _isStopping: 194 | _mutex.unlock() 195 | return 196 | _isStopping = true 197 | _mutex.unlock() 198 | 199 | for _i in range(MAX_THREADS): 200 | _semaphore.post() 201 | 202 | for thread in _saveThreads: 203 | if thread.is_alive(): 204 | thread.wait_to_finish() 205 | 206 | 207 | func _clearSelfRef() -> void: 208 | _selfRefToKeepAlive = null 209 | 210 | 211 | # force threads cleanup on instance freed 212 | # (preventing thread leaks if freed instance before it finished the job) 213 | func _notification(what: int) -> void: 214 | if what == NOTIFICATION_PREDELETE: 215 | _mutex.lock() 216 | _isStopping = true 217 | _mutex.unlock() 218 | 219 | # don't use separate func coz ref will be invalid 220 | for _i in range(MAX_THREADS): 221 | _semaphore.post() 222 | 223 | for thread in _saveThreads: 224 | if thread.is_started(): 225 | thread.wait_to_finish() 226 | -------------------------------------------------------------------------------- /addons/ThreadedResourceSaveLoadPlugin/ThreadedResourceSaver.gd.uid: -------------------------------------------------------------------------------- 1 | uid://c7s7lf5e21o8t 2 | -------------------------------------------------------------------------------- /addons/ThreadedResourceSaveLoadPlugin/plugin.cfg: -------------------------------------------------------------------------------- 1 | [plugin] 2 | 3 | name="ThreadedResourceSaveLoad" 4 | description="Threaded resource save-load utils" 5 | author="Mero" 6 | version="1.1.2" 7 | script="ThreadedResourceSaveLoadPlugin.gd" 8 | --------------------------------------------------------------------------------