├── app ├── gui │ ├── screens │ │ ├── help │ │ │ ├── style.tcss │ │ │ ├── HelpScreen.py │ │ │ └── help.md │ │ ├── database │ │ │ ├── style.tcss │ │ │ └── DatabaseScreen.py │ │ ├── popup │ │ │ ├── style.tcss │ │ │ └── PopupScreen.py │ │ ├── question │ │ │ ├── style.tcss │ │ │ └── QuestionScreen.py │ │ └── settings │ │ │ ├── style.tcss │ │ │ ├── PropertyWidget.py │ │ │ └── SettingsScreen.py │ └── MainApp.py └── util │ ├── Log.py │ ├── Config.py │ ├── Notion.py │ ├── Civitai.py │ └── Worker.py ├── .gitignore ├── .readme ├── app.png ├── civitai_api.png ├── notion_demo.png ├── notion_menu.png ├── notion_database.png ├── notion_card_demo.png ├── notion_capabilities.png └── notion_full_card_demo.png ├── requirements.txt ├── main.py ├── start.sh ├── start.bat ├── .vscode └── launch.json ├── LICENSE ├── README.md └── config.temp.json /app/gui/screens/help/style.tcss: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | .venv/ 3 | .cache/ 4 | config.user.json -------------------------------------------------------------------------------- /.readme/app.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VeyDlin/Civitai2notion/HEAD/.readme/app.png -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VeyDlin/Civitai2notion/HEAD/requirements.txt -------------------------------------------------------------------------------- /.readme/civitai_api.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VeyDlin/Civitai2notion/HEAD/.readme/civitai_api.png -------------------------------------------------------------------------------- /.readme/notion_demo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VeyDlin/Civitai2notion/HEAD/.readme/notion_demo.png -------------------------------------------------------------------------------- /.readme/notion_menu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VeyDlin/Civitai2notion/HEAD/.readme/notion_menu.png -------------------------------------------------------------------------------- /.readme/notion_database.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VeyDlin/Civitai2notion/HEAD/.readme/notion_database.png -------------------------------------------------------------------------------- /.readme/notion_card_demo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VeyDlin/Civitai2notion/HEAD/.readme/notion_card_demo.png -------------------------------------------------------------------------------- /.readme/notion_capabilities.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VeyDlin/Civitai2notion/HEAD/.readme/notion_capabilities.png -------------------------------------------------------------------------------- /.readme/notion_full_card_demo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/VeyDlin/Civitai2notion/HEAD/.readme/notion_full_card_demo.png -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | from app.gui.MainApp import MainApp 2 | from app.util.Config import Config 3 | 4 | 5 | 6 | if __name__ == "__main__": 7 | Config.init("config.user.json", "config.temp.json") 8 | 9 | app = MainApp() 10 | app.run() 11 | import sys 12 | 13 | sys.exit(app.return_code or 0) -------------------------------------------------------------------------------- /start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | if [ ! -d ".venv" ]; then 5 | echo "First install..." 6 | 7 | python3 -m venv .venv 8 | source "./.venv/bin/activate" 9 | pip install -r requirements.txt 10 | else 11 | source "./.venv/bin/activate" 12 | fi 13 | 14 | python3 main.py 15 | -------------------------------------------------------------------------------- /start.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | PUSHD "%~dp0" 3 | setlocal 4 | 5 | if not exist ".venv\" ( 6 | echo First install... 7 | 8 | call python -m venv .venv 9 | call .venv\Scripts\activate.bat & pip install -r requirements.txt 10 | ) 11 | 12 | call .venv\Scripts\activate.bat & python main.py 13 | 14 | pause 15 | exit /b 16 | 17 | -------------------------------------------------------------------------------- /app/gui/screens/database/style.tcss: -------------------------------------------------------------------------------- 1 | 2 | DatabaseScreen #top-content { 3 | height: 5; 4 | padding: 1 0; 5 | background: $boost; 6 | margin: 0; 7 | } 8 | 9 | DatabaseScreen #top-content Button { 10 | margin: 0 2; 11 | } 12 | 13 | DatabaseScreen #top-content #load { 14 | height: auto; 15 | width: auto; 16 | padding: 1 0; 17 | display: none; 18 | } 19 | 20 | DatabaseScreen #log { 21 | border: $accent-darken-3; 22 | } 23 | -------------------------------------------------------------------------------- /app/gui/screens/popup/style.tcss: -------------------------------------------------------------------------------- 1 | PopupScreen { 2 | align: center middle; 3 | } 4 | 5 | PopupScreen #dialog { 6 | grid-size: 1; 7 | grid-gutter: 1 2; 8 | grid-rows: 1fr 3; 9 | padding: 0 1; 10 | width: 60; 11 | height: 11; 12 | border: thick $background 80%; 13 | background: $surface; 14 | } 15 | 16 | PopupScreen #info { 17 | column-span: 2; 18 | height: 1fr; 19 | width: 1fr; 20 | content-align: center middle; 21 | } 22 | 23 | PopupScreen #okay { 24 | width: 60; 25 | } -------------------------------------------------------------------------------- /app/gui/screens/question/style.tcss: -------------------------------------------------------------------------------- 1 | QuestionScreen { 2 | align: center middle; 3 | } 4 | 5 | QuestionScreen #dialog { 6 | grid-size: 2; 7 | grid-gutter: 1 2; 8 | grid-rows: 1fr 3; 9 | padding: 0 1; 10 | width: 60; 11 | height: 11; 12 | border: thick $background 80%; 13 | background: $surface; 14 | } 15 | 16 | QuestionScreen #question { 17 | column-span: 2; 18 | height: 1fr; 19 | width: 1fr; 20 | content-align: center middle; 21 | } 22 | 23 | QuestionScreen Button { 24 | width: 100%; 25 | } -------------------------------------------------------------------------------- /app/gui/screens/help/HelpScreen.py: -------------------------------------------------------------------------------- 1 | from textual.app import ComposeResult 2 | from textual.screen import Screen 3 | from textual.widgets import Footer, Markdown 4 | import os 5 | 6 | 7 | 8 | class HelpScreen(Screen): 9 | CSS_PATH = "style.tcss" 10 | 11 | def compose(self) -> ComposeResult: 12 | current_dir = os.path.dirname(os.path.abspath(__file__)) 13 | file = open(os.path.join(current_dir, "help.md"), "r") 14 | help = file.read() 15 | file.close() 16 | 17 | yield Markdown(help) 18 | yield Footer() -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": "Python: Debug", 9 | "type": "python", 10 | "request": "launch", 11 | "program": "${workspaceFolder}/main.py", 12 | "console": "externalTerminal", 13 | "justMyCode": true 14 | } 15 | ] 16 | } -------------------------------------------------------------------------------- /app/gui/MainApp.py: -------------------------------------------------------------------------------- 1 | from textual.app import App 2 | 3 | from .screens.database.DatabaseScreen import DatabaseScreen 4 | from .screens.settings.SettingsScreen import SettingsScreen 5 | from .screens.help.HelpScreen import HelpScreen 6 | 7 | 8 | class MainApp(App[str]): 9 | BINDINGS = [ 10 | ("d", "switch_mode('database')", "Database"), 11 | ("s", "switch_mode('settings')", "Settings"), 12 | ("h", "switch_mode('help')", "Help"), 13 | ] 14 | MODES = { 15 | "database": DatabaseScreen, 16 | "settings": SettingsScreen, 17 | "help": HelpScreen 18 | } 19 | 20 | def on_mount(self) -> None: 21 | self.switch_mode("database") 22 | -------------------------------------------------------------------------------- /app/gui/screens/popup/PopupScreen.py: -------------------------------------------------------------------------------- 1 | 2 | from textual.app import ComposeResult 3 | from textual.containers import Grid 4 | from textual.screen import ModalScreen 5 | from textual.widgets import Button, Label 6 | 7 | 8 | 9 | class PopupScreen(ModalScreen): 10 | CSS_PATH = "style.tcss" 11 | 12 | 13 | def __init__(self, text): 14 | super().__init__() 15 | self.text = text 16 | 17 | 18 | def compose(self) -> ComposeResult: 19 | yield Grid( 20 | Label(self.text, id="info"), 21 | Button("Okay", variant="primary", id="okay"), 22 | id="dialog" 23 | ) 24 | 25 | 26 | 27 | def on_button_pressed(self, event: Button.Pressed) -> None: 28 | if event.button.id == "okay": 29 | self.app.pop_screen() -------------------------------------------------------------------------------- /app/gui/screens/settings/style.tcss: -------------------------------------------------------------------------------- 1 | PropertyWidget { 2 | background: $boost; 3 | height: 5; 4 | margin: 1; 5 | min-width: 50; 6 | padding: 1; 7 | } 8 | 9 | PropertyWidget > Label { 10 | dock: left; 11 | content-align: left middle; 12 | height: 3; 13 | min-width: 50; 14 | } 15 | 16 | PropertyWidget .right-dock { 17 | dock: right; 18 | width: 50%; 19 | } 20 | 21 | 22 | SettingsScreen #save { 23 | dock: right; 24 | width: 50%; 25 | } 26 | 27 | SettingsScreen #reset { 28 | dock: left; 29 | } 30 | 31 | SettingsScreen #save-content { 32 | padding: 1; 33 | height: 5; 34 | padding: 1 2; 35 | margin: 0; 36 | } 37 | 38 | SettingsScreen #group-content { 39 | height: auto; 40 | } 41 | SettingsScreen #settings-content { 42 | 43 | } -------------------------------------------------------------------------------- /app/gui/screens/question/QuestionScreen.py: -------------------------------------------------------------------------------- 1 | 2 | from textual.app import ComposeResult 3 | from textual.containers import Grid 4 | from textual.screen import ModalScreen 5 | from textual.widgets import Button, Label 6 | 7 | 8 | 9 | class QuestionScreen(ModalScreen): 10 | CSS_PATH = "style.tcss" 11 | 12 | 13 | def __init__(self, question, yes, no): 14 | super().__init__() 15 | self.question = question 16 | self.yes = yes 17 | self.no = no 18 | 19 | 20 | def compose(self) -> ComposeResult: 21 | yield Grid( 22 | Label(self.question, id="question"), 23 | Button(self.yes["text"], variant=self.yes["variant"], id="yes"), 24 | Button(self.no["text"], variant=self.no["variant"], id="no"), 25 | id="dialog", 26 | ) 27 | 28 | def on_button_pressed(self, event: Button.Pressed) -> None: 29 | if event.button.id == "yes": 30 | self.yes["call"]() 31 | if event.button.id == "no": 32 | self.no["call"]() 33 | 34 | self.app.pop_screen() 35 | 36 | 37 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. 20 | -------------------------------------------------------------------------------- /app/util/Log.py: -------------------------------------------------------------------------------- 1 | class Log(): 2 | INFO, OK, WARNING, ERROR = range(0, 4) 3 | 4 | __write_call = None 5 | 6 | @staticmethod 7 | def set_call(call): 8 | Log.__write_call = call 9 | 10 | @staticmethod 11 | def write(type, name, text): 12 | if Log.__write_call is not None: 13 | Log.__write_call(type, name, text) 14 | 15 | @staticmethod 16 | def info(name, text): 17 | Log.write(Log.INFO, name, text) 18 | 19 | @staticmethod 20 | def info_write(text): 21 | Log.write(Log.INFO, None, text) 22 | 23 | @staticmethod 24 | def ok(name, text): 25 | Log.write(Log.OK, name, text) 26 | 27 | @staticmethod 28 | def ok_write(text): 29 | Log.write(Log.OK, None, text) 30 | 31 | @staticmethod 32 | def warning(name, text): 33 | Log.write(Log.WARNING, name, text) 34 | 35 | @staticmethod 36 | def warning_write(text): 37 | Log.write(Log.WARNING, None, text) 38 | 39 | @staticmethod 40 | def error(name, text): 41 | Log.write(Log.ERROR, name, text) 42 | 43 | @staticmethod 44 | def error_write(text): 45 | Log.write(Log.ERROR, None, text) 46 | 47 | 48 | @staticmethod 49 | def exception(name, text, err, traceback_text): 50 | Log.write(Log.ERROR, name, f"{text} \r\n {err=} \r\n\r\n {traceback_text}") 51 | 52 | @staticmethod 53 | def exception_write(text, err, traceback_text): 54 | Log.write(Log.ERROR, None, f"{text} \r\n {err=} \r\n\r\n {traceback_text}") -------------------------------------------------------------------------------- /app/gui/screens/settings/PropertyWidget.py: -------------------------------------------------------------------------------- 1 | from textual.app import ComposeResult 2 | from textual.containers import Horizontal, Vertical 3 | from textual.widgets import Static, Label, Input, Switch 4 | from ....util.Config import Config 5 | 6 | 7 | class TimeDisplay(Static): 8 | """A widget to display elapsed time.""" 9 | 10 | 11 | class PropertyWidget(Static): 12 | def __init__(self, prop): 13 | super().__init__() 14 | self.prop = prop 15 | self.prop_path = prop.path 16 | self.control = None 17 | 18 | 19 | def compose(self) -> ComposeResult: 20 | yield Label(self.prop.title) 21 | 22 | if self.prop.type == "password": 23 | self.control = Input(password=True, value=str(self.prop.data), placeholder=self.prop.hint) 24 | if self.prop.type == "string": 25 | self.control = Input(password=False, value=str(self.prop.data), placeholder=self.prop.hint) 26 | if self.prop.type == "bool": 27 | self.control = Switch(animate=False, value=bool(self.prop.data)) 28 | if self.prop.type == "path": 29 | self.control = Input(password=False, value=str(self.prop.data), placeholder=self.prop.hint) 30 | 31 | yield Horizontal(self.control, classes="right-dock") 32 | 33 | 34 | 35 | def update_value(self): 36 | if self.prop.type == "bool": 37 | self.control.value = bool(Config.get(self.prop_path).data) 38 | else: 39 | self.control.value = str(Config.get(self.prop_path).data) 40 | 41 | 42 | 43 | def set_value(self): 44 | if self.prop.type == "bool": 45 | Config.get(self.prop_path).data = bool(self.control.value) 46 | else: 47 | Config.get(self.prop_path).data = str(self.control.value) -------------------------------------------------------------------------------- /app/gui/screens/settings/SettingsScreen.py: -------------------------------------------------------------------------------- 1 | from textual.app import ComposeResult 2 | from textual.screen import Screen 3 | from textual.containers import Vertical, VerticalScroll, Horizontal 4 | from textual.widgets import Footer, Collapsible, Button 5 | 6 | from .PropertyWidget import PropertyWidget 7 | from ..popup.PopupScreen import PopupScreen 8 | from ....util.Config import Config 9 | from ....util.Worker import Worker 10 | 11 | 12 | class SettingsScreen(Screen): 13 | CSS_PATH = "style.tcss" 14 | properties = [] 15 | 16 | 17 | def compose(self) -> ComposeResult: 18 | ettings_content = VerticalScroll(id="settings-content") 19 | 20 | for group in Config.groups: 21 | 22 | collapsible = Collapsible(title=group.title) 23 | group_content = Vertical(id="group-content") 24 | 25 | for prop in group.properties: 26 | property = PropertyWidget(prop) 27 | self.properties.append(property) 28 | group_content.compose_add_child(property) 29 | 30 | collapsible.compose_add_child(group_content) 31 | ettings_content.compose_add_child(collapsible) 32 | 33 | 34 | ettings_content.compose_add_child(Horizontal( 35 | Button("Reset", id="reset", variant="warning"), 36 | Button("Save", id="save", variant="success"), 37 | id="save-content" 38 | )) 39 | 40 | yield ettings_content 41 | 42 | yield Footer() 43 | 44 | 45 | def on_mount(self) -> None: 46 | pass 47 | 48 | 49 | def on_button_pressed(self, event: Button.Pressed) -> None: 50 | id = event.button.id 51 | if id == "save": 52 | self.save_config() 53 | if id == "reset": 54 | self.reset_config() 55 | 56 | 57 | 58 | def save_config(self): 59 | if Worker.busy(): 60 | self.app.push_screen(PopupScreen("The action is not possible until all processes are finished")) 61 | return 62 | 63 | for prop in self.properties: 64 | prop.set_value() 65 | Config.save() 66 | 67 | 68 | 69 | def reset_config(self): 70 | Config.reset() 71 | for prop in self.properties: 72 | prop.update_value() 73 | -------------------------------------------------------------------------------- /app/util/Config.py: -------------------------------------------------------------------------------- 1 | import json 2 | import shutil 3 | import os 4 | import copy 5 | 6 | 7 | 8 | class ConfigProperty(): 9 | def __init__(self, prop_json_data, group_json_data): 10 | self.id = prop_json_data["id"] 11 | self.type = prop_json_data["type"] 12 | self.data = prop_json_data["data"] 13 | self.title = prop_json_data["title"] 14 | self.hint = prop_json_data["hint"] 15 | self.help = prop_json_data["help"] 16 | self.path = group_json_data["id"]["val"] + '.' + self.id 17 | 18 | 19 | 20 | 21 | 22 | class ConfigPropertyGroup(): 23 | def __init__(self, group_json_data, properties): 24 | self.id = group_json_data["id"]["val"] 25 | self.title = group_json_data["id"]["title"] 26 | self.properties = properties 27 | 28 | 29 | 30 | 31 | 32 | class Config(): 33 | groups = [] 34 | __groups_temp = [] 35 | __file = None 36 | 37 | 38 | @staticmethod 39 | def init(file, temp_file = None): 40 | Config.__file = file 41 | 42 | if not os.path.exists(file) and temp_file is not None: 43 | shutil.copy2(temp_file, file) 44 | 45 | Config.load() 46 | 47 | 48 | 49 | @staticmethod 50 | def get(id): 51 | (group_id, prop_id) = id.split('.') 52 | group = next(((x for x in Config.groups if x.id == group_id)), None) 53 | prop = next(((x for x in group.properties if x.id == prop_id)), None) 54 | return prop 55 | 56 | 57 | 58 | @staticmethod 59 | def load(): 60 | raw_config = Config.json_load() 61 | for group in raw_config["config"]: 62 | properties = [] 63 | for prop in group["properties"]: 64 | properties.append(ConfigProperty(prop, group)) 65 | Config.groups.append(ConfigPropertyGroup(group, properties)) 66 | 67 | Config.__groups_temp = copy.deepcopy(Config.groups) 68 | 69 | 70 | 71 | @staticmethod 72 | def save(): 73 | config = [] 74 | 75 | for group in Config.groups: 76 | group_json = { 77 | "id": {"val": group.id, "title": group.title}, 78 | "properties": [] 79 | } 80 | for prop in group.properties: 81 | group_json["properties"].append( 82 | {"id": prop.id, "type": prop.type, "data": prop.data, "title": prop.title, "hint": prop.hint, "help": prop.help} 83 | ) 84 | config.append(group_json) 85 | 86 | Config.json_save({"config": config}) 87 | Config.__groups_temp = copy.deepcopy(Config.groups) 88 | 89 | 90 | @staticmethod 91 | def reset(): 92 | Config.groups = copy.deepcopy(Config.__groups_temp) 93 | 94 | 95 | 96 | @staticmethod 97 | def json_load(): 98 | with open(Config.__file) as f: 99 | return json.load(f) 100 | 101 | 102 | 103 | @staticmethod 104 | def json_save(data): 105 | with open(Config.__file, "w") as f: 106 | f.write(json.dumps(data, indent=4)) 107 | -------------------------------------------------------------------------------- /app/gui/screens/database/DatabaseScreen.py: -------------------------------------------------------------------------------- 1 | from textual.app import ComposeResult 2 | from textual.screen import Screen 3 | from textual.widgets import Footer, Button, RichLog, LoadingIndicator 4 | from textual.containers import Horizontal 5 | 6 | from ....util.Log import Log 7 | from ....util.Worker import Worker 8 | from ..popup.PopupScreen import PopupScreen 9 | from ..question.QuestionScreen import QuestionScreen 10 | 11 | 12 | class DatabaseScreen(Screen): 13 | CSS_PATH = "style.tcss" 14 | 15 | 16 | def compose(self) -> ComposeResult: 17 | yield Horizontal( 18 | Button("Import from civitai", id="ipmport", variant="success"), 19 | Button("Download from notion", id="download", variant="warning"), 20 | Button("Update notion database", id="update", variant="warning"), 21 | Button("Make all", id="make-all", variant="primary"), 22 | LoadingIndicator(id="load"), 23 | id="top-content" 24 | ) 25 | yield RichLog(wrap=True, markup=True, id="log") 26 | 27 | yield Footer() 28 | 29 | 30 | 31 | async def on_button_pressed(self, event: Button.Pressed) -> None: 32 | if Worker.busy(): 33 | self.app.push_screen(PopupScreen("The action is not possible until all processes are finished")) 34 | return 35 | 36 | id = event.button.id 37 | self.query_one("#load").styles.display = "block" 38 | 39 | if id == "ipmport": 40 | self.run_worker(Worker.ipmport_models_from_favorites(), exclusive=True) 41 | if id == "download": 42 | self.app.push_screen(QuestionScreen( 43 | "How do you want to download files from the database?", 44 | { 45 | "text": "Just download", 46 | "variant": "primary", 47 | "call": lambda: self.run_worker(Worker.download_models_from_notion_database(hash_check=False), exclusive=True) 48 | }, 49 | { 50 | "text": "Download with hash check", 51 | "variant": "warning", 52 | "call": lambda: self.run_worker(Worker.download_models_from_notion_database(hash_check=True), exclusive=True) 53 | } 54 | )) 55 | if id == "update": 56 | self.run_worker(Worker.update_notion_database(), exclusive=True) 57 | if id == "make-all": 58 | self.app.push_screen(QuestionScreen( 59 | "When the action reaches the file download stage, how do you want to download files from the database?", 60 | { 61 | "text": "Just download", 62 | "variant": "primary", 63 | "call": lambda: self.run_worker(Worker.make_all(False), exclusive=True) 64 | }, 65 | { 66 | "text": "Download with hash check", 67 | "variant": "warning", 68 | "call": lambda: self.run_worker(Worker.make_all(True), exclusive=True) 69 | } 70 | )) 71 | 72 | 73 | 74 | def on_mount(self): 75 | Worker.set_end_work_call(self.end_work) 76 | Log.set_call(self.write_log) 77 | 78 | 79 | 80 | def end_work(self): 81 | self.query_one("#load").styles.display = "none" 82 | 83 | 84 | 85 | def write_log(self, type, name, text): 86 | color = None 87 | if type == Log.INFO: 88 | color = "while" 89 | if type == Log.OK: 90 | color = "green" 91 | if type == Log.WARNING: 92 | color = "yellow" 93 | if type == Log.ERROR: 94 | color = "red" 95 | 96 | log = self.query_one(RichLog) 97 | 98 | if name is None: 99 | log.write(f"[bold {color}]{text}[/]") 100 | else: 101 | log.write(f"[bold {color}][{name.upper()}][/] {text}") -------------------------------------------------------------------------------- /app/gui/screens/help/help.md: -------------------------------------------------------------------------------- 1 | # Info 2 | Synchronize your Civitai LoRA, checkpoints and embeddings bookmarks and Notion and download them automatically 3 | 4 | `github` - https://github.com/VeyDlin/Civitai2notion 5 | 6 | # Using 7 | 8 | You need Python 3 version 9 | 10 | Just run `start.bat` if you are using Windows or `start.sh ` for Linux, the first installation may take some time 11 | 12 | Available utilities: 13 | 14 | 1. `Import from civitai` - `Import from civitai` - Add all bookmarks from Civitai to the Notion, only those models (LoRA, checkpoints or embeddings) for which you specified the database id in the settings will be added 15 | 16 | At this stage your models has not been downloaded yet, go to your database and edit your entries as desired, for example, sometimes you can add another name/title or change the image, as well as remove duplicates (see point 2) 17 | 18 | 2. `Download from notion` - Checks the notion database for duplicates, the check takes place by the `File` property, since it will then be used for the names of the files that you download, if duplicates were found, then edit the entries in your notion database, for this you can use the built-in notion search bar to quickly find duplicates by name 19 | 20 | You have the option to use the automatic duplicate conflict resolution setting, in which case the application will simply add a postfix with a digit until the file name becomes unique 21 | 22 | Please note that this utility can also check the hash in the database with already loaded models, this is necessary to fix failed downloads as well as to update models (see point 3) 23 | 24 | Due to the file hash calculation, it may take time if you have a slow hard drive 25 | 26 | 3. `Update notion database` - Check all models for new versions in Civitai, if new versions of models are found, the application will update the entries in the Notion database 27 | 28 | The action updates the fields `Trigger Words`, `Hash` and `Version`in the Notion database but does not download files 29 | 30 | To update the files you need to run the `Download from notion` utility and select `Download with hash check` 31 | 32 | This action will start checking the hash and loading models while outdated models will be updated due to a hash mismatch 33 | 34 | 4. `Make all` Runs in turn `Import from civitai` -> `Update notion database` -> `Download from notion` 35 | 36 | # Notion Config 37 | 38 | ### API Key 39 | 40 | 1) Go to the `integrations` (https://www.notion.so/my-integrations) and create a new integration 41 | 42 | 2) In the `Capabilities` menu set all permissions 43 | 44 | 3) Copy the API key and write it to `Settings` - `Tokens` - `Notion token` 45 | 46 | ### Database 47 | 48 | 1) Create a new notion database and name it whatever you want. Use the following properties: 49 | 50 | `Name` - `Type`: `Text` 51 | 52 | `URL` - `Type`: `URL` 53 | 54 | `File` - `Type`: `Text` 55 | 56 | `Trigger Words` - `Type`: `Text` 57 | 58 | `Tags` - `Type`: `Multi-select` 59 | 60 | `SD` - `Type`: `Select` 61 | 62 | `Version` - `Type`: `Text` 63 | 64 | `Model ID` - `Type`: `Text` 65 | 66 | `Hash` - `Type`: `Text` 67 | 68 | 2) Create a new database in the notion 69 | 70 | 3) Create a connection for your database so that the script can create records, to do this, select `...` from the top right and click `Add connections`, select your integration 71 | 72 | 4) Copy the `Database ID` while you are on the database page and write it to `Settings` - `LoRA settings` - `Notion database id` 73 | `Database ID` can be found in the browser line 74 | 75 | ``` 76 | https://www.notion.so/myworkspace/a8aec43384f447ed84390e8e42c2e089?v=... 77 | |--------- Database ID --------| 78 | ``` 79 | 80 | 5) Add at least one path to the folder to save in `Settings` - `LoRA settings` - `Path to save for 1.x versions` or `Settings` - `LoRA settings` - `Path to save for SDXL versions`. You don't have to do this and then the app will only be able to import your bookmarks, but not download them 81 | 82 | 6) Repeat steps 2 - 5 in order to create a database for embeddings (`Settings` - `Notion` - `Notion database id for embeddings`) and checkpoints (`Settings` - `Notion` - `Notion database id checkpoints`). You can skip this step, then the program will simply skip processing these categories 83 | 84 | # Civitai Config 85 | 86 | ### API Key 87 | 88 | 1) Go to the `Account Settings` (https://civitai.com/user/account) and create a new `API Key` 89 | 90 | 2) Copy the API key and write it to `Settings` - `Tokens` - `Civit AI token` -------------------------------------------------------------------------------- /app/util/Notion.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import json 3 | import aiofiles 4 | import os 5 | 6 | from notion_client import AsyncClient 7 | 8 | from .Log import Log 9 | 10 | 11 | 12 | class Notion: 13 | def __init__(self, token): 14 | self.notion = AsyncClient(auth=token) 15 | 16 | 17 | 18 | async def get_all(self, database_id, use_cache_file=False): 19 | cache_dir = ".cache" 20 | cache_file = os.path.join(cache_dir, f"notion_{database_id}.json") 21 | 22 | if use_cache_file: 23 | os.makedirs(cache_dir, exist_ok=True) 24 | if os.path.exists(cache_file): 25 | Log.warning("notion", f"Loading from cache file: {cache_file}") 26 | try: 27 | async with aiofiles.open(cache_file, 'r', encoding='utf-8') as f: 28 | content = await f.read() 29 | cached_data = json.loads(content) 30 | return cached_data 31 | except Exception as e: 32 | Log.error("notion", f"Failed to load cache: {e}") 33 | 34 | pages = await self.__get_all_raw(database_id) 35 | 36 | results = [] 37 | 38 | for page in pages: 39 | properties = {} 40 | for key, value in page["properties"].items(): 41 | type = value["type"] 42 | if type == "title": 43 | properties[key] = "".join([x["plain_text"] for x in value["title"]]) 44 | if type == "rich_text": 45 | properties[key] = "".join([x["plain_text"] for x in value["rich_text"]]) 46 | if type == "multi_select": 47 | properties[key] = [x["name"] for x in value["multi_select"]] 48 | if type == "select": 49 | properties[key] = value["select"]["name"] if value["select"] else "" 50 | if type == "url": 51 | properties[key] = value["url"] 52 | 53 | results.append({ "id": page["id"], "properties": properties }) 54 | 55 | if use_cache_file: 56 | try: 57 | async with aiofiles.open(cache_file, 'w', encoding='utf-8') as f: 58 | await f.write(json.dumps(results, ensure_ascii=False, indent=2)) 59 | Log.ok("notion", f"Cache saved to: {cache_file}") 60 | except Exception as e: 61 | Log.error("notion", f"Failed to save cache: {e}") 62 | 63 | return results 64 | 65 | 66 | 67 | async def add_page(self, database_id, properties, media_url = None): 68 | media_block = None 69 | 70 | if media_url: 71 | is_video_file = media_url.lower().endswith(('.mp4', '.webm', '.mov', '.avi', '.mkv')) 72 | 73 | if is_video_file: 74 | media_block = { 75 | "type": "embed", 76 | "embed": {"url": media_url} 77 | } 78 | else: 79 | media_block = { 80 | "type": "image", 81 | "image": {"type": "external", "external": {"url": media_url}} 82 | } 83 | 84 | 85 | query = await self.__query_retries(lambda: self.notion.pages.create( 86 | parent = { "database_id": database_id }, 87 | properties = properties, 88 | children = [media_block] if media_block else None 89 | )) 90 | return query 91 | 92 | 93 | 94 | async def update_page(self, page_id, properties): 95 | query = await self.__query_retries(lambda: self.notion.pages.update( 96 | page_id, 97 | properties = properties 98 | )) 99 | return query 100 | 101 | 102 | 103 | async def update_databases(self, database_id, properties): 104 | query = await self.__query_retries(lambda: self.notion.databases.update( 105 | database_id, 106 | properties = properties 107 | )) 108 | return query 109 | 110 | 111 | 112 | async def __get_all_raw(self, database_id): 113 | page_step = 100 114 | results = [] 115 | 116 | Log.info("notion", f"Load {page_step} pages") 117 | query = await self.__query_retries(lambda: self.notion.databases.query(database_id=database_id, page_size=page_step)) 118 | 119 | results.extend([o for o in query["results"]]) 120 | Log.ok("notion", f"Loaded") 121 | 122 | while query["next_cursor"]: 123 | Log.info("notion", f"Load {page_step} pages") 124 | query = await self.__query_retries(lambda: self.notion.databases.query(database_id=database_id, start_cursor=query["next_cursor"], page_size=page_step)) 125 | 126 | results.extend([o for o in query["results"]]) 127 | Log.ok("notion", f"Loaded") 128 | 129 | return results 130 | 131 | 132 | 133 | async def __query_retries(self, query_call, wait_time = 1, max_retries = 10): 134 | retries = 0 135 | while retries < max_retries: 136 | try: 137 | data = await query_call() 138 | return data 139 | except Exception as e: 140 | retries += 1 141 | if retries < max_retries: 142 | Log.warning("notion", f"{e}. Retrying in {wait_time} seconds...") 143 | await asyncio.sleep(wait_time) 144 | else: 145 | Log.error("notion", "Max retries reached") 146 | raise Exception(e) 147 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Info 2 | 3 | Synchronize your Civitai LoRA, checkpoints and embeddings bookmarks and Notion and download them automatically 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | # Using 13 | 14 | You need Python 3 version 15 | 16 | Just run `start.bat` if you are using Windows or `start.sh ` for Linux, the first installation may take some time 17 | 18 | Available utilities: 19 | 20 | 1) `Import from civitai` - `Import from civitai` - Add all bookmarks from Civitai to the Notion, only those models (LoRA, checkpoints or embeddings) for which you specified the database id in the settings will be added 21 | 22 | At this stage your models has not been downloaded yet, go to your database and edit your entries as desired, for example, sometimes you can add another name/title or change the image, as well as remove duplicates (see point 2) 23 | 24 | 2) `Download from notion` - Checks the notion database for duplicates, the check takes place by the `File` property, since it will then be used for the names of the files that you download, if duplicates were found, then edit the entries in your notion database, for this you can use the built-in notion search bar to quickly find duplicates by name 25 | 26 | You have the option to use the automatic duplicate conflict resolution setting, in which case the application will simply add a postfix with a digit until the file name becomes unique 27 | 28 | Please note that this utility can also check the hash in the database with already loaded models, this is necessary to fix failed downloads as well as to update models (see point 3) 29 | 30 | Due to the file hash calculation, it may take time if you have a slow hard drive 31 | 32 | 3. `Update notion database` - Check all models for new versions in Civitai, if new versions of models are found, the application will update the entries in the Notion database 33 | 34 | The action updates the fields `Trigger Words`, `Hash` and `Version`in the Notion database but does not download files 35 | 36 | To update the files you need to run the `Download from notion` utility and select `Download with hash check` 37 | 38 | This action will start checking the hash and loading models while outdated models will be updated due to a hash mismatch 39 | 40 | 4) `Make all` Runs in turn `Import from civitai` -> `Update notion database` -> `Download from notion` 41 | 42 | 43 | # Notion Config 44 | 45 | #### API Key 46 | 47 | 1) Go to the [integrations](https://www.notion.so/my-integrations) and create a new integration 48 | 49 | 2) In the `Capabilities` menu set all permissions 50 | 51 | 52 | 53 | 3) Copy the API key and write it to `Settings` - `Tokens` - `Notion token` 54 | 55 | #### Database 56 | 57 | 1) Create a new notion database and name it whatever you want. Use the following properties: 58 | 59 | `Name` - `Type`: `Text` 60 | 61 | `URL` - `Type`: `URL` 62 | 63 | `File` - `Type`: `Text` 64 | 65 | `Trigger Words` - `Type`: `Text` 66 | 67 | `Tags` - `Type`: `Multi-select` 68 | 69 | `SD` - `Type`: `Select` 70 | 71 | `Version` - `Type`: `Text` 72 | 73 | `Model ID` - `Type`: `Text` 74 | 75 | `Hash` - `Type`: `Text` 76 | 77 | 78 | 79 | 2) Create a new database in the notion 80 | 81 | 3) Create a connection for your database so that the script can create records, to do this, select `...` from the top right and click `Add connections`, select your integration 82 | 83 | 84 | 85 | 4) Copy the `Database ID` while you are on the database page and write it to `Settings` - `LoRA settings` - `Notion database id` 86 | `Database ID` can be found in the browser line 87 | 88 | ``` 89 | https://www.notion.so/myworkspace/a8aec43384f447ed84390e8e42c2e089?v=... 90 | |--------- Database ID --------| 91 | ``` 92 | 93 | 5) Add at least one path to the folder to save in `Settings` - `LoRA settings` - `Path to save for 1.x versions` or `Settings` - `LoRA settings` - `Path to save for SDXL versions`. You don't have to do this and then the app will only be able to import your bookmarks, but not download them 94 | 95 | 6) Repeat steps 2 - 5 in order to create a database for embeddings (`Settings` - `Notion` - `Notion database id for embeddings`) and checkpoints (`Settings` - `Notion` - `Notion database id checkpoints`). You can skip this step, then the program will simply skip processing these categories 96 | 97 | # Civitai 98 | 99 | #### API Key 100 | 101 | 1) Go to the [Account Settings](https://civitai.com/user/account) and create a new `API Key` 102 | 103 | 104 | 2) Copy the API key and write it to `Settings` - `Tokens` - `Civit AI token` 105 | 106 | 107 | # Additionally 108 | 109 | You can also use large image previews for LoRA cards in notion, for this you can use the `stylish` extension for your browser, add the following styles for the domain `notion.so ` 110 | 111 | 112 | 113 | 114 | ```css 115 | .notion-selectable.notion-page-block.notion-collection-item a > div:first-child > div:first-child > div:first-child { 116 | height: 350px !important; 117 | } 118 | 119 | .notion-selectable.notion-page-block.notion-collection-item a > div:first-child > div:first-child > div:first-child img { 120 | height: 350px !important; 121 | } 122 | ``` -------------------------------------------------------------------------------- /config.temp.json: -------------------------------------------------------------------------------- 1 | { 2 | "config": [ 3 | { 4 | "id": { 5 | "val": "token", 6 | "title": "Tokens" 7 | }, 8 | "properties": [ 9 | { 10 | "id": "notion", 11 | "type": "password", 12 | "data": "", 13 | "title": "Notion token", 14 | "hint": "'My integrations' in Notion", 15 | "help": "" 16 | }, 17 | { 18 | "id": "civitai", 19 | "type": "password", 20 | "data": "", 21 | "title": "Civit AI token", 22 | "hint": "'Account Settings' -> 'API Key'", 23 | "help": "" 24 | } 25 | ] 26 | }, 27 | { 28 | "id": { 29 | "val": "parser", 30 | "title": "Parser" 31 | }, 32 | "properties": [ 33 | { 34 | "id": "civitai_page_delay", 35 | "type": "string", 36 | "data": "1", 37 | "title": "Civitai page load delay", 38 | "hint": "Delay in seconds", 39 | "help": "" 40 | } 41 | ] 42 | }, 43 | { 44 | "id": { 45 | "val": "lora", 46 | "title": "LoRA settings" 47 | }, 48 | "properties": [ 49 | { 50 | "id": "enable", 51 | "type": "bool", 52 | "data": true, 53 | "title": "Enable", 54 | "hint": "", 55 | "help": "" 56 | }, 57 | { 58 | "id": "notion_database", 59 | "type": "string", 60 | "data": "", 61 | "title": "Notion database id", 62 | "hint": "Leave empty if not used", 63 | "help": "" 64 | }, 65 | { 66 | "id": "save_path_v1_x", 67 | "type": "path", 68 | "data": "", 69 | "title": "Path to save for 1.x versions", 70 | "hint": "Leave empty if not used", 71 | "help": "" 72 | }, 73 | { 74 | "id": "save_path_sdxl", 75 | "type": "path", 76 | "data": "", 77 | "title": "Path to save for SDXL versions", 78 | "hint": "Leave empty if not used", 79 | "help": "" 80 | }, 81 | { 82 | "id": "add_version_to_file_name", 83 | "type": "bool", 84 | "data": false, 85 | "title": "Add version to the file name", 86 | "hint": "", 87 | "help": "" 88 | }, 89 | { 90 | "id": "simple_titles", 91 | "type": "bool", 92 | "data": false, 93 | "title": "Use simple names in titles", 94 | "hint": "", 95 | "help": "" 96 | }, 97 | { 98 | "id": "file_name_clear", 99 | "type": "string", 100 | "data": "concept, lora, beta, locon, loha, lycoris, experimental, test", 101 | "title": "Remove words when using a file names", 102 | "hint": "test, beta, concept...", 103 | "help": "" 104 | }, 105 | { 106 | "id": "resolve_duplicates", 107 | "type": "bool", 108 | "data": true, 109 | "title": "Automatically resolve duplicates conflicts", 110 | "hint": "", 111 | "help": "" 112 | } 113 | ] 114 | }, 115 | { 116 | "id": { 117 | "val": "checkpoint", 118 | "title": "Checkpoint settings" 119 | }, 120 | "properties": [ 121 | { 122 | "id": "enable", 123 | "type": "bool", 124 | "data": true, 125 | "title": "Enable", 126 | "hint": "", 127 | "help": "" 128 | }, 129 | { 130 | "id": "notion_database", 131 | "type": "string", 132 | "data": "", 133 | "title": "Notion database id", 134 | "hint": "Leave empty if not used", 135 | "help": "" 136 | }, 137 | { 138 | "id": "save_path_v1_x", 139 | "type": "path", 140 | "data": "", 141 | "title": "Path to save for 1.x versions", 142 | "hint": "Leave empty if not used", 143 | "help": "" 144 | }, 145 | { 146 | "id": "save_path_sdxl", 147 | "type": "path", 148 | "data": "", 149 | "title": "Path to save for SDXL versions", 150 | "hint": "Leave empty if not used", 151 | "help": "" 152 | }, 153 | { 154 | "id": "add_version_to_file_name", 155 | "type": "bool", 156 | "data": false, 157 | "title": "Add version to the file name", 158 | "hint": "", 159 | "help": "" 160 | }, 161 | { 162 | "id": "simple_titles", 163 | "type": "bool", 164 | "data": false, 165 | "title": "Use simple names in titles", 166 | "hint": "", 167 | "help": "" 168 | }, 169 | { 170 | "id": "file_name_clear", 171 | "type": "string", 172 | "data": "beta, experimental, test", 173 | "title": "Remove words when using a file names", 174 | "hint": "test, beta, concept...", 175 | "help": "" 176 | }, 177 | { 178 | "id": "resolve_duplicates", 179 | "type": "bool", 180 | "data": false, 181 | "title": "Automatically resolve duplicates conflicts", 182 | "hint": "", 183 | "help": "" 184 | } 185 | ] 186 | }, 187 | { 188 | "id": { 189 | "val": "embedding", 190 | "title": "Embedding settings" 191 | }, 192 | "properties": [ 193 | { 194 | "id": "enable", 195 | "type": "bool", 196 | "data": true, 197 | "title": "Enable", 198 | "hint": "", 199 | "help": "" 200 | }, 201 | { 202 | "id": "notion_database", 203 | "type": "string", 204 | "data": "", 205 | "title": "Notion database id", 206 | "hint": "Leave empty if not used", 207 | "help": "" 208 | }, 209 | { 210 | "id": "save_path_v1_x", 211 | "type": "path", 212 | "data": "", 213 | "title": "Path to save for 1.x versions", 214 | "hint": "Leave empty if not used", 215 | "help": "" 216 | }, 217 | { 218 | "id": "save_path_sdxl", 219 | "type": "path", 220 | "data": "", 221 | "title": "Path to save for SDXL versions", 222 | "hint": "Leave empty if not used", 223 | "help": "" 224 | }, 225 | { 226 | "id": "add_version_to_file_name", 227 | "type": "bool", 228 | "data": false, 229 | "title": "Add version to the file name", 230 | "hint": "", 231 | "help": "" 232 | }, 233 | { 234 | "id": "simple_titles", 235 | "type": "bool", 236 | "data": false, 237 | "title": "Use simple names in titles", 238 | "hint": "", 239 | "help": "" 240 | }, 241 | { 242 | "id": "file_name_clear", 243 | "type": "string", 244 | "data": "beta, experimental, test", 245 | "title": "Remove words when using a file names", 246 | "hint": "test, beta, concept...", 247 | "help": "" 248 | }, 249 | { 250 | "id": "resolve_duplicates", 251 | "type": "bool", 252 | "data": false, 253 | "title": "Automatically resolve duplicates conflicts", 254 | "hint": "", 255 | "help": "" 256 | } 257 | ] 258 | } 259 | ] 260 | } -------------------------------------------------------------------------------- /app/util/Civitai.py: -------------------------------------------------------------------------------- 1 | import aiohttp 2 | import aiofiles 3 | import asyncio 4 | import hashlib 5 | from aiohttp import ClientError, ServerTimeoutError 6 | import os 7 | import re 8 | import traceback 9 | from .Log import Log 10 | 11 | 12 | 13 | class ModelsTypes: 14 | Checkpoint = "Checkpoint" 15 | TextualInversion = "TextualInversion" 16 | Hypernetwork = "Hypernetwork" 17 | AestheticGradient = "AestheticGradient" 18 | LORA = "LORA" 19 | LoCon = "LoCon" 20 | Controlnet = "Controlnet" 21 | Poses = "Poses" 22 | 23 | 24 | class ModelsSort: 25 | HighestRated = "Highest Rated" 26 | MostDownloaded = "Most Downloaded" 27 | Newest = "Newest" 28 | 29 | 30 | class ModelsPeriod: 31 | AllTime = "AllTime" 32 | Year = "Year" 33 | Month = "Month" 34 | Week = "Week" 35 | Day = "Day" 36 | 37 | 38 | 39 | class Civitai(): 40 | token = None 41 | types = [] 42 | favorites=False 43 | 44 | 45 | def __init__(self, token): 46 | self.token = token 47 | 48 | 49 | 50 | def set_filters(self, types=[], favorites=False): 51 | self.types = types 52 | self.favorites = favorites 53 | 54 | 55 | 56 | async def get_all(self, wait_time = 1): 57 | out_list = [] 58 | page = 1 59 | cursor = "" 60 | while True: 61 | data = await self.get_page_and_next(page, cursor) 62 | await asyncio.sleep(wait_time) 63 | if not data["models"] or data["cursor"] is None: 64 | break 65 | page += 1 66 | cursor = data["cursor"] 67 | out_list.extend(data["models"]) 68 | return out_list 69 | 70 | 71 | 72 | async def get_page_and_next(self, page, cursor, separate_model_request = False): 73 | Log.info("Civit AI", f"Load page {page}") 74 | 75 | out_list = [] 76 | models = await self.__get_models_json(favorites=self.favorites, types=self.types, cursor=cursor) 77 | 78 | Log.ok("Civit AI", f"Page Loaded ({len(models['items'])} models)") 79 | 80 | for model in models["items"]: 81 | try: 82 | if separate_model_request: 83 | out_list.append(await self.get_model_data(model["id"])) 84 | else: 85 | out_list.append(self.__convert_json_model_data(model)) 86 | except Exception as err: 87 | Log.exception("Civit AI", f"Load model info error. Model: https://civitai.com/models/{model['id']}", err, traceback.format_exc()) 88 | 89 | return { 90 | "models": out_list, 91 | "cursor": models.get("metadata", {}).get("nextCursor") 92 | } 93 | 94 | 95 | 96 | 97 | async def download_model(self, id, save_dir, name, hash=None, max_retries=3, retry_delay=5): 98 | model = await self.get_model_data(id) 99 | Log.info("Civit AI", f"Start download {id} model") 100 | 101 | for attempt in range(max_retries + 1): 102 | try: 103 | async with aiohttp.ClientSession() as session: 104 | timeout = aiohttp.ClientTimeout(total=3600, connect=30, sock_read=60) 105 | 106 | async with session.get( 107 | model["download"], 108 | params={'token': self.token}, 109 | timeout=timeout 110 | ) as response: 111 | response.raise_for_status() 112 | 113 | format = self.__get_extension_from_response(response) 114 | if format is None: 115 | Log.error("Civit AI", f"Download model {id} ({name}). Unable to download - unknown file extension") 116 | raise Exception(f"Unknown file extension. Headers: {response.headers}") 117 | 118 | if name == "": 119 | Log.error("Civit AI", f"Download model id: {id}. Unable to download - empty file name") 120 | raise Exception(f"Empty file name") 121 | 122 | file_path = os.path.join(save_dir, f'{name}.{format}') 123 | temp_file_path = file_path + '.tmp' 124 | 125 | try: 126 | async with aiofiles.open(temp_file_path, mode='wb') as f: 127 | downloaded = 0 128 | async for chunk in response.content.iter_chunked(1024 * 1024 * 10): 129 | await f.write(chunk) 130 | downloaded += len(chunk) 131 | if downloaded % (50 * 1024 * 1024) == 0: 132 | Log.info("Civit AI", f"Downloaded {downloaded / 1024 / 1024:.1f} MB for model {id}") 133 | Log.info("Civit AI", f"Download complete {downloaded / 1024 / 1024:.1f} MB for model {id}") 134 | 135 | if os.path.exists(file_path): 136 | os.remove(file_path) 137 | os.rename(temp_file_path, file_path) 138 | 139 | if hash: 140 | pass 141 | 142 | Log.ok("Civit AI", f"Model {id} ({name}) successfully downloaded") 143 | return file_path 144 | 145 | except Exception as e: 146 | if os.path.exists(temp_file_path): 147 | try: 148 | os.remove(temp_file_path) 149 | except: 150 | pass 151 | raise e 152 | 153 | except (ClientError, ServerTimeoutError, asyncio.TimeoutError, OSError) as e: 154 | if attempt < max_retries: 155 | Log.warning( 156 | "Civit AI", 157 | f"Download failed for model {id} (attempt {attempt + 1}/{max_retries + 1}): {str(e)}. " 158 | f"Retrying in {retry_delay} seconds..." 159 | ) 160 | await asyncio.sleep(retry_delay) 161 | retry_delay = min(retry_delay * 2, 60) 162 | else: 163 | Log.error("Civit AI", f"Failed to download model {id} after {max_retries + 1} attempts: {str(e)}") 164 | raise 165 | 166 | except Exception as e: 167 | Log.error("Civit AI", f"Unexpected error downloading model {id}: {str(e)}") 168 | raise 169 | 170 | 171 | async def download_model_from_url(self, url, save_dir, name, hash = None): 172 | id = re.search(r"models\/([0-9]+)", url)[1] 173 | return await self.download_model(id, save_dir, name, hash) 174 | 175 | 176 | 177 | async def get_model_data(self, id): 178 | Log.info("Civit AI", f"Load model info. Model ID: {id}") 179 | model = await self.__get_model_json(id) 180 | Log.ok("Civit AI", f"Model info loaded - {id} ({model['name']})") 181 | 182 | return self.__convert_json_model_data(model) 183 | 184 | 185 | 186 | def __convert_json_model_data(self, model): 187 | first_model = model["modelVersions"][0] 188 | first_model_file = next((item for item in first_model["files"] if item["type"] != "Training Data"), None) 189 | return { 190 | "url": f"https://civitai.com/models/{model['id']}", 191 | "tags": model["tags"], 192 | "id": model["id"], 193 | "type": model["type"], 194 | "triggers": first_model.get("trainedWords", ""), 195 | "download": first_model_file["downloadUrl"], 196 | "format": first_model_file["metadata"]["format"], 197 | "name": model["name"], 198 | "base_model": first_model["baseModel"], 199 | "version": first_model["name"], 200 | "images": [o["url"] for o in first_model["images"]], 201 | "SHA256": first_model_file["hashes"]["SHA256"] 202 | } 203 | 204 | 205 | 206 | async def __get_json(self, page, params = []): 207 | headers = {"Content-type": "application/json"} 208 | 209 | async with aiohttp.ClientSession().get(f'https://civitai.com/api/v1/{page}', params=params, headers=headers) as response: 210 | json_data = await response.json() 211 | 212 | if "error" in json_data: 213 | Log.error("Civit AI", f"JSON get error ({response.real_url}). Error info: {json_data['error']}") 214 | raise Exception() 215 | 216 | return json_data 217 | 218 | 219 | 220 | async def __get_models_json(self, cursor = "", types = [ModelsTypes.Checkpoint], sort = ModelsSort.Newest, period = ModelsPeriod.AllTime, query = "", username = "", favorites = False, nsfw = True, limit = 100): 221 | return await self.__get_json("models", { 222 | "cursor": cursor, 223 | "types": types, 224 | "sort": sort, 225 | "period": period, 226 | "query": query, 227 | "username": username, 228 | "favorites": "true" if favorites else "false", 229 | "nsfw": "true" if nsfw else "false", 230 | "limit": limit, 231 | "token": self.token 232 | }) 233 | 234 | 235 | 236 | async def __get_model_json(self, id): 237 | return await self.__get_json(f"models/{id}") 238 | 239 | 240 | 241 | async def get_model_from_url_json(self, url): 242 | id = re.search(r"models\/([0-9]+)", url)[1] 243 | return await self.__get_model_json(id) 244 | 245 | 246 | 247 | def __get_extension_from_response(self, response): 248 | cd = response.headers.get("content-disposition") 249 | if not cd: 250 | return None 251 | 252 | filename = re.findall("filename=\"(.+)\"", cd) 253 | if len(filename) == 0: 254 | return None 255 | 256 | return filename[0].split(".")[-1] 257 | 258 | 259 | 260 | @staticmethod 261 | async def sha256(filename): 262 | f = await aiofiles.open(filename, mode = "rb") 263 | bytes = await f.read() 264 | 265 | # TODO: make async hash 266 | sha256 = hashlib.sha256(bytes).hexdigest().upper() 267 | await f.close() 268 | return sha256 -------------------------------------------------------------------------------- /app/util/Worker.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | import uuid 4 | import traceback 5 | 6 | import asyncio 7 | import aiofiles 8 | import aiofiles.os 9 | 10 | from .Civitai import Civitai, ModelsTypes 11 | from .Notion import Notion 12 | from .Log import Log 13 | from .Config import Config 14 | 15 | 16 | 17 | class Worker: 18 | civitai = None 19 | notion = None 20 | __busy = False 21 | __civitai_cache = [] 22 | __notion_cache = {} 23 | __end_work_call = None 24 | __work_state_freeze = False 25 | 26 | 27 | @staticmethod 28 | def busy(): 29 | return Worker.__busy 30 | 31 | 32 | @staticmethod 33 | def set_end_work_call(end_work_call): 34 | Worker.__end_work_call = end_work_call 35 | 36 | 37 | 38 | @staticmethod 39 | async def ipmport_models_from_favorites(): 40 | if not Worker.__start_work(): 41 | return 42 | 43 | Log.info("App", "Starting ipmport models from Civit AI favorites to notion database") 44 | 45 | ability = Worker.__get_ability_from_config() 46 | 47 | if all(bool(item["database"]) is False for item in ability.values()): 48 | Log.error("App", "To work need at least one of settings \"notion database id\"") 49 | Worker.__end_work() 50 | return 51 | 52 | 53 | # Set filters 54 | filters = Worker.__get_civitai_filters_from_ability(ability, True) 55 | Worker.civitai.set_filters(filters, favorites = True) 56 | 57 | 58 | # Load all civitai data 59 | try: 60 | all_civitai_models = await Worker.__load_all_from_civitai() 61 | except Exception as err: 62 | Log.exception("App", f"Load models info from Civit AI error", err, traceback.format_exc()) 63 | Worker.__end_work() 64 | return 65 | 66 | 67 | # Add to notion 68 | civitai_filters = { 69 | "lora": [ModelsTypes.LORA, ModelsTypes.LoCon], 70 | "checkpoint": [ModelsTypes.Checkpoint], 71 | "embedding": [ModelsTypes.TextualInversion] 72 | } 73 | 74 | for key, val in ability.items(): 75 | if bool(val["database"]): 76 | await Worker.__add_models_to_notion(all_civitai_models, civitai_filters[key], val["database"], key) 77 | 78 | 79 | Log.ok("App", f"End ipmport models") 80 | 81 | Worker.__end_work() 82 | 83 | 84 | 85 | 86 | @staticmethod 87 | async def download_models_from_notion_database(hash_check): 88 | if not Worker.__start_work(): 89 | return 90 | 91 | ability = Worker.__get_ability_from_config() 92 | 93 | if all(item["active"] is False for item in ability.values()): 94 | Log.error("App", "To work need at least one combination of settings \"Path to save\" and \"notion database id\"") 95 | Worker.__end_work() 96 | return 97 | 98 | 99 | # Download from notion 100 | civitai_version = { "v1_x": ["SD 1.4", "SD 1.5"], "sdxl": ["SDXL 0.9", "SDXL 1.0", "Pony", "Illustrious"] } 101 | 102 | for key, val in ability.items(): 103 | if ability[key]["active"]: 104 | for version in ability[key]["path"]: 105 | path = ability[key]["path"][version] 106 | database_id = ability[key]["database"] 107 | 108 | await Worker.__download_models_from_notion(path, database_id, key, civitai_version[version], version, hash_check) 109 | 110 | Log.ok("App", f"End download models") 111 | 112 | Worker.__end_work() 113 | 114 | 115 | 116 | @staticmethod 117 | async def update_notion_database(): 118 | if not Worker.__start_work(): 119 | return 120 | 121 | ability = Worker.__get_ability_from_config() 122 | 123 | if all(bool(item["database"]) is False for item in ability.values()): 124 | Log.error("App", "To work need at least one of settings \"notion database id\"") 125 | Worker.__end_work() 126 | return 127 | 128 | 129 | Log.info("App", "Start update notion database") 130 | 131 | # Update from notion 132 | for key, val in ability.items(): 133 | if bool(val["database"]): 134 | await Worker.__update_notion_database(val["database"], key) 135 | 136 | Log.ok("App", f"End update models in notion database. Start \"Download from notion\" to update the model local files") 137 | 138 | Worker.__end_work() 139 | 140 | 141 | 142 | @staticmethod 143 | async def make_all(hash_check): 144 | if not Worker.__start_work(): 145 | return 146 | 147 | Worker.__work_state_freeze = True 148 | 149 | await Worker.ipmport_models_from_favorites() 150 | Worker.__clear_cache() 151 | 152 | await Worker.update_notion_database() 153 | Worker.__clear_cache() 154 | 155 | await Worker.download_models_from_notion_database(hash_check) 156 | Worker.__clear_cache() 157 | 158 | Worker.__work_state_freeze = False 159 | 160 | Worker.__end_work() 161 | 162 | 163 | 164 | @staticmethod 165 | async def __download_models_from_notion(path, database_id, type, filter_version, version, hash_check): 166 | Log.info("App", f"Starting download from {type} {version} notion database") 167 | 168 | if not await Worker.__check_notion_database_dublicate(database_id, use_cache = True): 169 | return 170 | 171 | try: 172 | all_notion_pages = await Worker.__load_all_from_notion(database_id, use_cache = True) 173 | except Exception as err: 174 | Log.exception("App", f"Load {type} database from notion error", err, traceback.format_exc()) 175 | return 176 | 177 | download_counter = 0 178 | for page in all_notion_pages: 179 | try: 180 | if not page["properties"]["SD"] in filter_version: 181 | continue 182 | 183 | hash = str(page["properties"]["Hash"]).upper() 184 | 185 | local_file = Worker.__file_exists_with_any_extension(path, page["properties"]["File"]) 186 | if local_file: 187 | if not hash_check: 188 | continue 189 | 190 | Log.info("App", f"Check hash for {page['properties']['File']}") 191 | if await Civitai.sha256(local_file) == hash: 192 | Log.ok("App", f"Hash for {page['properties']['File']} - ok") 193 | continue 194 | else: 195 | Log.warning("App", f"Invalid {page['properties']['File']} hash. Restart download") 196 | Log.warning("App", f"Delete old file") 197 | await aiofiles.os.remove(local_file) 198 | 199 | 200 | await Worker.civitai.download_model(page["properties"]["Model ID"], path, page["properties"]["File"], hash) 201 | download_counter += 1 202 | except Exception as err: 203 | Log.exception("App", f"Error model download", err, traceback.format_exc()) 204 | 205 | if not download_counter: 206 | Log.warning("App", "No new models to download") 207 | else: 208 | Log.ok("App", f"Download {download_counter} models") 209 | 210 | 211 | 212 | @staticmethod 213 | async def __update_notion_database(database_id, type): 214 | Log.info("App", f"Starting updating {type} notion database") 215 | 216 | if not await Worker.__check_notion_database_dublicate(database_id, use_cache = True): 217 | return 218 | 219 | try: 220 | all_notion_pages = await Worker.__load_all_from_notion(database_id, use_cache = True) 221 | except Exception as err: 222 | Log.exception("App", f"Load {type} database from notion error", err, traceback.format_exc()) 223 | return 224 | 225 | 226 | update_counter = 0 227 | for page in all_notion_pages: 228 | try: 229 | model = await Worker.civitai.get_model_data(page["properties"]["Model ID"]) 230 | 231 | page_hash = str(page["properties"]["Hash"]).upper() 232 | 233 | model_hash = str(model["SHA256"]).upper() 234 | trigger_words = ", ".join(model["triggers"]) 235 | model_version = model["version"] 236 | 237 | if page_hash == model_hash: 238 | Log.info("App", f"The latest version of the model has already been added - {page['properties']['File']} ({page['properties']['Name']})") 239 | continue 240 | 241 | Log.warning("App", f"Outdated model found - {page['properties']['File']} ({page['properties']['Name']})") 242 | 243 | await Worker.notion.update_page(page["id"], { 244 | "Trigger Words": {"rich_text": [{"text": {"content": trigger_words}}]}, 245 | "Hash": {"rich_text": [{"text": {"content": model_hash}}]}, 246 | "Version": {"rich_text": [{"text": {"content": model_version}}]}, 247 | }) 248 | 249 | Log.ok("App", f"Model updated - {page['properties']['File']} ({page['properties']['Name']})") 250 | 251 | update_counter += 1 252 | except Exception as err: 253 | Log.exception("App", f"Error model update in {type} database", err, traceback.format_exc()) 254 | 255 | if not update_counter: 256 | Log.warning("App", f"No models in {type} database to update") 257 | else: 258 | Log.ok("App", f"Update {update_counter} models in {type} database") 259 | 260 | 261 | 262 | @staticmethod 263 | async def __add_models_to_notion(civitai_models, model_type_filter, database_id, database_type): 264 | Log.info("App", f"Starting add {database_type} to notion database") 265 | 266 | try: 267 | all_notion_pages = await Worker.__load_all_from_notion(database_id) 268 | except Exception as err: 269 | Log.exception("App", f"Load {database_type} database from notion error", err, traceback.format_exc()) 270 | return 271 | 272 | # Delete models that have already been added to notion and models that do not fit the type 273 | filtered_models = [] 274 | for model in civitai_models: 275 | filter = [x for x in all_notion_pages if x["properties"]["Model ID"] == (str(model["id"]))] 276 | if not filter and model["type"] in model_type_filter: 277 | filtered_models.append(model) 278 | 279 | notion_file_list = [x["properties"]["File"] for x in all_notion_pages] 280 | 281 | # Add models to notion 282 | add_counter = 0 283 | for model in filtered_models: 284 | try: 285 | name = model["name"] 286 | url = model["url"] 287 | file_name = model["name"] 288 | trigger_words = ", ".join(model["triggers"]) 289 | sd_version = str(model["base_model"]).replace(",", ".") 290 | model_version = model["version"] 291 | model_id = str(model["id"]) 292 | hash = model["SHA256"] 293 | 294 | tags = ",".join(model["tags"]).split(",") 295 | tags = [t.strip() for t in tags if t.strip()] 296 | tags = [{"name": t.replace(",", " ")} for t in tags] 297 | 298 | if Config.get(f"{database_type}.add_version_to_file_name").data: 299 | file_name += f" {model_version}" 300 | file_name = Worker.__get_model_clear_name(file_name, database_type) 301 | 302 | if file_name == "": 303 | file_name = uuid.uuid4().hex 304 | 305 | if Config.get(f"{database_type}.simple_titles").data: 306 | name = Worker.__get_model_clear_name(model["name"], database_type, " ") 307 | 308 | if Config.get(f"{database_type}.resolve_duplicates").data: 309 | if file_name in notion_file_list: 310 | Log.warning("App", "Starting automatically resolve duplicates conflicts") 311 | new_file_name = file_name 312 | new_new_file_name_count = 2 313 | while new_file_name in notion_file_list: 314 | new_file_name = f"{file_name}_{new_new_file_name_count}" 315 | new_new_file_name_count += 1 316 | Log.ok("App", f"New file name selected - {new_file_name}") 317 | file_name = new_file_name 318 | 319 | 320 | await Worker.notion.add_page(database_id, { 321 | "Name": {"title": [{"text": {"content":name}}]}, 322 | "URL": {"type": "url", "url": url}, 323 | "File": {"rich_text": [{"text": {"content": file_name}}]}, 324 | "Trigger Words": {"rich_text": [{"text": {"content": trigger_words}}]}, 325 | "Model ID": {"rich_text": [{"text": {"content": model_id}}]}, 326 | "Hash": {"rich_text": [{"text": {"content": hash}}]}, 327 | "SD": { "select": { "name": sd_version } }, 328 | "Version": {"rich_text": [{"text": {"content": model_version}}]}, 329 | "Tags": { "type": "multi_select", "multi_select": tags } 330 | }, next(iter(model.get('images', [])), None)) 331 | 332 | notion_file_list.append(file_name) 333 | 334 | add_counter += 1 335 | Log.ok("App", f"Add model \"{model['name']}\" to notion database") 336 | except Exception as err: 337 | Log.exception("App", f"Add model \"{model['name']}\" (https://civitai.com/models/{model['id']}) to notion error", err, traceback.format_exc()) 338 | 339 | if not add_counter: 340 | Log.warning("App", "No new models to add") 341 | else: 342 | Log.ok("App", f"Added {add_counter} models") 343 | 344 | 345 | 346 | @staticmethod 347 | async def __check_notion_database_dublicate(database_id, use_cache = True): 348 | Log.info("App", "Check notion database for duplicates by file name") 349 | 350 | try: 351 | all_notion_pages = await Worker.__load_all_from_notion(database_id, use_cache) 352 | except Exception as err: 353 | Log.exception("App", f"Load database from notion error", err, traceback.format_exc()) 354 | return False 355 | 356 | 357 | duplicates = [] 358 | for page in all_notion_pages: 359 | filter = [x for x in all_notion_pages if x["properties"]["File"] == page["properties"]["File"]] 360 | if len(filter) > 1: 361 | duplicates.append(page) 362 | 363 | unique_duplicates = list(set([x["properties"]["File"] for x in duplicates])) 364 | 365 | if unique_duplicates: 366 | Log.error("App", "Duplicates found") 367 | 368 | for duplicate in unique_duplicates: 369 | Log.warning("App", f"File: {duplicate}") 370 | 371 | Log.info("App", "Rename the \"File\" fields in Notion so that they are unique") 372 | return False 373 | 374 | 375 | Log.ok("App", "No duplicates found") 376 | return True 377 | 378 | 379 | 380 | @staticmethod 381 | async def __load_all_from_civitai(use_cache = False): 382 | if use_cache and Worker.__civitai_cache: 383 | Log.warning("App", "Use Civit AI cache") 384 | return Worker.__civitai_cache 385 | 386 | all_models = await Worker.civitai.get_all(wait_time = 1) 387 | Worker.__civitai_cache = Worker.__combined_cache(all_models, Worker.__civitai_cache) 388 | 389 | return all_models 390 | 391 | 392 | 393 | @staticmethod 394 | async def __load_all_from_notion(database_id, use_cache = False): 395 | if use_cache and database_id in Worker.__notion_cache: 396 | Log.warning("App", "Use notion cache") 397 | return Worker.__notion_cache[database_id] 398 | 399 | all_pages = await Worker.notion.get_all(database_id) 400 | 401 | if not database_id in Worker.__notion_cache: 402 | Worker.__notion_cache[database_id] = [] 403 | Worker.__notion_cache[database_id] = Worker.__combined_cache(all_pages, Worker.__notion_cache[database_id]) 404 | 405 | return all_pages 406 | 407 | 408 | 409 | @staticmethod 410 | def __combined_cache(data, cache): 411 | combined = {obj["id"]: obj for obj in cache} 412 | combined.update({obj["id"]: obj for obj in data}) 413 | return list(combined.values()) 414 | 415 | 416 | 417 | @staticmethod 418 | def __get_ability_from_config(): 419 | ability = { 420 | "lora": {"active": False, "database": "", "path": { }}, 421 | "checkpoint": {"active": False, "database": "", "path": { }}, 422 | "embedding": {"active": False, "database": "", "path": { }} 423 | } 424 | 425 | for key, val in ability.items(): 426 | if not Config.get(f"{key}.enable").data: 427 | continue 428 | 429 | if database := Config.get(f"{key}.notion_database").data: 430 | ability[key]["database"] = database 431 | 432 | if data := Config.get(f"{key}.save_path_v1_x").data: 433 | ability[key]["path"]["v1_x"] = data 434 | 435 | if data := Config.get(f"{key}.save_path_sdxl").data: 436 | ability[key]["path"]["sdxl"] = data 437 | 438 | ability[key]["active"] = bool(ability[key]["path"]) 439 | 440 | return ability 441 | 442 | 443 | 444 | @staticmethod 445 | def __get_model_clear_name(name, model_type, separator = "_"): 446 | name = name.lower() 447 | 448 | if data := Config.get(f"{model_type}.file_name_clear").data: 449 | remove = [tag.strip() for tag in data.split(",")] 450 | name = " ".join([word for word in name.split() if word not in remove]) 451 | 452 | name = re.sub(r'[\W_]+', ' ', name) 453 | name = re.sub(r'[^\x00-\x7f]', ' ', name) 454 | name = re.sub(r' +', separator, name.strip()) 455 | return name 456 | 457 | 458 | 459 | @staticmethod 460 | def __get_civitai_filters_from_ability(ability, only_import = False): 461 | filters = [] 462 | 463 | id = "database" if only_import else "active" 464 | 465 | if bool(ability["lora"][id]): 466 | filters.extend([ModelsTypes.LORA, ModelsTypes.LoCon]) 467 | 468 | if bool(ability["checkpoint"][id]): 469 | filters.extend([ModelsTypes.Checkpoint]) 470 | 471 | if bool(ability["embedding"][id]): 472 | filters.extend([ModelsTypes.TextualInversion]) 473 | 474 | return filters 475 | 476 | 477 | 478 | @staticmethod 479 | def __file_exists_with_any_extension(directory, filename): 480 | for file in os.listdir(directory): 481 | file_name, file_extension = os.path.splitext(file) 482 | if file_name == filename: 483 | return os.path.join(directory, file) 484 | return False 485 | 486 | 487 | 488 | @staticmethod 489 | def __clear_cache(): 490 | Worker.__civitai_cache = [] 491 | Worker.__notion_cache = {} 492 | 493 | 494 | 495 | @staticmethod 496 | def __start_work(): 497 | if Worker.__work_state_freeze: 498 | return True 499 | 500 | if Worker.__busy: 501 | return False 502 | 503 | Worker.__clear_cache() 504 | Worker.__init_tokens() 505 | 506 | Worker.__busy = True 507 | return True 508 | 509 | 510 | 511 | @staticmethod 512 | def __end_work(): 513 | if Worker.__work_state_freeze: 514 | return 515 | 516 | Worker.__busy = False 517 | if Worker.__end_work_call: 518 | Worker.__end_work_call() 519 | 520 | 521 | @staticmethod 522 | def __init_tokens(): 523 | Worker.civitai = Civitai(Config.get("token.civitai").data) 524 | Worker.notion = Notion(Config.get("token.notion").data) --------------------------------------------------------------------------------