├── .dockerignore ├── .gitignore ├── AgeEncHandler.py ├── Dockerfile ├── LICENSE ├── buildNews.py ├── calc.py ├── change.log ├── config.py ├── config.sample.ini ├── dictionaries.py ├── docker-compose.yaml ├── flashcards.py ├── git.py ├── hypothesis.py ├── main.py ├── mindmap.py ├── news.json ├── readme.md ├── requirements.txt ├── sample.env ├── sm2.py └── utils.py /.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | __pycache__ 3 | __pypackages__ 4 | config.ini 5 | *.md 6 | GitDump.json 7 | persistence 8 | docker-compose.yaml 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Created by https://www.toptal.com/developers/gitignore/api/python 3 | # Edit at https://www.toptal.com/developers/gitignore?templates=python 4 | 5 | ### Python ### 6 | # Byte-compiled / optimized / DLL files 7 | __pycache__/ 8 | *.py[cod] 9 | *$py.class 10 | 11 | # C extensions 12 | *.so 13 | 14 | # Distribution / packaging 15 | .Python 16 | build/ 17 | develop-eggs/ 18 | dist/ 19 | downloads/ 20 | eggs/ 21 | .eggs/ 22 | lib/ 23 | lib64/ 24 | parts/ 25 | sdist/ 26 | var/ 27 | wheels/ 28 | pip-wheel-metadata/ 29 | share/python-wheels/ 30 | *.egg-info/ 31 | .installed.cfg 32 | *.egg 33 | MANIFEST 34 | 35 | # PyInstaller 36 | # Usually these files are written by a python script from a template 37 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 38 | *.manifest 39 | *.spec 40 | 41 | # Installer logs 42 | pip-log.txt 43 | pip-delete-this-directory.txt 44 | 45 | # Unit test / coverage reports 46 | htmlcov/ 47 | .tox/ 48 | .nox/ 49 | .coverage 50 | .coverage.* 51 | .cache 52 | nosetests.xml 53 | coverage.xml 54 | *.cover 55 | *.py,cover 56 | .hypothesis/ 57 | .pytest_cache/ 58 | pytestdebug.log 59 | 60 | # Translations 61 | *.mo 62 | *.pot 63 | 64 | # Django stuff: 65 | # *.log 66 | local_settings.py 67 | db.sqlite3 68 | db.sqlite3-journal 69 | 70 | # Flask stuff: 71 | instance/ 72 | .webassets-cache 73 | 74 | # Scrapy stuff: 75 | .scrapy 76 | 77 | # Sphinx documentation 78 | docs/_build/ 79 | doc/_build/ 80 | 81 | # PyBuilder 82 | target/ 83 | 84 | # Jupyter Notebook 85 | .ipynb_checkpoints 86 | 87 | # IPython 88 | profile_default/ 89 | ipython_config.py 90 | 91 | # pyenv 92 | .python-version 93 | 94 | # pipenv 95 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 96 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 97 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 98 | # install all needed dependencies. 99 | #Pipfile.lock 100 | 101 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 102 | __pypackages__/ 103 | 104 | # Celery stuff 105 | celerybeat-schedule 106 | celerybeat.pid 107 | 108 | # SageMath parsed files 109 | *.sage.py 110 | 111 | # Environments 112 | .env 113 | .venv 114 | env/ 115 | venv/ 116 | ENV/ 117 | env.bak/ 118 | venv.bak/ 119 | pythonenv* 120 | 121 | # Spyder project settings 122 | .spyderproject 123 | .spyproject 124 | 125 | # Rope project settings 126 | .ropeproject 127 | 128 | # mkdocs documentation 129 | /site 130 | 131 | # mypy 132 | .mypy_cache/ 133 | .dmypy.json 134 | dmypy.json 135 | 136 | # Pyre type checker 137 | .pyre/ 138 | 139 | # pytype static type analyzer 140 | .pytype/ 141 | 142 | # profiling data 143 | .prof 144 | 145 | # config.ini 146 | config.ini 147 | 148 | # persistence 149 | persistence 150 | 151 | # .archive/ 152 | .archive/ 153 | flashcards.db 154 | GitDump.json 155 | 156 | .venv 157 | 158 | # Secrets 159 | .env 160 | 161 | # Deta config 162 | .deta 163 | -------------------------------------------------------------------------------- /AgeEncHandler.py: -------------------------------------------------------------------------------- 1 | from age.cli import decrypt, encrypt 2 | import io 3 | import os 4 | import sys 5 | 6 | import config, utils 7 | 8 | def ageDecrypt(content): 9 | fname = os.path.expanduser("~/.config/age/" + utils.getTimestamp(True, True) + ".txt") 10 | 11 | content = content.encode('utf-8') 12 | 13 | f = open(fname, 'wb') 14 | decrypt(infile=io.BytesIO(content),outfile=f,ascii_armored=True) 15 | 16 | f = open(fname, 'r') 17 | out = f.read() 18 | f.close() 19 | os.remove(fname) 20 | return(out) 21 | 22 | 23 | def ageEncrypt(content): 24 | fname = os.path.expanduser("~/.config/age/" + utils.getTimestamp(True, True) + ".txt") 25 | 26 | f = open(fname, 'wb') 27 | 28 | encrypt(recipients=[config.getAgePublicKey()], infile=io.BytesIO(content.encode('utf-8')), outfile=f,ascii_armored=True) 29 | 30 | f = open(fname, 'r') 31 | out = f.read() 32 | f.close() 33 | os.remove(fname) 34 | return(out) 35 | 36 | def isAgeEncrypted(content): 37 | if content.startswith("-----BEGIN AGE ENCRYPTED FILE----- "): 38 | return 1 39 | elif content.startswith("-----BEGIN AGE ENCRYPTED FILE-----\n"): 40 | return 2 41 | else: 42 | return 0 43 | 44 | def convertToAgeString(content): 45 | s = "-----BEGIN AGE ENCRYPTED FILE-----\n" 46 | s += '\n'.join(content.split("FILE----- ")[1].split(" -----END")[0].split(' ')) 47 | s += "\n-----END AGE ENCRYPTED FILE-----" 48 | return s 49 | 50 | 51 | 52 | # encContent = """-----BEGIN AGE ENCRYPTED FILE----- 53 | # YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA1eWx4L2lHcXp6VEowWi9S 54 | # VjdSWnJlbzNkK0JwTVEzQU9UcndTKzdnVGw4Cll1MjJEczhJNGpTWWJBZWRKeEdm 55 | # VXlOL2J6WElRbmtkUE9ING9FUXRsQTgKLT4gZ1FdODdTMi1ncmVhc2UgTS0ofmM+ 56 | # ICZCPVMgRzNtJnlOIC5pSD11ClRKM1ZwRHRaeHNDR0Qwa0pKU0hmdElkWUlpSFdw 57 | # dzZOSGQ4U2xQM29JaDVpR3ZJS3IzYUdjUW5HdHh6TVV3Mm8KRmUxS2tCbHEzMWw4 58 | # V1NTNWZQSUczSTRXT293Ci0tLSB0dGhjR1Z4SnFCbXdLSEdZTXBsUTc3WDFlMVpX 59 | # a3QvQ0h1WkphU3ZsSnhRCo57XUKTWVOgacwUNCN81+T4nUKzLMwddOXYpvpa1QwI 60 | # SgimgEyvpVSBt06F6iQq34yc+4HgH40nrJr+v/V7zZE= 61 | # -----END AGE ENCRYPTED FILE-----""" 62 | 63 | # if isAgeEncrypted(encContent) == 1: 64 | # print(ageDecrypt(convertToAgeString(encContent))) 65 | # elif isAgeEncrypted(encContent) == 2: 66 | # print(ageDecrypt(encContent)) 67 | 68 | # CLEARTEXT = "Hello World!" 69 | 70 | # print(ageDecrypt(ageEncrypt(CLEARTEXT))) -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3-slim AS base 2 | 3 | FROM base AS build 4 | 5 | WORKDIR /app 6 | COPY requirements.txt . 7 | 8 | RUN apt-get update && \ 9 | apt-get install -y build-essential \ 10 | python3 libpython3-dev python3-venv \ 11 | libssl-dev libffi-dev cargo 12 | 13 | RUN pip3 install -U pip setuptools wheel && \ 14 | pip3 install --no-cache-dir -r requirements.txt -t /app 15 | RUN find . -name __pycache__ -exec rm -rf -v {} + 16 | COPY . . 17 | 18 | FROM base 19 | 20 | WORKDIR /app 21 | COPY --from=build /app . 22 | 23 | CMD ["python","-u","main.py"] 24 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 akhater 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /buildNews.py: -------------------------------------------------------------------------------- 1 | import json 2 | import urllib 3 | import requests 4 | 5 | newslist = {} 6 | newslist['news'] = [] 7 | newslist['news'].append({ 8 | 'newsid': 1, 9 | 'date': '2020-02-11', 10 | 'news': 'Verion 2.0.0 alpha updates\nLupin now supports Time Spaced Repetition based on SM2 algorithm!!\n Import your flashcards directly \ 11 | from your github by running /tsr import ... you can then go over your pending flashcards using /tsr x where \ 12 | x is the number of cards you\'d like to see. If you don\'t specify x the value set in your config.ini will be used.\ 13 | \nAlso Lupin can now pull these news updates after each significatif update. This will keep you posted about the latest news.\n\n \ 14 | This version requires new values in your config.ini file' 15 | }) 16 | 17 | newslist['news'].append({ 18 | 'newsid': 2, 19 | 'date': '2020-02-15', 20 | 'news': 'Verion 3.0.0 experimental updates\n \ 21 | A new version upgrade introducing MinMap capabilties among other thing. \ 22 | Send /getMM pageTitle and Lupin will generate a Markmap file containing a mindmap of that page and send it to you. \nCredits to https://markmap.js.org/\n \ 23 | /tsr command is now renamed to /srs' 24 | }) 25 | 26 | newslist['news'].append({ 27 | 'newsid': 3, 28 | 'date': '2020-02-18', 29 | 'news': 'Verion 3.1.0 experimental updates\n \ 30 | Thanks to @Piotr 3 months calendar can now be autogenerated and inserted in your right side-bar. Version releases also a new command /pullnow that forces refresh from GitHub & a scheduled pull that can be configured from the config.ini `GitHubUpdateFrequency` \ 31 | \n\nVersion requires new values in your config.ini file, please reffer to config.sample.ini' 32 | }) 33 | 34 | newslist['news'].append({ 35 | 'newsid': 4, 36 | 'date': '2020-02-20', 37 | 'news': 'Verion 3.2.0 & 3.3.0 experimental updates\n \ 38 | Timestamps in Journal entries can now be turned off from the config.ini file by setting `timestampEntries = false` \ 39 | TODOCommand is now decomissioned and replaced by CommandsMap for greater flexibility. Mapp your own sets of commands like \ 40 | TODO LATER etc...\n\n Version requires modifications in your config.ini file, please reffer to config.sample.ini' 41 | }) 42 | 43 | newslist['news'].append({ 44 | 'newsid': 5, 45 | 'date': '2020-02-26', 46 | 'news': 'Verion 3.4.0 experimental updates\n \ 47 | With all the great themes why have only one? put your choice of themes in the /logseq directory and name them ThemeName.custom.css\ 48 | then simply call Lupin with /themes to switch between one and the other.' 49 | }) 50 | 51 | newslist['news'].append({ 52 | 'newsid': 6, 53 | 'date': '2020-03-02', 54 | 'news': 'Verion 3.5.0 alpha updates\n \ 55 | Added feature to control how many months (0 to 3) you want LUPIN to generate calendars for, this is controlled thru generateMonths array of config.ini.\ 56 | first [0|1] specifies if you want LUPIN to generate prev month, second [0|1] specifies if you want LUPIN to generate next month\ 57 | 1,1 will generate prev,cur,next | 0,1 will generate curr,next | 0,0 will only generate curr' 58 | }) 59 | 60 | newslist['news'].append({ 61 | 'newsid': 7, 62 | 'date': '2020-03-22', 63 | 'news': 'Verion 3.8.0 experimental updates\n \ 64 | Lupin now supports doker thanks to Jon Molina https://github.com/digitalknk.\ 65 | Lupin also now supports Twitter embedding, Send a twitter URL to Lupin and see what happens.' 66 | }) 67 | 68 | with open('news.json', 'w') as outfile: 69 | json.dump(newslist, outfile) 70 | -------------------------------------------------------------------------------- /calc.py: -------------------------------------------------------------------------------- 1 | import calendar 2 | import datetime 3 | 4 | import utils 5 | from config import getfirstDayOfWeek 6 | 7 | 8 | def buildCalendar(year, month): 9 | firstDayOfWeek = getfirstDayOfWeek() 10 | 11 | cal = calendar.Calendar(firstDayOfWeek) 12 | dt = datetime.date(year, month, 1) 13 | dateFormatter = utils.getdateFormatter() 14 | 15 | calendar.setfirstweekday(firstDayOfWeek) 16 | daysOfWeek = (calendar.weekheader(2)).split(' ') 17 | HTMLOUT = ("""

{}

""").format(dt.strftime("%B %Y")) 18 | for dayOfWeek in daysOfWeek: 19 | HTMLOUT += ("").format(dayOfWeek) 20 | HTMLOUT += "" 21 | 22 | if(datetime.date.today().year == year and datetime.date.today().month == month): 23 | outofmonth = "" 24 | else: 25 | outofmonth = " outofmonth" 26 | 27 | dayClass = "" 28 | for week in cal.monthdays2calendar(year, month): 29 | HTMLOUT += "" 30 | for day in week: 31 | if(day[0]): 32 | calday = datetime.date(year, month, day[0]) 33 | dateFormat = utils.styleDateTime(datetime.date(year, month, day[0]), dateFormatter) 34 | if(datetime.date.today() == datetime.date(year, month, day[0])): 35 | dayClass = "page-ref today" 36 | elif utils.pageExists(utils.styleDateTime(calday, dateFormatter)): 37 | dayClass += "page-ref page-exists" + outofmonth 38 | else: 39 | dayClass = "page-ref" + outofmonth 40 | HTMLOUT += ("""""").format(dateFormat, dateFormat, dayClass, day[0]) 41 | else: 42 | HTMLOUT += "" 43 | HTMLOUT += "" 44 | 45 | HTMLOUT += "
{}
{}
" 46 | 47 | return HTMLOUT 48 | 49 | -------------------------------------------------------------------------------- /change.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pomdtr/Lupin/cc2434f47a55e51a1000338974b1a123ae59905a/change.log -------------------------------------------------------------------------------- /config.py: -------------------------------------------------------------------------------- 1 | import configparser 2 | import os 3 | 4 | config = configparser.RawConfigParser() 5 | config.optionxform = str #not to convert config to lowercase 6 | config.read('config.ini') 7 | 8 | __vMajor__ = '3' 9 | __vMinor__ = '8' 10 | __vPatch__ = '0' 11 | __vRel__ = 'e' 12 | __version__ = __vMajor__ + '.' + __vMinor__ + '.' + __vPatch__ + __vRel__ 13 | 14 | BotToken = os.getenv('BotToken') 15 | BotName = config.get('Bot','BotName') 16 | GitHubToken = os.getenv('GitHubToken') 17 | 18 | journalTemplate = config.get('Misc', 'journalTemplate', fallback="") 19 | dateFormat = config.get('Misc', 'dateFormat', fallback='%b {th}, %Y') 20 | GitHubBranch = config.get('GitHub','GitHubBranch') 21 | GitHubUser = config.get('GitHub','GitHubUser') 22 | GitHubRepo = config.get('GitHub','GitHubRepo') 23 | GitHubAuthor = config.get('GitHub','GitHubAuthor') 24 | GitHubEmail = config.get('GitHub','GitHubEmail') 25 | hour24 = (config.get('Misc','hour24')).lower() 26 | defaultIndentLevel = (config.get('Misc','defaultIndentLevel')) 27 | journalsFilesFormat = (config.get('Misc','journalsFilesFormat')) 28 | journalsFilesExtension = (config.get('Misc','journalsFilesExtension')) 29 | journalsFolder = (config.get('Misc','journalsFolder')) 30 | journalsPrefix = (config.get('Misc','journalsPrefix')) 31 | # TODOCommand = (config.get('Misc','TODOCommand')) 32 | BookmarkTag = (config.get('Misc','BookmarkTag')) 33 | assetsFolder = (config.get('Misc','assetsFolder')) 34 | hypothesisToken = os.getenv('hypothesisToken') 35 | hypothesisUsername = (config.get('hypothesis','hypothesisUsername')) 36 | manageHypothesisUpdates = (config.get('hypothesis','manageHypothesisUpdates')).lower() 37 | embedHypothesisAnnotations = (config.get('hypothesis','embedHypothesisAnnotations')).lower() 38 | hypothesisTagSpaceHandler = (config.get('hypothesis','hypothesisTagSpaceHandler')) 39 | 40 | def isBotAuthorized(chat_id): 41 | return any( 42 | str(chat_id) == str(BotAuthorizedId) 43 | for BotAuthorizedId in getBotAuthorizedIDs() 44 | ) 45 | 46 | def isNewer(): 47 | try: 48 | LastVersionRun = config.get('Bot', 'LastVersionRun') 49 | if(__version__ != LastVersionRun): 50 | config.set('Bot', 'LastVersionRun', __version__) 51 | # with open('config.ini', 'r') as configfile: 52 | # config.write(configfile) 53 | return True 54 | else: 55 | return False 56 | except: 57 | config.set('Bot', 'LastVersionRun', __version__) 58 | # with open('config.ini', 'r') as configfile: 59 | # config.write(configfile) 60 | return True 61 | 62 | def getBotVersion(): 63 | return __version__ 64 | 65 | def getBotAuthorizedIDs(): 66 | return config.get('Bot','BotAuthorizedIDs').split(',') 67 | 68 | def isManageHypothesis(): 69 | if manageHypothesisUpdates == 'true': 70 | return True 71 | else: 72 | return False 73 | 74 | def isHypothesisEmbedded(): 75 | if embedHypothesisAnnotations == 'true': 76 | return True 77 | else: 78 | return False 79 | 80 | def getHypothesisTagSpaceHandler(): 81 | return hypothesisTagSpaceHandler 82 | 83 | def getAssetsFolder(): 84 | return assetsFolder 85 | 86 | def getAssetsDestination(): 87 | return config.get('Bot','assetsDestination').lower() 88 | 89 | def getFirebaseBucketName(): 90 | return config.get('Firebase','BucketName') 91 | 92 | def getflashcardDailyGoal(): 93 | return int(config.get('TimeSpacedRepetion', 'flashcardDailyGoal')) 94 | 95 | def getflashcardsTag(): 96 | return config.get('TimeSpacedRepetion', 'flashcardTag') 97 | 98 | def getlastNewsDisplayed(): 99 | try: 100 | lastNewsDisplayed = config.get('Bot', 'lastNewsDisplayed') 101 | except: 102 | lastNewsDisplayed = 0 103 | config.set('Bot', 'lastNewsDisplayed', lastNewsDisplayed) 104 | # with open('config.ini', 'r') as configfile: 105 | # config.write(configfile) 106 | return lastNewsDisplayed 107 | 108 | def setlastNewsDisplayed(newsid): 109 | config.set('Bot', 'lastNewsDisplayed', newsid) 110 | # with open('config.ini', 'r') as configfile: 111 | # config.write(configfile) 112 | 113 | def getGitHubUpdateFrequency(): 114 | return int(config.get('GitHub', 'GitHubUpdateFrequency', fallback='720'))*60 115 | 116 | def isCalendarsAutogenerated(): 117 | # moveConfigSection('Bot','CalendarOptions', 'autoGenerateCalendars') 118 | return ( 119 | config.get( 120 | 'CalendarOptions', 'autoGenerateCalendars', fallback='false' 121 | ) 122 | == 'true' 123 | ) 124 | 125 | 126 | def getfirstDayOfWeek(): 127 | # moveConfigSection('Misc', 'CalendarOptions', 'firstDayOfWeek') 128 | return int(config.get('CalendarOptions', 'firstDayOfWeek')) 129 | 130 | def getcalendarFile(): 131 | # moveConfigSection('Misc','CalendarOptions', 'calendarFile') 132 | return config.get('CalendarOptions', 'calendarFile') 133 | 134 | def isEntryTimestamped(): 135 | return config.get('Bot', 'timestampEntries', fallback='true') == 'true' 136 | 137 | def getCommandsMap(): 138 | from ast import literal_eval 139 | return literal_eval(config.get('Misc', 'CommandsMap', fallback="{'TODO':'TODO', 'LATER':'LATER'}")) 140 | 141 | def getMonths2Generate(): 142 | return config.get('CalendarOptions', 'generateMonths', fallback="1,1").split(',') 143 | 144 | 145 | def moveConfigSection(oldSection, newSection, key): 146 | try: 147 | keyVal = config.get(oldSection, key) 148 | config.remove_option(oldSection,key) 149 | 150 | try: 151 | config.add_section(newSection) 152 | except: 153 | pass 154 | 155 | config.set(newSection, key, keyVal) 156 | # with open('config.ini', 'r') as configfile: 157 | # config.write(configfile) 158 | 159 | except: 160 | pass 161 | def getAgePublicKey(): 162 | return config.get('AgeEncryption', 'AgePublicKey') 163 | 164 | def generateAgeKeyFile(): 165 | keys_filename = os.path.expanduser("~/.config/age/keys.txt") 166 | KEYFILE = "# created: 2020-02-25T00:00:00\n# {}\n{}\n".format(getAgePublicKey(),config.get('AgeEncryption', 'AgePrivateKey')) 167 | 168 | # with open(keys_filename, 'w') as f: 169 | f = open(keys_filename, 'w') 170 | f.write(KEYFILE) 171 | f.close() 172 | 173 | def isGraphAgeEncrypted(): 174 | return config.get('AgeEncryption', 'AgeEncrypted', fallback='false') == 'true' 175 | 176 | def setGraphAgeEncrypted(state): 177 | config.set('AgeEncryption', 'AgeEncrypted', state) 178 | # with open('config.ini', 'r') as configfile: 179 | # config.write(configfile) 180 | -------------------------------------------------------------------------------- /config.sample.ini: -------------------------------------------------------------------------------- 1 | [Bot] 2 | ; Telegram Bot Toek 3 | BotToken = 4 | ; list of authorized Chat IDs to use this bot (comma separated) 5 | BotAuthorizedIDs = 6 | ; Give your bot a name 7 | BotName = Lupin 8 | ; Set to GitHub to upload to the same Gihub Rep embedded images will only be shown on the Desktop Application 9 | assetsDestination = GitHub 10 | ; set to false if you don't want your Journal Entries to have a timestamp 11 | timestampEntries = true 12 | 13 | [GitHub] 14 | GitHubToken = 15 | GitHubBranch = master 16 | ; Git hub username 17 | GitHubUser = 18 | ; LogSeq GitHub repo 19 | GitHubRepo = LogSeqPYBot 20 | ; Name and email used to push 21 | GitHubAuthor = 22 | GitHubEmail = 23 | ; Frequency, in minutes, you want to Lupin to pull new data from Github 24 | GitHubUpdateFrequency = 720 25 | 26 | [Misc] 27 | ; change to false to set time to AM / PM 28 | hour24 = true 29 | ; What indent level you want your entries to be at by default 30 | defaultIndentLevel = ## 31 | ; format of your journal files default 2021_02_06 32 | journalsFilesFormat = %Y_%m_%d 33 | ; journal files extension .md / .org 34 | journalsFilesExtension = .md 35 | ; change this value if you want your mobile Journal to reside in a different folder 36 | journalsFolder = journals 37 | ; change this if you want assests uploaded using Lupin to land in a different folder 38 | assetsFolder = assets 39 | ; fill this if you have a custom journal template 40 | journalTemplate = 41 | ; option to specify a custom date format 42 | dateFormat = %b {th}, %Y 43 | ; change this value if you want your mobile Journal entries not to go into your regular journal (default value none) 44 | journalsPrefix = none 45 | ; Change this to the tag you want to add to the links you feed to Lupin 46 | BookmarkTag = bookmark 47 | ; 48 | CommandsMap = {'T':'TODO', 'L':'LATER'} 49 | 50 | [hypothesis] 51 | ; you hypothes.is username add to it @hypothes.is 52 | hypothesisUsername = username@hypothes.is 53 | ; when set to [[]] tags with space will be imported as #[[some tag]] 54 | ; when set to - tags with space will be imported as #some-tag 55 | hypothesisTagSpaceHandler = [[]] 56 | ; change to false if you don't mange LUPIN to manage annotation updates 57 | manageHypothesisUpdates = true 58 | ; only valid if manageHypothesisUpdates == true; set to false to link instead of embed 59 | embedHypothesisAnnotations = true 60 | 61 | [Firebase] 62 | ; set this to your firebase bucket name 63 | BucketName = BucketName.appspot.com 64 | 65 | [AgeEncryption] 66 | ; change to true if your graph has encryption enabled 67 | AgeEncrypted = false 68 | ; Your Age Encryption Public Key 69 | AgePublicKey = age1LongPublicKey 70 | ; Your Age Encryption Private Key 71 | AgePrivateKey = AGE-SECRET-KEY-LongPrivateKey 72 | 73 | [CalendarOptions] 74 | ; set to false if you don't want Lupin to auto generate calendars for you 75 | autoGenerateCalendars = true 76 | ; Defines the first day of the week for the calendar 0: Mon | 1: Tue | ... | 6: Sun 77 | firstDayOfWeek = 0 78 | ; File where calendars should be inserted 79 | calendarFile = contents.md 80 | ; first [0|1] specifies if you want LUPIN to generate prev month, second [0|1] specifies if you want LUPIN to generate next month 81 | ; 1,1 will generate prev,cur,next | 0,1 will generate curr,next | 0,0 will only generate curr 82 | generateMonths = 1,1 83 | -------------------------------------------------------------------------------- /dictionaries.py: -------------------------------------------------------------------------------- 1 | bot_messages = { 2 | 'UNAUTHORIZED_MESSAGE' : 'Your ID is not Authorized, add {} to BotAuthorizedIDs inside config.ini & restart me', 3 | 'WELCOME_MESSAGE' : 'Hey, I\'m {} visit https://github.com/akhater/Lupin & get to know me better', 4 | 'JOURNALENTRY_MESSAGE' : 'Entry \"{}\" added to Today\'s Journal', 5 | 'IMAGEUPLOAD_MESSAGE' : 'The image is now uploaded and linked in Today\'s Journal', 6 | 'HYPOTHESIS_MESSAGE' : 'Annotations for \"{}\" added to Today\'s Journal', 7 | 'VER_MESSAGE' : 'Congrats, you are running {} version {}', 8 | 'HELP_MESSAGE' : 'List of Available Commands\n', 9 | 'VERCHANGE_MESSAGE' : 'Hey, I have detected a version change since our last encounter, your config.ini file may need updating. \ 10 | \nMake sure to visit https://github.com/akhater/Lupin for latest news', 11 | 'NOPENDIGCARDS_MESSAGE' : '🎊🎈🎉🥳 You have no pending cards in your queue 🥳🎉🎈🎊', 12 | 'NEXTROUND_MESSAGE' : 'Good work ! This card will reappear starting: ', 13 | 'IMPORTINGFC_MESSAGE' : 'Scanning and importing your Git Repo for flashcards, I\'ll let you know once done...', 14 | 'FILEREQ_MESSAGE' : 'Your file is being generated, I will send it to you once ready, hang-on ...', 15 | 'IMPORTEDFC_MESSAGE' : '{} flaschard(s) added & {} flaschard(s) updated.\nTest your knowledge now by running /tsr', 16 | 'FLASHCARD_OPTIONS' : 'Incorrect [ 😭:Hardest | 😖:Hard | 😕: Medium ], Correct [😊:Medium | 😄: Easy | 🥳: Easiest]', 17 | 'CANCELLED_MESSAGE' : 'Cancelled', 18 | 'SKIPPED_MESSAGE' : 'Skipped', 19 | 'FLASHCARD_SOURCE' : 'Flashcard source file ', 20 | 'FILENOTFOUND_MESSAGE' : '{} was not found, if you have recently created the resouce you are looking for please run /pullnow first', 21 | 'PULL_MESSAGE' : 'Pulling latest updates from your Git, I\'ll work in the background, you won\'t even notice me, I\'ll let you know once done...', 22 | 'PULLDONE_MESSAGE' : 'I\'ve updated the latest content from your Git', 23 | 'LINE_BREAK' : '_______________________________________', 24 | 'PICKTHEME_MESSAGE' : 'Pick your next theme ...', 25 | 'THEMESWITCHED_MESSAGE' : 'I\'ve set your theme to {}, make sure to pull & refresh', 26 | 'NOPAGEMM_MESSAGE' : 'I need the title of the page you want to convert to a MindMap, correct syntax is /getMM pageTitle', 27 | 'ENCRYPTINGRAPH_MESSAGE' : 'Encrypting your Graph, please be patient, this may take a little while, I\'ll let you know once it is done', 28 | 'DECRYPTINGRAPH_MESSAGE' : 'Decrypting your Graph, please be patient, this may take a little while, I\'ll let you know once it is done', 29 | 'ENCRYPTDONE_MESSAGE' : 'All your graph contents are now encrypted & I\'ve set AgeEncrypted to true in your config.ini', 30 | 'DECRYPTDONE_MESSAGE' : 'All your graph contents are now decrypted & I\'ve set AgeEncrypted to false in your config.ini' 31 | } 32 | 33 | git_messages = { 34 | 'COMMIT_MESSAGE' : 'Added by {} on {}' 35 | } 36 | 37 | btns = { 38 | 'SHOW_ANSWER' : 'Show Answer', 39 | 'SKIP' : 'Skip ...', 40 | 'CANCEL' : 'Cancel' 41 | } -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: "3.2" 2 | services: 3 | lupin: 4 | container_name: lupin 5 | image: digitalknk/lupin 6 | volumes: 7 | - type: bind 8 | source: ./config.ini 9 | target: /app/config.ini 10 | environment: 11 | TZ: America/New_York 12 | restart: unless-stopped 13 | -------------------------------------------------------------------------------- /flashcards.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import pickle 3 | from hashlib import md5 4 | from os import path 5 | 6 | import sm2 7 | 8 | from random import randint 9 | 10 | from config import getflashcardsTag 11 | import utils 12 | 13 | class Flashcard: 14 | 15 | def __init__(self, question, answer, source): 16 | self.question = question 17 | self.answer = answer 18 | self.source = source 19 | 20 | 21 | self.next = datetime.datetime(2021, 1, 1).timestamp() 22 | self.lastAnswered = datetime.datetime(2021, 1, 1).timestamp() 23 | self.history = [] 24 | 25 | def __repr__(self): 26 | a = [self.question, self.answer, self.next, self.source, self.history] 27 | return str(a) #"[% s ][% s]" % (self.question, self.answer) 28 | 29 | def updateProperties(self, next, history): 30 | self.next = next 31 | self.history = history 32 | 33 | SEPARATOR = "#" 34 | flashcardsDB = "flashcards.db" 35 | 36 | def scan4Flashcards(content): 37 | Qlist = [] 38 | buildFlashcardList(content, Qlist) 39 | return (Qlist) 40 | 41 | def countIdent(line): # TODO change that to .index and merge with utils function 42 | count = 0 43 | if(len(line) > 0): 44 | while line[count] == SEPARATOR: 45 | if(count == len(line) - 1): 46 | break 47 | count += 1 48 | return count 49 | 50 | def buildFlashcardList(content, Qlist): 51 | lines = content.split('\n') 52 | i = 0 53 | source = "" 54 | while i <= len(lines) - 1: 55 | if 'title:' in (lines[i]).lower(): 56 | source = lines[i].strip() 57 | if getflashcardsTag() in lines[i]: 58 | flashcardIndent = countIdent(lines[i]) 59 | isSub = True 60 | i += 1 61 | flashcard = Flashcard("-1", "-1", source) 62 | while isSub: 63 | if( i == len(lines)): 64 | break 65 | 66 | currentIdent = countIdent(lines[i]) 67 | if(currentIdent == flashcardIndent + 1): 68 | if(flashcard.question != "-1"): 69 | Qlist.append(flashcard) 70 | flashcard = Flashcard("-1", "-1", source) 71 | flashcard.question = lines[i][currentIdent:].strip() 72 | i += 1 73 | elif(currentIdent > flashcardIndent + 1): # scan for answer 74 | blockRef = utils.containsRefBlock(lines[i]) 75 | answer = "" 76 | if (blockRef): 77 | origLine = (lines[i][currentIdent:]).replace("(("+blockRef+"))","").strip() 78 | if(origLine): 79 | answer = origLine + " " 80 | answer += utils.findOrigBlock(blockRef) 81 | else: 82 | answer = lines[i][currentIdent:] 83 | if(flashcard.answer == "-1"): 84 | flashcard.answer = "" 85 | flashcard.answer += answer.strip() + "\n" 86 | i += 1 87 | else: 88 | isSub = False 89 | i -= 1 90 | if(flashcard.answer != "-1"): 91 | Qlist.append(flashcard) 92 | i += 1 93 | 94 | def saveFlashcardsDB(flashcardList, dump=False): 95 | if(dump): # don't check for differences 96 | with open(flashcardsDB, "wb") as fp: # Pickling 97 | pickle.dump(flashcardList, fp) 98 | print (str(len(flashcardList)) + " cards saved") 99 | elif(not(path.exists(flashcardsDB))): # new DB 100 | with open(flashcardsDB, "wb") as fp: # Pickling 101 | pickle.dump(flashcardList, fp) 102 | return [str(len(flashcardList)), "0" ] 103 | else: # updating exsiting DB 104 | savedDB = loadFlashcardsDB() 105 | tmpset = set((x.question) for x in savedDB) 106 | newFlashcards = [ x for x in flashcardList if (x.question) not in tmpset ] 107 | tmpset = set((x.question, x.answer) for x in savedDB) 108 | updatedFlashcards = [ x for x in flashcardList if ((x.question, x.answer) not in tmpset and x not in newFlashcards)] 109 | if updatedFlashcards: # no need to check for new since they will be saved with the updated ones (if any) 110 | for updatedFlashcard in updatedFlashcards: 111 | cardDetails = getFlashcardDetails(updatedFlashcard.question, savedDB) 112 | cardIndex = flashcardList.index(updatedFlashcard) 113 | print(cardDetails.index) 114 | flashcardList[cardIndex].updateProperties(cardDetails[0].next, cardDetails[0].history) 115 | print(flashcardList[cardIndex]) 116 | # print(updatedAnswers) 117 | saveFlashcardsDB(flashcardList, True) 118 | elif newFlashcards: 119 | savedDB += newFlashcards 120 | saveFlashcardsDB(savedDB, True) 121 | print(newFlashcards) 122 | 123 | return [str(len(newFlashcards)), str(len(updatedFlashcards)) ] 124 | 125 | def loadFlashcardsDB(): 126 | with open(flashcardsDB, "rb") as fp: # Unpickling 127 | db = pickle.load(fp) 128 | return db 129 | 130 | def getFlashcardDetails(question, flashcardList = ""): 131 | if (not(flashcardList)): 132 | flashcardList = loadFlashcardsDB() # in saved DB 133 | #print (flashcardList[0]) 134 | return [x for x in flashcardList if x.question == question] 135 | 136 | def updateFlashcard(flaschard): 137 | flashcardsDB = loadFlashcardsDB() 138 | flaschard.lastAnswered = (datetime.datetime.now()).timestamp() 139 | flaschard.next = flaschard.lastAnswered + sm2.supermemo_2(flaschard.history) * 86400 140 | cardIndex = next(i for i, x in enumerate(flashcardsDB) if x.question == flaschard.question) 141 | print(flashcardsDB[cardIndex]) 142 | print(datetime.datetime.fromtimestamp(flashcardsDB[cardIndex].next) ) 143 | flashcardsDB[cardIndex] = flaschard 144 | print(flashcardsDB[cardIndex]) 145 | print(datetime.datetime.fromtimestamp(flashcardsDB[cardIndex].next) ) 146 | saveFlashcardsDB(flashcardsDB, True) 147 | 148 | return datetime.datetime.fromtimestamp(flaschard.next).strftime("%Y-%m-%d") 149 | 150 | def getFlashcardFromPool(): 151 | flashcardsList = loadFlashcardsDB() 152 | tsToday = (datetime.datetime.now()).timestamp() 153 | overdueFC = [ x for x in flashcardsList if (x.next) <= tsToday ] # get overdue FC 154 | if overdueFC: 155 | return(overdueFC[randint(0,len(overdueFC)-1)]) 156 | else: 157 | return None 158 | 159 | 160 | -------------------------------------------------------------------------------- /git.py: -------------------------------------------------------------------------------- 1 | from github import Github, InputGitAuthor 2 | from pprint import pprint 3 | #import json 4 | 5 | import config 6 | import utils 7 | import AgeEncHandler 8 | 9 | from dictionaries import git_messages 10 | import flashcards 11 | 12 | GitHubToken = config.GitHubToken 13 | GitHubFullRepo = config.GitHubUser + "/" + config.GitHubRepo 14 | GitHubBranch = config.GitHubBranch 15 | BotName = config.BotName 16 | # TODOCommand = config.TODOCommand 17 | assetsFolder = config.getAssetsFolder() 18 | 19 | g = Github(GitHubToken) 20 | repo = g.get_repo(GitHubFullRepo) 21 | 22 | def push(path, message, content, branch, update=False): 23 | author = InputGitAuthor( 24 | config.GitHubAuthor, 25 | config.GitHubEmail 26 | ) 27 | #source = repo.get_Branch(Branch) 28 | #repo.create_git_ref(ref=f"refs/heads/{Branch}", sha=source.commit.sha) # Create new Branch from master 29 | if update: # If file already exists, update it 30 | #pass 31 | contents = repo.get_contents(path, ref=branch) # Retrieve old file to get its SHA and path 32 | repo.update_file(contents.path, message, content, contents.sha, branch=branch, author=author) # Add, commit and push Branch 33 | else: # If file doesn't exist, create it 34 | #pass 35 | repo.create_file(path, message, content, branch=branch, author=author) # Add, commit and push Branch 36 | 37 | def updateJournal(entry, needsBuilding=True, path=None, overwrite=False, alias='', ignoreURL=False, isJournalFile=True): 38 | if path == None: 39 | path = utils.getJournalPath() 40 | if needsBuilding: 41 | entry = buildJournalEntry(entry, ignoreURL) 42 | if(GitFileExists(path)): 43 | file = repo.get_contents(path, ref=GitHubBranch) # Get file from Branch 44 | if(overwrite): 45 | data = "---\ntitle: " + utils.getPageTitle(path) + "\nalias: " + alias + "\n---\n\n" 46 | else: 47 | data = file.decoded_content.decode("utf-8") # Get raw string data 48 | if(config.isGraphAgeEncrypted()): 49 | if AgeEncHandler.isAgeEncrypted(data) == 1: # encrypted but without \n requires transformation 50 | data = (AgeEncHandler.ageDecrypt(AgeEncHandler.convertToAgeString(data))) 51 | elif AgeEncHandler.isAgeEncrypted(data) == 2: # encrypted no transformation required 52 | data = (AgeEncHandler.ageDecrypt(data)) 53 | 54 | 55 | data += (entry).strip() + "\n" 56 | 57 | if(config.isGraphAgeEncrypted()): 58 | data = AgeEncHandler.ageEncrypt(data) 59 | 60 | push(path, git_messages['COMMIT_MESSAGE'].format(BotName, utils.getTimestamp()) , data, GitHubBranch, update=True) 61 | else: 62 | if isJournalFile: 63 | journalTemplate = config.journalTemplate 64 | if journalTemplate: 65 | data = "---\ntitle: " + utils.getJournalTitle() + "\n---\n\n" + journalTemplate + (entry).strip() + "\n" 66 | else: 67 | data = "---\ntitle: " + utils.getJournalTitle() + "\n---\n\n" + (entry).strip() + "\n" 68 | else: 69 | data = "---\ntitle: " + utils.getPageTitle(path) + "\nalias: " + alias + "\n---\n\n" + (entry).strip() + "\n" 70 | 71 | if(config.isGraphAgeEncrypted()): 72 | data = AgeEncHandler.ageEncrypt(data) 73 | push(path, git_messages['COMMIT_MESSAGE'].format(BotName, utils.getTimestamp()) , data, GitHubBranch, update=False) 74 | 75 | def GitFileExists(path): 76 | try: 77 | repo.get_contents(path, ref=GitHubBranch) # Get file from Branch 78 | return True 79 | except Exception as e: 80 | if (e.args[0] == 404): 81 | print (e.args[0]) 82 | return False 83 | 84 | def buildJournalEntry(entry, ignoreURL): 85 | journalEntry = "" 86 | 87 | currentTime = utils.getCurrentTime() 88 | if currentTime: 89 | currentTime += " " 90 | else: 91 | currentTime = "" 92 | 93 | # print(processCommandsMapping('21:40 some non todo entry T')) 94 | 95 | journalEntry = config.defaultIndentLevel + " " + utils.processCommandsMapping(currentTime + entry) 96 | # if(TODOCommand in entry): 97 | # journalEntry = config.defaultIndentLevel + " TODO " + currentTime + entry.replace(TODOCommand,'') 98 | # else: 99 | # journalEntry = config.defaultIndentLevel + " " + currentTime + entry 100 | 101 | if(not(ignoreURL)): 102 | # print(entry) 103 | journalEntryURL = utils.containsYTURL(entry) 104 | # print (journalEntryURL) 105 | if(journalEntryURL): 106 | #title = getWebPageTitle(journalEntryURL) 107 | journalEntry = journalEntry.replace(journalEntryURL, '{{youtube ' + journalEntryURL +'}}') 108 | else: 109 | journalEntryURL = utils.containsTWUrl(entry) 110 | if(journalEntryURL): 111 | journalEntry = utils.generateTwitterIframe(journalEntryURL) 112 | else: 113 | journalEntryURL = utils.containsURL(entry) 114 | if(journalEntryURL): 115 | title = utils.getWebPageTitle(journalEntryURL) 116 | if(config.journalsFilesExtension == '.md'): 117 | journalEntry = journalEntry.replace(journalEntryURL, '#' + config.BookmarkTag + ' [' + title + '](' + journalEntryURL + ')') 118 | elif(config.journalsFilesExtension == '.org'): 119 | journalEntry = journalEntry.replace(journalEntryURL, '#' + config.BookmarkTag + ' [[' + journalEntryURL + '][' + title + ']]') 120 | 121 | 122 | print (journalEntry) 123 | return journalEntry 124 | 125 | def updateAsset(data, fileType): 126 | print('u') 127 | path = assetsFolder + "/" + utils.getTimestamp(True,True) + "." + fileType 128 | print(config.getAssetsDestination()) 129 | if(config.getAssetsDestination() == 'github'): 130 | update = False 131 | if(GitFileExists(path)): 132 | update = True 133 | push(path, git_messages['COMMIT_MESSAGE'].format(BotName, utils.getTimestamp()) , data, GitHubBranch, update=update) 134 | path = ("![](./" + path + ")") 135 | elif(config.getAssetsDestination() == 'firebase'): 136 | path = ("![](" + utils.UploadToFirebase(data, path) + ")") 137 | 138 | return path 139 | 140 | def getGitFileContent(file, fetchContent = False): 141 | if (fetchContent): 142 | file = repo.get_contents(file, ref=GitHubBranch) # Get file from Branch 143 | try: 144 | content = file.decoded_content.decode("utf-8") # Get raw string data 145 | if(config.isGraphAgeEncrypted()): 146 | if AgeEncHandler.isAgeEncrypted(content) == 1: # encrypted but without \n requires transformation 147 | return (AgeEncHandler.ageDecrypt(AgeEncHandler.convertToAgeString(content))) 148 | elif AgeEncHandler.isAgeEncrypted(content) == 2: # encrypted no transformation required 149 | return (AgeEncHandler.ageDecrypt(content)) 150 | else: # not encrypted 151 | return content 152 | else: 153 | return content 154 | except Exception as e: 155 | print(e) 156 | return None 157 | 158 | def scanGit4Flashcards(path=""): 159 | contents = repo.get_contents(path) 160 | flashcardsList = [] 161 | #print (contents) 162 | 163 | while contents: 164 | content = contents.pop(0) 165 | # print(content.url) 166 | if '/assets/' not in content.url: #TODO change to assetsfolder 167 | if content.type == "dir": 168 | contents.extend(repo.get_contents(content.path)) 169 | else: 170 | #pass 171 | #file = content 172 | flashcardsList += flashcards.scan4Flashcards( getGitFileContent(content) ) 173 | return(flashcardsList) 174 | 175 | def updateFlashCards(): 176 | return flashcards.saveFlashcardsDB( scanGit4Flashcards() ) 177 | 178 | def Git2Json(path=""): 179 | AllFilesContent = [] 180 | contents = repo.get_contents(path) 181 | 182 | while contents: 183 | content = contents.pop(0) 184 | # print("fetching " + content.path) 185 | if '/assets/' not in content.url: 186 | if content.type == "dir": 187 | contents.extend(repo.get_contents(content.path)) 188 | else: 189 | gitFileContent = getGitFileContent(content) 190 | if gitFileContent: 191 | AllFilesContent.append(gitFileContent) 192 | 193 | utils.saveasJson(AllFilesContent,"/tmp/GitDump.json") 194 | 195 | def updateCalendarsFile(): 196 | path = "pages/" + config.getcalendarFile() 197 | contents = getGitFileContent(path, True) 198 | 199 | contents = utils.generateCalendarsFile(contents) 200 | if(config.isGraphAgeEncrypted()): 201 | contents = AgeEncHandler.ageEncrypt(contents) 202 | push(path, git_messages['COMMIT_MESSAGE'].format(BotName, utils.getTimestamp()) , contents, GitHubBranch, update=True) 203 | 204 | def getAllThemes(): 205 | AllThemes = [] 206 | contents = repo.get_contents('/logseq') 207 | while contents: 208 | content = contents.pop(0) 209 | if 'custom.css' in content.path: 210 | if content.path != "logseq/custom.css": 211 | entry = [content.path.replace('logseq/','').replace('.custom.css',''), content] 212 | AllThemes.append(entry) 213 | 214 | return(AllThemes) 215 | 216 | def switchTheme(cssFile): 217 | cssContent = getGitFileContent(cssFile) 218 | push('logseq/custom.css', git_messages['COMMIT_MESSAGE'].format(BotName, utils.getTimestamp()) , cssContent, GitHubBranch, update=True) 219 | 220 | 221 | # a = getAllThemes() 222 | # print(a[0][1]) 223 | # switchTheme(a[0][1]) 224 | def encryptGraph(): 225 | contents = repo.get_contents("") 226 | 227 | while contents: 228 | content = contents.pop(0) 229 | 230 | if '/assets/' not in content.url and '/logseq/' not in content.url: 231 | if content.type == "dir": 232 | contents.extend(repo.get_contents(content.path)) 233 | else: 234 | gitFileContent = getGitFileContent(content) 235 | if gitFileContent: 236 | print("encrypting " + content.path) 237 | try: 238 | encContents = (AgeEncHandler.ageEncrypt(gitFileContent)) 239 | push(content.path, git_messages['COMMIT_MESSAGE'].format(BotName, utils.getTimestamp()) , encContents, GitHubBranch, update=True) 240 | except: 241 | print("***********" + content.path + "*******************") 242 | # print(content.path) 243 | print("*********** All Files Decrytped *******************") 244 | config.setGraphAgeEncrypted('true') 245 | 246 | def decryptGraph(): 247 | contents = repo.get_contents("") 248 | 249 | while contents: 250 | content = contents.pop(0) 251 | 252 | if '/assets/' not in content.url and '/logseq/' not in content.url: 253 | if content.type == "dir": 254 | contents.extend(repo.get_contents(content.path)) 255 | else: 256 | gitFileContent = getGitFileContent(content) 257 | if gitFileContent: 258 | print("decrypting " + content.path) 259 | try: 260 | push(content.path, git_messages['COMMIT_MESSAGE'].format(BotName, utils.getTimestamp()) , gitFileContent, GitHubBranch, update=True) 261 | except: 262 | print("***********" + content.path + "*******************") 263 | # print(content.path) 264 | print("*********** All Files Encrypted *******************") 265 | config.setGraphAgeEncrypted('false') 266 | -------------------------------------------------------------------------------- /hypothesis.py: -------------------------------------------------------------------------------- 1 | import requests 2 | # add / to the uri to make exclude sub pages 3 | from config import hypothesisToken, hypothesisUsername, defaultIndentLevel, isManageHypothesis, getHypothesisTagSpaceHandler 4 | from utils import getWebPageTitle 5 | from itertools import groupby 6 | 7 | defaultIndentLevel = defaultIndentLevel + " " 8 | 9 | def byURI(data): 10 | return data['uri'] 11 | 12 | def getHypothesisAnnotations(targetURI): 13 | 14 | headers = {"Authorization": "Bearer " + hypothesisToken} 15 | # targetURI = "https://web.hypothes.is" 16 | endpoint = "https://api.hypothes.is/api/search?url=" + targetURI + "&limit=200&order=asc&user=acct:" + hypothesisUsername 17 | 18 | #print(endpoint) 19 | r = requests.get(endpoint, headers=headers).json() 20 | 21 | # headers = {"Authorization": "Bearer " + hypothesisToken} 22 | # targetURI = "https://web.hypothes.is" 23 | # endpoint = "https://api.hypothes.is/api/search?url=" + targetURI + "&limit=200&user=acct:" + hypothesisUsername 24 | 25 | grouped_byURI = groupby(sorted(r['rows'], key=byURI), byURI) 26 | 27 | groupedByURI = {} 28 | 29 | for k, v in grouped_byURI: 30 | groupedByURI[k] = list(v) 31 | 32 | # Title r['https://web.hypothes.is/'][0]['document']['title'] 33 | # Highlighted r['https://web.hypothes.is/'][0]['target'][0]['selector'][2]['exact'] 34 | # annotation r['https://web.hypothes.is/'][0]['text'] 35 | # tags r['https://web.hypothes.is/'][0]['tags'] 36 | 37 | #print(groupedByURI) 38 | outText = "" 39 | for row in groupedByURI: 40 | #print(row) 41 | outText += defaultIndentLevel + "[" + getWebPageTitle(row) + "](" + row + ")" + "\n" 42 | #print(title) 43 | # print((groupedByURI[row][0]['target'][0]['selector'][2]['exact']).strip()) 44 | for i in range(len(groupedByURI[row])): 45 | outText += "#" + defaultIndentLevel + (groupedByURI[row][i]['target'][0]['selector'][2]['exact']).strip() + "\n" 46 | 47 | if groupedByURI[row][i]['text']: 48 | annotation = groupedByURI[row][i]['text'] + "\n" 49 | else: 50 | annotation = "" 51 | 52 | tags = "" 53 | for tag in groupedByURI[row][i]['tags']: 54 | if (' ' in tag): 55 | if(getHypothesisTagSpaceHandler() == "[[]]"): 56 | tag = "[[" + tag + "]]" 57 | else: 58 | tag = tag.replace(' ', '-') 59 | tags += "#" + tag + " " 60 | 61 | if annotation: 62 | outText += "##" + defaultIndentLevel + annotation 63 | 64 | if tags: 65 | tags += "\n" 66 | 67 | if annotation: 68 | outText += "###" + defaultIndentLevel + tags 69 | else: 70 | outText += "##" + defaultIndentLevel + tags 71 | 72 | #print(outText) 73 | return outText 74 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | from telegram.ext import CallbackQueryHandler, CommandHandler, Filters, MessageHandler, PicklePersistence, Dispatcher 2 | from telegram import InlineKeyboardButton, InlineKeyboardMarkup, Update, Message, Bot 3 | from queue import Queue 4 | from io import BytesIO, StringIO 5 | from uuid import uuid4 6 | 7 | # from flashcards import updateFlashcard, getFlashcardFromPool 8 | import flashcards 9 | 10 | from config import ( 11 | BotToken, isBotAuthorized, BotName, GitHubBranch, getBotVersion, isNewer, getBotAuthorizedIDs, isGraphAgeEncrypted, generateAgeKeyFile, 12 | isManageHypothesis, isHypothesisEmbedded, getflashcardDailyGoal, getGitHubUpdateFrequency, isCalendarsAutogenerated 13 | ) 14 | 15 | from dictionaries import bot_messages, btns 16 | from git import updateJournal, updateAsset, updateCalendarsFile, encryptGraph, decryptGraph, getAllThemes, switchTheme, Git2Json 17 | from utils import getUptime, getAnnotationPath, getPageTitle, getWebPageTitle, getlatestNews, updateFlashCards, convert2MD, convert2Mindmap 18 | from hypothesis import getHypothesisAnnotations 19 | 20 | 21 | def start(update, context): 22 | if(not isBotAuthorized(update.effective_chat.id)): 23 | context.bot.send_message(chat_id=update.effective_chat.id, text=bot_messages['UNAUTHORIZED_MESSAGE'].format(update.effective_chat.id)) 24 | else: 25 | context.bot.send_message(chat_id=update.effective_chat.id, text=bot_messages['WELCOME_MESSAGE'].format(BotName)) 26 | 27 | def uptime(update, context): 28 | if(not isBotAuthorized(update.effective_chat.id)): 29 | context.bot.send_message(chat_id=update.effective_chat.id, text=bot_messages['UNAUTHORIZED_MESSAGE'].format(update.effective_chat.id)) 30 | else: 31 | message = "I've been up for %d days, %d hours, %d minutes, %d seconds" % getUptime() 32 | context.bot.send_message(chat_id=update.effective_chat.id, text=message) 33 | 34 | def addEntry(update, context): 35 | if(not isBotAuthorized(update.effective_chat.id)): 36 | context.bot.send_message(chat_id=update.effective_chat.id, text=bot_messages['UNAUTHORIZED_MESSAGE'].format(update.effective_chat.id)) 37 | else: 38 | updateJournal(update.message.text) 39 | context.bot.send_message(chat_id=update.effective_chat.id, 40 | text=bot_messages['JOURNALENTRY_MESSAGE'].format(update.message.text)) 41 | 42 | def version(update, context): 43 | if(not isBotAuthorized(update.effective_chat.id)): 44 | context.bot.send_message(chat_id=update.effective_chat.id, text=bot_messages['UNAUTHORIZED_MESSAGE']) 45 | else: 46 | context.bot.send_message(chat_id=update.effective_chat.id, text=bot_messages['VER_MESSAGE'].format(BotName,getBotVersion())) 47 | 48 | def help(update, context): 49 | if(not isBotAuthorized(update.effective_chat.id)): 50 | context.bot.send_message(chat_id=update.effective_chat.id, text=bot_messages['UNAUTHORIZED_MESSAGE']) 51 | else: 52 | commands = ["/help","/start","/uptime","/ver","/anno"] 53 | message = bot_messages['HELP_MESSAGE'] 54 | for command in commands: 55 | message += command + "\n" 56 | 57 | context.bot.send_message(chat_id=update.effective_chat.id, text=message) 58 | 59 | def hypothesis(update, context): 60 | if(not isBotAuthorized(update.effective_chat.id)): 61 | context.bot.send_message(chat_id=update.effective_chat.id, text=bot_messages['UNAUTHORIZED_MESSAGE'].format(update.effective_chat.id)) 62 | else: 63 | if(isManageHypothesis()): 64 | path = getAnnotationPath(context.args[0]) 65 | #print(path) 66 | pageAlias = getWebPageTitle(context.args[0]) 67 | 68 | updateJournal(entry=getHypothesisAnnotations(context.args[0]), needsBuilding=False, path=path, overwrite=True, alias=pageAlias, isJournalFile=False) 69 | 70 | if(isHypothesisEmbedded()): 71 | updateJournal(entry='{{embed [[' + getPageTitle(path) + ']]}}', isJournalFile=True) 72 | else: 73 | updateJournal(entry="Annotations of [" + pageAlias + "](" + getPageTitle(path) + ")", isJournalFile=True) 74 | else: 75 | updateJournal(getHypothesisAnnotations(context.args[0]), False, isJournalFile=False) 76 | 77 | context.bot.send_message(chat_id=update.effective_chat.id, 78 | text=bot_messages['HYPOTHESIS_MESSAGE'].format(context.args[0])) 79 | 80 | def image_handler(update, context): 81 | if(not isBotAuthorized(update.effective_chat.id)): 82 | context.bot.send_message(chat_id=update.effective_chat.id, text=bot_messages['UNAUTHORIZED_MESSAGE'].format(update.effective_chat.id)) 83 | else: 84 | # print ( context.bot.getFile(update.message.photo[-1])) 85 | file = context.bot.getFile(update.message.photo[-1].file_id) 86 | f = BytesIO(file.download_as_bytearray()) 87 | path = updateAsset(f.getvalue(),"jpg") 88 | updateJournal(path, ignoreURL=True, isJournalFile=False) 89 | 90 | context.bot.send_message(chat_id=update.effective_chat.id, text=bot_messages['IMAGEUPLOAD_MESSAGE'].format(BotName,getBotVersion())) 91 | 92 | def generateMD(update, context): 93 | if(not isBotAuthorized(update.effective_chat.id)): 94 | context.bot.send_message(chat_id=update.effective_chat.id, text=bot_messages['UNAUTHORIZED_MESSAGE'].format(update.effective_chat.id)) 95 | else: 96 | context.bot.send_message(chat_id=update.effective_chat.id, text=bot_messages['FILEREQ_MESSAGE']) 97 | PageName = str(' '.join(context.args)) 98 | 99 | s = StringIO() 100 | s.write(convert2MD(PageName)) 101 | s.seek(0) 102 | 103 | buf = BytesIO() 104 | buf.write(s.getvalue().encode()) 105 | buf.seek(0) 106 | #buf.name = f'PageName.md' 107 | buf.name = f'{PageName}.md' 108 | 109 | context.bot.send_document(chat_id=update.message.chat_id, document=buf) 110 | 111 | def generateMinmapHTML(update, context): 112 | if(not isBotAuthorized(update.effective_chat.id)): 113 | context.bot.send_message(chat_id=update.effective_chat.id, text=bot_messages['UNAUTHORIZED_MESSAGE'].format(update.effective_chat.id)) 114 | else: 115 | PageName = str(' '.join(context.args)) 116 | if PageName: 117 | context.bot.send_message(chat_id=update.effective_chat.id, text=bot_messages['FILEREQ_MESSAGE']) 118 | 119 | 120 | MindMap = convert2Mindmap(PageName) 121 | if(MindMap): 122 | HTMLOut = """ 123 | 124 | 125 | 126 | 127 | 128 | 129 | """ + PageName + """ Mindmap 130 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 149 | 150 | """ 151 | 152 | s = StringIO() 153 | s.write(HTMLOut) 154 | s.seek(0) 155 | 156 | buf = BytesIO() 157 | buf.write(s.getvalue().encode()) 158 | buf.seek(0) 159 | fileName = "mm_" + PageName.replace(' ','_').strip() 160 | buf.name = f'{fileName}.html' 161 | 162 | context.bot.send_document(chat_id=update.message.chat_id, document=buf) 163 | else: 164 | context.bot.send_message(chat_id=update.effective_chat.id, text=bot_messages['FILENOTFOUND_MESSAGE'].format(PageName)) 165 | else: 166 | context.bot.send_message(chat_id=update.effective_chat.id, text=bot_messages['NOPAGEMM_MESSAGE']) 167 | 168 | def listAllThemes(update, context): 169 | if(not isBotAuthorized(update.effective_chat.id)): 170 | context.bot.send_message(chat_id=update.effective_chat.id, text=bot_messages['UNAUTHORIZED_MESSAGE'].format(update.effective_chat.id)) 171 | else: 172 | AllThemes = getAllThemes() 173 | button_list = [] 174 | i = 0 175 | for theme in AllThemes: 176 | button_list.append([InlineKeyboardButton(theme[0],callback_data="ThemeSwitcher_"+str(i))]) 177 | i+=1 178 | button_list.append([InlineKeyboardButton(btns['CANCEL'], callback_data=btns['CANCEL'])]) 179 | reply_markup = InlineKeyboardMarkup(button_list) 180 | context.bot.send_message(chat_id=update.effective_chat.id, text=bot_messages['PICKTHEME_MESSAGE'], reply_markup=reply_markup) 181 | 182 | def ThemeSwitcher(update, context): 183 | args = update.callback_query.data.split('_') 184 | themeIndex = int(args[1]) 185 | AllThemes = getAllThemes() 186 | switchTheme(AllThemes[themeIndex][1]) 187 | context.bot.edit_message_text( 188 | message_id = update.callback_query.message.message_id, 189 | chat_id = update.callback_query.message.chat.id, 190 | text = bot_messages['THEMESWITCHED_MESSAGE'].format(AllThemes[themeIndex][0]), 191 | ) 192 | 193 | def ShowSkipCancelMenu(update, context, uid): 194 | button_list = [ 195 | [InlineKeyboardButton('😭',callback_data= "ansrfdbk_0_" + uid), 196 | InlineKeyboardButton('😖',callback_data= "ansrfdbk_1_" + uid), 197 | InlineKeyboardButton('😕',callback_data= "ansrfdbk_2_" + uid)], 198 | [InlineKeyboardButton('😊',callback_data= "ansrfdbk_3_" + uid), 199 | InlineKeyboardButton('😄',callback_data= "ansrfdbk_4_" + uid), 200 | InlineKeyboardButton('🥳',callback_data= "ansrfdbk_5_" + uid)], 201 | [InlineKeyboardButton(btns['SHOW_ANSWER'], callback_data=btns['SHOW_ANSWER'] + uid), 202 | InlineKeyboardButton(btns['SKIP'], callback_data=btns['SKIP'] + uid)], 203 | [InlineKeyboardButton(btns['CANCEL'], callback_data=btns['CANCEL'])] 204 | ] 205 | 206 | message = bot_messages['LINE_BREAK'] + "\n\n" + context.user_data[uid][0].question + "\n" + bot_messages['LINE_BREAK'] + "\n" 207 | message += bot_messages['FLASHCARD_OPTIONS'] 208 | reply_markup = InlineKeyboardMarkup(button_list) 209 | context.bot.send_message(chat_id=update.effective_chat.id, text=message, reply_markup=reply_markup) 210 | 211 | def ShowAnswer(update,context): 212 | uid = update.callback_query.data.replace(btns['SHOW_ANSWER'],'') 213 | flashcard = context.user_data[uid][0] 214 | message = flashcard.answer + "\n" + bot_messages['FLASHCARD_SOURCE'] + flashcard.source 215 | context.bot.send_message(chat_id=update.effective_chat.id, text=message) 216 | 217 | def AnswerHandler(update, context): 218 | args = update.callback_query.data.split('_') 219 | uid = args[2] 220 | answer = int(args[1]) 221 | flashcard = context.user_data[uid][0] 222 | flashcard.history.append(answer) 223 | roundCount = int(context.user_data[uid][1]) + 1 224 | roundGoal = context.user_data[uid][2] 225 | 226 | message = bot_messages['NEXTROUND_MESSAGE'] + flashcards.updateFlashcard(flashcard) 227 | context.bot.edit_message_text( 228 | message_id = update.callback_query.message.message_id, 229 | chat_id = update.callback_query.message.chat.id, 230 | text = message, 231 | ) 232 | if roundCount <= roundGoal: 233 | context.user_data[uid] = [flashcard, roundCount, roundGoal] 234 | TimeSpacedRepetition(update, context, uid) 235 | else: 236 | context.bot.send_message(chat_id=update.effective_chat.id, text="All Done for Today ") 237 | 238 | def Cancel(update,context): 239 | context.bot.edit_message_text( 240 | message_id = update.callback_query.message.message_id, 241 | chat_id = update.callback_query.message.chat.id, 242 | text = bot_messages['CANCELLED_MESSAGE'], 243 | ) 244 | 245 | def Skip(update, context): 246 | uid = update.callback_query.data.replace(btns['SKIP'],'') 247 | context.bot.edit_message_text( 248 | message_id = update.callback_query.message.message_id, 249 | chat_id = update.callback_query.message.chat.id, 250 | text = bot_messages['SKIPPED_MESSAGE'], 251 | ) 252 | TimeSpacedRepetition(update, context, uid) 253 | 254 | def TimeSpacedRepetition(update, context, uid=""): 255 | if(not isBotAuthorized(update.effective_chat.id)): 256 | context.bot.send_message(chat_id=update.effective_chat.id, text=bot_messages['UNAUTHORIZED_MESSAGE']) 257 | else: 258 | try: 259 | arg = context.args[0] 260 | except: 261 | arg = "" 262 | 263 | if arg == "import": 264 | importFlashCards(update, context) 265 | else: 266 | if(uid): 267 | roundCount = int(context.user_data[uid][1]) 268 | roundGoal = int(context.user_data[uid][2]) 269 | else: 270 | uid = str(uuid4()) 271 | roundCount = 1 272 | try: 273 | roundGoal = int(arg) 274 | except: 275 | roundGoal = getflashcardDailyGoal() 276 | 277 | flashcard = flashcards.getFlashcardFromPool() 278 | if(flashcard): 279 | message = "Card " + str(roundCount) + " out of " + str(roundGoal) + "\n" 280 | context.bot.send_message(chat_id=update.effective_chat.id, text=message) 281 | context.user_data[uid] = [flashcard, roundCount, roundGoal] 282 | return ShowSkipCancelMenu(update, context, uid) 283 | else: 284 | context.bot.send_message(chat_id=update.effective_chat.id, text=bot_messages['NOPENDIGCARDS_MESSAGE']) 285 | 286 | def tsrRetired(update, context): 287 | if(not isBotAuthorized(update.effective_chat.id)): 288 | context.bot.send_message(chat_id=update.effective_chat.id, text=bot_messages['UNAUTHORIZED_MESSAGE']) 289 | else: 290 | message = "command /tsr is being replace by /srs please use the latter from now on" 291 | context.bot.send_message(chat_id=update.effective_chat.id, text=message) 292 | 293 | def importFlashCards(update, context): 294 | if(not isBotAuthorized(update.effective_chat.id)): 295 | context.bot.send_message(chat_id=update.effective_chat.id, text=bot_messages['UNAUTHORIZED_MESSAGE'].format(update.effective_chat.id)) 296 | else: 297 | context.bot.send_message(chat_id=update.effective_chat.id, text=bot_messages['IMPORTINGFC_MESSAGE']) 298 | importResults = updateFlashCards() 299 | message = bot_messages['IMPORTEDFC_MESSAGE'].format(importResults[0],importResults[1]) 300 | context.bot.send_message(chat_id=update.effective_chat.id, text=message) 301 | 302 | def scheduleHousekeeping(chat_id, context): 303 | context.job_queue.run_repeating(scheduledHousekeeping, interval=getGitHubUpdateFrequency(), first=10, 304 | context=chat_id) 305 | # print(context.job_queue.jobs()) 306 | 307 | def scheduledHousekeeping(): 308 | print("Scheduled Housekeeping ...") 309 | # if isGraphAgeEncrypted(): 310 | # generateAgeKeyFile() 311 | 312 | # Git2Json() 313 | if isCalendarsAutogenerated(): 314 | updateCalendarsFile() 315 | 316 | def pullnow(update, context): 317 | if(not isBotAuthorized(update.effective_chat.id)): 318 | context.bot.send_message(chat_id=update.effective_chat.id, text=bot_messages['UNAUTHORIZED_MESSAGE']) 319 | else: 320 | print("Unscheduled Housekeeping ...") 321 | context.bot.send_message(chat_id=update.effective_chat.id, text=bot_messages['PULL_MESSAGE']) 322 | Git2Json() 323 | context.bot.send_message(chat_id=update.effective_chat.id, text=bot_messages['PULLDONE_MESSAGE']) 324 | 325 | def encryptall(update, context): 326 | if(not isBotAuthorized(update.effective_chat.id)): 327 | context.bot.send_message(chat_id=update.effective_chat.id, text=bot_messages['UNAUTHORIZED_MESSAGE']) 328 | else: 329 | context.bot.send_message(chat_id=update.effective_chat.id, text=bot_messages['ENCRYPTINGRAPH_MESSAGE']) 330 | encryptGraph() 331 | context.bot.send_message(chat_id=update.effective_chat.id, text=bot_messages['ENCRYPTDONE_MESSAGE']) 332 | 333 | 334 | def decryptall(update, context): 335 | if(not isBotAuthorized(update.effective_chat.id)): 336 | context.bot.send_message(chat_id=update.effective_chat.id, text=bot_messages['UNAUTHORIZED_MESSAGE']) 337 | else: 338 | context.bot.send_message(chat_id=update.effective_chat.id, text=bot_messages['DECRYPTINGRAPH_MESSAGE']) 339 | decryptGraph() 340 | context.bot.send_message(chat_id=update.effective_chat.id, text=bot_messages['DECRYPTDONE_MESSAGE']) 341 | 342 | def get_dispatcher(): 343 | # bot_persistence = PicklePersistence(filename='persistence') 344 | 345 | bot = Bot(BotToken) 346 | dispatcher = Dispatcher(bot=bot, update_queue=None, use_context=True) 347 | 348 | # if(isNewer()): 349 | # for BotAuthorizedId in getBotAuthorizedIDs(): 350 | # updater.bot.sendMessage(chat_id=BotAuthorizedId, text=bot_messages['VERCHANGE_MESSAGE']) 351 | 352 | # latestNews = getlatestNews() 353 | # for news in latestNews: 354 | # for BotAuthorizedId in getBotAuthorizedIDs(): 355 | # updater.bot.sendMessage(chat_id=BotAuthorizedId, text=news) 356 | 357 | # scheduleHousekeeping(getBotAuthorizedIDs()[0],updater) 358 | 359 | dispatcher.add_handler(CommandHandler('start', start)) 360 | dispatcher.add_handler(CommandHandler('uptime', uptime)) 361 | dispatcher.add_handler(CommandHandler('ver', version)) 362 | dispatcher.add_handler(CommandHandler('help', help)) 363 | dispatcher.add_handler(CommandHandler('anno', hypothesis)) 364 | # dispatcher.add_handler(CommandHandler('importFC', importFlashCards)) 365 | # dispatcher.add_handler(CommandHandler('tsr', tsrRetired)) 366 | # dispatcher.add_handler(CommandHandler('srs', TimeSpacedRepetition)) 367 | # dispatcher.add_handler(CommandHandler('getMD', generateMD)) 368 | # dispatcher.add_handler(CommandHandler('getMM', generateMinmapHTML)) TODO 369 | # dispatcher.add_handler(CommandHandler('pullnow', pullnow)) 370 | dispatcher.add_handler(CommandHandler('themes', listAllThemes)) 371 | dispatcher.add_handler(CommandHandler('encryptall', encryptall)) 372 | dispatcher.add_handler(CommandHandler('decryptall', decryptall)) 373 | 374 | dispatcher.add_handler(MessageHandler(Filters.text, addEntry)) 375 | dispatcher.add_handler(MessageHandler(Filters.photo, image_handler)) 376 | 377 | # dispatcher.add_handler(CallbackQueryHandler(ShowAnswer,pattern=btns['SHOW_ANSWER'])) 378 | # dispatcher.add_handler(CallbackQueryHandler(AnswerHandler,pattern="ansrfdbk")) 379 | # dispatcher.add_handler(CallbackQueryHandler(Skip,pattern=btns['SKIP'])) 380 | # dispatcher.add_handler(CallbackQueryHandler(Cancel,pattern=btns['CANCEL'])) 381 | dispatcher.add_handler(CallbackQueryHandler(ThemeSwitcher,pattern="ThemeSwitcher")) 382 | 383 | return dispatcher 384 | 385 | dispatcher = get_dispatcher() 386 | # a POST route for our webhook events 387 | from fastapi import FastAPI, Request 388 | from deta import App 389 | 390 | app = App(FastAPI()) 391 | 392 | @app.get('/') 393 | def hello_world(): 394 | return "Hello World!" 395 | 396 | @app.post("/webhook") 397 | async def webhook_handler(req: Request): 398 | import os 399 | 400 | data = await req.json() 401 | update = Update.de_json(data, dispatcher.bot) 402 | dispatcher.process_update(update) 403 | 404 | @app.lib.cron() 405 | def updateCalendar(event): 406 | Git2Json() 407 | scheduledHousekeeping() 408 | dispatcher.bot.send_message(chat_id=1751933281, text="Updated!") 409 | -------------------------------------------------------------------------------- /mindmap.py: -------------------------------------------------------------------------------- 1 | class Node: 2 | def __init__(self, indented_line): 3 | self.t = 'list_item' 4 | self.d = indented_line.index('# ') # len(indented_line) - len(indented_line.lstrip()) 5 | self.p = {} 6 | self.v = indented_line[self.d + 1:].strip() 7 | self.c = [] 8 | 9 | def add_children(self, nodes): 10 | childlevel = nodes[0].d 11 | while nodes: 12 | node = nodes.pop(0) 13 | if node.d == childlevel: # add node as a child 14 | self.c.append(node) 15 | elif node.d > childlevel: # add nodes as grandchildren of the last child 16 | nodes.insert(0,node) 17 | self.c[-1].add_children(nodes) 18 | elif node.d <= self.d: # this node is a sibling, no more children 19 | nodes.insert(0,node) 20 | return 21 | 22 | def get_leaf_nodes(self): 23 | leafs = [] 24 | def _get_leaf_nodes(node): 25 | if node is not None: 26 | if len(node.c) == 0: 27 | leafs.append(node) 28 | for n in node.c: 29 | _get_leaf_nodes(n) 30 | _get_leaf_nodes(self) 31 | return leafs 32 | 33 | def pruneLeafs(self): 34 | for node in self.get_leaf_nodes(): 35 | delattr(node,'c') 36 | 37 | 38 | def buildMindmapTree(content, pageTitle): 39 | import json 40 | mmTree = Node('# ' + pageTitle) 41 | mmTree.add_children([Node(line) for line in content.splitlines() if line.strip() and line.startswith('#')]) 42 | mmTree.pruneLeafs() 43 | return mmTree 44 | 45 | -------------------------------------------------------------------------------- /news.json: -------------------------------------------------------------------------------- 1 | {"news": [{"newsid": 1, "date": "2020-02-11", "news": "Verion 2.0.0 alpha updates\nLupin now supports Time Spaced Repetition based on SM2 algorithm!!\n Import your flashcards directly from your github by running /tsr import ... you can then go over your pending flashcards using /tsr x where x is the number of cards you'd like to see. If you don't specify x the value set in your config.ini will be used.\nAlso Lupin can now pull these news updates after each significatif update. This will keep you posted about the latest news.\n\n This version requires new values in your config.ini file"}, {"newsid": 2, "date": "2020-02-15", "news": "Verion 3.0.0 experimental updates\n A new version upgrade introducing MinMap capabilties among other thing. Send /getMM pageTitle and Lupin will generate a Markmap file containing a mindmap of that page and send it to you. \nCredits to https://markmap.js.org/\n /tsr command is now renamed to /srs"}, {"newsid": 3, "date": "2020-02-18", "news": "Verion 3.1.0 experimental updates\n Thanks to @Piotr 3 months calendar can now be autogenerated and inserted in your right side-bar. Version releases also a new command /pullnow that forces refresh from GitHub & a scheduled pull that can be configured from the config.ini `GitHubUpdateFrequency` \n\nVersion requires new values in your config.ini file, please reffer to config.sample.ini"}, {"newsid": 4, "date": "2020-02-20", "news": "Verion 3.2.0 & 3.3.0 experimental updates\n Timestamps in Journal entries can now be turned off from the config.ini file by setting `timestampEntries = false` TODOCommand is now decomissioned and replaced by CommandsMap for greater flexibility. Mapp your own sets of commands like TODO LATER etc...\n\n Version requires modifications in your config.ini file, please reffer to config.sample.ini"}, {"newsid": 5, "date": "2020-02-26", "news": "Verion 3.4.0 experimental updates\n With all the great themes why have only one? put your choice of themes in the /logseq directory and name them ThemeName.custom.cssthen simply call Lupin with /themes to switch between one and the other."}, {"newsid": 6, "date": "2020-03-02", "news": "Verion 3.5.0 alpha updates\n Added feature to control how many months (0 to 3) you want LUPIN to generate calendars for, this is controlled thru generateMonths array of config.ini.first [0|1] specifies if you want LUPIN to generate prev month, second [0|1] specifies if you want LUPIN to generate next month1,1 will generate prev,cur,next | 0,1 will generate curr,next | 0,0 will only generate curr"}, {"newsid": 7, "date": "2020-03-22", "news": "Verion 3.8.0 experimental updates\n Lupin now supports doker thanks to Jon Molina https://github.com/digitalknk.Lupin also now supports Twitter embedding, Send a twitter URL to Lupin and see what happens."}]} -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # Deta Runner for Lupin 2 | 3 | ## Notable changes 4 | 5 | Some features of Lupin requires to persist a dump of you Github repository. Deta only provides a read-only file system for your apps, and the worker on which your app run frequently switches. 6 | 7 | Due to these limitations, some features were disabled: 8 | 9 | - Generation of MindMaps 10 | - Markdown Export (but this is natively supported by Logseq in 0.0.14) 11 | - Space repetition 12 | 13 | I did not test the encryption because I am not interested in this feature. 14 | 15 | ## Installation Guide for Deta Deployement 16 | 17 | 1. Clone me: https://github.com/pomdtr/Lupin 18 | 1. Fill your secrets 19 | 1. [Create a telegram bot](https://core.telegram.org/bots#creating-a-new-bot) 20 | 1. Generate a Github token from https://github.com/settings/tokens 21 | 1. `mv sample.env .env` 22 | 1. Add your tokens to the envfile 23 | 1. Configure Lupin using the config.ini file (`mv config.sample.ini config.ini`) 24 | 1. Install deta and deploy your app 25 | 1. Create an account in Deta 26 | 1. [Install the deta cli](https://docs.deta.sh/docs/cli/install) 27 | 1. Create a new micro: 'deta new lupin' 28 | 1. Add your credentials to your micro: `deta update -e .env` 29 | 1. Deploy you app: `deta deploy` 30 | 1. Configure the webhook 31 | 1. `curl --header 'Content-Type: application/json' --data '{"url": "/webhook"}' 'https://api.telegram.org/bot/setWebhook'` 32 | 1. Add a cron to deta if you want an automatically generated calendar 33 | 1. `deta cron set "1 day"` 34 | 35 | 36 | # Original Readme 37 | 38 | See https://github.com/akhater/Lupin/blob/master/readme.md 39 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | requests==2.25.1 2 | age==0.4.1 3 | python_telegram_bot==12.4.2 4 | beautifulsoup4==4.9.3 5 | PyGithub==1.54.1 6 | telegram==0.0.1 7 | fastapi 8 | -------------------------------------------------------------------------------- /sample.env: -------------------------------------------------------------------------------- 1 | BotToken= 2 | GitHubToken= 3 | hypothesisToken= 4 | -------------------------------------------------------------------------------- /sm2.py: -------------------------------------------------------------------------------- 1 | # credits https://gist.github.com/doctorpangloss/13ab29abd087dc1927475e560f876797 2 | 3 | def supermemo_2(x: [int], a=6.0, b=-0.8, c=0.28, d=0.02, assumed_score=2.5, min_score=1.3, theta=0.1) -> float: 4 | """ 5 | Returns the number of days until seeing a problem again based on the 6 | history of answers x to the problem, where the meaning of x is: 7 | x == 0: Incorrect, Hardest 8 | x == 1: Incorrect, Hard 9 | x == 2: Incorrect, Medium 10 | x == 3: Correct, Medium 11 | x == 4: Correct, Easy 12 | x == 5: Correct, Easiest 13 | @param x The history of answers in the above scoring. 14 | @param theta When larger, the delays for correct answers will increase. 15 | """ 16 | assert all(0 <= x_i <= 5 for x_i in x) 17 | correct = [x_i >= 3 for x_i in x] 18 | # If you got the last question incorrect, just return 1 19 | if not correct[-1]: 20 | return 1.0 21 | 22 | # Calculate the latest consecutive answer streak 23 | r = 0 24 | for c_i in reversed(correct): 25 | if c_i: 26 | r+=1 27 | else: 28 | break 29 | 30 | return a*(max(min_score, assumed_score + sum(b+c*x_i+d*x_i*x_i for x_i in x)))**(theta*r) -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | import re 3 | import json 4 | import requests 5 | import hashlib 6 | from os.path import basename 7 | from bs4 import BeautifulSoup 8 | import urllib.parse 9 | 10 | 11 | from config import ( hour24, journalsFilesFormat, journalsFilesExtension, journalsFolder, isEntryTimestamped, 12 | journalsPrefix,getFirebaseBucketName, getlastNewsDisplayed, setlastNewsDisplayed, getcalendarFile, 13 | getCommandsMap, getMonths2Generate 14 | ) 15 | 16 | import flashcards 17 | from mindmap import buildMindmapTree 18 | from calc import buildCalendar 19 | 20 | 21 | bootTime = datetime.now() 22 | 23 | def getJournalPath(): 24 | dateTimeObj = datetime.now() 25 | 26 | if (journalsPrefix == "none"): 27 | return journalsFolder + "/" + dateTimeObj.strftime(journalsFilesFormat) + journalsFilesExtension 28 | else: 29 | return journalsFolder + "/" + journalsPrefix + dateTimeObj.strftime(journalsFilesFormat) + journalsFilesExtension 30 | 31 | def getAnnotationPath(uri): 32 | return 'annotations/' + getURIHash(uri) + journalsFilesExtension 33 | 34 | def getCurrentTime(): 35 | if(not(isEntryTimestamped())): 36 | return '' 37 | else: 38 | dateTimeObj = datetime.now() 39 | 40 | if(hour24 == "true"): 41 | return dateTimeObj.strftime("%H:%M") 42 | else: 43 | return dateTimeObj.strftime("%I:%M %p") 44 | 45 | def getTimestamp(isoFormat=False,withSeconds=False): 46 | dateTimeObj = datetime.now() 47 | 48 | if isoFormat: 49 | if withSeconds: 50 | return dateTimeObj.strftime("%Y%m%d%H%M%S%f") 51 | else: 52 | return dateTimeObj.strftime("%Y%m%d%H%M") 53 | elif (hour24 == "true"): 54 | return dateTimeObj.strftime("%Y-%m-%d %H:%M") 55 | else: 56 | return dateTimeObj.strftime("%Y-%m-%d %I:%M %p") 57 | 58 | def getUptime(): 59 | seconds = date_diff_in_seconds(datetime.now(), bootTime) 60 | 61 | minutes, seconds = divmod(seconds, 60) 62 | hours, minutes = divmod(minutes, 60) 63 | days, hours = divmod(hours, 24) 64 | return (days, hours, minutes, seconds) 65 | 66 | def date_diff_in_seconds(dt2, dt1): 67 | timedelta = dt2 - dt1 68 | return timedelta.days * 24 * 3600 + timedelta.seconds 69 | 70 | def containsURL(s): 71 | url = re.search('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', s) 72 | if url: 73 | return url.group() 74 | else: 75 | return False 76 | 77 | def containsRefBlock(s): 78 | try: 79 | return (re.search(r"\(\((.*?)\)\)", s)).group(1) 80 | except: 81 | return False 82 | 83 | def getWebPageTitle(url): 84 | r = requests.get(url, headers={'User-Agent': 'Mozilla/5.0'}) 85 | if r.status_code == 200: 86 | html_text = r.text 87 | soup = BeautifulSoup(html_text, 'html.parser') 88 | if soup.title: 89 | return soup.title.text.strip() 90 | else: 91 | return r.url.strip() 92 | raise Exception(r.status_code) 93 | 94 | def containsYTURL(s): 95 | url = re.search('((?:https?:)?//)?((?:www|m).)?((?:youtube.com|youtu.be))(/(?:[\\w-]+\\?v=|embed/|v/)?)([\\w-]+)(\\S+)?',s) 96 | if url: 97 | return url.group() 98 | else: 99 | return False 100 | 101 | def getMD5Hash(s): 102 | byte_s = s.encode('utf-8') 103 | return hashlib.md5(byte_s).hexdigest() 104 | 105 | def stripURI(uri): 106 | regex = re.compile(r"https?://?") 107 | return regex.sub('', uri).strip().strip('/') 108 | 109 | def getURIHash(uri): 110 | return getMD5Hash(stripURI(uri)) 111 | 112 | def getPageTitle(path): 113 | return basename(path).replace(journalsFilesExtension, '') 114 | 115 | def UploadToFirebase(data, path): 116 | APIRUI = 'https://firebasestorage.googleapis.com/v0/b/' + getFirebaseBucketName() + "/o/" + path.replace('/', '%2F') 117 | 118 | headers = {"Content-Type": "img/jpg"} 119 | 120 | result = requests.post(APIRUI, 121 | headers=headers, 122 | data=data) 123 | return (APIRUI + "?alt=media&token=" + result.json()['downloadTokens']) 124 | 125 | def getlatestNews(): 126 | url = 'https://github.com/akhater/Lupin/raw/master/news.json' 127 | 128 | newslist = (requests.get(url)).json() 129 | lastNewsDisplayed = getlastNewsDisplayed() 130 | recentNews = [] 131 | lstNewsID = 0 132 | for news in newslist['news']: 133 | # print(news) 134 | if(news['newsid'] > int(lastNewsDisplayed)): 135 | recentNews.append(news['news']) 136 | lstNewsID = news['newsid'] 137 | print(lstNewsID) 138 | setlastNewsDisplayed(lstNewsID) 139 | return recentNews 140 | 141 | def saveasJson(content, file): 142 | with open(file, 'w') as outfile: 143 | json.dump(content, outfile) 144 | 145 | def findOrigBlock(ref): 146 | with open('/tmp/GitDump.json') as json_file: 147 | AllFilesContent = json.load(json_file) 148 | 149 | origBlock = "" 150 | for fileContent in AllFilesContent: 151 | if ":id: " + ref in fileContent.lower() or ":id:" + ref in fileContent.lower(): #add no space after id: 152 | lines = fileContent.split('\n') 153 | i = 0 154 | while i <= len(lines) - 1: 155 | if ":id: " + ref in lines[i].lower() or ":id:" + ref in lines[i].lower(): 156 | break 157 | i += 1 158 | origLine = lines[i-2] 159 | origBlock = origLine[origLine.index(' '):].strip() 160 | break 161 | return(origBlock) 162 | 163 | def scanJson4Flashcards(): 164 | # from git import Git2Json 165 | # Git2Json() 166 | with open('/tmp/GitDump.json') as json_file: 167 | AllFilesContent = json.load(json_file) 168 | 169 | flashcardsList = [] 170 | 171 | for fileContent in AllFilesContent: 172 | flashcardsList += flashcards.scan4Flashcards( fileContent ) 173 | 174 | return(flashcardsList) 175 | 176 | def updateFlashCards(): 177 | return flashcards.saveFlashcardsDB( scanJson4Flashcards() ) 178 | 179 | def convert2MD(pageTitle): 180 | # from git import Git2Json 181 | # Git2Json() 182 | 183 | with open('/tmp/GitDump.json') as json_file: 184 | AllFilesContent = json.load(json_file) 185 | 186 | for content in AllFilesContent: 187 | lines = content.split('\n') 188 | lvl1 = -1 189 | 190 | i = 0 191 | out = "" 192 | continueScan = True 193 | while i <= len(lines) - 1: 194 | line = lines[i] 195 | if(line): 196 | # print(line[0]) 197 | if 'title:' in line.lower() and pageTitle.lower() not in line.lower(): # not correct page 198 | continueScan = False 199 | if not continueScan: 200 | break 201 | if 'title:' in line.lower() and pageTitle.lower() in line.lower(): 202 | out += "# " + line.replace('title:', '').strip() + "\n" 203 | elif line[0] == "#": 204 | outln = "" 205 | blockRef = containsRefBlock(line) 206 | if (blockRef): 207 | origLine = (line[line.index(' '):]).replace("(("+blockRef+"))","").strip() 208 | if(origLine): 209 | outln = origLine + " " 210 | outln += findOrigBlock(blockRef) 211 | else: 212 | outln = line[line.index(' '):] 213 | 214 | if(lvl1 == -1): 215 | lvl1 = line.index(' ') 216 | space = " " 217 | for _ in range((line.index(' ')-lvl1)* 4): 218 | space += " " 219 | out += space + "- " + outln.strip() + "\n" #+ (line[line.index(' '):]) + '\n' # .replace('#','-') + "\n" 220 | # print(out) 221 | i +=1 222 | return(out) 223 | 224 | def convert2Mindmap(pageTitle): 225 | # from git import Git2Json 226 | # Git2Json() 227 | 228 | with open('/tmp/GitDump.json') as json_file: 229 | AllFilesContent = json.load(json_file) 230 | 231 | for content in AllFilesContent: 232 | if ('---\ntitle: ' + pageTitle.lower()) in content.lower(): 233 | return json.dumps(buildMindmapTree(content, pageTitle), default=lambda x: x.__dict__) 234 | # return json.dumps(buildMindmapTree(content, pageTitle).c[0], default=lambda x: x.__dict__) 235 | 236 | def pageExists(pageTitle): 237 | # from git import Git2Json 238 | # Git2Json() 239 | 240 | with open('/tmp/GitDump.json') as json_file: 241 | AllFilesContent = json.load(json_file) 242 | 243 | for content in AllFilesContent: 244 | if ('---\ntitle: ' + pageTitle.lower()) in content.lower(): 245 | return True 246 | 247 | return False 248 | 249 | 250 | def getdateFormatter(): 251 | with open('GitDump.json') as json_file: 252 | AllFilesContent = json.load(json_file) 253 | 254 | for content in AllFilesContent: 255 | dateFormatter = re.findall("\n :date-formatter.*",content) # ,re.MULTILINE) 256 | if dateFormatter: 257 | break 258 | 259 | if not(dateFormatter): 260 | dateFormatter = '%b {th}, %Y' 261 | else: 262 | mapping = {'yyyy': '%Y', 'yy': '%y', 'MM': '%m', 'MMM': '%b', 'MMMM': '%B', 'dd': '%d', 'do': '{th}', 'EE': '%a', 'EEE': '%a', 'EEEEEE': '%A'} 263 | 264 | def replace(match): 265 | return mapping[match.group(0)] 266 | 267 | dateFormatter = dateFormatter[0].split(':date-formatter')[1] 268 | dateFormatter = dateFormatter[1:].replace('\"', ' ').rstrip() 269 | 270 | dateFormatter = (re.sub('|'.join(r'\b%s\b' % re.escape(s) for s in mapping), 271 | replace, dateFormatter) ) 272 | 273 | print(dateFormatter.strip()) 274 | return (dateFormatter.strip()) 275 | 276 | def generateCalendarsFile(contents): 277 | import datetime as dt 278 | today = dt.date.today() 279 | 280 | months2Generate = getMonths2Generate() 281 | 282 | out = "##\n" 283 | if int(months2Generate[0]) == 1: 284 | lastMonth = today.replace(day=1) - dt.timedelta(days=1) 285 | out += buildCalendar(lastMonth.year, lastMonth.month) + "\n##\n" 286 | 287 | out += buildCalendar(today.year, today.month) + "\n" 288 | 289 | if int(months2Generate[1]) == 1: 290 | nextMonth = today.replace(day=28) + dt.timedelta(days=4) 291 | out += "##\n" + buildCalendar(nextMonth.year, nextMonth.month) + "\n" 292 | 293 | # out = "##\n" + buildCalendar(lastMonth.year, lastMonth.month) + "\n##\n" + buildCalendar(today.year, today.month) + "\n##\n" + buildCalendar(nextMonth.year, nextMonth.month) + "\n" 294 | 295 | t = (re.sub('(##[\\s\n]).*?()', '', contents, flags=re.DOTALL)).strip() 296 | 297 | out += t 298 | 299 | return out 300 | 301 | def processCommandsMapping(entry): 302 | # import re 303 | CommandsMap = getCommandsMap() 304 | 305 | def replace(match): 306 | return CommandsMap[match.group(0)] 307 | 308 | s = (re.sub('|'.join(r'\b%s\b' % re.escape(s) for s in CommandsMap), 309 | replace, entry) ) 310 | 311 | rValue = "" 312 | for _, value in CommandsMap.items(): 313 | if value in s: 314 | rValue = s.split(value) 315 | rValue = value + ' ' + (' '.join([x.strip() for x in rValue])).strip() 316 | if rValue: 317 | return rValue 318 | else: 319 | return entry 320 | 321 | def ord(n): 322 | return str(n)+("th" if 4<=n%100<=20 else {1:"st",2:"nd",3:"rd"}.get(n%10, "th")) 323 | 324 | def styleDateTime(dt,f): 325 | return dt.strftime(f).replace("{th}", ord(dt.day)) 326 | 327 | 328 | def getJournalTitle(): 329 | import config 330 | return styleDateTime(datetime.now(), config.dateFormat) 331 | 332 | 333 | def getJournalTemplate(): 334 | with open('/tmp/GitDump.json') as json_file: 335 | AllFilesContent = json.load(json_file) 336 | 337 | for content in AllFilesContent: 338 | JournalTemplate = re.findall("\n :default-templates\n {:journals.*",content) # ,re.MULTILINE) 339 | if JournalTemplate: 340 | break 341 | 342 | if (JournalTemplate): 343 | return JournalTemplate[0].split('\n :default-templates\n {:journals "')[1][:len(JournalTemplate)-3].replace("\\n","\n") 344 | else: 345 | return None 346 | 347 | def generateTwitterIframe(TwitterUrl): 348 | endpoint = 'https://publish.twitter.com/oembed?url=' 349 | 350 | TargetUrl = endpoint + TwitterUrl 351 | 352 | r = requests.get(TargetUrl, headers={'User-Agent': 'Mozilla/5.0'}) 353 | 354 | if r.status_code == 200: 355 | jsonObj = json.loads(r.text) 356 | srcCode = urllib.parse.quote(jsonObj['html']) 357 | 358 | iframeCode = ("""""").format(TwitterUrl,srcCode) 359 | 360 | return iframeCode 361 | 362 | def containsTWUrl(s): 363 | # url = re.search('((?:https?:)?//)?((?:www|m).)?((?:youtube.com|youtu.be))(/(?:[\\w-]+\\?v=|embed/|v/)?)([\\w-]+)(\\S+)?',s) 364 | # url = re.search('(?:http://)?(?:www.)?twitter.com/(?:(?:\\w)*#!/)?(?:pages/)?(?:[\\w-]*/)*([\\w-]*)',s) 365 | url = re.search('((?:https?:)?//)?(?:www.)?twitter.com/(?:(?:\\w)*#!/)?(?:pages/)?(?:[\\w-]*/)*([\\w-]*)',s) 366 | if url: 367 | return url.group() 368 | else: 369 | return False 370 | --------------------------------------------------------------------------------