├── wallabag2.koplugin ├── _meta.lua └── main.lua └── README.md /wallabag2.koplugin/_meta.lua: -------------------------------------------------------------------------------- 1 | local _ = require("gettext") 2 | return { 3 | name = "wallabag2", 4 | fullname = _("Wallabag2"), 5 | description = _([[Synchronises articles with a Wallabag version 2.x (2.4+) server.]]), 6 | } 7 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # wallabag2.koplugin 2 | 3 | Home page https://github.com/clach04/wallabag2.koplugin 4 | 5 | A KoReader plugin for Wallabag (version 2) servers. 6 | 7 | Makes use of the REST API documented at https://app.wallabag.it/api/doc/ 8 | 9 | See https://github.com/koreader/koreader/issues/10738 for background, 10 | basically this plugin should be faster and better suited to modern wallabag instances. 11 | 12 | ## Installation 13 | 14 | Copy the `wallabag2.koplugin` to the koreader `plugins` directory. 15 | 16 | * Under Android, local/koreader/plugins 17 | * Under Kindle, /koreader/plugins (under Windows mount `D:\koreader\plugins`) 18 | 19 | Requires KoReader to be restarted. 20 | 21 | Plugin will show up as, `NEW: Wallabag2`, on (top) left hand menu for: 22 | 23 | * for file browser view, file system 24 | * for book reader view, bookmarks 25 | 26 | ## Usage 27 | 28 | Same as Wallabag(1) plugin, see https://github.com/koreader/koreader/wiki/Wallabag 29 | 30 | NOTE Uses the **same** config file (at least for now) as existing wallabag plugin! 31 | I.e. `/koreader/settings/wallabag.lua`. 32 | 33 | ## Debugging 34 | 35 | * need either a [real Wallabag server](https://github.com/wallabag/wallabag) or (semi) compatible one such as: 36 | * https://github.com/clach04/whatabagacack 37 | * https://github.com/LordEidi/wombag 38 | * https://koreader.rocks/doc/topics/Hacking.md.html 39 | * enable debug log https://github.com/koreader/koreader/issues/2530#issuecomment-500866288 - `Developer options`, `Enable debug logging` - goes to `crash.log` 40 | -------------------------------------------------------------------------------- /wallabag2.koplugin/main.lua: -------------------------------------------------------------------------------- 1 | --[[-- 2 | @module koplugin.wallabag2 3 | ]] 4 | 5 | -- wallabag2 - for Wallabag 2.4+ (or compatible) API servers 6 | -- clach04 7 | -- NOTE for now; uses the same wallabag settings that Wallabag (Wallabag1) uses 8 | -- NOTE for now; has new text Wallabag2 - may want to remove and revert back to reduce(/remove) translation effort 9 | 10 | 11 | local BD = require("ui/bidi") 12 | local DataStorage = require("datastorage") 13 | local Dispatcher = require("dispatcher") 14 | local DocSettings = require("docsettings") 15 | local DocumentRegistry = require("document/documentregistry") 16 | local Event = require("ui/event") 17 | local FFIUtil = require("ffi/util") 18 | local FileManager = require("apps/filemanager/filemanager") 19 | local InfoMessage = require("ui/widget/infomessage") 20 | local InputDialog = require("ui/widget/inputdialog") 21 | local JSON = require("json") 22 | local LuaSettings = require("frontend/luasettings") 23 | local Math = require("optmath") 24 | local MultiConfirmBox = require("ui/widget/multiconfirmbox") 25 | local MultiInputDialog = require("ui/widget/multiinputdialog") 26 | local NetworkMgr = require("ui/network/manager") 27 | local ReadHistory = require("readhistory") 28 | local UIManager = require("ui/uimanager") 29 | local WidgetContainer = require("ui/widget/container/widgetcontainer") 30 | local filemanagerutil = require("apps/filemanager/filemanagerutil") 31 | local http = require("socket.http") 32 | local lfs = require("libs/libkoreader-lfs") 33 | local logger = require("logger") 34 | local ltn12 = require("ltn12") 35 | local socket = require("socket") 36 | local socketutil = require("socketutil") 37 | local util = require("util") 38 | local _ = require("gettext") 39 | local T = FFIUtil.template 40 | 41 | -- constants 42 | local article_id_prefix = "[w-id_" 43 | local article_id_postfix = "] " 44 | local failed, skipped, downloaded = 1, 2, 3 45 | 46 | local Wallabag2 = WidgetContainer:extend{ 47 | name = "wallabag2", 48 | } 49 | 50 | function Wallabag2:onDispatcherRegisterActions() 51 | Dispatcher:registerAction("wallabag_download", { category="none", event="SynchronizeWallabag2", title=_("Wallabag2 retrieval"), general=true,}) 52 | end 53 | 54 | function Wallabag2:init() 55 | self.token_expiry = 0 56 | -- default values so that user doesn't have to explicitely set them 57 | self.is_delete_finished = true 58 | self.is_delete_read = false 59 | self.is_auto_delete = false 60 | self.is_sync_remote_delete = false 61 | self.is_archiving_deleted = true 62 | self.send_review_as_tags = false 63 | self.filter_tag = "" 64 | self.ignore_tags = "" 65 | self.auto_tags = "" 66 | self.articles_per_sync = 30 -- this is the max number of articles to get metadata for, this impacts number of epubs downloaded BUT potentially could skip ones already downloaded and not pick up new article metadata 67 | 68 | self:onDispatcherRegisterActions() 69 | self.ui.menu:registerToMainMenu(self) 70 | self.wb_settings = self.readSettings() 71 | self.server_url = self.wb_settings.data.wallabag.server_url 72 | self.client_id = self.wb_settings.data.wallabag.client_id 73 | self.client_secret = self.wb_settings.data.wallabag.client_secret 74 | self.username = self.wb_settings.data.wallabag.username 75 | self.password = self.wb_settings.data.wallabag.password 76 | self.directory = self.wb_settings.data.wallabag.directory 77 | if self.wb_settings.data.wallabag.is_delete_finished ~= nil then 78 | self.is_delete_finished = self.wb_settings.data.wallabag.is_delete_finished 79 | end 80 | if self.wb_settings.data.wallabag.send_review_as_tags ~= nil then 81 | self.send_review_as_tags = self.wb_settings.data.wallabag.send_review_as_tags 82 | end 83 | if self.wb_settings.data.wallabag.is_delete_read ~= nil then 84 | self.is_delete_read = self.wb_settings.data.wallabag.is_delete_read 85 | end 86 | if self.wb_settings.data.wallabag.is_auto_delete ~= nil then 87 | self.is_auto_delete = self.wb_settings.data.wallabag.is_auto_delete 88 | end 89 | if self.wb_settings.data.wallabag.is_sync_remote_delete ~= nil then 90 | self.is_sync_remote_delete = self.wb_settings.data.wallabag.is_sync_remote_delete 91 | end 92 | if self.wb_settings.data.wallabag.is_archiving_deleted ~= nil then 93 | self.is_archiving_deleted = self.wb_settings.data.wallabag.is_archiving_deleted 94 | end 95 | if self.wb_settings.data.wallabag.filter_tag then 96 | self.filter_tag = self.wb_settings.data.wallabag.filter_tag 97 | end 98 | if self.wb_settings.data.wallabag.ignore_tags then 99 | self.ignore_tags = self.wb_settings.data.wallabag.ignore_tags 100 | end 101 | if self.wb_settings.data.wallabag.auto_tags then 102 | self.auto_tags = self.wb_settings.data.wallabag.auto_tags 103 | end 104 | if self.wb_settings.data.wallabag.articles_per_sync ~= nil then 105 | self.articles_per_sync = self.wb_settings.data.wallabag.articles_per_sync 106 | end 107 | self.remove_finished_from_history = self.wb_settings.data.wallabag.remove_finished_from_history or false 108 | self.download_queue = self.wb_settings.data.wallabag.download_queue or {} 109 | 110 | -- workaround for dateparser only available if newsdownloader is active 111 | self.is_dateparser_available = false 112 | self.is_dateparser_checked = false 113 | 114 | -- workaround for dateparser, only once 115 | -- the parser is in newsdownloader.koplugin, check if it is available 116 | if not self.is_dateparser_checked then 117 | local res 118 | res, self.dateparser = pcall(require, "lib.dateparser") 119 | if res then self.is_dateparser_available = true end 120 | self.is_dateparser_checked = true 121 | end 122 | 123 | if self.ui and self.ui.link then 124 | self.ui.link:addToExternalLinkDialog("25_wallabag2", function(this, link_url) 125 | return { 126 | text = _("Add to Wallabag2"), 127 | callback = function() 128 | UIManager:close(this.external_link_dialog) 129 | this.ui:handleEvent(Event:new("AddWallabag2Article", link_url)) 130 | end, 131 | } 132 | end) 133 | end 134 | end 135 | 136 | function Wallabag2:addToMainMenu(menu_items) 137 | menu_items.wallabag2 = { 138 | text = _("Wallabag2"), 139 | sub_item_table = { 140 | { 141 | text = _("Retrieve new articles from server"), 142 | callback = function() 143 | self.ui:handleEvent(Event:new("SynchronizeWallabag2")) 144 | end, 145 | }, 146 | { 147 | text = _("Delete finished articles remotely"), 148 | callback = function() 149 | local connect_callback = function() 150 | local num_deleted = self:processLocalFiles("manual") 151 | UIManager:show(InfoMessage:new{ 152 | text = T(_("Articles processed.\nDeleted: %1"), num_deleted) 153 | }) 154 | self:refreshCurrentDirIfNeeded() 155 | end 156 | NetworkMgr:runWhenOnline(connect_callback) 157 | end, 158 | enabled_func = function() 159 | return self.is_delete_finished or self.is_delete_read 160 | end, 161 | }, 162 | { 163 | text = _("Go to download folder"), 164 | callback = function() 165 | if self.ui.document then 166 | self.ui:onClose() 167 | end 168 | if FileManager.instance then 169 | FileManager.instance:reinit(self.directory) 170 | else 171 | FileManager:showFiles(self.directory) 172 | end 173 | end, 174 | }, 175 | { 176 | text = _("Settings"), 177 | callback_func = function() 178 | return nil 179 | end, 180 | separator = true, 181 | sub_item_table = { 182 | { 183 | text = _("Configure Wallabag2 server"), 184 | keep_menu_open = true, 185 | callback = function() 186 | self:editServerSettings() 187 | end, 188 | }, 189 | { 190 | text = _("Configure Wallabag2 client"), 191 | keep_menu_open = true, 192 | callback = function() 193 | self:editClientSettings() 194 | end, 195 | }, 196 | { 197 | text_func = function() 198 | local path 199 | if not self.directory or self.directory == "" then 200 | path = _("Not set") 201 | else 202 | path = filemanagerutil.abbreviate(self.directory) 203 | end 204 | return T(_("Set download folder: %1"), BD.dirpath(path)) 205 | end, 206 | keep_menu_open = true, 207 | callback = function(touchmenu_instance) 208 | self:setDownloadDirectory(touchmenu_instance) 209 | end, 210 | separator = true, 211 | }, 212 | { 213 | text_func = function() 214 | local filter 215 | if not self.filter_tag or self.filter_tag == "" then 216 | filter = _("All articles") 217 | else 218 | filter = self.filter_tag 219 | end 220 | return T(_("Filter articles by tag: %1"), filter) 221 | end, 222 | keep_menu_open = true, 223 | callback = function(touchmenu_instance) 224 | self:setFilterTag(touchmenu_instance) 225 | end, 226 | }, 227 | { 228 | text_func = function() 229 | if not self.ignore_tags or self.ignore_tags == "" then 230 | return _("Ignore tags") 231 | end 232 | return T(_("Ignore tags (%1)"), self.ignore_tags) 233 | end, 234 | keep_menu_open = true, 235 | callback = function(touchmenu_instance) 236 | self:setTagsDialog(touchmenu_instance, 237 | _("Tags to ignore"), 238 | _("Enter a comma-separated list of tags to ignore."), 239 | self.ignore_tags, 240 | function(tags) 241 | self.ignore_tags = tags 242 | end 243 | ) 244 | end, 245 | }, 246 | { 247 | text_func = function() 248 | if not self.auto_tags or self.auto_tags == "" then 249 | return _("Automatic tags") 250 | end 251 | return T(_("Automatic tags (%1)"), self.auto_tags) 252 | end, 253 | keep_menu_open = true, 254 | callback = function(touchmenu_instance) 255 | self:setTagsDialog(touchmenu_instance, 256 | _("Tags to automatically add"), 257 | _("Enter a comma-separated list of tags to automatically add to new articles."), 258 | self.auto_tags, 259 | function(tags) 260 | self.auto_tags = tags 261 | end 262 | ) 263 | end, 264 | separator = true, 265 | }, 266 | { 267 | text = _("Article deletion"), 268 | separator = true, 269 | sub_item_table = { 270 | { 271 | text = _("Remotely delete finished articles"), 272 | checked_func = function() return self.is_delete_finished end, 273 | callback = function() 274 | self.is_delete_finished = not self.is_delete_finished 275 | self:saveSettings() 276 | end, 277 | }, 278 | { 279 | text = _("Remotely delete 100% read articles"), 280 | checked_func = function() return self.is_delete_read end, 281 | callback = function() 282 | self.is_delete_read = not self.is_delete_read 283 | self:saveSettings() 284 | end, 285 | separator = true, 286 | }, 287 | { 288 | text = _("Mark as finished instead of deleting"), -- FIXME (and translations). "Mark as finished (archive) instead of deleting" 289 | checked_func = function() return self.is_archiving_deleted end, 290 | callback = function() 291 | self.is_archiving_deleted = not self.is_archiving_deleted 292 | self:saveSettings() 293 | end, 294 | separator = true, 295 | }, 296 | { 297 | text = _("Process deletions when downloading"), 298 | checked_func = function() return self.is_auto_delete end, 299 | callback = function() 300 | self.is_auto_delete = not self.is_auto_delete 301 | self:saveSettings() 302 | end, 303 | }, 304 | { 305 | text = _("Synchronize remotely deleted files"), 306 | checked_func = function() return self.is_sync_remote_delete end, 307 | callback = function() 308 | self.is_sync_remote_delete = not self.is_sync_remote_delete 309 | self:saveSettings() 310 | end, 311 | }, 312 | }, 313 | }, 314 | { 315 | text = _("Send review as tags"), 316 | help_text = _("This allow you to write tags in the review field, separated by commas, which can then be sent to Wallabag2."), 317 | keep_menu_open = true, 318 | checked_func = function() 319 | return self.send_review_as_tags or false 320 | end, 321 | callback = function() 322 | self.send_review_as_tags = not self.send_review_as_tags 323 | self:saveSettings() 324 | end, 325 | }, 326 | { 327 | text = _("Remove finished articles from history"), 328 | keep_menu_open = true, 329 | checked_func = function() 330 | return self.remove_finished_from_history or false 331 | end, 332 | callback = function() 333 | self.remove_finished_from_history = not self.remove_finished_from_history 334 | self:saveSettings() 335 | end, 336 | }, 337 | { 338 | text = _("Remove 100% read articles from history"), 339 | keep_menu_open = true, 340 | checked_func = function() 341 | return self.remove_read_from_history or false 342 | end, 343 | callback = function() 344 | self.remove_read_from_history = not self.remove_read_from_history 345 | self:saveSettings() 346 | end, 347 | separator = true, 348 | }, 349 | { 350 | text = _("Help"), 351 | keep_menu_open = true, 352 | callback = function() 353 | UIManager:show(InfoMessage:new{ 354 | text = _([[Download directory: use a directory that is exclusively used by the Wallabag2 plugin. Existing files in this directory risk being deleted. 355 | 356 | Articles marked as finished or 100% read can be deleted from the server. Those articles can also be deleted automatically when downloading new articles if the 'Process deletions during download' option is enabled. 357 | 358 | The 'Synchronize remotely deleted files' option will remove local files that no longer exist on the server.]]) 359 | }) 360 | end, 361 | } 362 | } 363 | }, 364 | { 365 | text = _("Info"), 366 | keep_menu_open = true, 367 | callback = function() 368 | UIManager:show(InfoMessage:new{ 369 | text = T(_([[Wallabag2 is an open source read-it-later service. This plugin synchronizes with a Wallabag2 server. 370 | 371 | More details: https://wallabag.org 372 | 373 | Downloads to folder: %1]]), BD.dirpath(filemanagerutil.abbreviate(self.directory))) 374 | }) 375 | end, 376 | }, 377 | }, 378 | } 379 | end 380 | 381 | function Wallabag2:getBearerToken() 382 | 383 | -- Check if the configuration is complete 384 | local function isempty(s) 385 | return s == nil or s == "" 386 | end 387 | 388 | local server_empty = isempty(self.server_url) or isempty(self.username) or isempty(self.password) or isempty(self.client_id) or isempty(self.client_secret) 389 | local directory_empty = isempty(self.directory) 390 | if server_empty or directory_empty then 391 | UIManager:show(MultiConfirmBox:new{ 392 | text = _("Please configure the server settings and set a download folder."), 393 | choice1_text_func = function() 394 | if server_empty then 395 | return _("Server (★)") 396 | else 397 | return _("Server") 398 | end 399 | end, 400 | choice1_callback = function() self:editServerSettings() end, 401 | choice2_text_func = function() 402 | if directory_empty then 403 | return _("Folder (★)") 404 | else 405 | return _("Folder") 406 | end 407 | end, 408 | choice2_callback = function() self:setDownloadDirectory() end, 409 | }) 410 | return false 411 | end 412 | 413 | -- Check if the download directory is valid 414 | local dir_mode = lfs.attributes(self.directory, "mode") 415 | if dir_mode ~= "directory" then 416 | UIManager:show(InfoMessage:new{ 417 | text = _("The download directory is not valid.\nPlease configure it in the settings.") 418 | }) 419 | return false 420 | end 421 | if string.sub(self.directory, -1) ~= "/" then 422 | self.directory = self.directory .. "/" 423 | end 424 | 425 | local now = os.time() 426 | if self.token_expiry - now > 300 then 427 | -- token still valid for a while, no need to renew 428 | return true 429 | end 430 | 431 | local login_url = "/oauth/v2/token" 432 | 433 | local body = { 434 | grant_type = "password", 435 | client_id = self.client_id, 436 | client_secret = self.client_secret, 437 | username = self.username, 438 | password = self.password 439 | } 440 | 441 | local bodyJSON = JSON.encode(body) 442 | 443 | local headers = { 444 | ["Content-type"] = "application/json", 445 | ["Accept"] = "application/json, */*", 446 | ["Content-Length"] = tostring(#bodyJSON), 447 | } 448 | local result = self:callAPI("POST", login_url, headers, bodyJSON, "") 449 | 450 | if result then 451 | self.access_token = result.access_token 452 | self.token_expiry = now + result.expires_in 453 | return true 454 | else 455 | UIManager:show(InfoMessage:new{ 456 | text = _("Could not login to Wallabag2 server."), }) 457 | return false 458 | end 459 | end 460 | 461 | --- Get a JSON formatted list of articles from the server. 462 | -- The list should have self.article_per_sync item, or less if an error occured. 463 | -- If filter_tag is set, only articles containing this tag are queried. 464 | -- If ignore_tags is defined, articles containing either of the tags are skipped. 465 | function Wallabag2:getArticleList() 466 | local filtering = "" 467 | if self.filter_tag ~= "" then 468 | filtering = "&tags=" .. self.filter_tag 469 | end 470 | 471 | local article_list = {} 472 | local page = 1 473 | -- query the server for articles until we hit our target number 474 | -- FIXME this is horrible, use the 2.4+ API to know when we've ran out 475 | -- https://github.com/koreader/koreader/issues/10738 issue #10738 476 | while #article_list < self.articles_per_sync do 477 | -- get the JSON containing the article list 478 | local articles_url = "/api/entries?archive=0" 479 | .. "&detail=metadata" 480 | .. "&page=" .. page 481 | .. "&perPage=" .. self.articles_per_sync 482 | .. filtering 483 | local articles_json, err, code = self:callAPI("GET", articles_url, nil, "", "", true) 484 | 485 | if err == "http_error" and code == 404 then 486 | -- we may have hit the last page, there are no more articles 487 | logger.dbg("Wallabag2: couldn't get page #", page) 488 | break -- exit while loop 489 | elseif err or articles_json == nil then 490 | -- another error has occured. Don't proceed with downloading 491 | -- or deleting articles 492 | logger.warn("Wallabag2: download of page #", page, "failed with", err, code) 493 | UIManager:show(InfoMessage:new{ 494 | text = _("Requesting article list failed."), }) 495 | return 496 | end 497 | 498 | -- We're only interested in the actual articles in the JSON 499 | -- build an array of those so it's easier to manipulate later 500 | local new_article_list = {} 501 | for _, article in ipairs(articles_json._embedded.items) do 502 | table.insert(new_article_list, article) 503 | end 504 | 505 | -- Apply the filters 506 | new_article_list = self:filterIgnoredTags(new_article_list) 507 | 508 | -- Append the filtered list to the final article list 509 | for _, article in ipairs(new_article_list) do 510 | if #article_list == self.articles_per_sync then 511 | logger.dbg("Wallabag2: hit the article target", self.articles_per_sync) 512 | break 513 | end 514 | table.insert(article_list, article) 515 | end 516 | 517 | -- TODO add sanity check for articles_json.total field being present and a number 518 | -- logger.warn("Wallabag2: download of page #", page, "failed with", err, code) 519 | -- UIManager:show(InfoMessage:new{ text = _("Requesting article list failed."), }) 520 | -- return 521 | if #article_list >= articles_json.total then 522 | -- we now have more articles than the server has available 523 | logger.dbg("Wallabag2: no more articles to query server for", articles_json.total) 524 | break; 525 | end 526 | if page >= articles_json.pages then 527 | -- no more pages of results on server 528 | logger.dbg("Wallabag2: no more pages of articles to query server for", articles_json.total) 529 | break; 530 | end 531 | page = page + 1 532 | end 533 | 534 | return article_list 535 | end 536 | 537 | --- Remove all the articles from the list containing one of the ignored tags. 538 | -- article_list: array containing a json formatted list of articles 539 | -- returns: same array, but without any articles that contain an ignored tag. 540 | function Wallabag2:filterIgnoredTags(article_list) 541 | -- decode all tags to ignore 542 | local ignoring = {} 543 | if self.ignore_tags ~= "" then 544 | for tag in util.gsplit(self.ignore_tags, "[,]+", false) do 545 | ignoring[tag] = true 546 | end 547 | end 548 | 549 | -- rebuild a list without the ignored articles 550 | local filtered_list = {} 551 | for _, article in ipairs(article_list) do 552 | local skip_article = false 553 | for _, tag in ipairs(article.tags) do 554 | if ignoring[tag.label] then 555 | skip_article = true 556 | logger.dbg("Wallabag2: ignoring tag", tag.label, "in article", 557 | article.id, ":", article.title) 558 | break -- no need to look for other tags 559 | end 560 | end 561 | if not skip_article then 562 | table.insert(filtered_list, article) 563 | end 564 | end 565 | 566 | return filtered_list 567 | end 568 | 569 | --- Download Wallabag2 article. 570 | -- @string article 571 | -- @treturn int 1 failed, 2 skipped, 3 downloaded 572 | function Wallabag2:download(article) 573 | local skip_article = false 574 | local title = util.getSafeFilename(article.title, self.directory, 230, 0) 575 | local file_ext = ".epub" 576 | local item_url = "/api/entries/" .. article.id .. "/export.epub" 577 | 578 | -- If the article links to a supported file, we will download it directly. 579 | -- All webpages are HTML. Ignore them since we want the Wallabag2 EPUB instead! 580 | if article.mimetype ~= "text/html" then 581 | if DocumentRegistry:hasProvider(nil, article.mimetype) then 582 | file_ext = "."..DocumentRegistry:mimeToExt(article.mimetype) 583 | item_url = article.url 584 | -- A function represents `null` in our JSON.decode, because `nil` would just disappear. 585 | -- In that case, fall back to the file extension. 586 | elseif type(article.mimetype) == "function" and DocumentRegistry:hasProvider(article.url) then 587 | file_ext = "" 588 | item_url = article.url 589 | end 590 | end 591 | 592 | local local_path = self.directory .. article_id_prefix .. article.id .. article_id_postfix .. title .. file_ext 593 | logger.dbg("Wallabag2: DOWNLOAD: id: ", article.id) 594 | logger.dbg("Wallabag2: DOWNLOAD: title: ", article.title) 595 | logger.dbg("Wallabag2: DOWNLOAD: filename: ", local_path) 596 | 597 | local attr = lfs.attributes(local_path) 598 | if attr then 599 | -- File already exists, skip it. Preferably only skip if the date of local file is newer than server's. 600 | -- newsdownloader.koplugin has a date parser but it is available only if the plugin is activated. 601 | --- @todo find a better solution 602 | if self.is_dateparser_available then 603 | local server_date = self.dateparser.parse(article.updated_at) 604 | if server_date < attr.modification then 605 | skip_article = true 606 | logger.dbg("Wallabag2: skipping file (date checked): ", local_path) 607 | end 608 | else 609 | skip_article = true 610 | logger.dbg("Wallabag2: skipping file: ", local_path) 611 | end 612 | end 613 | 614 | if skip_article == false then 615 | if self:callAPI("GET", item_url, nil, "", local_path) then 616 | return downloaded 617 | else 618 | return failed 619 | end 620 | end 621 | return skipped 622 | end 623 | 624 | -- method: (mandatory) GET, POST, DELETE, PATCH, etc... 625 | -- apiurl: (mandatory) API call excluding the server path, or full URL to a file 626 | -- headers: defaults to auth if given nil value, provide all headers necessary if in use 627 | -- body: empty string if not needed 628 | -- filepath: downloads the file if provided, returns JSON otherwise 629 | -- @treturn result or (nil, "network_error") or (nil, "json_error") 630 | -- or (nil, "http_error", code) 631 | ---- @todo separate call to internal API from the download on external server 632 | function Wallabag2:callAPI(method, apiurl, headers, body, filepath, quiet) 633 | local sink = {} 634 | local request = {} 635 | 636 | -- Is it an API call, or a regular file direct download? 637 | if apiurl:sub(1, 1) == "/" then 638 | -- API call to our server, has the form "/random/api/call" 639 | request.url = self.server_url .. apiurl 640 | if headers == nil then 641 | headers = { 642 | ["Authorization"] = "Bearer " .. self.access_token, 643 | } 644 | end 645 | else 646 | -- regular url link to a foreign server 647 | local file_url = apiurl 648 | request.url = file_url 649 | if headers == nil then 650 | -- no need for a token here 651 | headers = {} 652 | end 653 | end 654 | 655 | request.method = method 656 | if filepath ~= "" then 657 | request.sink = ltn12.sink.file(io.open(filepath, "w")) 658 | socketutil:set_timeout(socketutil.FILE_BLOCK_TIMEOUT, socketutil.FILE_TOTAL_TIMEOUT) 659 | else 660 | request.sink = ltn12.sink.table(sink) 661 | socketutil:set_timeout(socketutil.LARGE_BLOCK_TIMEOUT, socketutil.LARGE_TOTAL_TIMEOUT) 662 | end 663 | request.headers = headers 664 | if body ~= "" then 665 | request.source = ltn12.source.string(body) 666 | end 667 | logger.dbg("Wallabag2: URL ", request.url) 668 | logger.dbg("Wallabag2: method ", method) 669 | 670 | local code, resp_headers, status = socket.skip(1, http.request(request)) 671 | socketutil:reset_timeout() 672 | -- raise error message when network is unavailable 673 | if resp_headers == nil then 674 | logger.dbg("Wallabag2: Server error:", status or code) 675 | return nil, "network_error" 676 | end 677 | if code == 200 then 678 | if filepath ~= "" then 679 | logger.dbg("Wallabag2: file downloaded to", filepath) 680 | return true 681 | else 682 | local content = table.concat(sink) 683 | if content ~= "" and string.sub(content, 1,1) == "{" then 684 | local ok, result = pcall(JSON.decode, content) 685 | if ok and result then 686 | -- Only enable this log when needed, the output can be large 687 | --logger.dbg("Wallabag2: result ", result) 688 | return result 689 | else 690 | UIManager:show(InfoMessage:new{ 691 | text = _("Server response is not valid."), }) 692 | end 693 | else 694 | UIManager:show(InfoMessage:new{ 695 | text = _("Server response is not valid."), }) 696 | end 697 | return nil, "json_error" 698 | end 699 | else 700 | if filepath ~= "" then 701 | local entry_mode = lfs.attributes(filepath, "mode") 702 | if entry_mode == "file" then 703 | os.remove(filepath) 704 | logger.dbg("Wallabag2: Removed failed download:", filepath) 705 | end 706 | elseif not quiet then 707 | UIManager:show(InfoMessage:new{ 708 | text = _("Communication with server failed."), }) 709 | end 710 | logger.dbg("Wallabag2: Request failed:", status or code) 711 | logger.dbg("Wallabag2: Response headers:", resp_headers) 712 | return nil, "http_error", code 713 | end 714 | end 715 | 716 | function Wallabag2:synchronize() 717 | local info = InfoMessage:new{ text = _("Connecting…") } 718 | UIManager:show(info) 719 | UIManager:forceRePaint() 720 | UIManager:close(info) 721 | 722 | if self:getBearerToken() == false then 723 | return false 724 | end 725 | if self.download_queue and next(self.download_queue) ~= nil then 726 | info = InfoMessage:new{ text = _("Adding articles from queue…") } 727 | UIManager:show(info) 728 | UIManager:forceRePaint() 729 | for _, articleUrl in ipairs(self.download_queue) do 730 | self:addArticle(articleUrl) 731 | end 732 | self.download_queue = {} 733 | self:saveSettings() 734 | UIManager:close(info) 735 | end 736 | 737 | local deleted_count = self:processLocalFiles() 738 | 739 | info = InfoMessage:new{ text = _("Getting article list…") } 740 | UIManager:show(info) 741 | UIManager:forceRePaint() 742 | UIManager:close(info) 743 | 744 | local remote_article_ids = {} 745 | local downloaded_count = 0 746 | local failed_count = 0 747 | if self.access_token ~= "" then 748 | local articles = self:getArticleList() 749 | if articles then 750 | logger.dbg("Wallabag2: number of articles:", #articles) 751 | 752 | info = InfoMessage:new{ text = _("Downloading articles…") } 753 | UIManager:show(info) 754 | UIManager:forceRePaint() 755 | UIManager:close(info) 756 | for _, article in ipairs(articles) do 757 | logger.dbg("Wallabag2: processing article ID: ", article.id) 758 | remote_article_ids[ tostring(article.id) ] = true 759 | local res = self:download(article) 760 | if res == downloaded then 761 | downloaded_count = downloaded_count + 1 762 | elseif res == failed then 763 | failed_count = failed_count + 1 764 | end 765 | end 766 | -- synchronize remote deletions 767 | deleted_count = deleted_count + self:processRemoteDeletes(remote_article_ids) 768 | 769 | local msg 770 | if failed_count ~= 0 then 771 | msg = _("Processing finished.\n\nArticles downloaded: %1\nDeleted: %2\nFailed: %3") 772 | info = InfoMessage:new{ text = T(msg, downloaded_count, deleted_count, failed_count) } 773 | else 774 | msg = _("Processing finished.\n\nArticles downloaded: %1\nDeleted: %2") 775 | info = InfoMessage:new{ text = T(msg, downloaded_count, deleted_count) } 776 | end 777 | UIManager:show(info) 778 | end -- articles 779 | end -- access_token 780 | end 781 | 782 | function Wallabag2:processRemoteDeletes(remote_article_ids) 783 | if not self.is_sync_remote_delete then 784 | logger.dbg("Wallabag2: Processing of remote file deletions disabled.") 785 | return 0 786 | end 787 | logger.dbg("Wallabag2: articles IDs from server: ", remote_article_ids) 788 | 789 | local info = InfoMessage:new{ text = _("Synchronizing remote deletions…") } 790 | UIManager:show(info) 791 | UIManager:forceRePaint() 792 | UIManager:close(info) 793 | local deleted_count = 0 794 | for entry in lfs.dir(self.directory) do 795 | if entry ~= "." and entry ~= ".." then 796 | local entry_path = self.directory .. "/" .. entry 797 | local id = self:getArticleID(entry_path) 798 | if not remote_article_ids[ id ] then 799 | logger.dbg("Wallabag2: Deleting local file (deleted on server): ", entry_path) 800 | self:deleteLocalArticle(entry_path) 801 | deleted_count = deleted_count + 1 802 | end 803 | end 804 | end -- for entry 805 | return deleted_count 806 | end 807 | 808 | function Wallabag2:processLocalFiles(mode) 809 | if mode then 810 | if self.is_auto_delete == false and mode ~= "manual" then 811 | logger.dbg("Wallabag2: Automatic processing of local files disabled.") 812 | return 0, 0 813 | end 814 | end 815 | 816 | if self:getBearerToken() == false then 817 | return 0, 0 818 | end 819 | 820 | local num_deleted = 0 821 | if self.is_delete_finished or self.is_delete_read then 822 | local info = InfoMessage:new{ text = _("Processing local files…") } 823 | UIManager:show(info) 824 | UIManager:forceRePaint() 825 | UIManager:close(info) 826 | for entry in lfs.dir(self.directory) do 827 | if entry ~= "." and entry ~= ".." then 828 | local entry_path = self.directory .. "/" .. entry 829 | if DocSettings:hasSidecarFile(entry_path) then 830 | if self.send_review_as_tags then 831 | self:addTags(entry_path) 832 | end 833 | local doc_settings = DocSettings:open(entry_path) 834 | local summary = doc_settings:readSetting("summary") 835 | local status = summary and summary.status 836 | local percent_finished = doc_settings:readSetting("percent_finished") 837 | if status == "complete" or status == "abandoned" then 838 | if self.is_delete_finished then 839 | self:removeArticle(entry_path) 840 | num_deleted = num_deleted + 1 841 | end 842 | elseif percent_finished == 1 then -- 100% read 843 | if self.is_delete_read then 844 | self:removeArticle(entry_path) 845 | num_deleted = num_deleted + 1 846 | end 847 | end 848 | end -- has sidecar 849 | end -- not . and .. 850 | end -- for entry 851 | end -- flag checks 852 | return num_deleted 853 | end 854 | 855 | function Wallabag2:addArticle(article_url) 856 | logger.dbg("Wallabag2: adding article ", article_url) 857 | 858 | if not article_url or self:getBearerToken() == false then 859 | return false 860 | end 861 | 862 | local body = { 863 | url = article_url, 864 | tags = self.auto_tags, 865 | } 866 | 867 | local body_JSON = JSON.encode(body) 868 | 869 | local headers = { 870 | ["Content-type"] = "application/json", 871 | ["Accept"] = "application/json, */*", 872 | ["Content-Length"] = tostring(#body_JSON), 873 | ["Authorization"] = "Bearer " .. self.access_token, 874 | } 875 | 876 | return self:callAPI("POST", "/api/entries.json", headers, body_JSON, "") 877 | end 878 | 879 | function Wallabag2:addTags(path) 880 | logger.dbg("Wallabag2: managing tags for article ", path) 881 | local id = self:getArticleID(path) 882 | if id then 883 | local doc_settings = DocSettings:open(path) 884 | local summary = doc_settings:readSetting("summary") 885 | local tags = summary and summary.note 886 | if tags and tags ~= "" then 887 | logger.dbg("Wallabag2: sending tags ", tags, " for ", path) 888 | 889 | local body = { 890 | tags = tags, 891 | } 892 | 893 | local bodyJSON = JSON.encode(body) 894 | 895 | local headers = { 896 | ["Content-type"] = "application/json", 897 | ["Accept"] = "application/json, */*", 898 | ["Content-Length"] = tostring(#bodyJSON), 899 | ["Authorization"] = "Bearer " .. self.access_token, 900 | } 901 | 902 | self:callAPI("POST", "/api/entries/" .. id .. "/tags.json", headers, bodyJSON, "") 903 | else 904 | logger.dbg("Wallabag2: no tags to send for ", path) 905 | end 906 | end 907 | end 908 | 909 | function Wallabag2:removeArticle(path) 910 | logger.dbg("Wallabag2: removing article ", path) 911 | local id = self:getArticleID(path) 912 | if id then 913 | if self.is_archiving_deleted then 914 | local body = { 915 | archive = 1 916 | } 917 | local bodyJSON = JSON.encode(body) 918 | 919 | local headers = { 920 | ["Content-type"] = "application/json", 921 | ["Accept"] = "application/json, */*", 922 | ["Content-Length"] = tostring(#bodyJSON), 923 | ["Authorization"] = "Bearer " .. self.access_token, 924 | } 925 | 926 | self:callAPI("PATCH", "/api/entries/" .. id .. ".json", headers, bodyJSON, "") 927 | else 928 | self:callAPI("DELETE", "/api/entries/" .. id .. ".json", nil, "", "") 929 | end 930 | self:deleteLocalArticle(path) 931 | end 932 | end 933 | 934 | function Wallabag2:deleteLocalArticle(path) 935 | if lfs.attributes(path, "mode") == "file" then 936 | FileManager:deleteFile(path, true) 937 | end 938 | end 939 | 940 | function Wallabag2:getArticleID(path) 941 | -- extract the Wallabag2 ID from the file name 942 | local offset = self.directory:len() + 2 -- skip / and advance to the next char 943 | local prefix_len = article_id_prefix:len() 944 | if path:sub(offset , offset + prefix_len - 1) ~= article_id_prefix then 945 | logger.warn("Wallabag2: getArticleID: no match! ", path:sub(offset , offset + prefix_len - 1)) 946 | return 947 | end 948 | local endpos = path:find(article_id_postfix, offset + prefix_len) 949 | if endpos == nil then 950 | logger.warn("Wallabag2: getArticleID: no match! ") 951 | return 952 | end 953 | local id = path:sub(offset + prefix_len, endpos - 1) 954 | return id 955 | end 956 | 957 | function Wallabag2:refreshCurrentDirIfNeeded() 958 | if FileManager.instance then 959 | FileManager.instance:onRefresh() 960 | end 961 | end 962 | 963 | function Wallabag2:setFilterTag(touchmenu_instance) 964 | self.tag_dialog = InputDialog:new { 965 | title = _("Set a single tag to filter articles on"), 966 | input = self.filter_tag, 967 | buttons = { 968 | { 969 | { 970 | text = _("Cancel"), 971 | id = "close", 972 | callback = function() 973 | UIManager:close(self.tag_dialog) 974 | end, 975 | }, 976 | { 977 | text = _("OK"), 978 | is_enter_default = true, 979 | callback = function() 980 | self.filter_tag = self.tag_dialog:getInputText() 981 | self:saveSettings() 982 | touchmenu_instance:updateItems() 983 | UIManager:close(self.tag_dialog) 984 | end, 985 | } 986 | } 987 | }, 988 | } 989 | UIManager:show(self.tag_dialog) 990 | self.tag_dialog:onShowKeyboard() 991 | end 992 | 993 | function Wallabag2:setTagsDialog(touchmenu_instance, title, description, value, callback) 994 | self.tags_dialog = InputDialog:new { 995 | title = title, 996 | description = description, 997 | input = value, 998 | buttons = { 999 | { 1000 | { 1001 | text = _("Cancel"), 1002 | id = "close", 1003 | callback = function() 1004 | UIManager:close(self.tags_dialog) 1005 | end, 1006 | }, 1007 | { 1008 | text = _("Set tags"), 1009 | is_enter_default = true, 1010 | callback = function() 1011 | callback(self.tags_dialog:getInputText()) 1012 | self:saveSettings() 1013 | touchmenu_instance:updateItems() 1014 | UIManager:close(self.tags_dialog) 1015 | end, 1016 | } 1017 | } 1018 | }, 1019 | } 1020 | UIManager:show(self.tags_dialog) 1021 | self.tags_dialog:onShowKeyboard() 1022 | end 1023 | 1024 | function Wallabag2:editServerSettings() 1025 | local text_info = T(_([[ 1026 | Enter the details of your Wallabag2 server and account. 1027 | 1028 | Client ID and client secret are long strings so you might prefer to save the empty settings and edit the config file directly in your installation directory: 1029 | %1/wallabag.lua 1030 | 1031 | Restart KOReader after editing the config file.]]), BD.dirpath(DataStorage:getSettingsDir())) 1032 | 1033 | self.settings_dialog = MultiInputDialog:new { 1034 | title = _("Wallabag2 settings"), 1035 | fields = { 1036 | { 1037 | text = self.server_url, 1038 | --description = T(_("Server URL:")), 1039 | hint = _("Server URL") 1040 | }, 1041 | { 1042 | text = self.client_id, 1043 | --description = T(_("Client ID and secret")), 1044 | hint = _("Client ID") 1045 | }, 1046 | { 1047 | text = self.client_secret, 1048 | hint = _("Client secret") 1049 | }, 1050 | { 1051 | text = self.username, 1052 | --description = T(_("Username and password")), 1053 | hint = _("Username") 1054 | }, 1055 | { 1056 | text = self.password, 1057 | text_type = "password", 1058 | hint = _("Password") 1059 | }, 1060 | }, 1061 | buttons = { 1062 | { 1063 | { 1064 | text = _("Cancel"), 1065 | id = "close", 1066 | callback = function() 1067 | UIManager:close(self.settings_dialog) 1068 | end 1069 | }, 1070 | { 1071 | text = _("Info"), 1072 | callback = function() 1073 | UIManager:show(InfoMessage:new{ text = text_info }) 1074 | end 1075 | }, 1076 | { 1077 | text = _("Apply"), 1078 | callback = function() 1079 | local myfields = self.settings_dialog:getFields() 1080 | self.server_url = myfields[1]:gsub("/*$", "") -- remove all trailing "/" slashes 1081 | self.client_id = myfields[2] 1082 | self.client_secret = myfields[3] 1083 | self.username = myfields[4] 1084 | self.password = myfields[5] 1085 | self:saveSettings() 1086 | UIManager:close(self.settings_dialog) 1087 | end 1088 | }, 1089 | }, 1090 | }, 1091 | } 1092 | UIManager:show(self.settings_dialog) 1093 | self.settings_dialog:onShowKeyboard() 1094 | end 1095 | 1096 | function Wallabag2:editClientSettings() 1097 | self.client_settings_dialog = MultiInputDialog:new { 1098 | title = _("Wallabag2 client settings"), 1099 | fields = { 1100 | { 1101 | text = self.articles_per_sync, 1102 | description = _("Number of articles"), 1103 | input_type = "number", 1104 | hint = _("Number of articles to download per sync") 1105 | }, 1106 | }, 1107 | buttons = { 1108 | { 1109 | { 1110 | text = _("Cancel"), 1111 | id = "close", 1112 | callback = function() 1113 | UIManager:close(self.client_settings_dialog) 1114 | end 1115 | }, 1116 | { 1117 | text = _("Apply"), 1118 | callback = function() 1119 | local myfields = self.client_settings_dialog:getFields() 1120 | self.articles_per_sync = math.max(1, tonumber(myfields[1]) or self.articles_per_sync) 1121 | self:saveSettings(myfields) 1122 | UIManager:close(self.client_settings_dialog) 1123 | end 1124 | }, 1125 | }, 1126 | }, 1127 | } 1128 | UIManager:show(self.client_settings_dialog) 1129 | self.client_settings_dialog:onShowKeyboard() 1130 | end 1131 | 1132 | function Wallabag2:setDownloadDirectory(touchmenu_instance) 1133 | require("ui/downloadmgr"):new{ 1134 | onConfirm = function(path) 1135 | logger.dbg("Wallabag2: set download directory to: ", path) 1136 | self.directory = path 1137 | self:saveSettings() 1138 | if touchmenu_instance then 1139 | touchmenu_instance:updateItems() 1140 | end 1141 | end, 1142 | }:chooseDir() 1143 | end 1144 | 1145 | function Wallabag2:saveSettings() 1146 | local tempsettings = { 1147 | server_url = self.server_url, 1148 | client_id = self.client_id, 1149 | client_secret = self.client_secret, 1150 | username = self.username, 1151 | password = self.password, 1152 | directory = self.directory, 1153 | filter_tag = self.filter_tag, 1154 | ignore_tags = self.ignore_tags, 1155 | auto_tags = self.auto_tags, 1156 | is_delete_finished = self.is_delete_finished, 1157 | is_delete_read = self.is_delete_read, 1158 | is_archiving_deleted = self.is_archiving_deleted, 1159 | is_auto_delete = self.is_auto_delete, 1160 | is_sync_remote_delete = self.is_sync_remote_delete, 1161 | articles_per_sync = self.articles_per_sync, 1162 | send_review_as_tags = self.send_review_as_tags, 1163 | remove_finished_from_history = self.remove_finished_from_history, 1164 | remove_read_from_history = self.remove_read_from_history, 1165 | download_queue = self.download_queue, 1166 | } 1167 | self.wb_settings:saveSetting("wallabag", tempsettings) 1168 | self.wb_settings:flush() 1169 | 1170 | end 1171 | 1172 | function Wallabag2:readSettings() 1173 | local wb_settings = LuaSettings:open(DataStorage:getSettingsDir().."/wallabag.lua") 1174 | wb_settings:readSetting("wallabag", {}) 1175 | return wb_settings 1176 | end 1177 | 1178 | function Wallabag2:saveWBSettings(setting) 1179 | if not self.wb_settings then self.wb_settings = self:readSettings() end 1180 | self.wb_settings:saveSetting("wallabag", setting) 1181 | self.wb_settings:flush() 1182 | end 1183 | 1184 | function Wallabag2:onAddWallabag2Article(article_url) 1185 | if not NetworkMgr:isOnline() then 1186 | self:addToDownloadQueue(article_url) 1187 | UIManager:show(InfoMessage:new{ 1188 | text = T(_("Article added to download queue:\n%1"), BD.url(article_url)), 1189 | timeout = 1, 1190 | }) 1191 | return 1192 | end 1193 | 1194 | local wallabag_result = self:addArticle(article_url) 1195 | if wallabag_result then 1196 | UIManager:show(InfoMessage:new{ 1197 | text = T(_("Article added to Wallabag2:\n%1"), BD.url(article_url)), 1198 | }) 1199 | else 1200 | UIManager:show(InfoMessage:new{ 1201 | text = T(_("Error adding link to Wallabag2:\n%1"), BD.url(article_url)), 1202 | }) 1203 | end 1204 | 1205 | -- stop propagation 1206 | return true 1207 | end 1208 | 1209 | function Wallabag2:onSynchronizeWallabag2() 1210 | local connect_callback = function() 1211 | self:synchronize() 1212 | self:refreshCurrentDirIfNeeded() 1213 | end 1214 | NetworkMgr:runWhenOnline(connect_callback) 1215 | 1216 | -- stop propagation 1217 | return true 1218 | end 1219 | 1220 | function Wallabag2:getLastPercent() 1221 | local percent = self.ui.paging and self.ui.paging:getLastPercent() or self.ui.rolling:getLastPercent() 1222 | return Math.roundPercent(percent) 1223 | end 1224 | 1225 | function Wallabag2:addToDownloadQueue(article_url) 1226 | table.insert(self.download_queue, article_url) 1227 | self:saveSettings() 1228 | end 1229 | 1230 | function Wallabag2:onCloseDocument() 1231 | if self.remove_finished_from_history or self.remove_read_from_history then 1232 | local document_full_path = self.ui.document.file 1233 | local summary = self.ui.doc_settings:readSetting("summary") 1234 | local status = summary and summary.status 1235 | local is_finished = status == "complete" or status == "abandoned" 1236 | local is_read = self:getLastPercent() == 1 1237 | 1238 | if document_full_path 1239 | and self.directory 1240 | and ( (self.remove_finished_from_history and is_finished) or (self.remove_read_from_history and is_read) ) 1241 | and self.directory == string.sub(document_full_path, 1, string.len(self.directory)) then 1242 | ReadHistory:removeItemByPath(document_full_path) 1243 | self.ui:setLastDirForFileBrowser(self.directory) 1244 | end 1245 | end 1246 | end 1247 | 1248 | return Wallabag2 1249 | --------------------------------------------------------------------------------