From 69ec6c07c56be7d70ebd1c2921fce00e5c3e77ba Mon Sep 17 00:00:00 2001 From: poire-z Date: Wed, 20 Dec 2017 15:26:01 +0100 Subject: [PATCH] Wikipedia Save as EPUB: fix some wiki links and speedup download (#3551) - Fix links to wikipedia pages containing '?oldid=123', that wouldn't work when used back thru the API. - Make external links to other wikipedias with a different langage work (they didn't work when non-ascii page titles) - Speedup download of images by using a newly added 'fast_refresh' option to Trapper:info() to bypass UIManager. --- frontend/ui/trapper.lua | 50 ++++++++++++++++++++++++++------------- frontend/ui/wikipedia.lua | 24 ++++++++++++++++++- 2 files changed, 56 insertions(+), 18 deletions(-) diff --git a/frontend/ui/trapper.lua b/frontend/ui/trapper.lua index 0f2e4f5a5..f69f685f4 100644 --- a/frontend/ui/trapper.lua +++ b/frontend/ui/trapper.lua @@ -105,14 +105,19 @@ steps of the work, to provide good responsiveness. Trapper:info() is a shortcut to get dismiss info while keeping the existing InfoMessage displayed. +Optional fast_refresh parameter should only be used when +displaying an InfoMessage over a previous InfoMessage of the +exact same size. + @string text text to display as an InfoMessage (or nil to keep existing one) +@boolean fast_refresh[opt=false] true for faster refresh @treturn boolean true if InfoMessage was not dismissed, false if dismissed @usage Trapper:info("some text about step or progress") go_on = Trapper:info() ]] -function Trapper:info(text) +function Trapper:info(text, fast_refresh) local _coroutine = coroutine.running() if not _coroutine then logger.info("unwrapped info:", text) @@ -181,23 +186,34 @@ function Trapper:info(text) -- events won't be considered action on the yet to be displayed -- widget - -- We're going to display a new widget, close previous one - if self.current_widget then - UIManager:close(self.current_widget) - -- no repaint here, we'll do that below when a new one is shown - end + -- If fast_refresh option, avoid UIManager refresh overhead + if fast_refresh and self.current_widget and self.current_widget.is_infomessage then + self.current_widget:free() + self.current_widget.text = text + self.current_widget:init() + local Screen = require("device").screen + self.current_widget:paintTo(Screen.bb, 0,0) + local d = self.current_widget[1][1].dimen + Screen.refreshUI(Screen, d.x, d.y, d.w, d.h) + else + -- We're going to display a new widget, close previous one + if self.current_widget then + UIManager:close(self.current_widget) + -- no repaint here, we'll do that below when a new one is shown + end - -- dismiss_callback will be checked for at start of next call - self.current_widget = InfoMessage:new{ - text = text, - dismiss_callback = function() - coroutine.resume(_coroutine, false) - end, - is_infomessage = true -- flag on our InfoMessages - } - logger.dbg("Showing InfoMessage:", text) - UIManager:show(self.current_widget) - UIManager:forceRePaint() + -- dismiss_callback will be checked for at start of next call + self.current_widget = InfoMessage:new{ + text = text, + dismiss_callback = function() + coroutine.resume(_coroutine, false) + end, + is_infomessage = true -- flag on our InfoMessages + } + logger.dbg("Showing InfoMessage:", text) + UIManager:show(self.current_widget) + UIManager:forceRePaint() + end return true end diff --git a/frontend/ui/wikipedia.lua b/frontend/ui/wikipedia.lua index 11916d488..09e0141bb 100644 --- a/frontend/ui/wikipedia.lua +++ b/frontend/ui/wikipedia.lua @@ -685,7 +685,18 @@ time, abbr, sup { -- when we should have get "\xc3\xa7" ... -- We can avoid that by putting in the url plain unencoded UTF8 local hex_to_char = function(x) return string.char(tonumber(x, 16)) end + -- Do that first (need to be done first) for full links to other language wikipedias + local fixEncodedOtherLangWikiPageTitle = function(wiki_lang, wiki_page) + -- First, remove any "?otherkey=othervalue" from url (a real "?" part of the wiki_page word + -- would be encoded as %3f), that could cause problem when used. + wiki_page = wiki_page:gsub("%?.*", "") + wiki_page = wiki_page:gsub("%%(%x%x)", hex_to_char) + return string.format([[href="https://%s.wikipedia.org/wiki/%s"]], wiki_lang, wiki_page) + end + html = html:gsub([[href="https?://([^%.]+).wikipedia.org/wiki/([^"]*)"]], fixEncodedOtherLangWikiPageTitle) + -- Now, do it for same wikipedia short urls local fixEncodedWikiPageTitle = function(wiki_page) + wiki_page = wiki_page:gsub("%?.*", "") wiki_page = wiki_page:gsub("%%(%x%x)", hex_to_char) return string.format([[href="%s/wiki/%s"]], wiki_base_url, wiki_page) end @@ -747,6 +758,11 @@ time, abbr, sup { ]], page_cleaned, page_htmltitle, lang:upper(), saved_on, see_online_version, html)) + -- Force a GC to free the memory we used till now (the second call may + -- help reclaim more memory). + collectgarbage() + collectgarbage() + -- ---------------------------------------------------------------- -- OEBPS/images/* if include_images then @@ -754,7 +770,8 @@ time, abbr, sup { for inum, img in ipairs(images) do -- Process can be interrupted at this point between each image download -- by tapping while the InfoMessage is displayed - local go_on = UI:info(T(_("Fetching image %1 / %2 …"), inum, nb_images)) + -- We use the fast_refresh option from image #2 for a quicker download + local go_on = UI:info(T(_("Fetching image %1 / %2 …"), inum, nb_images), inum >= 2) if not go_on then cancelled = true break @@ -810,6 +827,11 @@ time, abbr, sup { -- Finally move the .tmp to the final file os.rename(epub_path_tmp, epub_path) logger.info("successfully created:", epub_path) + + -- Force a GC to free the memory we used (the second call may help + -- reclaim more memory). + collectgarbage() + collectgarbage() return true end