From 5a4f5b4d59fda3cf2633a6783d28c4f37a15758a Mon Sep 17 00:00:00 2001 From: poire-z Date: Mon, 24 Feb 2020 15:25:08 +0100 Subject: [PATCH] bump crengine, migrate books to normalized xpointers Enable new rendering feature COMPLETE_INCOMPLETE_TABLES on all enhanced rendering mode, but have it disabled for earlier cre_dom_version. Also increase default_cre_storage_size_factor from 20 to 40. --- base | 2 +- .../apps/reader/modules/readerrolling.lua | 231 +++++++++++++++++- .../apps/reader/modules/readertypeset.lua | 25 +- frontend/docsettings.lua | 8 +- frontend/document/credocument.lua | 18 +- 5 files changed, 276 insertions(+), 8 deletions(-) diff --git a/base b/base index 6a88ab496..4913a4697 160000 --- a/base +++ b/base @@ -1 +1 @@ -Subproject commit 6a88ab496b7390885b78b7210b6bfddc1508f744 +Subproject commit 4913a469722a4826d0251de1c050d609dd48a477 diff --git a/frontend/apps/reader/modules/readerrolling.lua b/frontend/apps/reader/modules/readerrolling.lua index 4ed154bf6..7df665126 100644 --- a/frontend/apps/reader/modules/readerrolling.lua +++ b/frontend/apps/reader/modules/readerrolling.lua @@ -152,6 +152,26 @@ function ReaderRolling:onReadSettings(config) end end self.ui.document:requestDomVersion(config:readSetting("cre_dom_version")) + -- If we're using a DOM version without normalized XPointers, some stuff + -- may need tweaking: + if config:readSetting("cre_dom_version") < cre.getDomVersionWithNormalizedXPointers() then + -- Show some warning when styles "display:" have changed that + -- bookmarks may break + self.using_non_normalized_xpointers = true + -- Also tell ReaderTypeset, which ensures block rendering mode, + -- that we'd rather have some of its BLOCK_RENDERING_FLAGS disabled + -- if an old DOM version is requested, as some flags may "box" + -- (into inserted internal elements) long fragment of text, + -- which may break previous highlights. + self.ui.typeset:ensureSanerBlockRenderingFlags() + -- And check if we can migrate to a newest DOM version after + -- the book is loaded (unless the user told us not to). + if not config:readSetting("cre_keep_old_dom_version") then + self.ui:registerPostReadyCallback(function() + self:checkXPointersAndProposeDOMVersionUpgrade() + end) + end + end local last_xp = config:readSetting("last_xpointer") local last_per = config:readSetting("last_percent") @@ -243,7 +263,8 @@ end function ReaderRolling:onCheckDomStyleCoherence() if self.ui.document and self.ui.document:isBuiltDomStale() then local has_bookmarks_warn_txt = "" - if self.ui.bookmark:hasBookmarks() then + -- When using an older DOM version, bookmarks may break + if self.using_non_normalized_xpointers and self.ui.bookmark:hasBookmarks() then has_bookmarks_warn_txt = _("\nNote that this change in styles may render your bookmarks or highlights no more valid.\nIf some of them do not show anymore, you can just revert the change you just made to have them shown again.\n\n") end UIManager:show(ConfirmBox:new{ @@ -1086,4 +1107,212 @@ function ReaderRolling:showEngineProgress(percent) end end +function ReaderRolling:checkXPointersAndProposeDOMVersionUpgrade() + if self.ui.document and self.ui.document:isBuiltDomStale() then + -- DOM is not in sync, and some message "Styles have changed + -- in such a way" is going to be displayed. + -- Wait for things to be saner to migrate. + return + end + + -- Loop thru all known xpointers holders, and apply + -- func(object, key, info_text) to each of them + local applyFuncToXPointersSlots = function(func) + -- Last position + func(self, "xpointer", "last position in book") + -- Bookmarks + if self.ui.bookmark and self.ui.bookmark.bookmarks and #self.ui.bookmark.bookmarks > 0 then + local slots = { "page", "pos0", "pos1" } + for _, bookmark in ipairs(self.ui.bookmark.bookmarks) do + for _, slot in ipairs(slots) do + func(bookmark, slot, bookmark.notes or "bookmark") + end + end + end + -- Highlights + if self.view.highlight and self.view.highlight.saved then + local slots = { "pos0", "pos1" } + for page, items in pairs(self.view.highlight.saved) do + if items and #items > 0 then + for _, highlight in ipairs(items) do + for _, slot in ipairs(slots) do + func(highlight, slot, highlight.text or "highlight") + end + end + end + end + end + end + + -- Cache and counters + local normalized_xpointers = {} + local lost_xpointer_info = {} + local nb_xpointers = 0 + local nb_xpointers_found = 0 + local nb_xpointers_changed = 0 + local nb_xpointers_lost = 0 + + -- To be provided to applyFuncToXPointersSlots() + local checkAndCount = function(obj, slot, info) + local xp = obj[slot] + if not xp then + return + end + if normalized_xpointers[xp] ~= nil then -- already seen + return + end + nb_xpointers = nb_xpointers + 1 + local nxp = self.ui.document:getNormalizedXPointer(xp) + normalized_xpointers[xp] = nxp -- cache it + if nxp then + nb_xpointers_found = nb_xpointers_found + 1 + if nxp ~= xp then + nb_xpointers_changed = nb_xpointers_changed + 1 + end + else + nb_xpointers_lost = nb_xpointers_lost + 1 + lost_xpointer_info[xp] = info + end + end + + -- To be provided to applyFuncToXPointersSlots() + local migrateXPointer = function(obj, slot, info) + local xp = obj[slot] + if not xp then + return + end + local new_xp = normalized_xpointers[xp] + if new_xp then + obj[slot] = new_xp + else + -- Let lost/not-found XPointer be. There is a small chance that + -- it will be found (it it was made before the boxing code moved + -- it into a box, it might be a normalized xpointer) but there is + -- also a smaller chance that it will map to something completely + -- different... + -- Flag it, so one can investigate and fix it manually + if slot ~= "xpointer" then -- (not for last_xpointer) + obj["not_found_not_migrated"] = true + end + end + end + + -- Do the actual xpointers migration, and related changes + local upgradeToLatestDOMVersion = function() + logger.info("Upgrading book to latest DOM version:") + + -- Backup metadata.lua + local cur_dom_version = self.ui.doc_settings:readSetting("cre_dom_version") or "unknown" + if self.ui.doc_settings.filepath then + local backup_filepath = self.ui.doc_settings.filepath .. ".old_dom" .. tostring(cur_dom_version) + if not lfs.attributes(backup_filepath) then -- backup does not yet exist + os.rename(self.ui.doc_settings.filepath, backup_filepath) + logger.info(" previous docsetting file saved as", backup_filepath) + end + end + + -- Migrate all XPointers + applyFuncToXPointersSlots(migrateXPointer) + logger.info(T(" xpointers updated: %1 unchanged, %2 modified, %3 not found let as-is", + nb_xpointers_found - nb_xpointers_changed, nb_xpointers_changed, nb_xpointers_lost)) + + -- Set latest DOM version, to be used at next load + local latest_dom_version = self.ui.document:getLatestDomVersion() + self.ui.doc_settings:saveSetting("cre_dom_version", latest_dom_version) + logger.info(" cre_dom_version updated to", latest_dom_version) + + -- Switch to default block rendering mode if this book has it set to "legacy", + -- unless the user had set the global mode to be "legacy". + -- (see ReaderTypeset:onReadSettings() for the logic of block_rendering_mode) + local g_block_rendering_mode = G_reader_settings:readSetting("copt_block_rendering_mode") + if g_block_rendering_mode ~= 0 then -- default is not "legacy" + if not g_block_rendering_mode then -- nil means: use default + g_block_rendering_mode = 3 -- default in ReaderTypeset:onReadSettings() + end + -- This setting is actually saved by self.ui.document.configurable + local block_rendering_mode = self.ui.document.configurable.block_rendering_mode + if block_rendering_mode == 0 then + self.ui.document.configurable.block_rendering_mode = g_block_rendering_mode + logger.info(" block_rendering_mode switched to", g_block_rendering_mode) + end + end + + -- No need for "if doc:hasCacheFile() then doc:invalidateCacheFile()", as + -- a change in gDOMVersionRequested has crengine trash previous cache file. + end + + -- Check all xpointers + applyFuncToXPointersSlots(checkAndCount) + logger.info(T("%1 xpointers checked: %2 found (%3 changed) - %4 lost", + nb_xpointers, nb_xpointers_found, nb_xpointers_changed, nb_xpointers_lost)) + if nb_xpointers_lost > 0 then + logger.warn("Lost xpointers:") + for k, v in pairs(lost_xpointer_info) do + logger.warn(" ", k, ":", v) + end + end + + local text = _([[ +This book was first opened, and has been handled since, by an older version of the rendering code. +Bookmarks and highlights can be upgraded to the latest version of the code. + +%1 + +Proceed with this upgrade and reload the book?]]) + local details = {} + if nb_xpointers_lost == 0 then + table.insert(details, _([[All your bookmarks and highlights are valid and will be available after the migration.]])) + else + table.insert(details, T(_([[ +Note that %1 (out of %2) xpaths from your bookmarks and highlights aren't currently found in the book, and may have been lost. You might want to toggle Rendering mode between 'legacy' and 'flat', and re-open this book, and see if they are found again, before proceeding.]]), + nb_xpointers_lost, nb_xpointers)) + end + if nb_xpointers_changed > 0 then + table.insert(details, T(_([[ +Note that %1 (out of %2) xpaths from your bookmarks and highlights have been normalized, and may not work on previous KOReader versions (if you're synchronizing your reading between multiple devices, you'll need to update KOReader on all of them).]]), + nb_xpointers_changed, nb_xpointers)) + end + text = T(text, table.concat(details, "\n\n")) + + UIManager:show(ConfirmBox:new{ + text = text, + other_buttons = {{ + { + -- this is the real cancel/do nothing + text = _("Not now"), + } + }}, + cancel_text = _("Not for this book"), + cancel_callback = function() + self.ui.doc_settings:saveSetting("cre_keep_old_dom_version", true) + end, + ok_text = _("Upgrade now"), + ok_callback = function() + -- Allow for ConfirmBox to be closed before migrating + UIManager:scheduleIn(0.5, function () + -- And check we haven't quit reader in these 0.5s + if self.ui.document then + -- We'd rather not have any painting between the upgrade + -- and the document reloading (readerview might draw + -- highlights from the migrated xpointers, that would + -- not be found in the document... + local InfoMessage = require("ui/widget/infomessage") + local infomsg = InfoMessage:new{ + text = _("Upgrading and reloading book…"), + } + UIManager:show(infomsg) + -- Let this message be shown + UIManager:scheduleIn(2, function () + UIManager:close(infomsg) + if self.ui.document then + upgradeToLatestDOMVersion() + self.ui:reloadDocument() + end + end) + end + end) + end, + }) +end + return ReaderRolling diff --git a/frontend/apps/reader/modules/readertypeset.lua b/frontend/apps/reader/modules/readertypeset.lua index 507aedde0..2dca2f6d7 100644 --- a/frontend/apps/reader/modules/readertypeset.lua +++ b/frontend/apps/reader/modules/readertypeset.lua @@ -344,11 +344,12 @@ end -- ALLOW_EXACT_FLOATS_FOOTPRINTS 0x00200000 x x -- -- BOX_INLINE_BLOCKS 0x01000000 x x x +-- COMPLETE_INCOMPLETE_TABLES 0x02000000 x x x local BLOCK_RENDERING_FLAGS = { 0x00000000, -- legacy block rendering - 0x01030031, -- flat mode (with prepared floatBoxes, so inlined, to avoid display hash mismatch) - 0x01375131, -- book mode (floating floatBoxes, limited widths support) + 0x03030031, -- flat mode (with prepared floatBoxes, so inlined, to avoid display hash mismatch) + 0x03375131, -- book mode (floating floatBoxes, limited widths support) 0x7FFFFFFF, -- web mode, all features/flags } @@ -364,10 +365,30 @@ function ReaderTypeset:setBlockRenderingMode(mode) return end self.block_rendering_mode = mode + if self.ensure_saner_block_rendering_flags then -- see next function + -- Don't enable BOX_INLINE_BLOCKS + -- inlineBoxes have been around and allowed on older DOM_VERSION + -- for a few weeks - let's disable it: it may break highlights + -- made during this time, but may resurrect others made during + -- a longer previous period of time. + flags = bit.band(flags, bit.bnot(0x01000000)) + -- Don't enable COMPLETE_INCOMPLETE_TABLES, as they may add + -- many boxing elements around huge amount of text, and break + -- some past highlights made on the non-boxed elements. + flags = bit.band(flags, bit.bnot(0x02000000)) + end self.ui.document:setBlockRenderingFlags(flags) self.ui:handleEvent(Event:new("UpdatePos")) end +function ReaderTypeset:ensureSanerBlockRenderingFlags(mode) + -- Called by ReaderRolling:onReadSettings() when old + -- DOM version requested, before normalized xpointers, + -- asking us to unset some of the flags set previously. + self.ensure_saner_block_rendering_flags = true + self:setBlockRenderingMode(self.block_rendering_mode) +end + function ReaderTypeset:toggleImageScaling(toggle) if toggle and (toggle == true or toggle == 1) then self.smooth_scaling = true diff --git a/frontend/docsettings.lua b/frontend/docsettings.lua index 136427ac9..66223f54b 100644 --- a/frontend/docsettings.lua +++ b/frontend/docsettings.lua @@ -132,7 +132,7 @@ function DocSettings:open(docfile) return l[2] > r[2] end end) - local ok, stored + local ok, stored, filepath for _, k in pairs(candidates) do -- Ignore empty files if lfs.attributes(k[1], "size") > 0 then @@ -140,6 +140,7 @@ function DocSettings:open(docfile) -- Ignore the empty table. if ok and next(stored) ~= nil then logger.dbg("data is read from ", k[1]) + filepath = k[1] break end end @@ -149,6 +150,7 @@ function DocSettings:open(docfile) if ok and stored then new.data = stored new.candidates = candidates + new.filepath = filepath else new.data = {} end @@ -236,6 +238,10 @@ function DocSettings:close() self:flush() end +function DocSettings:getFilePath() + return self.filepath +end + --- Purges (removes) sidecar directory. function DocSettings:purge() if self.history_file then diff --git a/frontend/document/credocument.lua b/frontend/document/credocument.lua index 7dfa7c3c6..9e000bca5 100644 --- a/frontend/document/credocument.lua +++ b/frontend/document/credocument.lua @@ -55,13 +55,13 @@ function CreDocument:cacheInit() -- the less recently used ones when this limit is reached local default_cre_disk_cache_max_size = 64 -- in MB units -- crengine various in-memory caches max-sizes are rather small - -- (2.5 / 4.5 / 1.5 / 1 MB), and we can avoid some bugs if we - -- increase them. Let's multiply them by 20 (each cache would + -- (2.5 / 4.5 / 4.5 / 1 MB), and we can avoid some bugs if we + -- increase them. Let's multiply them by 40 (each cache would -- grow only when needed, depending on book characteristics). -- People who would get out of memory crashes with big books on -- older devices can decrease that with setting: -- "cre_storage_size_factor"=1 (or 2, or 5) - local default_cre_storage_size_factor = 20 + local default_cre_storage_size_factor = 40 cre.initCache(DataStorage:getDataDir() .. "/cache/cr3cache", (G_reader_settings:readSetting("cre_disk_cache_max_size") or default_cre_disk_cache_max_size)*1024*1024, G_reader_settings:nilOrTrue("cre_compress_cached_data"), @@ -138,6 +138,10 @@ function CreDocument:init() self:setupCallCache() end +function CreDocument:getDomVersionWithNormalizedXPointers() + return cre.getDomVersionWithNormalizedXPointers() +end + function CreDocument:getLatestDomVersion() return cre.getLatestDomVersion() end @@ -522,6 +526,13 @@ function CreDocument:getHTMLFromXPointers(xp0, xp1, flags, from_root_node) end end +function CreDocument:getNormalizedXPointer(xp) + -- Returns false when xpointer is not found in the DOM. + -- When requested DOM version >= getDomVersionWithNormalizedXPointers, + -- should return xp unmodified when found. + return self._document:getNormalizedXPointer(xp) +end + function CreDocument:gotoPos(pos) logger.dbg("CreDocument: goto position", pos) self._document:gotoPos(pos) @@ -1152,6 +1163,7 @@ function CreDocument:setupCallCache() elseif name == "getNextVisibleChar" then no_wrap = true elseif name == "getCacheFilePath" then no_wrap = true elseif name == "getStatistics" then no_wrap = true + elseif name == "getNormalizedXPointer" then no_wrap = true -- Some get* have different results by page/pos elseif name == "getLinkFromPosition" then cache_by_tag = true