bump crengine, migrate books to normalized xpointers

Enable new rendering feature COMPLETE_INCOMPLETE_TABLES
on all enhanced rendering mode, but have it disabled for
earlier cre_dom_version.
Also increase default_cre_storage_size_factor from 20 to 40.
reviewable/pr5898/r1
poire-z 4 years ago
parent 7d7781b768
commit 5a4f5b4d59

@ -1 +1 @@
Subproject commit 6a88ab496b7390885b78b7210b6bfddc1508f744
Subproject commit 4913a469722a4826d0251de1c050d609dd48a477

@ -152,6 +152,26 @@ function ReaderRolling:onReadSettings(config)
end
end
self.ui.document:requestDomVersion(config:readSetting("cre_dom_version"))
-- If we're using a DOM version without normalized XPointers, some stuff
-- may need tweaking:
if config:readSetting("cre_dom_version") < cre.getDomVersionWithNormalizedXPointers() then
-- Show some warning when styles "display:" have changed that
-- bookmarks may break
self.using_non_normalized_xpointers = true
-- Also tell ReaderTypeset, which ensures block rendering mode,
-- that we'd rather have some of its BLOCK_RENDERING_FLAGS disabled
-- if an old DOM version is requested, as some flags may "box"
-- (into inserted internal elements) long fragment of text,
-- which may break previous highlights.
self.ui.typeset:ensureSanerBlockRenderingFlags()
-- And check if we can migrate to a newest DOM version after
-- the book is loaded (unless the user told us not to).
if not config:readSetting("cre_keep_old_dom_version") then
self.ui:registerPostReadyCallback(function()
self:checkXPointersAndProposeDOMVersionUpgrade()
end)
end
end
local last_xp = config:readSetting("last_xpointer")
local last_per = config:readSetting("last_percent")
@ -243,7 +263,8 @@ end
function ReaderRolling:onCheckDomStyleCoherence()
if self.ui.document and self.ui.document:isBuiltDomStale() then
local has_bookmarks_warn_txt = ""
if self.ui.bookmark:hasBookmarks() then
-- When using an older DOM version, bookmarks may break
if self.using_non_normalized_xpointers and self.ui.bookmark:hasBookmarks() then
has_bookmarks_warn_txt = _("\nNote that this change in styles may render your bookmarks or highlights no more valid.\nIf some of them do not show anymore, you can just revert the change you just made to have them shown again.\n\n")
end
UIManager:show(ConfirmBox:new{
@ -1086,4 +1107,212 @@ function ReaderRolling:showEngineProgress(percent)
end
end
function ReaderRolling:checkXPointersAndProposeDOMVersionUpgrade()
if self.ui.document and self.ui.document:isBuiltDomStale() then
-- DOM is not in sync, and some message "Styles have changed
-- in such a way" is going to be displayed.
-- Wait for things to be saner to migrate.
return
end
-- Loop thru all known xpointers holders, and apply
-- func(object, key, info_text) to each of them
local applyFuncToXPointersSlots = function(func)
-- Last position
func(self, "xpointer", "last position in book")
-- Bookmarks
if self.ui.bookmark and self.ui.bookmark.bookmarks and #self.ui.bookmark.bookmarks > 0 then
local slots = { "page", "pos0", "pos1" }
for _, bookmark in ipairs(self.ui.bookmark.bookmarks) do
for _, slot in ipairs(slots) do
func(bookmark, slot, bookmark.notes or "bookmark")
end
end
end
-- Highlights
if self.view.highlight and self.view.highlight.saved then
local slots = { "pos0", "pos1" }
for page, items in pairs(self.view.highlight.saved) do
if items and #items > 0 then
for _, highlight in ipairs(items) do
for _, slot in ipairs(slots) do
func(highlight, slot, highlight.text or "highlight")
end
end
end
end
end
end
-- Cache and counters
local normalized_xpointers = {}
local lost_xpointer_info = {}
local nb_xpointers = 0
local nb_xpointers_found = 0
local nb_xpointers_changed = 0
local nb_xpointers_lost = 0
-- To be provided to applyFuncToXPointersSlots()
local checkAndCount = function(obj, slot, info)
local xp = obj[slot]
if not xp then
return
end
if normalized_xpointers[xp] ~= nil then -- already seen
return
end
nb_xpointers = nb_xpointers + 1
local nxp = self.ui.document:getNormalizedXPointer(xp)
normalized_xpointers[xp] = nxp -- cache it
if nxp then
nb_xpointers_found = nb_xpointers_found + 1
if nxp ~= xp then
nb_xpointers_changed = nb_xpointers_changed + 1
end
else
nb_xpointers_lost = nb_xpointers_lost + 1
lost_xpointer_info[xp] = info
end
end
-- To be provided to applyFuncToXPointersSlots()
local migrateXPointer = function(obj, slot, info)
local xp = obj[slot]
if not xp then
return
end
local new_xp = normalized_xpointers[xp]
if new_xp then
obj[slot] = new_xp
else
-- Let lost/not-found XPointer be. There is a small chance that
-- it will be found (it it was made before the boxing code moved
-- it into a box, it might be a normalized xpointer) but there is
-- also a smaller chance that it will map to something completely
-- different...
-- Flag it, so one can investigate and fix it manually
if slot ~= "xpointer" then -- (not for last_xpointer)
obj["not_found_not_migrated"] = true
end
end
end
-- Do the actual xpointers migration, and related changes
local upgradeToLatestDOMVersion = function()
logger.info("Upgrading book to latest DOM version:")
-- Backup metadata.lua
local cur_dom_version = self.ui.doc_settings:readSetting("cre_dom_version") or "unknown"
if self.ui.doc_settings.filepath then
local backup_filepath = self.ui.doc_settings.filepath .. ".old_dom" .. tostring(cur_dom_version)
if not lfs.attributes(backup_filepath) then -- backup does not yet exist
os.rename(self.ui.doc_settings.filepath, backup_filepath)
logger.info(" previous docsetting file saved as", backup_filepath)
end
end
-- Migrate all XPointers
applyFuncToXPointersSlots(migrateXPointer)
logger.info(T(" xpointers updated: %1 unchanged, %2 modified, %3 not found let as-is",
nb_xpointers_found - nb_xpointers_changed, nb_xpointers_changed, nb_xpointers_lost))
-- Set latest DOM version, to be used at next load
local latest_dom_version = self.ui.document:getLatestDomVersion()
self.ui.doc_settings:saveSetting("cre_dom_version", latest_dom_version)
logger.info(" cre_dom_version updated to", latest_dom_version)
-- Switch to default block rendering mode if this book has it set to "legacy",
-- unless the user had set the global mode to be "legacy".
-- (see ReaderTypeset:onReadSettings() for the logic of block_rendering_mode)
local g_block_rendering_mode = G_reader_settings:readSetting("copt_block_rendering_mode")
if g_block_rendering_mode ~= 0 then -- default is not "legacy"
if not g_block_rendering_mode then -- nil means: use default
g_block_rendering_mode = 3 -- default in ReaderTypeset:onReadSettings()
end
-- This setting is actually saved by self.ui.document.configurable
local block_rendering_mode = self.ui.document.configurable.block_rendering_mode
if block_rendering_mode == 0 then
self.ui.document.configurable.block_rendering_mode = g_block_rendering_mode
logger.info(" block_rendering_mode switched to", g_block_rendering_mode)
end
end
-- No need for "if doc:hasCacheFile() then doc:invalidateCacheFile()", as
-- a change in gDOMVersionRequested has crengine trash previous cache file.
end
-- Check all xpointers
applyFuncToXPointersSlots(checkAndCount)
logger.info(T("%1 xpointers checked: %2 found (%3 changed) - %4 lost",
nb_xpointers, nb_xpointers_found, nb_xpointers_changed, nb_xpointers_lost))
if nb_xpointers_lost > 0 then
logger.warn("Lost xpointers:")
for k, v in pairs(lost_xpointer_info) do
logger.warn(" ", k, ":", v)
end
end
local text = _([[
This book was first opened, and has been handled since, by an older version of the rendering code.
Bookmarks and highlights can be upgraded to the latest version of the code.
%1
Proceed with this upgrade and reload the book?]])
local details = {}
if nb_xpointers_lost == 0 then
table.insert(details, _([[All your bookmarks and highlights are valid and will be available after the migration.]]))
else
table.insert(details, T(_([[
Note that %1 (out of %2) xpaths from your bookmarks and highlights aren't currently found in the book, and may have been lost. You might want to toggle Rendering mode between 'legacy' and 'flat', and re-open this book, and see if they are found again, before proceeding.]]),
nb_xpointers_lost, nb_xpointers))
end
if nb_xpointers_changed > 0 then
table.insert(details, T(_([[
Note that %1 (out of %2) xpaths from your bookmarks and highlights have been normalized, and may not work on previous KOReader versions (if you're synchronizing your reading between multiple devices, you'll need to update KOReader on all of them).]]),
nb_xpointers_changed, nb_xpointers))
end
text = T(text, table.concat(details, "\n\n"))
UIManager:show(ConfirmBox:new{
text = text,
other_buttons = {{
{
-- this is the real cancel/do nothing
text = _("Not now"),
}
}},
cancel_text = _("Not for this book"),
cancel_callback = function()
self.ui.doc_settings:saveSetting("cre_keep_old_dom_version", true)
end,
ok_text = _("Upgrade now"),
ok_callback = function()
-- Allow for ConfirmBox to be closed before migrating
UIManager:scheduleIn(0.5, function ()
-- And check we haven't quit reader in these 0.5s
if self.ui.document then
-- We'd rather not have any painting between the upgrade
-- and the document reloading (readerview might draw
-- highlights from the migrated xpointers, that would
-- not be found in the document...
local InfoMessage = require("ui/widget/infomessage")
local infomsg = InfoMessage:new{
text = _("Upgrading and reloading book…"),
}
UIManager:show(infomsg)
-- Let this message be shown
UIManager:scheduleIn(2, function ()
UIManager:close(infomsg)
if self.ui.document then
upgradeToLatestDOMVersion()
self.ui:reloadDocument()
end
end)
end
end)
end,
})
end
return ReaderRolling

@ -344,11 +344,12 @@ end
-- ALLOW_EXACT_FLOATS_FOOTPRINTS 0x00200000 x x
--
-- BOX_INLINE_BLOCKS 0x01000000 x x x
-- COMPLETE_INCOMPLETE_TABLES 0x02000000 x x x
local BLOCK_RENDERING_FLAGS = {
0x00000000, -- legacy block rendering
0x01030031, -- flat mode (with prepared floatBoxes, so inlined, to avoid display hash mismatch)
0x01375131, -- book mode (floating floatBoxes, limited widths support)
0x03030031, -- flat mode (with prepared floatBoxes, so inlined, to avoid display hash mismatch)
0x03375131, -- book mode (floating floatBoxes, limited widths support)
0x7FFFFFFF, -- web mode, all features/flags
}
@ -364,10 +365,30 @@ function ReaderTypeset:setBlockRenderingMode(mode)
return
end
self.block_rendering_mode = mode
if self.ensure_saner_block_rendering_flags then -- see next function
-- Don't enable BOX_INLINE_BLOCKS
-- inlineBoxes have been around and allowed on older DOM_VERSION
-- for a few weeks - let's disable it: it may break highlights
-- made during this time, but may resurrect others made during
-- a longer previous period of time.
flags = bit.band(flags, bit.bnot(0x01000000))
-- Don't enable COMPLETE_INCOMPLETE_TABLES, as they may add
-- many boxing elements around huge amount of text, and break
-- some past highlights made on the non-boxed elements.
flags = bit.band(flags, bit.bnot(0x02000000))
end
self.ui.document:setBlockRenderingFlags(flags)
self.ui:handleEvent(Event:new("UpdatePos"))
end
function ReaderTypeset:ensureSanerBlockRenderingFlags(mode)
-- Called by ReaderRolling:onReadSettings() when old
-- DOM version requested, before normalized xpointers,
-- asking us to unset some of the flags set previously.
self.ensure_saner_block_rendering_flags = true
self:setBlockRenderingMode(self.block_rendering_mode)
end
function ReaderTypeset:toggleImageScaling(toggle)
if toggle and (toggle == true or toggle == 1) then
self.smooth_scaling = true

@ -132,7 +132,7 @@ function DocSettings:open(docfile)
return l[2] > r[2]
end
end)
local ok, stored
local ok, stored, filepath
for _, k in pairs(candidates) do
-- Ignore empty files
if lfs.attributes(k[1], "size") > 0 then
@ -140,6 +140,7 @@ function DocSettings:open(docfile)
-- Ignore the empty table.
if ok and next(stored) ~= nil then
logger.dbg("data is read from ", k[1])
filepath = k[1]
break
end
end
@ -149,6 +150,7 @@ function DocSettings:open(docfile)
if ok and stored then
new.data = stored
new.candidates = candidates
new.filepath = filepath
else
new.data = {}
end
@ -236,6 +238,10 @@ function DocSettings:close()
self:flush()
end
function DocSettings:getFilePath()
return self.filepath
end
--- Purges (removes) sidecar directory.
function DocSettings:purge()
if self.history_file then

@ -55,13 +55,13 @@ function CreDocument:cacheInit()
-- the less recently used ones when this limit is reached
local default_cre_disk_cache_max_size = 64 -- in MB units
-- crengine various in-memory caches max-sizes are rather small
-- (2.5 / 4.5 / 1.5 / 1 MB), and we can avoid some bugs if we
-- increase them. Let's multiply them by 20 (each cache would
-- (2.5 / 4.5 / 4.5 / 1 MB), and we can avoid some bugs if we
-- increase them. Let's multiply them by 40 (each cache would
-- grow only when needed, depending on book characteristics).
-- People who would get out of memory crashes with big books on
-- older devices can decrease that with setting:
-- "cre_storage_size_factor"=1 (or 2, or 5)
local default_cre_storage_size_factor = 20
local default_cre_storage_size_factor = 40
cre.initCache(DataStorage:getDataDir() .. "/cache/cr3cache",
(G_reader_settings:readSetting("cre_disk_cache_max_size") or default_cre_disk_cache_max_size)*1024*1024,
G_reader_settings:nilOrTrue("cre_compress_cached_data"),
@ -138,6 +138,10 @@ function CreDocument:init()
self:setupCallCache()
end
function CreDocument:getDomVersionWithNormalizedXPointers()
return cre.getDomVersionWithNormalizedXPointers()
end
function CreDocument:getLatestDomVersion()
return cre.getLatestDomVersion()
end
@ -522,6 +526,13 @@ function CreDocument:getHTMLFromXPointers(xp0, xp1, flags, from_root_node)
end
end
function CreDocument:getNormalizedXPointer(xp)
-- Returns false when xpointer is not found in the DOM.
-- When requested DOM version >= getDomVersionWithNormalizedXPointers,
-- should return xp unmodified when found.
return self._document:getNormalizedXPointer(xp)
end
function CreDocument:gotoPos(pos)
logger.dbg("CreDocument: goto position", pos)
self._document:gotoPos(pos)
@ -1152,6 +1163,7 @@ function CreDocument:setupCallCache()
elseif name == "getNextVisibleChar" then no_wrap = true
elseif name == "getCacheFilePath" then no_wrap = true
elseif name == "getStatistics" then no_wrap = true
elseif name == "getNormalizedXPointer" then no_wrap = true
-- Some get* have different results by page/pos
elseif name == "getLinkFromPosition" then cache_by_tag = true

Loading…
Cancel
Save