diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index a28465f78..fd43de881 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -29,3 +29,12 @@ Android won't have a crash.log file because Google restricts what apps can log, Please try to include the relevant sections in your issue description. You can upload the whole `crash.log` file on GitHub by dragging and dropping it onto this textbox. + +If you instead opt to inline it, please do so behind a spoiler tag: +
+ crash.log + +``` + +``` +
diff --git a/defaults.lua b/defaults.lua index 88062ad41..55d808092 100644 --- a/defaults.lua +++ b/defaults.lua @@ -27,13 +27,13 @@ DHINTCOUNT = 1 DRENDER_MODE = 0 -- 0 is COLOUR -- minimum cache size -DGLOBAL_CACHE_SIZE_MINIMUM = 1024*1024*10 +DGLOBAL_CACHE_SIZE_MINIMUM = 1024*1024*16 -- proportion of system free memory used as global cache DGLOBAL_CACHE_FREE_PROPORTION = 0.4 -- maximum cache size -DGLOBAL_CACHE_SIZE_MAXIMUM = 1024*1024*60 +DGLOBAL_CACHE_SIZE_MAXIMUM = 1024*1024*64 -- background colour in non scroll mode: 8 = gray, 0 = white, 15 = black DBACKGROUND_COLOR = 0 diff --git a/frontend/apps/reader/modules/readerzooming.lua b/frontend/apps/reader/modules/readerzooming.lua index 8464750fc..ecc66f262 100644 --- a/frontend/apps/reader/modules/readerzooming.lua +++ b/frontend/apps/reader/modules/readerzooming.lua @@ -458,9 +458,9 @@ function ReaderZooming:getZoom(pageno) or self.zoom_factor zoom = zoom_w * zoom_factor end - if zoom and zoom > 10 and not Cache:willAccept(zoom * (self.dimen.w * self.dimen.h + 64)) then + if zoom and zoom > 10 and not Cache:willAccept(zoom * (self.dimen.w * self.dimen.h + 512)) then logger.dbg("zoom too large, adjusting") - while not Cache:willAccept(zoom * (self.dimen.w * self.dimen.h + 64)) do + while not Cache:willAccept(zoom * (self.dimen.w * self.dimen.h + 512)) do if zoom > 100 then zoom = zoom - 50 elseif zoom > 10 then diff --git a/frontend/apps/reader/readerui.lua b/frontend/apps/reader/readerui.lua index 4217054f2..dc6521c03 100644 --- a/frontend/apps/reader/readerui.lua +++ b/frontend/apps/reader/readerui.lua @@ -733,13 +733,13 @@ function ReaderUI:onClose(full_refresh) if self.dialog ~= self then self:saveSettings() end + -- serialize last used items for later launch + Cache:serialize() if self.document ~= nil then logger.dbg("closing document") self:notifyCloseDocument() end UIManager:close(self.dialog, full_refresh and "full") - -- serialize last used items for later launch - Cache:serialize() if _running_instance == self then _running_instance = nil end diff --git a/frontend/cache.lua b/frontend/cache.lua index 5f075cf72..e1f4ece73 100644 --- a/frontend/cache.lua +++ b/frontend/cache.lua @@ -12,28 +12,99 @@ if CanvasContext.should_restrict_JIT then jit.off(true, true) end +-- For documentation purposes, here's a battle-tested shell version of calcFreeMem +--[[ + if grep -q 'MemAvailable' /proc/meminfo ; then + # We'll settle for 85% of available memory to leave a bit of breathing room + tmpfs_size="$(awk '/MemAvailable/ {printf "%d", $2 * 0.85}' /proc/meminfo)" + elif grep -q 'Inactive(file)' /proc/meminfo ; then + # Basically try to emulate the kernel's computation, c.f., https://unix.stackexchange.com/q/261247 + # Again, 85% of available memory + tmpfs_size="$(awk -v low=$(grep low /proc/zoneinfo | awk '{k+=$2}END{printf "%d", k}') \ + '{a[$1]=$2} + END{ + printf "%d", (a["MemFree:"]+a["Active(file):"]+a["Inactive(file):"]+a["SReclaimable:"]-(12*low))*0.85; + }' /proc/meminfo)" + else + # Ye olde crap workaround of Free + Buffers + Cache... + # Take it with a grain of salt, and settle for 80% of that... + tmpfs_size="$(awk \ + '{a[$1]=$2} + END{ + printf "%d", (a["MemFree:"]+a["Buffers:"]+a["Cached:"])*0.80; + }' /proc/meminfo)" + fi +--]] + +-- And here's our simplified Lua version... local function calcFreeMem() + local memtotal, memfree, memavailable, buffers, cached + local meminfo = io.open("/proc/meminfo", "r") - local freemem = 0 if meminfo then for line in meminfo:lines() do - local free, buffer, cached, n - free, n = line:gsub("^MemFree:%s-(%d+) kB", "%1") - if n ~= 0 then freemem = freemem + tonumber(free)*1024 end - buffer, n = line:gsub("^Buffers:%s-(%d+) kB", "%1") - if n ~= 0 then freemem = freemem + tonumber(buffer)*1024 end - cached, n = line:gsub("^Cached:%s-(%d+) kB", "%1") - if n ~= 0 then freemem = freemem + tonumber(cached)*1024 end + if not memtotal then + memtotal = line:match("^MemTotal:%s-(%d+) kB") + if memtotal then + -- Next! + goto continue + end + end + + if not memfree then + memfree = line:match("^MemFree:%s-(%d+) kB") + if memfree then + -- Next! + goto continue + end + end + + if not memavailable then + memavailable = line:match("^MemAvailable:%s-(%d+) kB") + if memavailable then + -- Best case scenario, we're done :) + break + end + end + + if not buffers then + buffers = line:match("^Buffers:%s-(%d+) kB") + if buffers then + -- Next! + goto continue + end + end + + if not cached then + cached = line:match("^Cached:%s-(%d+) kB") + if cached then + -- Ought to be the last entry we care about, we're done + break + end + end + + ::continue:: end meminfo:close() + else + -- Not on Linux? + return 0, 0 + end + + if memavailable then + -- Leave a bit of margin, and report 85% of that... + return math.floor(memavailable * 0.85) * 1024, memtotal * 1024 + else + -- Crappy Free + Buffers + Cache version, because the zoneinfo approach is a tad hairy... + -- So, leave an even larger margin, and only report 75% of that... + return math.floor((memfree + buffers + cached) * 0.75) * 1024, memtotal * 1024 end - return freemem end local function calcCacheMemSize() local min = DGLOBAL_CACHE_SIZE_MINIMUM local max = DGLOBAL_CACHE_SIZE_MAXIMUM - local calc = calcFreeMem()*(DGLOBAL_CACHE_FREE_PROPORTION or 0) + local calc = calcFreeMem() * (DGLOBAL_CACHE_FREE_PROPORTION or 0) return math.min(max, math.max(min, calc)) end @@ -45,7 +116,7 @@ local cache_path = DataStorage:getDataDir() .. "/cache/" local function getDiskCache() local cached = {} for key_md5 in lfs.dir(cache_path) do - local file = cache_path..key_md5 + local file = cache_path .. key_md5 if lfs.attributes(file, "mode") == "file" then cached[key_md5] = file end @@ -78,13 +149,13 @@ function Cache:_unref(key) for i = #self.cache_order, 1, -1 do if self.cache_order[i] == key then table.remove(self.cache_order, i) + break end end end -- internal: free cache item function Cache:_free(key) - if not self.cache[key] then return end self.current_memsize = self.current_memsize - self.cache[key].size self.cache[key]:onFree() self.cache[key] = nil @@ -92,6 +163,8 @@ end -- drop an item named via key from the cache function Cache:drop(key) + if not self.cache[key] then return end + self:_unref(key) self:_free(key) end @@ -99,31 +172,37 @@ end function Cache:insert(key, object) -- make sure that one key only exists once: delete existing self:drop(key) - -- guarantee that we have enough memory in cache - if (object.size > self.max_memsize) then - logger.warn("too much memory claimed for", key) + -- If this object is single-handledly too large for the cache, we're done + if object.size > self.max_memsize then + logger.warn("Too much memory would be claimed by caching", key) return end - -- delete objects that least recently used + -- If inserting this obect would blow the cache's watermark, + -- start dropping least recently used items first. -- (they are at the end of the cache_order array) while self.current_memsize + object.size > self.max_memsize do local removed_key = table.remove(self.cache_order) - self:_free(removed_key) + if removed_key then + self:_free(removed_key) + else + logger.warn("Cache accounting is broken") + break + end end - -- insert new object in front of the LRU order + -- Insert new object in front of the LRU order table.insert(self.cache_order, 1, key) self.cache[key] = object self.current_memsize = self.current_memsize + object.size end --[[ --- check for cache item for key +-- check for cache item by key -- if ItemClass is given, disk cache is also checked. --]] function Cache:check(key, ItemClass) if self.cache[key] then if self.cache_order[1] ~= key then - -- put key in front of the LRU list + -- Move key in front of the LRU list (i.e., MRU) self:_unref(key) table.insert(self.cache_order, 1, key) end @@ -137,57 +216,77 @@ function Cache:check(key, ItemClass) self:insert(key, item) return item else - logger.warn("discard cache", msg) + logger.warn("Failed to load on-disk cache:", msg) + --- It's apparently unusable, purge it and refresh the snapshot. + os.remove(cached) + self:refreshSnapshot() end end end end function Cache:willAccept(size) - -- we only allow single objects to fill 75% of the cache - if size*4 < self.max_memsize*3 then - return true - end + -- We only allow single objects to fill 75% of the cache + return size*4 < self.max_memsize*3 end function Cache:serialize() - -- calculate disk cache size + -- Calculate the current disk cache size local cached_size = 0 local sorted_caches = {} for _, file in pairs(self.cached) do table.insert(sorted_caches, {file=file, time=lfs.attributes(file, "access")}) cached_size = cached_size + (lfs.attributes(file, "size") or 0) end - table.sort(sorted_caches, function(v1,v2) return v1.time > v2.time end) - -- only serialize the most recently used cache - local cache_size = 0 + table.sort(sorted_caches, function(v1, v2) return v1.time > v2.time end) + + -- Only serialize the second most recently used cache item (as the MRU would be the *hinted* page). + local mru_key + local mru_found = 0 for _, key in ipairs(self.cache_order) do local cache_item = self.cache[key] - -- only dump cache item that requests serialization explicitly + -- Only dump cache items that actually request persistence if cache_item.persistent and cache_item.dump then - local cache_full_path = cache_path..md5(key) - local cache_file_exists = lfs.attributes(cache_full_path) - - if cache_file_exists then break end + mru_key = key + mru_found = mru_found + 1 + if mru_found >= 2 then + -- We found the second MRU item, i.e., the *displayed* page + break + end + end + end + if mru_key then + local cache_full_path = cache_path .. md5(mru_key) + local cache_file_exists = lfs.attributes(cache_full_path) - logger.dbg("dump cache item", key) - cache_size = cache_item:dump(cache_full_path) or 0 - if cache_size > 0 then break end + if not cache_file_exists then + logger.dbg("Dumping cache item", mru_key) + local cache_item = self.cache[mru_key] + local cache_size = cache_item:dump(cache_full_path) + if cache_size then + cached_size = cached_size + cache_size + end end end - -- set disk cache the same limit as memory cache - while cached_size + cache_size - self.max_memsize > 0 do + + -- Allocate the same amount of storage to the disk cache than the memory cache + while cached_size > self.max_memsize do -- discard the least recently used cache local discarded = table.remove(sorted_caches) - cached_size = cached_size - lfs.attributes(discarded.file, "size") - os.remove(discarded.file) + if discarded then + cached_size = cached_size - lfs.attributes(discarded.file, "size") + os.remove(discarded.file) + else + logger.warn("Cache accounting is broken") + break + end end - -- disk cache may have changes so need to refresh disk cache snapshot - self.cached = getDiskCache() + -- We may have updated the disk cache's content, so refresh its state + self:refreshSnapshot() end --- blank the cache +-- Blank the cache function Cache:clear() for k, _ in pairs(self.cache) do self.cache[k]:onFree() @@ -197,9 +296,41 @@ function Cache:clear() self.current_memsize = 0 end +-- Terribly crappy workaround: evict half the cache if we appear to be redlining on free RAM... +function Cache:memoryPressureCheck() + local memfree, memtotal = calcFreeMem() + + -- Nonsensical values? (!Linux), skip this. + if memtotal == 0 then + return + end + + -- If less that 20% of the total RAM is free, drop half the Cache... + if memfree / memtotal < 0.20 then + logger.warn("Running low on memory, evicting half of the cache...") + for i = #self.cache_order / 2, 1, -1 do + local removed_key = table.remove(self.cache_order) + self:_free(removed_key) + end + + -- And finish by forcing a GC sweep now... + collectgarbage() + collectgarbage() + end +end + -- Refresh the disk snapshot (mainly used by ui/data/onetime_migration) function Cache:refreshSnapshot() self.cached = getDiskCache() end +-- Evict the disk cache (ditto) +function Cache:clearDiskCache() + for _, file in pairs(self.cached) do + os.remove(file) + end + + self:refreshSnapshot() +end + return Cache diff --git a/frontend/cacheitem.lua b/frontend/cacheitem.lua index 2ca58928c..3b009c4d1 100644 --- a/frontend/cacheitem.lua +++ b/frontend/cacheitem.lua @@ -3,8 +3,11 @@ Inheritable abstraction for cache items --]] local CacheItem = { - size = 64, -- some reasonable default for simple Lua values / small tables + size = 128, -- some reasonable default for a small table. } +--- NOTE: As far as size estimations go, the assumption is that a key, value pair should roughly take two words, +--- and the most common items we cache are Geom-like tables (i.e., 4 key-value pairs). +--- That's generally a low estimation, especially for larger tables, where memory allocation trickery may be happening. function CacheItem:new(o) o = o or {} diff --git a/frontend/configurable.lua b/frontend/configurable.lua index 97a4c1802..123106449 100644 --- a/frontend/configurable.lua +++ b/frontend/configurable.lua @@ -1,3 +1,5 @@ +local ffiUtil = require("ffi/util") + local Configurable = {} function Configurable:new(o) @@ -8,7 +10,7 @@ function Configurable:new(o) end function Configurable:reset() - for key,value in pairs(self) do + for key, value in pairs(self) do local value_type = type(value) if value_type == "number" or value_type == "string" then self[key] = nil @@ -18,7 +20,7 @@ end function Configurable:hash(sep) local hash = "" - for key,value in pairs(self) do + for key, value in ffiUtil.orderedPairs(self) do local value_type = type(value) if value_type == "number" or value_type == "string" then hash = hash..sep..value @@ -31,7 +33,7 @@ function Configurable:loadDefaults(config_options) -- reset configurable before loading new options self:reset() local prefix = config_options.prefix.."_" - for i=1,#config_options do + for i=1, #config_options do local options = config_options[i].options for j=1,#options do local key = options[j].name @@ -46,7 +48,7 @@ function Configurable:loadDefaults(config_options) end function Configurable:loadSettings(settings, prefix) - for key,value in pairs(self) do + for key, value in pairs(self) do local value_type = type(value) if value_type == "number" or value_type == "string" or value_type == "table" then @@ -59,7 +61,7 @@ function Configurable:loadSettings(settings, prefix) end function Configurable:saveSettings(settings, prefix) - for key,value in pairs(self) do + for key, value in pairs(self) do local value_type = type(value) if value_type == "number" or value_type == "string" or value_type == "table" then diff --git a/frontend/document/document.lua b/frontend/document/document.lua index 2008f96a7..9036fa0fd 100644 --- a/frontend/document/document.lua +++ b/frontend/document/document.lua @@ -93,9 +93,19 @@ end -- this might be overridden by a document implementation function Document:close() local DocumentRegistry = require("document/documentregistry") - if self.is_open and DocumentRegistry:closeDocument(self.file) == 0 then - self.is_open = false - self._document:close() + if self.is_open then + if DocumentRegistry:closeDocument(self.file) == 0 then + self.is_open = false + self._document:close() + self._document = nil + + -- NOTE: DocumentRegistry:openDocument will force a GC sweep the next time we open a Document. + -- MµPDF will also do a bit of spring cleaning of its internal cache when opening a *different* document. + else + logger.warn("Tried to close a document with *multiple* remaining hot references") + end + else + logger.warn("Tried to close an already closed document") end end @@ -375,7 +385,7 @@ function Document:renderPage(pageno, rect, zoom, rotation, gamma, render_mode) -- this will be the size we actually render local size = page_size -- we prefer to render the full page, if it fits into cache - if not Cache:willAccept(size.w * size.h + 64) then + if not Cache:willAccept(size.w * size.h * (self.render_color and 4 or 1) + 512) then -- whole page won't fit into cache logger.dbg("rendering only part of the page") --- @todo figure out how to better segment the page @@ -392,11 +402,11 @@ function Document:renderPage(pageno, rect, zoom, rotation, gamma, render_mode) -- prepare cache item with contained blitbuffer tile = TileCacheItem:new{ persistent = true, - size = size.w * size.h + 64, -- estimation excerpt = size, pageno = pageno, bb = Blitbuffer.new(size.w, size.h, self.render_color and self.color_bb_type or nil) } + tile.size = tonumber(tile.bb.stride) * tile.bb.h + 512 -- estimation -- create a draw context local dc = DrawContext.new() @@ -429,6 +439,9 @@ end -- a hint for the cache engine to paint a full page to the cache --- @todo this should trigger a background operation function Document:hintPage(pageno, zoom, rotation, gamma, render_mode) + --- @note: Crappy safeguard around memory issues like in #7627: if we're eating too much RAM, drop half the cache... + Cache:memoryPressureCheck() + logger.dbg("hinting page", pageno) self:renderPage(pageno, nil, zoom, rotation, gamma, render_mode) end diff --git a/frontend/document/koptinterface.lua b/frontend/document/koptinterface.lua index 863a18c8e..ab08ad7a2 100644 --- a/frontend/document/koptinterface.lua +++ b/frontend/document/koptinterface.lua @@ -10,9 +10,9 @@ local DEBUG = require("dbg") local Document = require("document/document") local Geom = require("ui/geometry") local KOPTContext = require("ffi/koptcontext") +local Persist = require("persist") local TileCacheItem = require("document/tilecacheitem") local logger = require("logger") -local serial = require("serialize") local util = require("ffi/util") local KoptInterface = { @@ -36,16 +36,41 @@ end function ContextCacheItem:dump(filename) if self.kctx:isPreCache() == 0 then - logger.dbg("dumping koptcontext to", filename) - return serial.dump(self.size, KOPTContext.totable(self.kctx), filename) + logger.dbg("Dumping KOPTContext to", filename) + + local cache_file = Persist:new{ + path = filename, + codec = "zstd", + } + + local t = KOPTContext.totable(self.kctx) + t.cache_size = self.size + + local ok, size = cache_file:save(t) + if ok then + return size + else + logger.warn("Failed to dump KOPTContext") + return nil + end end end function ContextCacheItem:load(filename) - logger.dbg("loading koptcontext from", filename) - local size, kc_table = serial.load(filename) - self.size = size - self.kctx = KOPTContext.fromtable(kc_table) + logger.dbg("Loading KOPTContext from", filename) + + local cache_file = Persist:new{ + path = filename, + codec = "zstd", + } + + local t = cache_file:load(filename) + if t then + self.size = t.cache_size + self.kctx = KOPTContext.fromtable(t) + else + logger.warn("Failed to load KOPTContext") + end end local OCREngine = CacheItem:new{} @@ -154,7 +179,8 @@ Auto detect bbox. function KoptInterface:getAutoBBox(doc, pageno) local native_size = Document.getNativePageDimensions(doc, pageno) local bbox = { - x0 = 0, y0 = 0, + x0 = 0, + y0 = 0, x1 = native_size.w, y1 = native_size.h, } @@ -172,7 +198,7 @@ function KoptInterface:getAutoBBox(doc, pageno) else bbox = Document.getPageBBox(doc, pageno) end - Cache:insert(hash, CacheItem:new{ autobbox = bbox }) + Cache:insert(hash, CacheItem:new{ autobbox = bbox, size = 160 }) page:close() kc:free() return bbox @@ -207,7 +233,7 @@ function KoptInterface:getSemiAutoBBox(doc, pageno) auto_bbox = bbox end page:close() - Cache:insert(hash, CacheItem:new{ semiautobbox = auto_bbox }) + Cache:insert(hash, CacheItem:new{ semiautobbox = auto_bbox, size = 160 }) kc:free() return auto_bbox else @@ -240,7 +266,7 @@ function KoptInterface:getCachedContext(doc, pageno) --self:logReflowDuration(pageno, dur) local fullwidth, fullheight = kc:getPageDim() logger.dbg("reflowed page", pageno, "fullwidth:", fullwidth, "fullheight:", fullheight) - self.last_context_size = fullwidth * fullheight + 128 -- estimation + self.last_context_size = fullwidth * fullheight + 3072 -- estimation Cache:insert(kctx_hash, ContextCacheItem:new{ persistent = true, size = self.last_context_size, @@ -251,7 +277,7 @@ function KoptInterface:getCachedContext(doc, pageno) -- wait for background thread local kc = self:waitForContext(cached.kctx) local fullwidth, fullheight = kc:getPageDim() - self.last_context_size = fullwidth * fullheight + 128 -- estimation + self.last_context_size = fullwidth * fullheight + 3072 -- estimation return kc end end @@ -312,20 +338,20 @@ function KoptInterface:renderReflowedPage(doc, pageno, rect, zoom, rotation, ren local cached = Cache:check(renderpg_hash) if not cached then - -- do the real reflowing if kctx is not been cached yet + -- do the real reflowing if kctx has not been cached yet local kc = self:getCachedContext(doc, pageno) local fullwidth, fullheight = kc:getPageDim() - if not Cache:willAccept(fullwidth * fullheight / 2) then + if not Cache:willAccept(fullwidth * fullheight) then -- whole page won't fit into cache error("aborting, since we don't have enough cache for this page") end -- prepare cache item with contained blitbuffer local tile = TileCacheItem:new{ - size = fullwidth * fullheight + 64, -- estimation excerpt = Geom:new{ w = fullwidth, h = fullheight }, pageno = pageno, } tile.bb = kc:dstToBlitBuffer() + tile.size = tonumber(tile.bb.stride) * tile.bb.h + 512 -- estimation Cache:insert(renderpg_hash, tile) return tile else @@ -363,7 +389,6 @@ function KoptInterface:renderOptimizedPage(doc, pageno, rect, zoom, rotation, re -- prepare cache item with contained blitbuffer local tile = TileCacheItem:new{ persistent = true, - size = fullwidth * fullheight + 64, -- estimation excerpt = Geom:new{ x = 0, y = 0, w = fullwidth, @@ -372,6 +397,7 @@ function KoptInterface:renderOptimizedPage(doc, pageno, rect, zoom, rotation, re pageno = pageno, } tile.bb = kc:dstToBlitBuffer() + tile.size = tonumber(tile.bb.stride) * tile.bb.h + 512 -- estimation kc:free() Cache:insert(renderpg_hash, tile) return tile @@ -478,8 +504,8 @@ function KoptInterface:getReflowedTextBoxes(doc, pageno) local kc = self:waitForContext(cached.kctx) --kc:setDebug() local fullwidth, fullheight = kc:getPageDim() - local boxes = kc:getReflowedWordBoxes("dst", 0, 0, fullwidth, fullheight) - Cache:insert(hash, CacheItem:new{ rfpgboxes = boxes }) + local boxes, nr_word = kc:getReflowedWordBoxes("dst", 0, 0, fullwidth, fullheight) + Cache:insert(hash, CacheItem:new{ rfpgboxes = boxes, size = 192 * nr_word }) -- estimation return boxes end else @@ -502,8 +528,8 @@ function KoptInterface:getNativeTextBoxes(doc, pageno) local kc = self:waitForContext(cached.kctx) --kc:setDebug() local fullwidth, fullheight = kc:getPageDim() - local boxes = kc:getNativeWordBoxes("dst", 0, 0, fullwidth, fullheight) - Cache:insert(hash, CacheItem:new{ nativepgboxes = boxes }) + local boxes, nr_word = kc:getNativeWordBoxes("dst", 0, 0, fullwidth, fullheight) + Cache:insert(hash, CacheItem:new{ nativepgboxes = boxes, size = 192 * nr_word }) -- estimation return boxes end else @@ -529,8 +555,8 @@ function KoptInterface:getReflowedTextBoxesFromScratch(doc, pageno) local fullwidth, fullheight = reflowed_kc:getPageDim() local kc = self:createContext(doc, pageno) kc:copyDestBMP(reflowed_kc) - local boxes = kc:getNativeWordBoxes("dst", 0, 0, fullwidth, fullheight) - Cache:insert(hash, CacheItem:new{ scratchrfpgboxes = boxes }) + local boxes, nr_word = kc:getNativeWordBoxes("dst", 0, 0, fullwidth, fullheight) + Cache:insert(hash, CacheItem:new{ scratchrfpgboxes = boxes, size = 192 * nr_word }) -- estimation kc:free() return boxes end @@ -575,8 +601,8 @@ function KoptInterface:getNativeTextBoxesFromScratch(doc, pageno) kc:setZoom(1.0) local page = doc._document:openPage(pageno) page:getPagePix(kc) - local boxes = kc:getNativeWordBoxes("src", 0, 0, page_size.w, page_size.h) - Cache:insert(hash, CacheItem:new{ scratchnativepgboxes = boxes }) + local boxes, nr_word = kc:getNativeWordBoxes("src", 0, 0, page_size.w, page_size.h) + Cache:insert(hash, CacheItem:new{ scratchnativepgboxes = boxes, size = 192 * nr_word }) -- estimation page:close() kc:free() return boxes @@ -607,7 +633,7 @@ function KoptInterface:getPageBlock(doc, pageno, x, y) local page = doc._document:openPage(pageno) page:getPagePix(kc) kc:findPageBlocks() - Cache:insert(hash, CacheItem:new{ kctx = kc }) + Cache:insert(hash, CacheItem:new{ kctx = kc, size = 3072 }) -- estimation page:close() kctx = kc else @@ -621,7 +647,7 @@ Get word from OCR providing selected word box. --]] function KoptInterface:getOCRWord(doc, pageno, wbox) if not Cache:check(self.ocrengine) then - Cache:insert(self.ocrengine, OCREngine:new{ ocrengine = KOPTContext.new() }) + Cache:insert(self.ocrengine, OCREngine:new{ ocrengine = KOPTContext.new(), size = 3072 }) -- estimation end if doc.configurable.text_wrap == 1 then return self:getReflewOCRWord(doc, pageno, wbox.sbox) @@ -648,7 +674,7 @@ function KoptInterface:getReflewOCRWord(doc, pageno, rect) kc.getTOCRWord, kc, "dst", rect.x, rect.y, rect.w, rect.h, self.tessocr_data, self.ocr_lang, self.ocr_type, 0, 1) - Cache:insert(hash, CacheItem:new{ rfocrword = word }) + Cache:insert(hash, CacheItem:new{ rfocrword = word, size = #word + 64 }) -- estimation return word end else @@ -681,7 +707,7 @@ function KoptInterface:getNativeOCRWord(doc, pageno, rect) kc.getTOCRWord, kc, "src", 0, 0, word_w, word_h, self.tessocr_data, self.ocr_lang, self.ocr_type, 0, 1) - Cache:insert(hash, CacheItem:new{ ocrword = word }) + Cache:insert(hash, CacheItem:new{ ocrword = word, size = #word + 64 }) -- estimation logger.dbg("word", word) page:close() kc:free() @@ -696,7 +722,7 @@ Get text from OCR providing selected text boxes. --]] function KoptInterface:getOCRText(doc, pageno, tboxes) if not Cache:check(self.ocrengine) then - Cache:insert(self.ocrengine, OCREngine:new{ ocrengine = KOPTContext.new() }) + Cache:insert(self.ocrengine, OCREngine:new{ ocrengine = KOPTContext.new(), size = 3072 }) -- estimation end logger.info("Not implemented yet") end diff --git a/frontend/document/pdfdocument.lua b/frontend/document/pdfdocument.lua index d13746f4b..fcb00f8a1 100644 --- a/frontend/document/pdfdocument.lua +++ b/frontend/document/pdfdocument.lua @@ -152,9 +152,9 @@ function PdfDocument:getUsedBBox(pageno) if used.x1 > pwidth then used.x1 = pwidth end if used.y0 < 0 then used.y0 = 0 end if used.y1 > pheight then used.y1 = pheight end - --- @todo Give size for cacheitem? 02.12 2012 (houqp) Cache:insert(hash, CacheItem:new{ ubbox = used, + size = 256, -- might be closer to 160 }) page:close() return used @@ -170,6 +170,7 @@ function PdfDocument:getPageLinks(pageno) local links = page:getPageLinks() Cache:insert(hash, CacheItem:new{ links = links, + size = 64 + (8 * 32 * #links), }) page:close() return links diff --git a/frontend/document/tilecacheitem.lua b/frontend/document/tilecacheitem.lua index 7246fe6fa..fdbf81446 100644 --- a/frontend/document/tilecacheitem.lua +++ b/frontend/document/tilecacheitem.lua @@ -1,6 +1,6 @@ local Blitbuffer = require("ffi/blitbuffer") local CacheItem = require("cacheitem") -local serial = require("serialize") +local Persist = require("persist") local logger = require("logger") local TileCacheItem = CacheItem:new{} @@ -12,19 +12,65 @@ function TileCacheItem:onFree() end end +--- @note: Perhaps one day we'll be able to teach bitser or string.buffer about custom structs with pointers to buffers, +--- so we won't have to do the BB tostring/fromstring dance anymore... +function TileCacheItem:totable() + local t = { + size = self.size, + pageno = self.pageno, + excerpt = self.excerpt, + persistent = self.persistent, + bb = { + w = self.bb.w, + h = self.bb.h, + stride = tonumber(self.bb.stride), + fmt = self.bb:getType(), + data = Blitbuffer.tostring(self.bb), + }, + } + + return t +end + function TileCacheItem:dump(filename) - logger.dbg("dumping tile cache to", filename, self.excerpt) - return serial.dump(self.size, self.excerpt, self.pageno, - self.bb.w, self.bb.h, tonumber(self.bb.stride), self.bb:getType(), - Blitbuffer.tostring(self.bb), filename) + logger.dbg("Dumping tile cache to", filename, self.excerpt) + + local cache_file = Persist:new{ + path = filename, + codec = "zstd", + } + + local ok, size = cache_file:save(self:totable()) + if ok then + return size + else + logger.warn("Failed to dump tile cache") + return nil + end +end + +function TileCacheItem:fromtable(t) + self.size = t.size + self.pageno = t.pageno + self.excerpt = t.excerpt + self.persistent = t.persistent + self.bb = Blitbuffer.fromstring(t.bb.w, t.bb.h, t.bb.fmt, t.bb.data, t.bb.stride) end function TileCacheItem:load(filename) - local w, h, stride, bb_type, bb_data - self.size, self.excerpt, self.pageno, - w, h, stride, bb_type, bb_data = serial.load(filename) - self.bb = Blitbuffer.fromstring(w, h, bb_type, bb_data, stride) - logger.dbg("loading tile cache from", filename, self) + local cache_file = Persist:new{ + path = filename, + codec = "zstd", + } + + local t = cache_file:load(filename) + if t then + self:fromtable(t) + + logger.dbg("Loaded tile cache from", filename, self) + else + logger.warn("Failed to load tile cache from", filename) + end end return TileCacheItem diff --git a/frontend/persist.lua b/frontend/persist.lua index 9de62c5ce..fedac5937 100644 --- a/frontend/persist.lua +++ b/frontend/persist.lua @@ -94,7 +94,8 @@ local codecs = { C.fclose(f) C.free(cbuff) - return true + --- @note: Slight API extension for TileCacheItem, which needs to know the on-disk size, and saves us a :size() call + return true, clen end, deserialize = function(path) @@ -216,6 +217,9 @@ function Persist:save(t, as_bytecode) if not ok then return nil, err end + + -- c.f., note above, err is the on-disk size + return true, err else local str, err = codecs[self.codec].serialize(t, as_bytecode) if not str then diff --git a/frontend/ui/data/onetime_migration.lua b/frontend/ui/data/onetime_migration.lua index 3fb03d245..a27d5661c 100644 --- a/frontend/ui/data/onetime_migration.lua +++ b/frontend/ui/data/onetime_migration.lua @@ -7,7 +7,7 @@ local lfs = require("libs/libkoreader-lfs") local logger = require("logger") -- Date at which the last migration snippet was added -local CURRENT_MIGRATION_DATE = 20210414 +local CURRENT_MIGRATION_DATE = 20210503 -- Retrieve the date of the previous migration, if any local last_migration_date = G_reader_settings:readSetting("last_migration_date", 0) @@ -209,5 +209,13 @@ if last_migration_date < 20210414 then end end +-- Cache, migration to Persist, https://github.com/koreader/koreader/pull/7624 +if last_migration_date < 20210503 then + logger.info("Performing one-time migration for 20210503") + + local Cache = require("cache") + Cache:clearDiskCache() +end + -- We're done, store the current migration date G_reader_settings:saveSetting("last_migration_date", CURRENT_MIGRATION_DATE) diff --git a/frontend/ui/rendertext.lua b/frontend/ui/rendertext.lua index e062ef23c..dc496ca53 100644 --- a/frontend/ui/rendertext.lua +++ b/frontend/ui/rendertext.lua @@ -113,7 +113,7 @@ function RenderText:getGlyph(face, charcode, bold) return end glyph = CacheItem:new{rendered_glyph} - glyph.size = glyph[1].bb:getWidth() * glyph[1].bb:getHeight() / 2 + 32 + glyph.size = tonumber(glyph[1].bb.stride) * glyph[1].bb.h + 320 GlyphCache:insert(hash, glyph) return rendered_glyph end @@ -314,7 +314,7 @@ function RenderText:getGlyphByIndex(face, glyphindex, bold) return end glyph = CacheItem:new{rendered_glyph} - glyph.size = glyph[1].bb:getWidth() * glyph[1].bb:getHeight() / 2 + 32 + glyph.size = tonumber(glyph[1].bb.stride) * glyph[1].bb.h + 320 GlyphCache:insert(hash, glyph) return rendered_glyph end diff --git a/frontend/ui/widget/imagewidget.lua b/frontend/ui/widget/imagewidget.lua index 77b8cea7f..c6f75565c 100644 --- a/frontend/ui/widget/imagewidget.lua +++ b/frontend/ui/widget/imagewidget.lua @@ -39,7 +39,7 @@ end local DPI_SCALE = get_dpi_scale() local ImageCache = Cache:new{ - max_memsize = 5*1024*1024, -- 5M of image cache + max_memsize = 8*1024*1024, -- 8M of image cache current_memsize = 0, cache = {}, -- this will hold the LRU order of the cache diff --git a/plugins/opds.koplugin/opdsbrowser.lua b/plugins/opds.koplugin/opdsbrowser.lua index c36b97905..fc15f384e 100644 --- a/plugins/opds.koplugin/opdsbrowser.lua +++ b/plugins/opds.koplugin/opdsbrowser.lua @@ -30,7 +30,7 @@ local CatalogCacheItem = CacheItem:new{ -- cache catalog parsed from feed xml local CatalogCache = Cache:new{ - max_memsize = 20*1024, -- keep only 20 cache items + max_memsize = 20*1024, -- keep only 20 items current_memsize = 0, cache = {}, cache_order = {},