From 47c59e0e5a48f825ad78a7405736ea5fe0267a70 Mon Sep 17 00:00:00 2001 From: NiLuJe Date: Tue, 13 Apr 2021 18:11:39 +0200 Subject: [PATCH] Persist: Add the new native LuaJIT serializer to the list of supported codecs (#7543) And swap the Calibre metadata cache to it. --- frontend/persist.lua | 29 +++++++++++++++++++++++--- frontend/ui/data/onetime_migration.lua | 13 +++++++++++- plugins/calibre.koplugin/search.lua | 3 ++- spec/unit/persist_spec.lua | 27 +++++++++++++++++------- 4 files changed, 59 insertions(+), 13 deletions(-) diff --git a/frontend/persist.lua b/frontend/persist.lua index 3bb59cbb3..9100d73fc 100644 --- a/frontend/persist.lua +++ b/frontend/persist.lua @@ -1,4 +1,5 @@ local bitser = require("ffi/bitser") +local buffer = require("string.buffer") local dump = require("dump") local lfs = require("libs/libkoreader-lfs") local logger = require("logger") @@ -18,7 +19,7 @@ local function readFile(file, bytes) end local codecs = { - -- bitser: binary form, fast encode/decode, low size. Not human readable. + -- bitser: binary format, fast encode/decode, low size. Not human readable. bitser = { id = "bitser", reads_from_file = false, @@ -26,7 +27,7 @@ local codecs = { serialize = function(t) local ok, str = pcall(bitser.dumps, t) if not ok then - return nil, "cannot serialize " .. tostring(t) + return nil, "cannot serialize " .. tostring(t) .. " (" .. str .. ")" end return str end, @@ -34,7 +35,29 @@ local codecs = { deserialize = function(str) local ok, t = pcall(bitser.loads, str) if not ok then - return nil, "malformed serialized data" + return nil, "malformed serialized data: " .. t + end + return t + end, + }, + -- luajit: binary format, optimized for speed, not size (combine w/ zstd if necessary). Not human readable. + -- Slightly larger on-disk representation than bitser, *much* faster to decode, slightly faster to encode. + luajit = { + id = "luajit", + reads_from_file = false, + + serialize = function(t) + local ok, str = pcall(buffer.encode, t) + if not ok then + return nil, "cannot serialize " .. tostring(t) .. " (" .. str .. ")" + end + return str + end, + + deserialize = function(str) + local ok, t = pcall(buffer.decode, str) + if not ok then + return nil, "malformed serialized data (" .. t .. ")" end return t end, diff --git a/frontend/ui/data/onetime_migration.lua b/frontend/ui/data/onetime_migration.lua index a7b61c9bb..1db519194 100644 --- a/frontend/ui/data/onetime_migration.lua +++ b/frontend/ui/data/onetime_migration.lua @@ -7,7 +7,7 @@ local lfs = require("libs/libkoreader-lfs") local logger = require("logger") -- Date at which the last migration snippet was added -local CURRENT_MIGRATION_DATE = 20210413 +local CURRENT_MIGRATION_DATE = 20210414 -- Retrieve the date of the previous migration, if any local last_migration_date = G_reader_settings:readSetting("last_migration_date", 0) @@ -198,5 +198,16 @@ if last_migration_date < 20210412 then Cache:refreshSnapshot() end +-- Calibre, cache encoding format change, https://github.com/koreader/koreader/pull/7543 +if last_migration_date < 20210414 then + logger.info("Performing one-time migration for 20210414") + + local cache_path = DataStorage:getDataDir() .. "/cache/calibre" + ok, err = os.remove(cache_path .. "/books.dat") + if not ok then + logger.warn("os.remove:", err) + end +end + -- We're done, store the current migration date G_reader_settings:saveSetting("last_migration_date", CURRENT_MIGRATION_DATE) diff --git a/plugins/calibre.koplugin/search.lua b/plugins/calibre.koplugin/search.lua index 61587269e..0aee44429 100644 --- a/plugins/calibre.koplugin/search.lua +++ b/plugins/calibre.koplugin/search.lua @@ -176,7 +176,7 @@ local CalibreSearch = InputContainer:new{ }, cache_books = Persist:new{ path = DataStorage:getDataDir() .. "/cache/calibre/books.dat", - codec = "bitser", + codec = "luajit", }, } @@ -590,6 +590,7 @@ function CalibreSearch:getMetadata() local cache, err = self.cache_books:load() if not cache then logger.warn("invalid cache:", err) + self:invalidateCache() else local is_newer = true for path, enabled in pairs(self.libraries) do diff --git a/spec/unit/persist_spec.lua b/spec/unit/persist_spec.lua index 879a98251..0fc656a7c 100644 --- a/spec/unit/persist_spec.lua +++ b/spec/unit/persist_spec.lua @@ -1,7 +1,7 @@ describe("Persist module", function() local Persist local sample - local bitserInstance, dumpInstance + local bitserInstance, luajitInstance, dumpInstance local ser, deser, str, tab local fail = { a = function() end, } @@ -29,12 +29,14 @@ describe("Persist module", function() require("commonrequire") Persist = require("persist") bitserInstance = Persist:new{ path = "test.dat", codec = "bitser" } + luajitInstance = Persist:new{ path = "testj.dat", codec = "luajit" } dumpInstance = Persist:new { path = "test.txt", codec = "dump" } sample = arrayOf(1000) end) it("should save a table to file", function() assert.is_true(bitserInstance:save(sample)) + assert.is_true(luajitInstance:save(sample)) assert.is_true(dumpInstance:save(sample)) end) @@ -42,23 +44,30 @@ describe("Persist module", function() assert.is_true(bitserInstance:exists()) assert.is_true(bitserInstance:size() > 0) assert.is_true(type(bitserInstance:timestamp()) == "number") + + assert.is_true(luajitInstance:exists()) + assert.is_true(luajitInstance:size() > 0) + assert.is_true(type(luajitInstance:timestamp()) == "number") end) it("should load a table from file", function() assert.are.same(sample, bitserInstance:load()) + assert.are.same(sample, luajitInstance:load()) assert.are.same(sample, dumpInstance:load()) end) it("should delete the file", function() bitserInstance:delete() + luajitInstance:delete() dumpInstance:delete() assert.is_nil(bitserInstance:exists()) + assert.is_nil(luajitInstance:exists()) assert.is_nil(dumpInstance:exists()) end) it("should return standalone serializers/deserializers", function() tab = sample - for _, codec in ipairs({"dump", "bitser"}) do + for _, codec in ipairs({"dump", "bitser", "luajit"}) do assert.is_true(Persist.getCodec(codec).id == codec) ser = Persist.getCodec(codec).serialize deser = Persist.getCodec(codec).deserialize @@ -69,15 +78,17 @@ describe("Persist module", function() end) it("should work with huge tables", function() - tab = arrayOf(100000) - ser = Persist.getCodec("bitser").serialize - deser = Persist.getCodec("bitser").deserialize - str = ser(tab) - assert.are.same(deser(str), tab) + for _, codec in ipairs({"bitser", "luajit"}) do + tab = arrayOf(100000) + ser = Persist.getCodec(codec).serialize + deser = Persist.getCodec(codec).deserialize + str = ser(tab) + assert.are.same(deser(str), tab) + end end) it ("should fail to serialize functions", function() - for _, codec in ipairs({"dump", "bitser"}) do + for _, codec in ipairs({"dump", "bitser", "luajit"}) do assert.is_true(Persist.getCodec(codec).id == codec) ser = Persist.getCodec(codec).serialize deser = Persist.getCodec(codec).deserialize