Persist: Add the new native LuaJIT serializer to the list of supported codecs (#7543)

And swap the Calibre metadata cache to it.

frontend/persist.lua

@@ -1,4 +1,5 @@
 local bitser = require("ffi/bitser")
+local buffer = require("string.buffer")
 local dump = require("dump")
 local lfs = require("libs/libkoreader-lfs")
 local logger = require("logger")
@@ -18,7 +19,7 @@ local function readFile(file, bytes)
 end
 
 local codecs = {
-    -- bitser: binary form, fast encode/decode, low size. Not human readable.
+    -- bitser: binary format, fast encode/decode, low size. Not human readable.
     bitser = {
         id = "bitser",
         reads_from_file = false,
@@ -26,7 +27,7 @@ local codecs = {
         serialize = function(t)
             local ok, str = pcall(bitser.dumps, t)
             if not ok then
-                return nil, "cannot serialize " .. tostring(t)
+                return nil, "cannot serialize " .. tostring(t) .. " (" .. str .. ")"
             end
             return str
         end,
@@ -34,7 +35,29 @@ local codecs = {
         deserialize = function(str)
             local ok, t = pcall(bitser.loads, str)
             if not ok then
-                return nil, "malformed serialized data"
+                return nil, "malformed serialized data: " .. t
+            end
+            return t
+        end,
+    },
+    -- luajit: binary format, optimized for speed, not size (combine w/ zstd if necessary). Not human readable.
+    --         Slightly larger on-disk representation than bitser, *much* faster to decode, slightly faster to encode.
+    luajit = {
+        id = "luajit",
+        reads_from_file = false,
+
+        serialize = function(t)
+            local ok, str = pcall(buffer.encode, t)
+            if not ok then
+                return nil, "cannot serialize " .. tostring(t) .. " (" .. str .. ")"
+            end
+            return str
+        end,
+
+        deserialize = function(str)
+            local ok, t = pcall(buffer.decode, str)
+            if not ok then
+                return nil, "malformed serialized data (" .. t .. ")"
             end
             return t
         end,

frontend/ui/data/onetime_migration.lua

@@ -7,7 +7,7 @@ local lfs = require("libs/libkoreader-lfs")
 local logger = require("logger")
 
 -- Date at which the last migration snippet was added
-local CURRENT_MIGRATION_DATE = 20210413
+local CURRENT_MIGRATION_DATE = 20210414
 
 -- Retrieve the date of the previous migration, if any
 local last_migration_date = G_reader_settings:readSetting("last_migration_date", 0)
@@ -198,5 +198,16 @@ if last_migration_date < 20210412 then
     Cache:refreshSnapshot()
 end
 
+-- Calibre, cache encoding format change, https://github.com/koreader/koreader/pull/7543
+if last_migration_date < 20210414 then
+    logger.info("Performing one-time migration for 20210414")
+
+    local cache_path = DataStorage:getDataDir() .. "/cache/calibre"
+    ok, err = os.remove(cache_path .. "/books.dat")
+    if not ok then
+       logger.warn("os.remove:", err)
+    end
+end
+
 -- We're done, store the current migration date
 G_reader_settings:saveSetting("last_migration_date", CURRENT_MIGRATION_DATE)

plugins/calibre.koplugin/search.lua

@@ -176,7 +176,7 @@ local CalibreSearch = InputContainer:new{
     },
     cache_books = Persist:new{
         path = DataStorage:getDataDir() .. "/cache/calibre/books.dat",
-        codec = "bitser",
+        codec = "luajit",
     },
 }
 
@@ -590,6 +590,7 @@ function CalibreSearch:getMetadata()
         local cache, err = self.cache_books:load()
         if not cache then
             logger.warn("invalid cache:", err)
+            self:invalidateCache()
         else
             local is_newer = true
             for path, enabled in pairs(self.libraries) do

spec/unit/persist_spec.lua

@@ -1,7 +1,7 @@
 describe("Persist module", function()
     local Persist
     local sample
-    local bitserInstance, dumpInstance
+    local bitserInstance, luajitInstance, dumpInstance
     local ser, deser, str, tab
     local fail = { a = function() end, }
 
@@ -29,12 +29,14 @@ describe("Persist module", function()
         require("commonrequire")
         Persist = require("persist")
         bitserInstance = Persist:new{ path = "test.dat", codec = "bitser" }
+        luajitInstance = Persist:new{ path = "testj.dat", codec = "luajit" }
         dumpInstance = Persist:new { path = "test.txt", codec = "dump" }
         sample = arrayOf(1000)
     end)
 
     it("should save a table to file", function()
         assert.is_true(bitserInstance:save(sample))
+        assert.is_true(luajitInstance:save(sample))
         assert.is_true(dumpInstance:save(sample))
     end)
 
@@ -42,23 +44,30 @@ describe("Persist module", function()
         assert.is_true(bitserInstance:exists())
         assert.is_true(bitserInstance:size() > 0)
         assert.is_true(type(bitserInstance:timestamp()) == "number")
+
+        assert.is_true(luajitInstance:exists())
+        assert.is_true(luajitInstance:size() > 0)
+        assert.is_true(type(luajitInstance:timestamp()) == "number")
     end)
 
     it("should load a table from file", function()
         assert.are.same(sample, bitserInstance:load())
+        assert.are.same(sample, luajitInstance:load())
         assert.are.same(sample, dumpInstance:load())
     end)
 
     it("should delete the file", function()
         bitserInstance:delete()
+        luajitInstance:delete()
         dumpInstance:delete()
         assert.is_nil(bitserInstance:exists())
+        assert.is_nil(luajitInstance:exists())
         assert.is_nil(dumpInstance:exists())
     end)
 
     it("should return standalone serializers/deserializers", function()
         tab = sample
-        for _, codec in ipairs({"dump", "bitser"}) do
+        for _, codec in ipairs({"dump", "bitser", "luajit"}) do
             assert.is_true(Persist.getCodec(codec).id == codec)
             ser = Persist.getCodec(codec).serialize
             deser = Persist.getCodec(codec).deserialize
@@ -69,15 +78,17 @@ describe("Persist module", function()
     end)
 
     it("should work with huge tables", function()
-        tab = arrayOf(100000)
+        for _, codec in ipairs({"bitser", "luajit"}) do
-        ser = Persist.getCodec("bitser").serialize
+            tab = arrayOf(100000)
-        deser = Persist.getCodec("bitser").deserialize
+            ser = Persist.getCodec(codec).serialize
-        str = ser(tab)
+            deser = Persist.getCodec(codec).deserialize
-        assert.are.same(deser(str), tab)
+            str = ser(tab)
+            assert.are.same(deser(str), tab)
+        end
     end)
 
     it ("should fail to serialize functions", function()
-        for _, codec in ipairs({"dump", "bitser"}) do
+        for _, codec in ipairs({"dump", "bitser", "luajit"}) do
             assert.is_true(Persist.getCodec(codec).id == codec)
             ser = Persist.getCodec(codec).serialize
             deser = Persist.getCodec(codec).deserialize