Use fsync() for more robust setting files saving

Bump base for util.fsyncOpenedFile() and util.fsyncDirectory().

Use these to force flush to storage device when
saving luasetting, docsetting, and history.lua.
Also dont rotate them to .old until they are at least
60 seconds old.
Also make auto_save_paging_count count right.

Also bump crengine: open (as text) small or empty files
pull/5684/head
poire-z 4 years ago
parent 1e1ceedd4d
commit 04d9a557aa

@ -1 +1 @@
Subproject commit 10cfb8379d6badfaa4a9599d6a659cc1983a0535 Subproject commit f94e953b32858a02aa2d0a6b780ee28b88717ced

@ -928,11 +928,10 @@ function ReaderView:onReaderReady()
end end
else else
self.autoSaveSettings = function() self.autoSaveSettings = function()
self.auto_save_paging_count = self.auto_save_paging_count + 1
if self.auto_save_paging_count == DAUTO_SAVE_PAGING_COUNT then if self.auto_save_paging_count == DAUTO_SAVE_PAGING_COUNT then
self.ui:saveSettings() self.ui:saveSettings()
self.auto_save_paging_count = 0 self.auto_save_paging_count = 0
else
self.auto_save_paging_count = self.auto_save_paging_count + 1
end end
end end
end end

@ -6,9 +6,9 @@ in the so-called sidecar directory
local DataStorage = require("datastorage") local DataStorage = require("datastorage")
local dump = require("dump") local dump = require("dump")
local ffiutil = require("ffi/util")
local lfs = require("libs/libkoreader-lfs") local lfs = require("libs/libkoreader-lfs")
local logger = require("logger") local logger = require("logger")
local purgeDir = require("ffi/util").purgeDir
local DocSettings = {} local DocSettings = {}
@ -187,9 +187,19 @@ function DocSettings:flush()
local s_out = dump(self.data) local s_out = dump(self.data)
os.setlocale('C', 'numeric') os.setlocale('C', 'numeric')
for _, f in pairs(serials) do for _, f in pairs(serials) do
local directory_updated = false
if lfs.attributes(f, "mode") == "file" then if lfs.attributes(f, "mode") == "file" then
logger.dbg("Rename ", f, " to ", f .. ".old") -- As an additional safety measure (to the ffiutil.fsync* calls
os.rename(f, f .. ".old") -- used below), we only backup the file to .old when it has
-- not been modified in the last 60 seconds. This should ensure
-- in the case the fsync calls are not supported that the OS
-- may have itself sync'ed that file content in the meantime.
local mtime = lfs.attributes(f, "modification")
if mtime < os.time() - 60 then
logger.dbg("Rename ", f, " to ", f .. ".old")
os.rename(f, f .. ".old")
directory_updated = true -- fsync directory content too below
end
end end
logger.dbg("Write to ", f) logger.dbg("Write to ", f)
local f_out = io.open(f, "w") local f_out = io.open(f, "w")
@ -197,6 +207,7 @@ function DocSettings:flush()
f_out:write("-- we can read Lua syntax here!\nreturn ") f_out:write("-- we can read Lua syntax here!\nreturn ")
f_out:write(s_out) f_out:write(s_out)
f_out:write("\n") f_out:write("\n")
ffiutil.fsyncOpenedFile(f_out) -- force flush to the storage device
f_out:close() f_out:close()
if self.candidates ~= nil if self.candidates ~= nil
@ -212,6 +223,10 @@ function DocSettings:flush()
end end
end end
if directory_updated then
-- Ensure the file renaming is flushed to storage device
ffiutil.fsyncDirectory(f)
end
break break
end end
end end
@ -227,7 +242,7 @@ function DocSettings:purge()
os.remove(self.history_file) os.remove(self.history_file)
end end
if lfs.attributes(self.sidecar, "mode") == "directory" then if lfs.attributes(self.sidecar, "mode") == "directory" then
purgeDir(self.sidecar) ffiutil.purgeDir(self.sidecar)
end end
self.data = {} self.data = {}
end end

@ -51,21 +51,27 @@ function LuaData:open(file_path, o) -- luacheck: ignore 312
end end
end end
local ok = pcall(dofile, new.file) local ok = false
if lfs.attributes(new.file, "mode") == "file" then
if ok then ok = pcall(dofile, new.file)
logger.dbg("data is read from ", new.file) if ok then
else logger.dbg("data is read from ", new.file)
logger.dbg(new.file, " is invalid, remove.") else
os.remove(new.file) logger.dbg(new.file, " is invalid, remove.")
os.remove(new.file)
end
end
if not ok then
for i=1, self.max_backups, 1 do for i=1, self.max_backups, 1 do
local backup_file = new.file..".old."..i local backup_file = new.file..".old."..i
if pcall(dofile, backup_file) then if lfs.attributes(backup_file, "mode") == "file" then
logger.dbg("data is read from ", backup_file) if pcall(dofile, backup_file) then
break logger.dbg("data is read from ", backup_file)
else break
logger.dbg(backup_file, " is invalid, remove.") else
os.remove(backup_file) logger.dbg(backup_file, " is invalid, remove.")
os.remove(backup_file)
end
end end
end end
end end

@ -3,6 +3,7 @@ This module handles generic settings as well as KOReader's global settings syste
]] ]]
local dump = require("dump") local dump = require("dump")
local ffiutil = require("ffi/util")
local lfs = require("libs/libkoreader-lfs") local lfs = require("libs/libkoreader-lfs")
local logger = require("logger") local logger = require("logger")
@ -180,8 +181,18 @@ end
--- Writes settings to disk. --- Writes settings to disk.
function LuaSettings:flush() function LuaSettings:flush()
if not self.file then return end if not self.file then return end
local directory_updated = false
if lfs.attributes(self.file, "mode") == "file" then if lfs.attributes(self.file, "mode") == "file" then
os.rename(self.file, self.file .. ".old") -- As an additional safety measure (to the ffiutil.fsync* calls
-- used below), we only backup the file to .old when it has
-- not been modified in the last 60 seconds. This should ensure
-- in the case the fsync calls are not supported that the OS
-- may have itself sync'ed that file content in the meantime.
local mtime = lfs.attributes(self.file, "modification")
if mtime < os.time() - 60 then
os.rename(self.file, self.file .. ".old")
directory_updated = true -- fsync directory content too below
end
end end
local f_out = io.open(self.file, "w") local f_out = io.open(self.file, "w")
if f_out ~= nil then if f_out ~= nil then
@ -189,8 +200,13 @@ function LuaSettings:flush()
f_out:write("-- we can read Lua syntax here!\nreturn ") f_out:write("-- we can read Lua syntax here!\nreturn ")
f_out:write(dump(self.data)) f_out:write(dump(self.data))
f_out:write("\n") f_out:write("\n")
ffiutil.fsyncOpenedFile(f_out) -- force flush to the storage device
f_out:close() f_out:close()
end end
if directory_updated then
-- Ensure the file renaming is flushed to storage device
ffiutil.fsyncDirectory(self.file)
end
return self return self
end end

@ -1,10 +1,11 @@
local DataStorage = require("datastorage") local DataStorage = require("datastorage")
local DocSettings = require("docsettings") local DocSettings = require("docsettings")
local dump = require("dump") local dump = require("dump")
local ffiutil = require("ffi/util")
local getFriendlySize = require("util").getFriendlySize local getFriendlySize = require("util").getFriendlySize
local joinPath = require("ffi/util").joinPath local joinPath = ffiutil.joinPath
local lfs = require("libs/libkoreader-lfs") local lfs = require("libs/libkoreader-lfs")
local realpath = require("ffi/util").realpath local realpath = ffiutil.realpath
local history_file = joinPath(DataStorage:getDataDir(), "history.lua") local history_file = joinPath(DataStorage:getDataDir(), "history.lua")
@ -91,6 +92,7 @@ function ReadHistory:_flush()
end end
local f = io.open(history_file, "w") local f = io.open(history_file, "w")
f:write("return " .. dump(content) .. "\n") f:write("return " .. dump(content) .. "\n")
ffiutil.fsyncOpenedFile(f) -- force flush to the storage device
f:close() f:close()
end end

@ -100,6 +100,12 @@ describe("docsettings module", function()
d:flush() d:flush()
-- metadata.pdf.lua should be generated. -- metadata.pdf.lua should be generated.
assert.Equals("file", lfs.attributes(d.sidecar_file, "mode")) assert.Equals("file", lfs.attributes(d.sidecar_file, "mode"))
d:flush()
-- metadata.pdf.lua.old should not yet be generated.
assert.are.not_equal("file", lfs.attributes(d.sidecar_file .. ".old", "mode"))
-- make metadata.pdf.lua older to bypass 60s age needed for .old rotation
local minutes_ago = os.time() - 120
lfs.touch(d.sidecar_file, minutes_ago)
d:close() d:close()
-- metadata.pdf.lua and metadata.pdf.lua.old should be generated. -- metadata.pdf.lua and metadata.pdf.lua.old should be generated.
assert.Equals("file", lfs.attributes(d.sidecar_file, "mode")) assert.Equals("file", lfs.attributes(d.sidecar_file, "mode"))
@ -152,6 +158,9 @@ describe("docsettings module", function()
d:flush() d:flush()
-- metadata.pdf.lua should be generated. -- metadata.pdf.lua should be generated.
assert.Equals("file", lfs.attributes(d.sidecar_file, "mode")) assert.Equals("file", lfs.attributes(d.sidecar_file, "mode"))
-- make metadata.pdf.lua older to bypass 60s age needed for .old rotation
local minutes_ago = os.time() - 120
lfs.touch(d.sidecar_file, minutes_ago)
d:close() d:close()
-- metadata.pdf.lua and metadata.pdf.lua.old should be generated. -- metadata.pdf.lua and metadata.pdf.lua.old should be generated.
assert.Equals("file", lfs.attributes(d.sidecar_file, "mode")) assert.Equals("file", lfs.attributes(d.sidecar_file, "mode"))
@ -182,6 +191,9 @@ describe("docsettings module", function()
d:flush() d:flush()
-- metadata.pdf.lua should be generated. -- metadata.pdf.lua should be generated.
assert.Equals("file", lfs.attributes(d.sidecar_file, "mode")) assert.Equals("file", lfs.attributes(d.sidecar_file, "mode"))
-- make metadata.pdf.lua older to bypass 60s age needed for .old rotation
local minutes_ago = os.time() - 120
lfs.touch(d.sidecar_file, minutes_ago)
d:close() d:close()
-- metadata.pdf.lua and metadata.pdf.lua.old should be generated. -- metadata.pdf.lua and metadata.pdf.lua.old should be generated.
assert.Equals("file", lfs.attributes(d.sidecar_file, "mode")) assert.Equals("file", lfs.attributes(d.sidecar_file, "mode"))

Loading…
Cancel
Save