Statistics: Attempt to prevent rampant replication of null id_books (#10749)

And add an OTM block to do a cleanup pass on existing DBs (which might take a while if you're severely affected, because we've seen reports of DBs north of 2GB).

frontend/ui/data/onetime_migration.lua

@@ -5,10 +5,12 @@ Centralizes any and all one time migration concerns.
 local DataStorage = require("datastorage")
 local lfs = require("libs/libkoreader-lfs")
 local logger = require("logger")
+local SQ3 = require("lua-ljsqlite3/init")
+local util = require("util")
 local _ = require("gettext")
 
 -- Date at which the last migration snippet was added
-local CURRENT_MIGRATION_DATE = 20230731
+local CURRENT_MIGRATION_DATE = 20230802
 
 -- Retrieve the date of the previous migration, if any
 local last_migration_date = G_reader_settings:readSetting("last_migration_date", 0)
@@ -597,5 +599,27 @@ if last_migration_date < 20230731 then
     end
 end
 
+-- 20230802, Statistics plugin null id_book in page_stat_data
+if last_migration_date < 20230802 then
+    logger.info("Performing one-time migration for 20230802")
+    local db_location = DataStorage:getSettingsDir() .. "/statistics.sqlite3"
+    if util.fileExists(db_location) then
+        local conn = SQ3.open(db_location)
+        local ok, value = pcall(conn.exec, conn, "PRAGMA table_info('page_stat_data')")
+        if ok and value then
+            -- Has table
+            conn:exec("DELETE FROM page_stat_data WHERE id_book IS null;")
+            local ok2, errmsg = pcall(conn.exec, conn, "VACUUM;")
+            if not ok2 then
+                logger.warn("Failed compacting statistics database when fixing null id_book:", errmsg)
+            end
+        else
+            logger.warn("db not compatible when performing onetime migration:", ok, value)
+        end
+    else
+        logger.info("statistics.sqlite3 not found.")
+    end
+end
+
 -- We're done, store the current migration date
 G_reader_settings:saveSetting("last_migration_date", CURRENT_MIGRATION_DATE)

plugins/statistics.koplugin/main.lua

@@ -3104,9 +3104,9 @@ function ReaderStatistics.onSync(local_path, cached_path, income_path)
         INSERT INTO page_stat_data (id_book, page, start_time, duration, total_pages)
             SELECT map.mid, page, start_time, duration, total_pages
             FROM income_db.page_stat_data
-            LEFT JOIN book_id_map as map
+            INNER JOIN book_id_map as map
             ON id_book = map.iid
-            WHERE true
+            WHERE map.mid IS NOT null
         ON CONFLICT(id_book, page, start_time) DO UPDATE SET
         duration = MAX(duration, excluded.duration);