Statistics: speed up Calendar view (#6807)

Just by tweaking the SQL queries, and adding an index
on page_stat_data(start_time).
Also, in the hourly histogram, show at least a 1px bar
when there was any reading this hour.

plugins/statistics.koplugin/calendarview.lua

@@ -96,6 +96,9 @@ function HistogramWidget:paintTo(bb, x, y)
         local i_w = self.item_widths[n]
         local ratio = self.ratios and self.ratios[n] or 0
         local i_h = Math.round(ratio * self.height)
+        if i_h == 0 and ratio > 0 then -- show at least 1px
+            i_h = 1
+        end
         local i_y = self.height - i_h
         if i_h > 0 then
             bb:paintRect(x + i_x, y + i_y, i_w, i_h, self.color)

plugins/statistics.koplugin/main.lua

@@ -403,6 +403,7 @@ local STATISTICS_DB_PAGE_STAT_DATA_SCHEMA = [[
             UNIQUE (id_book, page, start_time),
             FOREIGN KEY(id_book) REFERENCES book(id)
         );
+    CREATE INDEX IF NOT EXISTS page_stat_data_start_time ON page_stat_data(start_time);
 ]]
 
 local STATISTICS_DB_PAGE_STAT_VIEW_SCHEMA = [[
@@ -509,6 +510,13 @@ function ReaderStatistics:upgradeDB(conn)
     ]]
     conn:exec(sql_stmt)
 
+    -- Get back the space taken by the deleted page_stat table
+    conn:exec("PRAGMA temp_store = 2;") -- use memory for temp files
+    local ok, errmsg = pcall(conn.exec, conn, "VACUUM;") -- this may take some time
+    if not ok then
+        logger.warn("Failed compacting statistics database:", errmsg)
+    end
+
     -- Create the new page_stat view stuff
     conn:exec(STATISTICS_DB_PAGE_STAT_VIEW_SCHEMA)
 
@@ -2285,13 +2293,22 @@ function ReaderStatistics:getFirstTimestamp()
 end
 
 function ReaderStatistics:getReadingRatioPerHourByDay(month)
+    -- We used to have in the SQL statement (with ? = 'YYYY-MM'):
+    --   WHERE  strftime('%Y-%m', start_time, 'unixepoch', 'localtime') = ?
+    -- but strftime()ing all start_time is slow.
+    -- Comverting the month into timestamp boundaries, and just comparing
+    -- integers, can be 5 times faster.
+    -- We let SQLite compute these timestamp boundaries from the provided
+    -- month; we need the start of the month to be a real date:
+    month = month.."-01"
     local sql_stmt = [[
         SELECT
             strftime('%Y-%m-%d', start_time, 'unixepoch', 'localtime') day,
             strftime('%H', start_time, 'unixepoch', 'localtime') hour,
             sum(duration)/3600.0 ratio
         FROM   page_stat
-        WHERE  strftime('%Y-%m', start_time, 'unixepoch', 'localtime') = ?
+        WHERE  start_time BETWEEN strftime('%s', ?, 'utc')
+                              AND strftime('%s', ?, 'utc', '+33 days', 'start of month', '-1 second')
         GROUP  BY
             strftime('%Y-%m-%d', start_time, 'unixepoch', 'localtime'),
             strftime('%H', start_time, 'unixepoch', 'localtime')
@@ -2299,7 +2316,7 @@ function ReaderStatistics:getReadingRatioPerHourByDay(month)
     ]]
     local conn = SQ3.open(db_location)
     local stmt = conn:prepare(sql_stmt)
-    local res, nb = stmt:reset():bind(month):resultset("i")
+    local res, nb = stmt:reset():bind(month, month):resultset("i")
     stmt:close()
     conn:close()
     local per_day = {}
@@ -2315,6 +2332,7 @@ function ReaderStatistics:getReadingRatioPerHourByDay(month)
 end
 
 function ReaderStatistics:getReadBookByDay(month)
+    month = month.."-01"
     local sql_stmt = [[
         SELECT
             strftime('%Y-%m-%d', start_time, 'unixepoch', 'localtime') day,
@@ -2323,7 +2341,8 @@ function ReaderStatistics:getReadBookByDay(month)
             book.title book_title
         FROM   page_stat
         JOIN   book ON book.id = page_stat.id_book
-        WHERE  strftime('%Y-%m', start_time, 'unixepoch', 'localtime') = ?
+        WHERE  start_time BETWEEN strftime('%s', ?, 'utc')
+                              AND strftime('%s', ?, 'utc', '+33 days', 'start of month', '-1 second')
         GROUP  BY
             strftime('%Y-%m-%d', start_time, 'unixepoch', 'localtime'),
             id_book,
@@ -2332,7 +2351,7 @@ function ReaderStatistics:getReadBookByDay(month)
     ]]
     local conn = SQ3.open(db_location)
     local stmt = conn:prepare(sql_stmt)
-    local res, nb = stmt:reset():bind(month):resultset("i")
+    local res, nb = stmt:reset():bind(month, month):resultset("i")
     stmt:close()
     conn:close()
     local per_day = {}