Visual feedback around tar/zsync on Kobo during OTA updates (#4043)

* With a bonus version normalizer fix.
  * And a zsync bump/fix to avoid softlocks with OpenStack HTTP frontends (also, pull those from the OTA mirror list).

Makefile

@@ -166,6 +166,7 @@ kindleupdate: all
 	ln -sf ../../$(KINDLE_DIR)/koreader.sh $(INSTALL_DIR)/koreader
 	ln -sf ../../$(KINDLE_DIR)/libkohelper.sh $(INSTALL_DIR)/koreader
 	ln -sf ../../$(KINDLE_DIR)/kotar_cpoint $(INSTALL_DIR)/koreader
+	ln -sf ../../$(KINDLE_DIR)/zsync_status.sh $(INSTALL_DIR)/koreader
 	# create new package
 	cd $(INSTALL_DIR) && pwd && \
 		zip -9 -r \
@@ -197,6 +198,7 @@ koboupdate: all
 	cp $(KOBO_DIR)/koreader.png $(INSTALL_DIR)/koreader.png
 	cp $(KOBO_DIR)/fmon/README.txt $(INSTALL_DIR)/README_kobo.txt
 	cp $(KOBO_DIR)/*.sh $(INSTALL_DIR)/koreader
+	cp $(KOBO_DIR)/kotar_cpoint $(INSTALL_DIR)/koreader
 	# create new package
 	cd $(INSTALL_DIR) && \
 		zip -9 -r \

README.md

@@ -318,7 +318,7 @@ KOReader's build system. To install ccache:
 [nb-script]:https://gitlab.com/koreader/nightly-builds/blob/master/build_release.sh
 [circleci-badge]:https://circleci.com/gh/koreader/koreader.svg?style=shield
 [circleci-link]:https://circleci.com/gh/koreader/koreader
-[circleci-conf]:https://github.com/koreader/koreader-base/blob/master/.circleci/config.yml
+[circleci-conf]:https://github.com/koreader/koreader/blob/master/.circleci/config.yml
 [linux-vm]:http://www.howtogeek.com/howto/11287/how-to-run-ubuntu-in-windows-7-with-vmware-player/
 [l10n-readme]:https://github.com/koreader/koreader/blob/master/l10n/README.md
 [koreader-transifex]:https://www.transifex.com/projects/p/koreader/

base

@@ -1 +1 @@
-Subproject commit 438014b1d901cacf8591354e8060b220b8a964bd
+Subproject commit abe91abb0f4a2146a65ca131d323eb1d47760068

frontend/device/kindle/device.lua

@@ -548,6 +548,13 @@ function KindleOasis2:init()
         }
     }
 
+    -- FIXME: When starting KOReader with the device upside down ("D"), touch input is registered wrong
+    --        (i.e., probably upside down).
+    --        If it's started upright ("U"), everything's okay, and turning it upside down after that works just fine.
+    --        See #2206 & #2209 for the original KOA implementation, which obviously doesn't quite cut it here...
+    --        See also https://www.mobileread.com/forums/showthread.php?t=298302&page=5
+    -- NOTE: It'd take some effort to actually start KOReader while in a LANDSCAPE orientation,
+    --       since they're only exposed inside the stock reader, and not the Home/KUAL Booklets.
     local haslipc, lipc = pcall(require, "liblipclua")
     if haslipc and lipc then
         local lipc_handle = lipc.init("com.github.koreader.screen")

frontend/ui/otamanager.lua

@@ -16,9 +16,16 @@ local OTAManager = {
     ota_servers = {
         "http://ota.koreader.rocks:80/",
         "http://vislab.bjmu.edu.cn:80/apps/koreader/ota/",
+        --[[
+        -- NOTE: Because we can't have nice things,
+        --       the HTTP frontend of these OpenStack storage containers doesn't actually properly support
+        --       HTTP/1.1 Range requests when multiple byte ranges are requested: they return bogus data when doing so,
+        --       which confuses zsync, causing it to retry indefinitely instead of aborting...
+        --       c.f., https://github.com/koreader/koreader-base/pull/699
         "http://koreader-fr.ak-team.com:80/",
         "http://koreader-pl.ak-team.com:80/",
         "http://koreader-na.ak-team.com:80/",
+        --]]
         "http://koreader.ak-team.com:80/",
     },
     ota_channels = {
@@ -174,17 +181,30 @@ function OTAManager:_buildLocalPackage()
     end
     if Device:isAndroid() then
         return os.execute(string.format(
-            "./tar cvf %s -T %s --no-recursion",
+            "./tar cf %s -T %s --no-recursion",
             self.installed_package, self.package_indexfile))
     else
-        return os.execute(string.format(
-            "./tar cvf %s -C .. -T %s --no-recursion",
-            self.installed_package, self.package_indexfile))
+        -- With visual feedback if supported...
+        if lfs.attributes("./kotar_cpoint", "mode") == "file" then
+            return os.execute(string.format(
+                "./tar cf %s -C .. -T %s --no-recursion --checkpoint=200 --checkpoint-action=exec='./kotar_cpoint $TAR_CHECKPOINT create'",
+                self.installed_package, self.package_indexfile))
+        else
+            return os.execute(string.format(
+                "./tar cf %s -C .. -T %s --no-recursion",
+                self.installed_package, self.package_indexfile))
+        end
     end
 end
 
 function OTAManager:zsync()
     if self:_buildLocalPackage() == 0 then
+        -- Make it clear that it's now zsync churning CPU time, instead of tar churning IO ;).
+        if Device:isKindle() then
+            os.execute("./zsync_status.sh")
+        elseif Device:isKobo() then
+            os.execute("./fbink -q -y -7 -pmh 'Computing zsync delta . . .'")
+        end
         return os.execute(
             ("./zsync -i %s -o %s -u %s %s%s"):format(
                 self.installed_package,

frontend/version.lua

@@ -5,7 +5,7 @@ This module helps with retrieving version information.
 local Version = {}
 
 --- Returns current KOReader git-rev.
--- @treturn string full KOReader git-rev such `v2015.11-982-g704d4238`
+-- @treturn string full KOReader git-rev such as `v2015.11-982-g704d4238`
 function Version:getCurrentRevision()
     if not self.rev then
         local rev_file = io.open("git-rev", "r")
@@ -22,18 +22,19 @@ function Version:getCurrentRevision()
 end
 
 --- Returns normalized version of KOReader git-rev input string.
--- @string rev full KOReader git-rev such `v2015.11-982-g704d4238`
--- @treturn int version in the form of a number such as `201511982`
+-- @string rev full KOReader git-rev such as `v2015.11-982-g704d4238`
+-- @treturn int version in the form of a 10 digit number such as `2015110982`
 -- @treturn string short git commit version hash such as `704d4238`
 function Version:getNormalizedVersion(rev)
     if not rev then return end
     local year, month, revision = rev:match("v(%d%d%d%d)%.(%d%d)-?(%d*)")
-    local commit = rev:match("-%d*-g(.*)")
-    return ((year or 0) * 100 + (month or 0)) * 1000 + (revision or 0), commit
+    local commit = rev:match("-%d*-g(%x*)[%d_%-]*")
+    -- NOTE: * 10000 to handle at most 9999 commits since last tag ;).
+    return ((year or 0) * 100 + (month or 0)) * 10000 + (revision or 0), commit
 end
 
 --- Returns current version of KOReader.
--- @treturn int version in the form of a number such as `201511982`
+-- @treturn int version in the form of a 10 digit number such as `2015110982`
 -- @treturn string short git commit version hash such as `704d4238`
 -- @see normalized_version
 function Version:getNormalizedCurrentVersion()

platform/kindle/koreader.sh

@@ -94,25 +94,27 @@ if [ -f "${NEWUPDATE}" ]; then
     GNUTAR_BIN="${KOREADER_DIR}/tar"
     if [ -x "${GNUTAR_BIN}" ]; then
         # Let our checkpoint script handle the detailed visual feedback...
-        eips_print_bottom_centered "Updating koreader" 1
+        eips_print_bottom_centered "Updating KOReader" 3
         # shellcheck disable=SC2016
         ${GNUTAR_BIN} -C "/mnt/us" --no-same-owner --no-same-permissions --checkpoint=200 --checkpoint-action=exec='./kotar_cpoint $TAR_CHECKPOINT' -xf "${NEWUPDATE}"
         fail=$?
     else
         # Fall back to busybox tar
-        eips_print_bottom_centered "Updating koreader . . ." 1
+        eips_print_bottom_centered "Updating KOReader . . ." 3
         tar -C "/mnt/us" -xf "${NEWUPDATE}"
         fail=$?
     fi
     # Cleanup behind us...
-    if [ $fail -eq 0 ]; then
+    if [ "${fail}" -eq 0 ]; then
         mv "${NEWUPDATE}" "${INSTALLED}"
-        logmsg "Update sucessful :)"
-        eips_print_bottom_centered "Update successful :)" 1
+        logmsg "Update successful :)"
+        eips_print_bottom_centered "Update successful :)" 2
+        eips_print_bottom_centered "KOReader will start momentarily . . ." 1
     else
         # Huh ho...
         logmsg "Update failed :("
-        eips_print_bottom_centered "Update failed :(" 1
+        eips_print_bottom_centered "Update failed :(" 2
+        eips_print_bottom_centered "KOReader may fail to function properly" 1
     fi
     rm -f "${NEWUPDATE}" # always purge newupdate in all cases to prevent update loop
 fi

platform/kindle/kotar_cpoint

@@ -15,17 +15,30 @@ else
     exit 1
 fi
 
-## First arg is the chekpoint number, and we get one every 200 checkpoints.
+## First arg is the checkpoint number, and we get one every 200 checkpoints.
 CHECKPOINT_NUM="${1}"
 CHECKPOINT_GRANULARITY="200"
 
-# Use that to build a poor man's progress bar, with dots.
+# Use that to build a progress bar
 PROGRESS_AMOUNT="$((CHECKPOINT_NUM / CHECKPOINT_GRANULARITY))"
-PROGRESS_STRING="Updating koreader "
+
+# Which text do we want to use?
+case "${2}" in
+    "create")
+        PROGRESS_BASE_STRING="Preparing local OTA package"
+        ;;
+    *)
+        PROGRESS_BASE_STRING="Updating KOReader"
+        ;;
+esac
+
+# And start drawing the progress bar...
+PROGRESS_STRING=""
 for _ in $(seq 1 ${PROGRESS_AMOUNT}); do
-    # Append a dot until we hit the needed amount
+    # Fill the progress bar...
     PROGRESS_STRING="${PROGRESS_STRING}."
 done
 
 # Print our progress :)
-eips_print_bottom_centered "${PROGRESS_STRING}" 1
+eips_print_bottom_centered "${PROGRESS_BASE_STRING}" 3
+eips_print_bottom_centered "${PROGRESS_STRING}" 2

platform/kindle/zsync_status.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+
+# KOReader's working directory
+KOREADER_DIR="/mnt/us/koreader"
+
+# We do NOT want to sleep during eips calls!
+export EIPS_NO_SLEEP="true"
+
+# Load our helper functions...
+if [ -f "${KOREADER_DIR}/libkohelper.sh" ]; then
+    # shellcheck source=/dev/null
+    . "${KOREADER_DIR}/libkohelper.sh"
+else
+    echo "Can't source helper functions, aborting!"
+    exit 1
+fi
+
+eips_print_bottom_centered "Computing zsync delta . . ." 3
+

platform/kobo/koreader.sh

@@ -11,9 +11,19 @@ cd "${KOREADER_DIR}" || exit
 NEWUPDATE="${KOREADER_DIR}/ota/koreader.updated.tar"
 INSTALLED="${KOREADER_DIR}/ota/koreader.installed.tar"
 if [ -f "${NEWUPDATE}" ]; then
-    # TODO: any graphic indication for the updating progress?
-    ./tar xf "${NEWUPDATE}" --strip-components=1 --no-same-permissions --no-same-owner \
-        && mv "${NEWUPDATE}" "${INSTALLED}"
+    # shellcheck disable=SC2016
+    ./tar xf "${NEWUPDATE}" --strip-components=1 --no-same-permissions --no-same-owner --checkpoint=200 --checkpoint-action=exec='./kotar_cpoint $TAR_CHECKPOINT'
+    fail=$?
+    # Cleanup behind us...
+    if [ "${fail}" -eq 0 ]; then
+        mv "${NEWUPDATE}" "${INSTALLED}"
+        ./fbink -q -y -6 -pm "Update successful :)"
+        ./fbink -q -y -5 -pm "KOReader will start momentarily . . ."
+    else
+        # Huh ho...
+        ./fbink -q -y -6 -pmh "Update failed :("
+        ./fbink -q -y -5 -pm "KOReader may fail to function properly!"
+    fi
     rm -f "${NEWUPDATE}" # always purge newupdate in all cases to prevent update loop
 fi
 

platform/kobo/kotar_cpoint

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+## First arg is the checkpoint number, and we get one every 200 checkpoints.
+CHECKPOINT_NUM="${1}"
+CHECKPOINT_GRANULARITY="200"
+
+# Use that to build a progress bar
+PROGRESS_AMOUNT="$((CHECKPOINT_NUM / CHECKPOINT_GRANULARITY))"
+
+# Which text do we want to use?
+case "${2}" in
+    "create")
+        PROGRESS_BASE_STRING="Preparing local OTA package"
+        ;;
+    *)
+        PROGRESS_BASE_STRING="Updating KOReader"
+        ;;
+esac
+
+# And start drawing the progress bar...
+PROGRESS_STRING=""
+for _ in $(seq 1 ${PROGRESS_AMOUNT}); do
+    # Fill the progress bar...
+    PROGRESS_STRING="${PROGRESS_STRING}."
+done
+
+# Print our progress :)
+./fbink -q -y -7 -pmh "${PROGRESS_BASE_STRING}" "${PROGRESS_STRING}"

spec/unit/readerlink_spec.lua

@@ -17,8 +17,8 @@ describe("ReaderLink module", function()
             document = DocumentRegistry:openDocument(sample_epub),
         }
         readerui.rolling:onGotoPage(5)
-        readerui.link:onTap(nil, {pos = {x = 326, y = 220}})
-        assert.is.same(44, readerui.rolling.current_page)
+        readerui.link:onTap(nil, {pos = {x = 340, y = 70}})
+        assert.is.same(41, readerui.rolling.current_page)
     end)
 
     it("should jump to links in pdf page mode", function()
@@ -54,8 +54,8 @@ describe("ReaderLink module", function()
             document = DocumentRegistry:openDocument(sample_epub),
         }
         readerui.rolling:onGotoPage(5)
-        readerui.link:onTap(nil, {pos = {x = 326, y = 220}})
-        assert.is.same(44, readerui.rolling.current_page)
+        readerui.link:onTap(nil, {pos = {x = 340, y = 70}})
+        assert.is.same(41, readerui.rolling.current_page)
         readerui.link:onGoBackLink()
         assert.is.same(5, readerui.rolling.current_page)
     end)

spec/unit/version_spec.lua

@@ -7,27 +7,35 @@ describe("Version module", function()
     it("should get current revision", function()
         local rev = Version:getCurrentRevision()
         local year, month, revision = rev:match("v(%d%d%d%d)%.(%d%d)-?(%d*)")
-        local commit = rev:match("-%d*-g(.*)")
+        local commit = rev:match("-%d*-g(%x*)[%d_%-]*")
         assert.is_truthy(year)
         assert.is_truthy(month)
         assert.is_truthy(revision)
         assert.is_truthy(commit)
-        assert.is_true(4 <= year:len())
+        assert.is_true(4 == year:len())
         assert.is_true(2 == month:len())
         assert.is_true(1 <= revision:len())
-        assert.is_true(5 <= commit:len())
+        assert.is_true(7 <= commit:len())
     end)
     it("should get normalized current version", function()
-        assert.is_true(7 <= tostring(Version:getNormalizedCurrentVersion()):len())
+        assert.is_true(10 == tostring(Version:getNormalizedCurrentVersion()):len())
     end)
     it("should get normalized version", function()
         local rev = "v2015.11-982-g704d4238"
         local version, commit = Version:getNormalizedVersion(rev)
-        local expected_version = 201511982
+        local expected_version = 2015110982
         local expected_commit = "704d4238"
         assert.are.same(expected_version, version)
         assert.are.same(expected_commit, commit)
     end)
+    it("should also get normalized version", function()
+        local rev = "v2015.11-1755-gecd7b5b_2018-07-02"
+        local version, commit = Version:getNormalizedVersion(rev)
+        local expected_version = 2015111755
+        local expected_commit = "ecd7b5b"
+        assert.are.same(expected_version, version)
+        assert.are.same(expected_commit, commit)
+    end)
     it("should fail gracefully", function()
         local version, commit = Version:getNormalizedVersion()
         local expected_version = nil