[feat] Add dictionary download option (#3176)

You can now download pretty much all of the easily available freely licensed dictionaries I could find.

.luacheckrc

@@ -122,4 +122,5 @@ files["spec/unit/*"].globals = {
 ignore = {
     "211/__*",
     "631",
+    "dummy",
 }

frontend/apps/reader/modules/readerdictionary.lua

@@ -8,8 +8,10 @@ local InputContainer = require("ui/widget/container/inputcontainer")
 local JSON = require("json")
 local KeyValuePage = require("ui/widget/keyvaluepage")
 local LuaData = require("luadata")
+local NetworkMgr = require("ui/network/manager")
 local Trapper = require("ui/trapper")
 local UIManager = require("ui/uimanager")
+local ffiUtil  = require("ffi/util")
 local logger = require("logger")
 local util  = require("util")
 local _ = require("gettext")
@@ -238,6 +240,10 @@ function ReaderDictionary:addToMainMenu(menu_items)
                     })
                 end,
             },
+            {
+                text = _("Download dictionaries"),
+                sub_item_table = self:_genDownloadDictionariesMenu()
+            },
             {
                 text = _("Enable fuzzy search"),
                 checked_func = function()
@@ -370,6 +376,50 @@ function ReaderDictionary:getNumberOfDictionaries()
     return nb_available, nb_enabled, nb_disabled
 end
 
+function ReaderDictionary:_genDownloadDictionariesMenu()
+    local downloadable_dicts = require("ui/data/dictionaries")
+    local languages = {}
+
+    for i = 1, #downloadable_dicts do
+        local dict = downloadable_dicts[i]
+        local dict_lang_in = dict.lang_in
+        local dict_lang_out = dict.lang_out
+        if not languages[dict_lang_in] then
+            languages[dict_lang_in] = {}
+        end
+        table.insert(languages[dict_lang_in], dict)
+        if not languages[dict_lang_out] then
+            languages[dict_lang_out] = {}
+        end
+        table.insert(languages[dict_lang_out], dict)
+    end
+
+    -- remove duplicates
+    for lang_key,lang in pairs(languages) do
+        local hash = {}
+        local res = {}
+        for k,v in ipairs(lang) do
+           if not hash[v.name] then
+               res[#res+1] = v
+               hash[v.name] = true
+           end
+        end
+        languages[lang_key] = res
+    end
+
+    local menu_items = {}
+    for lang_key, available_langs in ffiUtil.orderedPairs(languages) do
+        table.insert(menu_items, {
+            text = lang_key,
+            callback = function()
+                self:showDownload(available_langs)
+            end
+        })
+    end
+
+    return menu_items
+end
+
 function ReaderDictionary:genDictionariesMenu()
     local items = {}
     for _, ifo in pairs(available_ifos) do
@@ -677,6 +727,128 @@ function ReaderDictionary:showDict(word, results, box, link)
     end
 end
 
+function ReaderDictionary:showDownload(downloadable_dicts)
+    local kv_pairs = {}
+    table.insert(kv_pairs, {_("Tap dictionary name to download"), ""})
+    table.insert(kv_pairs, "----------------------------")
+    for dummy, dict in ipairs(downloadable_dicts) do
+        table.insert(kv_pairs, {dict.name, "",
+            callback = function()
+                if not NetworkMgr:isOnline() then
+                    NetworkMgr:promptWifiOn()
+                    return
+                end
+                self:downloadDictionaryPrep(dict)
+            end})
+        local lang
+        if dict.lang_in == dict.lang_out then
+            lang = string.format("    %s", dict.lang_in)
+        else
+            lang = string.format("    %s–%s", dict.lang_in, dict.lang_out)
+        end
+        table.insert(kv_pairs, {lang, ""})
+        table.insert(kv_pairs, {"    ".._("License"), dict.license})
+        table.insert(kv_pairs, {"    ".._("Entries"), dict.entries})
+        table.insert(kv_pairs, "----------------------------")
+    end
+    self.download_window = KeyValuePage:new{
+        title = _("Download dictionaries"),
+        kv_pairs = kv_pairs,
+    }
+    UIManager:show(self.download_window)
+end
+
+function ReaderDictionary:downloadDictionaryPrep(dict, size)
+    local dummy, filename = util.splitFilePathName(dict.url)
+    local download_location = string.format("%s/%s", self.data_dir, filename)
+
+    local lfs = require("libs/libkoreader-lfs")
+    if lfs.attributes(download_location) then
+        UIManager:show(ConfirmBox:new{
+            text =  _("File already exists. Overwrite?"),
+            ok_text =  _("Overwrite"),
+            ok_callback = function()
+                self:downloadDictionary(dict, download_location)
+            end,
+        })
+    else
+        self:downloadDictionary(dict, download_location)
+    end
+end
+
+function ReaderDictionary:downloadDictionary(dict, download_location, continue)
+    continue = continue or false
+    local socket = require("socket")
+    local http = socket.http
+    local https = require("ssl.https")
+    local ltn12 = require("ltn12")
+    local url = socket.url
+
+    local parsed = url.parse(dict.url)
+    local httpRequest = parsed.scheme == "http" and http.request or https.request
+
+    if not continue then
+        local file_size
+        --local r, c, h = httpRequest {
+        local dummy, headers, dummy = socket.skip(1, httpRequest{
+            method = "HEAD",
+            url = dict.url,
+            --redirect = true,
+        })
+        --logger.dbg(status)
+        --logger.dbg(headers)
+        --logger.dbg(code)
+        file_size = headers and headers["content-length"]
+
+        UIManager:show(ConfirmBox:new{
+            text =  T(_("Dictionary filesize is %1 (%2 bytes). Continue with download?"), util.getFriendlySize(file_size), util.getFormattedSize(file_size)),
+            ok_text =  _("Download"),
+            ok_callback = function()
+                -- call ourselves with continue = true
+                self:downloadDictionary(dict, download_location, true)
+            end,
+        })
+        return
+    else
+        UIManager:nextTick(function()
+            UIManager:show(InfoMessage:new{
+                text = _("Downloading…"),
+                timeout = 3,
+            })
+        end)
+    end
+
+    local dummy, c, dummy = httpRequest{
+        url = dict.url,
+        sink = ltn12.sink.file(io.open(download_location, "w")),
+    }
+    if c == 200 then
+        logger.dbg("file downloaded to", download_location)
+    else
+        UIManager:show(InfoMessage:new{
+            text = _("Could not save file to:\n") .. download_location,
+            --timeout = 3,
+        })
+        return false
+    end
+
+    local ok, error = util.unpackArchive(download_location, self.data_dir)
+
+    if ok then
+        available_ifos = false
+        self:init()
+        UIManager:show(InfoMessage:new{
+            text = _("Dictionary downloaded:\n") .. dict.name,
+        })
+        return true
+    else
+        UIManager:show(InfoMessage:new{
+            text = _("Dictionary failed to download:\n") .. string.format("%s\n%s", dict.name, error),
+        })
+        return false
+    end
+end
+
 function ReaderDictionary:onUpdateDefaultDict(dict)
     logger.dbg("make default dictionary:", dict)
     self.default_dictionary = dict

frontend/ui/data/dictionaries.lua

@@ -0,0 +1,127 @@
+local _ = require("gettext")
+
+-- largely thanks to https://tuxor1337.github.io/firedict/dictionaries.html
+local dictionaries = {
+    {
+        name = "CIA World Factbook 2014",
+        lang_in = "English",
+        lang_out = "English",
+        entries = 2577,
+        license = _("Public Domain"),
+        url = "http://build.koreader.rocks/download/dict/factbook.tar.lz",
+    },
+    {
+        name = "GNU Collaborative International Dictionary of English",
+        lang_in = "English",
+        lang_out = "English",
+        entries = 108121,
+        license = "GPLv3+",
+        url = "http://build.koreader.rocks/download/dict/gcide.tar.lz",
+    },
+    {
+        name = "Douglas Harper's Online Etymology Dictionary",
+        lang_in = "English",
+        lang_out = "English",
+        entries = 46133,
+        license = "Unknown/©Douglas Harper",
+        url = "https://gitlab.com/koreader/stardict-dictionaries/uploads/1f3a66fe3f776e718590a66958df1b9b/etymonline.tar.lz",
+    },
+    {
+        name = "Folkets lexikon",
+        lang_in = "English",
+        lang_out = "Swedish",
+        entries = 53618,
+        license = "CC-BY-SA 2.5",
+        url = "https://gitlab.com/koreader/stardict-dictionaries/uploads/fad990cc29c6dd448f22469ce8648825/folkets_en-sv.tar.lz",
+    },
+    {
+        name = "Folkets lexikon",
+        lang_in = "Swedish",
+        lang_out = "English",
+        entries = 36513,
+        license = "CC-BY-SA 2.5",
+        url = "https://gitlab.com/koreader/stardict-dictionaries/uploads/fdc4265bbfc9af27fa4b1742ce3bdadd/folkets_sv-en.tar.lz",
+    },
+    {
+        name = "Dictionnaire Littré (xmlittre)",
+        lang_in = "French",
+        lang_out = "French",
+        entries = 78428,
+        license = "CC-BY-SA 3.0",
+        url = "http://http.debian.net/debian/pool/main/s/stardict-xmlittre/stardict-xmlittre_1.0.orig.tar.gz",
+    },
+    {
+        name = "Dictionnaire de l'Académie Française: 8ème edition",
+        lang_in = "French",
+        lang_out = "French",
+        entries = 31934,
+        license = _("Public Domain (copyright expired, published 1935)"),
+        url = "https://gitlab.com/koreader/stardict-dictionaries/uploads/a8b3027e84f344f64723fb7fe5c63c04/acadfran.tar.lz",
+    },
+    {
+        name = "Pape: Handwörterbuch der griechischen Sprache",
+        lang_in = "Ancient Greek",
+        lang_out = "German",
+        entries = 98893,
+        license = _("Public Domain (copyright expired, published 1880)"),
+        url = "https://gitlab.com/koreader/stardict-dictionaries/uploads/3ee9cb3acaca679bb9a95d845b813673/pape_gr-de.tar.lz",
+    },
+    {
+        name = "Georges: Ausführliches lateinisch-deutsches Handwörterbuch",
+        lang_in = "Latin",
+        lang_out = "German",
+        entries = 54831,
+        license = _("Public Domain (copyright expired, published 1913)"),
+        url = "https://gitlab.com/koreader/stardict-dictionaries/uploads/8d1e52d6c28d3b6865415979bc221fa6/georges_lat-de.tar.lz",
+    },
+    {
+        name = "Georges: Kleines deutsch-lateinisches Handwörterbuch",
+        lang_in = "German",
+        lang_out = "Latin",
+        entries = 26608,
+        license = _("Public Domain (copyright expired, published 1910)"),
+        url = "https://gitlab.com/koreader/stardict-dictionaries/uploads/a66b2a7655a4b102150fb5c6f1789cc5/georges_de-lat.tar.lz",
+    },
+    {
+        name = "Dicionário Aberto",
+        lang_in = "Portuguese",
+        lang_out = "Portuguese",
+        entries = 128521,
+        license = _("CC-BY-SA 2.5"),
+        url = "http://www.dicionario-aberto.net/stardict-DicAberto.tar.bz2",
+    },
+    {
+        name = "GNU/FDL Anglicko/Český slovník",
+        lang_in = "English",
+        lang_out = "Czech",
+        entries = 178904, -- ~90000 each way
+        license = _("GNU/FDL"),
+        url = "http://http.debian.net/debian/pool/non-free/s/stardict-english-czech/stardict-english-czech_20161201.orig.tar.gz",
+    },
+    {
+        name = "GNU/FDL Anglicko/Český slovník",
+        lang_in = "Czech",
+        lang_out = "English",
+        entries = 178904, -- ~90000 each way
+        license = _("GNU/FDL"),
+        url = "http://http.debian.net/debian/pool/non-free/s/stardict-english-czech/stardict-english-czech_20161201.orig.tar.gz",
+    },
+    {
+        name = "GNU/FDL Německo/Český slovník",
+        lang_in = "German",
+        lang_out = "Czech",
+        entries = 2341, -- ~1200 each way
+        license = _("GNU/FDL"),
+        url = "http://http.debian.net/debian/pool/non-free/s/stardict-german-czech/stardict-german-czech_20161201.orig.tar.gz",
+    },
+    {
+        name = "GNU/FDL Německo/Český slovník",
+        lang_in = "Czech",
+        lang_out = "German",
+        entries = 2341, -- ~1200 each way
+        license = _("GNU/FDL"),
+        url = "http://http.debian.net/debian/pool/non-free/s/stardict-german-czech/stardict-german-czech_20161201.orig.tar.gz",
+    },
+}
+
+return dictionaries

frontend/util.lua

@@ -3,6 +3,10 @@ This module contains miscellaneous helper functions for the KOReader frontend.
 ]]
 
 local BaseUtil = require("ffi/util")
+local dbg = require("dbg")
+local _ = require("gettext")
+local T = BaseUtil.template
+
 local util = {}
 
 --- Strips all punctuation and spaces from a string.
@@ -136,11 +140,11 @@ function util.tableEquals(o1, o2, ignore_mt)
 end
 
 --- Returns number of keys in a table.
----- @param T Lua table
----- @treturn int number of keys in table T
-function util.tableSize(T)
+---- @param t Lua table
+---- @treturn int number of keys in table t
+function util.tableSize(t)
     local count = 0
-    for _ in pairs(T) do count = count + 1 end
+    for _ in pairs(t) do count = count + 1 end
     return count
 end
 
@@ -431,6 +435,7 @@ end
 ---- @int size (bytes)
 ---- @treturn string
 function util.getFriendlySize(size)
+    size = tonumber(size)
     if not size or type(size) ~= "number" then return end
     local s
     if size > 1024*1024*1024 then
@@ -691,4 +696,25 @@ function util.checkLuaSyntax(lua_text)
     return err
 end
 
+--- Unpack an archive.
+-- Extract the contents of an archive, detecting its format by
+-- filename extension. Inspired by luarocks archive_unpack()
+-- @param archive string: Filename of archive.
+-- @param extract_to string: Destination directory.
+-- @return boolean or (boolean, string): true on success, false and an error message on failure.
+function util.unpackArchive(archive, extract_to)
+    dbg.dassert(type(archive) == "string")
+
+    local ok
+    if archive:match("%.tar%.bz2$") or archive:match("%.tar%.gz$") or archive:match("%.tar%.lz$") or archive:match("%.tgz$") then
+        ok = os.execute(("./tar xf %q -C %q"):format(archive, extract_to))
+    else
+        return false, T(_("Couldn't extract archive:\n\n%1\n\nUnrecognized filename extension."), archive)
+    end
+    if not ok then
+        return false, T(_("Extracting archive failed:\n\n%1", archive))
+    end
+    return true
+end
+
 return util