From b261a647c24a9e830a4646fb74b845e7a26244d2 Mon Sep 17 00:00:00 2001 From: Frans de Jonge Date: Thu, 13 Dec 2018 06:27:49 +0000 Subject: [PATCH] [feat] Add dictionary download option (#3176) You can now download pretty much all of the easily available freely licensed dictionaries I could find. --- .luacheckrc | 1 + .../apps/reader/modules/readerdictionary.lua | 172 ++++++++++++++++++ frontend/ui/data/dictionaries.lua | 127 +++++++++++++ frontend/util.lua | 34 +++- 4 files changed, 330 insertions(+), 4 deletions(-) create mode 100644 frontend/ui/data/dictionaries.lua diff --git a/.luacheckrc b/.luacheckrc index 5797babbc..f2c267a51 100644 --- a/.luacheckrc +++ b/.luacheckrc @@ -122,4 +122,5 @@ files["spec/unit/*"].globals = { ignore = { "211/__*", "631", + "dummy", } diff --git a/frontend/apps/reader/modules/readerdictionary.lua b/frontend/apps/reader/modules/readerdictionary.lua index 00f1005d4..2d055c113 100644 --- a/frontend/apps/reader/modules/readerdictionary.lua +++ b/frontend/apps/reader/modules/readerdictionary.lua @@ -8,8 +8,10 @@ local InputContainer = require("ui/widget/container/inputcontainer") local JSON = require("json") local KeyValuePage = require("ui/widget/keyvaluepage") local LuaData = require("luadata") +local NetworkMgr = require("ui/network/manager") local Trapper = require("ui/trapper") local UIManager = require("ui/uimanager") +local ffiUtil = require("ffi/util") local logger = require("logger") local util = require("util") local _ = require("gettext") @@ -238,6 +240,10 @@ function ReaderDictionary:addToMainMenu(menu_items) }) end, }, + { + text = _("Download dictionaries"), + sub_item_table = self:_genDownloadDictionariesMenu() + }, { text = _("Enable fuzzy search"), checked_func = function() @@ -370,6 +376,50 @@ function ReaderDictionary:getNumberOfDictionaries() return nb_available, nb_enabled, nb_disabled end +function ReaderDictionary:_genDownloadDictionariesMenu() + local downloadable_dicts = require("ui/data/dictionaries") + local languages = {} + + for i = 1, #downloadable_dicts do + local dict = downloadable_dicts[i] + local dict_lang_in = dict.lang_in + local dict_lang_out = dict.lang_out + if not languages[dict_lang_in] then + languages[dict_lang_in] = {} + end + table.insert(languages[dict_lang_in], dict) + if not languages[dict_lang_out] then + languages[dict_lang_out] = {} + end + table.insert(languages[dict_lang_out], dict) + end + + -- remove duplicates + for lang_key,lang in pairs(languages) do + local hash = {} + local res = {} + for k,v in ipairs(lang) do + if not hash[v.name] then + res[#res+1] = v + hash[v.name] = true + end + end + languages[lang_key] = res + end + + local menu_items = {} + for lang_key, available_langs in ffiUtil.orderedPairs(languages) do + table.insert(menu_items, { + text = lang_key, + callback = function() + self:showDownload(available_langs) + end + }) + end + + return menu_items +end + function ReaderDictionary:genDictionariesMenu() local items = {} for _, ifo in pairs(available_ifos) do @@ -677,6 +727,128 @@ function ReaderDictionary:showDict(word, results, box, link) end end +function ReaderDictionary:showDownload(downloadable_dicts) + local kv_pairs = {} + table.insert(kv_pairs, {_("Tap dictionary name to download"), ""}) + table.insert(kv_pairs, "----------------------------") + for dummy, dict in ipairs(downloadable_dicts) do + table.insert(kv_pairs, {dict.name, "", + callback = function() + if not NetworkMgr:isOnline() then + NetworkMgr:promptWifiOn() + return + end + self:downloadDictionaryPrep(dict) + end}) + local lang + if dict.lang_in == dict.lang_out then + lang = string.format(" %s", dict.lang_in) + else + lang = string.format(" %s–%s", dict.lang_in, dict.lang_out) + end + table.insert(kv_pairs, {lang, ""}) + table.insert(kv_pairs, {" ".._("License"), dict.license}) + table.insert(kv_pairs, {" ".._("Entries"), dict.entries}) + table.insert(kv_pairs, "----------------------------") + end + self.download_window = KeyValuePage:new{ + title = _("Download dictionaries"), + kv_pairs = kv_pairs, + } + UIManager:show(self.download_window) +end + +function ReaderDictionary:downloadDictionaryPrep(dict, size) + local dummy, filename = util.splitFilePathName(dict.url) + local download_location = string.format("%s/%s", self.data_dir, filename) + + local lfs = require("libs/libkoreader-lfs") + if lfs.attributes(download_location) then + UIManager:show(ConfirmBox:new{ + text = _("File already exists. Overwrite?"), + ok_text = _("Overwrite"), + ok_callback = function() + self:downloadDictionary(dict, download_location) + end, + }) + else + self:downloadDictionary(dict, download_location) + end +end + +function ReaderDictionary:downloadDictionary(dict, download_location, continue) + continue = continue or false + local socket = require("socket") + local http = socket.http + local https = require("ssl.https") + local ltn12 = require("ltn12") + local url = socket.url + + local parsed = url.parse(dict.url) + local httpRequest = parsed.scheme == "http" and http.request or https.request + + if not continue then + local file_size + --local r, c, h = httpRequest { + local dummy, headers, dummy = socket.skip(1, httpRequest{ + method = "HEAD", + url = dict.url, + --redirect = true, + }) + --logger.dbg(status) + --logger.dbg(headers) + --logger.dbg(code) + file_size = headers and headers["content-length"] + + UIManager:show(ConfirmBox:new{ + text = T(_("Dictionary filesize is %1 (%2 bytes). Continue with download?"), util.getFriendlySize(file_size), util.getFormattedSize(file_size)), + ok_text = _("Download"), + ok_callback = function() + -- call ourselves with continue = true + self:downloadDictionary(dict, download_location, true) + end, + }) + return + else + UIManager:nextTick(function() + UIManager:show(InfoMessage:new{ + text = _("Downloading…"), + timeout = 3, + }) + end) + end + + local dummy, c, dummy = httpRequest{ + url = dict.url, + sink = ltn12.sink.file(io.open(download_location, "w")), + } + if c == 200 then + logger.dbg("file downloaded to", download_location) + else + UIManager:show(InfoMessage:new{ + text = _("Could not save file to:\n") .. download_location, + --timeout = 3, + }) + return false + end + + local ok, error = util.unpackArchive(download_location, self.data_dir) + + if ok then + available_ifos = false + self:init() + UIManager:show(InfoMessage:new{ + text = _("Dictionary downloaded:\n") .. dict.name, + }) + return true + else + UIManager:show(InfoMessage:new{ + text = _("Dictionary failed to download:\n") .. string.format("%s\n%s", dict.name, error), + }) + return false + end +end + function ReaderDictionary:onUpdateDefaultDict(dict) logger.dbg("make default dictionary:", dict) self.default_dictionary = dict diff --git a/frontend/ui/data/dictionaries.lua b/frontend/ui/data/dictionaries.lua new file mode 100644 index 000000000..eae9adb79 --- /dev/null +++ b/frontend/ui/data/dictionaries.lua @@ -0,0 +1,127 @@ +local _ = require("gettext") + +-- largely thanks to https://tuxor1337.github.io/firedict/dictionaries.html +local dictionaries = { + { + name = "CIA World Factbook 2014", + lang_in = "English", + lang_out = "English", + entries = 2577, + license = _("Public Domain"), + url = "http://build.koreader.rocks/download/dict/factbook.tar.lz", + }, + { + name = "GNU Collaborative International Dictionary of English", + lang_in = "English", + lang_out = "English", + entries = 108121, + license = "GPLv3+", + url = "http://build.koreader.rocks/download/dict/gcide.tar.lz", + }, + { + name = "Douglas Harper's Online Etymology Dictionary", + lang_in = "English", + lang_out = "English", + entries = 46133, + license = "Unknown/©Douglas Harper", + url = "https://gitlab.com/koreader/stardict-dictionaries/uploads/1f3a66fe3f776e718590a66958df1b9b/etymonline.tar.lz", + }, + { + name = "Folkets lexikon", + lang_in = "English", + lang_out = "Swedish", + entries = 53618, + license = "CC-BY-SA 2.5", + url = "https://gitlab.com/koreader/stardict-dictionaries/uploads/fad990cc29c6dd448f22469ce8648825/folkets_en-sv.tar.lz", + }, + { + name = "Folkets lexikon", + lang_in = "Swedish", + lang_out = "English", + entries = 36513, + license = "CC-BY-SA 2.5", + url = "https://gitlab.com/koreader/stardict-dictionaries/uploads/fdc4265bbfc9af27fa4b1742ce3bdadd/folkets_sv-en.tar.lz", + }, + { + name = "Dictionnaire Littré (xmlittre)", + lang_in = "French", + lang_out = "French", + entries = 78428, + license = "CC-BY-SA 3.0", + url = "http://http.debian.net/debian/pool/main/s/stardict-xmlittre/stardict-xmlittre_1.0.orig.tar.gz", + }, + { + name = "Dictionnaire de l'Académie Française: 8ème edition", + lang_in = "French", + lang_out = "French", + entries = 31934, + license = _("Public Domain (copyright expired, published 1935)"), + url = "https://gitlab.com/koreader/stardict-dictionaries/uploads/a8b3027e84f344f64723fb7fe5c63c04/acadfran.tar.lz", + }, + { + name = "Pape: Handwörterbuch der griechischen Sprache", + lang_in = "Ancient Greek", + lang_out = "German", + entries = 98893, + license = _("Public Domain (copyright expired, published 1880)"), + url = "https://gitlab.com/koreader/stardict-dictionaries/uploads/3ee9cb3acaca679bb9a95d845b813673/pape_gr-de.tar.lz", + }, + { + name = "Georges: Ausführliches lateinisch-deutsches Handwörterbuch", + lang_in = "Latin", + lang_out = "German", + entries = 54831, + license = _("Public Domain (copyright expired, published 1913)"), + url = "https://gitlab.com/koreader/stardict-dictionaries/uploads/8d1e52d6c28d3b6865415979bc221fa6/georges_lat-de.tar.lz", + }, + { + name = "Georges: Kleines deutsch-lateinisches Handwörterbuch", + lang_in = "German", + lang_out = "Latin", + entries = 26608, + license = _("Public Domain (copyright expired, published 1910)"), + url = "https://gitlab.com/koreader/stardict-dictionaries/uploads/a66b2a7655a4b102150fb5c6f1789cc5/georges_de-lat.tar.lz", + }, + { + name = "Dicionário Aberto", + lang_in = "Portuguese", + lang_out = "Portuguese", + entries = 128521, + license = _("CC-BY-SA 2.5"), + url = "http://www.dicionario-aberto.net/stardict-DicAberto.tar.bz2", + }, + { + name = "GNU/FDL Anglicko/Český slovník", + lang_in = "English", + lang_out = "Czech", + entries = 178904, -- ~90000 each way + license = _("GNU/FDL"), + url = "http://http.debian.net/debian/pool/non-free/s/stardict-english-czech/stardict-english-czech_20161201.orig.tar.gz", + }, + { + name = "GNU/FDL Anglicko/Český slovník", + lang_in = "Czech", + lang_out = "English", + entries = 178904, -- ~90000 each way + license = _("GNU/FDL"), + url = "http://http.debian.net/debian/pool/non-free/s/stardict-english-czech/stardict-english-czech_20161201.orig.tar.gz", + }, + { + name = "GNU/FDL Německo/Český slovník", + lang_in = "German", + lang_out = "Czech", + entries = 2341, -- ~1200 each way + license = _("GNU/FDL"), + url = "http://http.debian.net/debian/pool/non-free/s/stardict-german-czech/stardict-german-czech_20161201.orig.tar.gz", + }, + { + name = "GNU/FDL Německo/Český slovník", + lang_in = "Czech", + lang_out = "German", + entries = 2341, -- ~1200 each way + license = _("GNU/FDL"), + url = "http://http.debian.net/debian/pool/non-free/s/stardict-german-czech/stardict-german-czech_20161201.orig.tar.gz", + }, +} + +return dictionaries diff --git a/frontend/util.lua b/frontend/util.lua index 1a84bf47c..c95e293fe 100644 --- a/frontend/util.lua +++ b/frontend/util.lua @@ -3,6 +3,10 @@ This module contains miscellaneous helper functions for the KOReader frontend. ]] local BaseUtil = require("ffi/util") +local dbg = require("dbg") +local _ = require("gettext") +local T = BaseUtil.template + local util = {} --- Strips all punctuation and spaces from a string. @@ -136,11 +140,11 @@ function util.tableEquals(o1, o2, ignore_mt) end --- Returns number of keys in a table. ----- @param T Lua table ----- @treturn int number of keys in table T -function util.tableSize(T) +---- @param t Lua table +---- @treturn int number of keys in table t +function util.tableSize(t) local count = 0 - for _ in pairs(T) do count = count + 1 end + for _ in pairs(t) do count = count + 1 end return count end @@ -431,6 +435,7 @@ end ---- @int size (bytes) ---- @treturn string function util.getFriendlySize(size) + size = tonumber(size) if not size or type(size) ~= "number" then return end local s if size > 1024*1024*1024 then @@ -691,4 +696,25 @@ function util.checkLuaSyntax(lua_text) return err end +--- Unpack an archive. +-- Extract the contents of an archive, detecting its format by +-- filename extension. Inspired by luarocks archive_unpack() +-- @param archive string: Filename of archive. +-- @param extract_to string: Destination directory. +-- @return boolean or (boolean, string): true on success, false and an error message on failure. +function util.unpackArchive(archive, extract_to) + dbg.dassert(type(archive) == "string") + + local ok + if archive:match("%.tar%.bz2$") or archive:match("%.tar%.gz$") or archive:match("%.tar%.lz$") or archive:match("%.tgz$") then + ok = os.execute(("./tar xf %q -C %q"):format(archive, extract_to)) + else + return false, T(_("Couldn't extract archive:\n\n%1\n\nUnrecognized filename extension."), archive) + end + if not ok then + return false, T(_("Extracting archive failed:\n\n%1", archive)) + end + return true +end + return util