diff --git a/Makefile b/Makefile index 2dca6defb..0846f171c 100644 --- a/Makefile +++ b/Makefile @@ -466,7 +466,8 @@ XGETTEXT_BIN=xgettext pot: mkdir -p $(TEMPLATE_DIR) - $(XGETTEXT_BIN) --from-code=utf-8 --keyword=C_:1c,2 \ + $(XGETTEXT_BIN) --from-code=utf-8 \ + --keyword=C_:1c,2 --keyword=N_:1,2 --keyword=NC_:1c,2,3 \ --add-comments=@translators \ reader.lua `find frontend -iname "*.lua"` \ `find plugins -iname "*.lua"` \ diff --git a/frontend/gettext.lua b/frontend/gettext.lua index ec4ec6620..95f13abcc 100644 --- a/frontend/gettext.lua +++ b/frontend/gettext.lua @@ -5,7 +5,8 @@ local GetText = { translation = {}, current_lang = "C", dirname = "l10n", - textdomain = "koreader" + textdomain = "koreader", + plural_default = "n != 1", } local GetText_mt = { @@ -31,6 +32,87 @@ local function c_escape(what) end end +--- Converts C logical operators to Lua. +local function logicalCtoLua(logical_str) + logical_str = logical_str:gsub("&&", "and") + logical_str = logical_str:gsub("!=", "~=") + logical_str = logical_str:gsub("||", "or") + return logical_str +end + +--- Default getPlural function. +local function getDefaultPlural(n) + if n ~= 1 then + return 1 + else + return 0 + end +end + +--- Generates a proper Lua function out of logical gettext math tests. +local function getPluralFunc(pl_tests, nplurals, plural_default) + -- the return function() stuff is a bit of loadstring trickery + local plural_func_str = "return function(n) if " + + if #pl_tests > 1 then + for i = 1, #pl_tests do + local pl_test = pl_tests[i] + pl_test = logicalCtoLua(pl_test) + + if i > 1 and not (tonumber(pl_test) ~= nil) then + pl_test = " elseif "..pl_test + end + if tonumber(pl_test) ~= nil then + -- no condition, just a number + pl_test = " else return "..pl_test + end + pl_test = pl_test:gsub("?", " then return") + + -- append to plural function + plural_func_str = plural_func_str..pl_test + end + plural_func_str = plural_func_str.." end end" + else + local pl_test = pl_tests[1] + -- Ensure JIT compiled function if we're dealing with one of the many simpler languages. + -- After all, loadstring won't be. + -- Potential workaround: write to file and use require. + if pl_test == plural_default then + return getDefaultPlural + end + pl_test = logicalCtoLua(pl_test) + plural_func_str = "return function(n) if "..pl_test.." then return 1 else return 0 end end" + end + return loadstring(plural_func_str)() +end + +local function addTranslation(msgctxt, msgid, msgstr, n) + -- translated string + local unescaped_string = string.gsub(msgstr, "\\(.)", c_escape) + if msgctxt and msgctxt ~= "" then + if not GetText.context[msgctxt] then + GetText.context[msgctxt] = {} + end + if n then + if not GetText.context[msgctxt][msgid] then + GetText.context[msgctxt][msgid] = {} + end + GetText.context[msgctxt][msgid][n] = unescaped_string + else + GetText.context[msgctxt][msgid] = unescaped_string + end + else + if n then + if not GetText.translation[msgid] then + GetText.translation[msgid] = {} + end + GetText.translation[msgid][n] = unescaped_string + else + GetText.translation[msgid] = unescaped_string + end + end +end + -- for PO file syntax, see -- https://www.gnu.org/software/gettext/manual/html_node/PO-Files.html -- we only implement a sane subset for now @@ -57,20 +139,43 @@ function GetText_mt.__index.changeLang(new_lang) end local data = {} + local headers local what = nil while true do local line = po:read("*l") if line == nil or line == "" then - if data.msgid and data.msgstr and data.msgstr ~= "" then - local unescaped_string = string.gsub(data.msgstr, "\\(.)", c_escape) - if data.msgctxt and data.msgctxt ~= "" then - if not GetText.context[data.msgctxt] then - GetText.context[data.msgctxt] = {} + if data.msgid and data.msgid_plural and data["msgstr[0]"] then + for k, v in pairs(data) do + local n = tonumber(k:match("msgstr%[([0-9]+)%]")) + local msgstr = v + + if n and msgstr then + addTranslation(data.msgctxt, data.msgid, msgstr, n) + end + end + elseif data.msgid and data.msgstr and data.msgstr ~= "" then + -- header + if not headers and data.msgid == "" then + local util = require("util") + headers = data.msgstr + local plural_forms = data.msgstr:match("Plural%-Forms: (.*);") + local nplurals = plural_forms:match("nplurals=([0-9]+);") or 2 + local plurals = plural_forms:match("%((.*)%)") + + if plurals:find("[^n!=%%<>&:%(%)|?0-9 ]") then + -- we don't trust this input, go with default instead + plurals = GetText.plural_default + end + + local pl_tests = util.splitToArray(plurals, " : ") + + GetText.getPlural = getPluralFunc(pl_tests, nplurals, GetText.plural_default) + if not GetText.getPlural then + GetText.getPlural = getDefaultPlural end - GetText.context[data.msgctxt][data.msgid] = unescaped_string - else - GetText.translation[data.msgid] = unescaped_string end + + addTranslation(data.msgctxt, data.msgid, data.msgstr) end -- stop at EOF: if line == nil then break end @@ -80,7 +185,7 @@ function GetText_mt.__index.changeLang(new_lang) -- comment if not line:match("^#") then -- new data item (msgid, msgstr, ... - local w, s = line:match("^%s*(%a+)%s+\"(.*)\"%s*$") + local w, s = line:match("^%s*([%a_%[%]0-9]+)%s+\"(.*)\"%s*$") if w then what = w else @@ -100,8 +205,30 @@ function GetText_mt.__index.changeLang(new_lang) GetText.current_lang = new_lang end -function GetText_mt.__index.pgettext(msgctxt, msgstr) - return GetText.context[msgctxt] and GetText.context[msgctxt][msgstr] or msgstr +GetText_mt.__index.getPlural = getDefaultPlural + +function GetText_mt.__index.ngettext(msgid, msgid_plural, n) + local plural = GetText.getPlural(n) + + if plural == 0 then + return GetText.translation[msgid] and GetText.translation[msgid][plural] or msgid + else + return GetText.translation[msgid] and GetText.translation[msgid][plural] or msgid_plural + end +end + +function GetText_mt.__index.npgettext(msgctxt, msgid, msgid_plural, n) + local plural = GetText.getPlural(n) + + if plural == 0 then + return GetText.context[msgctxt] and GetText.context[msgctxt][msgid] and GetText.context[msgctxt][msgid][plural] or msgid + else + return GetText.context[msgctxt] and GetText.context[msgctxt][msgid] and GetText.context[msgctxt][msgid][plural] or msgid_plural + end +end + +function GetText_mt.__index.pgettext(msgctxt, msgid) + return GetText.context[msgctxt] and GetText.context[msgctxt][msgid] or msgid end setmetatable(GetText, GetText_mt) diff --git a/frontend/ui/widget/filechooser.lua b/frontend/ui/widget/filechooser.lua index 8d5aea412..078008480 100644 --- a/frontend/ui/widget/filechooser.lua +++ b/frontend/ui/widget/filechooser.lua @@ -12,6 +12,7 @@ local ffiUtil = require("ffi/util") local T = ffiUtil.template local C = ffi.C local _ = require("gettext") +local N_ = _.ngettext local Screen = Device.screen local util = require("util") local getFileNameSuffix = util.getFileNameSuffix @@ -210,13 +211,7 @@ function FileChooser:genItemTableFromPath(path) local subdir_path = self.path.."/"..dir.name self.list(subdir_path, sub_dirs, dir_files) local num_items = #sub_dirs + #dir_files - local istr - if num_items == 1 then - istr = _("1 item") - else - -- @translators %1 is a placeholder for a plural number. So "%1 items" will automatically show up as "2 items", "3 items", etc. - istr = ffiUtil.template(_("%1 items"), num_items) - end + local istr = ffiUtil.template(N_("1 item", "%1 items", num_items), num_items) local text if dir.name == ".." then text = up_folder_arrow diff --git a/plugins/coverbrowser.koplugin/listmenu.lua b/plugins/coverbrowser.koplugin/listmenu.lua index 863f02dfb..44fae8fc3 100644 --- a/plugins/coverbrowser.koplugin/listmenu.lua +++ b/plugins/coverbrowser.koplugin/listmenu.lua @@ -28,6 +28,7 @@ local lfs = require("libs/libkoreader-lfs") local logger = require("logger") local util = require("util") local _ = require("gettext") +local N_ = _.ngettext local Screen = Device.screen local T = require("ffi/util").template @@ -381,7 +382,7 @@ function ListMenuItem:update() end else if pages then - pages_str = T(_("%1 pages"), pages) + pages_str = T(N_("1 page", "%1 pages", pages), pages) end end diff --git a/plugins/evernote.koplugin/main.lua b/plugins/evernote.koplugin/main.lua index 5d31f28e0..c1a66208d 100644 --- a/plugins/evernote.koplugin/main.lua +++ b/plugins/evernote.koplugin/main.lua @@ -12,6 +12,7 @@ local Device = require("device") local DEBUG = require("dbg") local T = require("ffi/util").template local _ = require("gettext") +local N_ = _.ngettext local slt2 = require('slt2') local MyClipping = require("clip") local realpath = require("ffi/util").realpath @@ -409,22 +410,20 @@ function EvernoteExporter:exportClippings(clippings) local all_count = export_count + error_count if export_count > 0 and error_count == 0 then if all_count == 1 then - msg = T(_("Exported notes from the book:\n%1"), export_title) - else msg = T( - -- @translators %1 is the title of a book and %2 a number of 2 or higher. To track better handling of plurals please see https://github.com/koreader/koreader/issues/5249 - _("Exported notes from the book:\n%1\nand %2 others."), + N_("Exported notes from the book:\n%1", + "Exported notes from the book:\n%1\nand %2 others.", + all_count-1), export_title, all_count-1 ) end elseif error_count > 0 then if all_count == 1 then - msg = T(_("An error occurred while trying to export notes from the book:\n%1"), error_title) - else msg = T( - -- @translators %1 is the title of a book and %2 a number of 2 or higher. To track better handling of plurals please see https://github.com/koreader/koreader/issues/5249 - _("Multiple errors occurred while trying to export notes from the book:\n%1\nand %2 others."), + N_("An error occurred while trying to export notes from the book:\n%1", + "Multiple errors occurred while trying to export notes from the book:\n%1\nand %2 others.", + error_count-1), error_title, error_count-1 ) diff --git a/spec/unit/gettext_spec.lua b/spec/unit/gettext_spec.lua index 10e362f49..f4ec2c590 100644 --- a/spec/unit/gettext_spec.lua +++ b/spec/unit/gettext_spec.lua @@ -1,9 +1,107 @@ +local test_po_part1 = [[ +# KOReader PATH/TO/FILE.PO +# Copyright (C) 2005-2019 KOReader Development Team +# +# Translators: +# Frans de Jonge , 2014-2019 +# Markismus , 2014 +msgid "" +msgstr "" +"Project-Id-Version: KOReader\n" +"Report-Msgid-Bugs-To: https://github.com/koreader/koreader-base/issues\n" +"POT-Creation-Date: 2019-08-10 06:01+0000\n" +"PO-Revision-Date: 2019-08-08 06:34+0000\n" +"Last-Translator: Frans de Jonge \n" +"Language-Team: Dutch (Netherlands) (http://www.transifex.com/houqp/koreader/language/nl_NL/)\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Language: nl_NL\n" +]] + +local test_plurals_nl = [[ +"Plural-Forms: nplurals=2; plural=(n != 1);\n" +]] +local test_plurals_ru = [[ +"Plural-Forms: nplurals=4; plural=(n%10==1 && n%100!=11 ? 0 : n%10>=2 && n%10<=4 && (n%100<12 || n%100>14) ? 1 : n%10==0 || (n%10>=5 && n%10<=9) || (n%100>=11 && n%100<=14)? 2 : 3);\n" +]] + +local test_po_part2 = [[ + +#: frontend/ui/widget/configdialog.lua:1016 +msgid "" +"\n" +"message" +msgstr "\nbericht" + +#: frontend/device/android/device.lua:259 +msgid "1 item" +msgid_plural "%1 items" +msgstr[0] "1 ding" +msgstr[1] "%1 dingen" +msgstr[2] "%1 dingen 2" + +#: frontend/ui/data/css_tweaks.lua:17 +msgctxt "Style tweaks category" +msgid "Pages" +msgstr "Pagina's" + +#: frontend/ui/data/css_tweaks.lua:20 +msgctxt "Other pages" +msgid "Pages" +msgstr "Pages different context" + +#: frontend/ui/data/css_tweaks.lua:30 +msgctxt "Context 1" +msgid "Page" +msgid_plural "Pages" +msgstr[0] "Pagina" +msgstr[1] "Pagina's" +msgstr[2] "Pagina's plural 2" + +#: frontend/ui/data/css_tweaks.lua:40 +msgctxt "Context 2" +msgid "Page" +msgid_plural "Pages" +msgstr[0] "Pagina context 2 plural 0" +msgstr[1] "Pagina's context 2 plural 1" +msgstr[2] "Pagina's context 2 plural 2" +]] + describe("GetText module", function() local GetText + local test_po_nl, test_po_ru + setup(function() require("commonrequire") GetText = require("gettext") + GetText.dirname = "i18n-test" + + local lfs = require("libs/libkoreader-lfs") + lfs.mkdir(GetText.dirname) + lfs.mkdir(GetText.dirname.."/nl_NL") + lfs.mkdir(GetText.dirname.."/ru") + + test_po_nl = GetText.dirname.."/nl_NL/koreader.po" + local f = io.open(test_po_nl, "w") + f:write(test_po_part1, test_plurals_nl, test_po_part2) + f:close() + + -- same file, just different plural for testing + test_po_ru = GetText.dirname.."/ru/koreader.po" + f = io.open(test_po_ru, "w") + f:write(test_po_part1, test_plurals_ru, test_po_part2) + f:close() end) + + teardown(function() + os.remove(test_po_nl) + os.remove(test_po_ru) + os.remove(GetText.dirname.."/nl_NL") + os.remove(GetText.dirname.."/ru") + os.remove(GetText.dirname) + end) + describe("changeLang", function() it("should return nil when passing newlang = C", function() assert.is_nil(GetText.changeLang("C")) @@ -22,4 +120,67 @@ describe("GetText module", function() assert.is_false(GetText.changeLang("more_NONSENSE")) end) end) + + describe("cannot find string", function() + it("gettext should return input string", function() + assert.is_equal("bla", GetText("bla")) + end) + it("ngettext should return input string", function() + assert.is_equal("bla", GetText.ngettext("bla", "blabla", 1)) + assert.is_equal("blabla", GetText.ngettext("bla", "blabla", 2)) + end) + it("pgettext should return input string", function() + assert.is_equal("bla", GetText.pgettext("some context", "bla")) + end) + it("npgettext should return input string", function() + assert.is_equal("bla", GetText.npgettext("some context", "bla", "blabla", 1)) + assert.is_equal("blabla", GetText.npgettext("some context", "bla", "blabla", 2)) + end) + end) + + describe("language with standard plurals", function() + GetText.changeLang("nl_NL") + it("gettext should translate multiline string", function() + assert.is_equal("\nbericht", GetText("\nmessage")) + end) + it("ngettext should translate plurals", function() + assert.is_equal("1 ding", GetText.ngettext("1 item", "%1 items", 1)) + assert.is_equal("%1 dingen", GetText.ngettext("1 item", "%1 items", 2)) + assert.is_equal("%1 dingen", GetText.ngettext("1 item", "%1 items", 5)) + end) + it("pgettext should distinguish context", function() + assert.is_equal("Pagina's", GetText.pgettext("Style tweaks category", "Pages")) + assert.is_equal("Pages different context", GetText.pgettext("Other pages", "Pages")) + end) + it("npgettext should translate plurals and distinguish context", function() + assert.is_equal("Pagina", GetText.npgettext("Context 1", "Page", "Pages", 1)) + assert.is_equal("Pagina's", GetText.npgettext("Context 1", "Page", "Pages", 2)) + assert.is_equal("Pagina context 2 plural 0", GetText.npgettext("Context 2", "Page", "Pages", 1)) + assert.is_equal("Pagina's context 2 plural 1", GetText.npgettext("Context 2", "Page", "Pages", 2)) + end) + end) + + describe("language with complex plurals", function() + GetText.changeLang("ru") + it("gettext should translate multiline string", function() + assert.is_equal("\nbericht", GetText("\nmessage")) + end) + it("ngettext should translate plurals", function() + assert.is_equal("1 ding", GetText.ngettext("1 item", "%1 items", 1)) + assert.is_equal("%1 dingen", GetText.ngettext("1 item", "%1 items", 2)) + assert.is_equal("%1 dingen 2", GetText.ngettext("1 item", "%1 items", 5)) + end) + it("pgettext should distinguish context", function() + assert.is_equal("Pagina's", GetText.pgettext("Style tweaks category", "Pages")) + assert.is_equal("Pages different context", GetText.pgettext("Other pages", "Pages")) + end) + it("npgettext should translate plurals and distinguish context", function() + assert.is_equal("Pagina", GetText.npgettext("Context 1", "Page", "Pages", 1)) + assert.is_equal("Pagina's", GetText.npgettext("Context 1", "Page", "Pages", 2)) + assert.is_equal("Pagina's plural 2", GetText.npgettext("Context 1", "Page", "Pages", 5)) + assert.is_equal("Pagina context 2 plural 0", GetText.npgettext("Context 2", "Page", "Pages", 1)) + assert.is_equal("Pagina's context 2 plural 1", GetText.npgettext("Context 2", "Page", "Pages", 2)) + assert.is_equal("Pagina's context 2 plural 2", GetText.npgettext("Context 2", "Page", "Pages", 5)) + end) + end) end)