[feat, i18n] Implement ngettext (#5257)

Fixes <https://github.com/koreader/koreader/issues/5249>. See https://www.gnu.org/software/gettext/manual/html_node/Plural-forms.html and https://www.gnu.org/software/gettext/manual/html_node/Translating-plural-forms.html for more information. Usage: ```lua local T = ffiUtil.template local _ = require("gettext") local N_ = _.ngettext local items_string = T(N_("1 item", "%1 items", num_items), num_items) ```
5 years ago · 2c555830f9
parent 45a0f285f1
commit 2c555830f9
6 changed files with 313 additions and 29 deletions
--- a/3
+++ b/3
@ -466,7 +466,8 @@ XGETTEXT_BIN=xgettext

 pot:
 	mkdir -p $(TEMPLATE_DIR)
-	$(XGETTEXT_BIN) --from-code=utf-8 --keyword=C_:1c,2 \
+	$(XGETTEXT_BIN) --from-code=utf-8 \
+		--keyword=C_:1c,2 --keyword=N_:1,2 --keyword=NC_:1c,2,3 \
 		--add-comments=@translators \
 		reader.lua `find frontend -iname "*.lua"` \
 		`find plugins -iname "*.lua"` \
--- a/frontend/gettext.lua
+++ b/frontend/gettext.lua
@ -5,7 +5,8 @@ local GetText = {
    translation = {},
    current_lang = "C",
    dirname = "l10n",
-    textdomain = "koreader"
+    textdomain = "koreader",
+    plural_default = "n != 1",
 }

 local GetText_mt = {
@ -31,6 +32,87 @@ local function c_escape(what)
    end
 end

+--- Converts C logical operators to Lua.
+local function logicalCtoLua(logical_str)
+    logical_str = logical_str:gsub("&&", "and")
+    logical_str = logical_str:gsub("!=", "~=")
+    logical_str = logical_str:gsub("||", "or")
+    return logical_str
+end
+
+--- Default getPlural function.
+local function getDefaultPlural(n)
+    if n ~= 1 then
+        return 1
+    else
+        return 0
+    end
+end
+
+--- Generates a proper Lua function out of logical gettext math tests.
+local function getPluralFunc(pl_tests, nplurals, plural_default)
+    -- the return function() stuff is a bit of loadstring trickery
+    local plural_func_str = "return function(n) if "
+
+    if #pl_tests > 1 then
+        for i = 1, #pl_tests do
+            local pl_test = pl_tests[i]
+            pl_test = logicalCtoLua(pl_test)
+
+            if i > 1 and not (tonumber(pl_test) ~= nil) then
+                pl_test = " elseif "..pl_test
+            end
+            if tonumber(pl_test) ~= nil then
+                -- no condition, just a number
+                pl_test = " else return "..pl_test
+            end
+            pl_test = pl_test:gsub("?", " then return")
+
+            -- append to plural function
+            plural_func_str = plural_func_str..pl_test
+        end
+        plural_func_str = plural_func_str.." end end"
+    else
+        local pl_test = pl_tests[1]
+        -- Ensure JIT compiled function if we're dealing with one of the many simpler languages.
+        -- After all, loadstring won't be.
+        -- Potential workaround: write to file and use require.
+        if pl_test == plural_default then
+            return getDefaultPlural
+        end
+        pl_test = logicalCtoLua(pl_test)
+        plural_func_str = "return function(n) if "..pl_test.." then return 1 else return 0 end end"
+    end
+    return loadstring(plural_func_str)()
+end
+
+local function addTranslation(msgctxt, msgid, msgstr, n)
+    -- translated string
+    local unescaped_string = string.gsub(msgstr, "\\(.)", c_escape)
+    if msgctxt and msgctxt ~= "" then
+        if not GetText.context[msgctxt] then
+            GetText.context[msgctxt] = {}
+        end
+        if n then
+            if not GetText.context[msgctxt][msgid] then
+                GetText.context[msgctxt][msgid] = {}
+            end
+            GetText.context[msgctxt][msgid][n] = unescaped_string
+        else
+            GetText.context[msgctxt][msgid] = unescaped_string
+        end
+    else
+        if n then
+            if not GetText.translation[msgid] then
+                GetText.translation[msgid] = {}
+            end
+            GetText.translation[msgid][n] = unescaped_string
+        else
+            GetText.translation[msgid] = unescaped_string
+        end
+    end
+end
+
 -- for PO file syntax, see
 -- https://www.gnu.org/software/gettext/manual/html_node/PO-Files.html
 -- we only implement a sane subset for now
@ -57,20 +139,43 @@ function GetText_mt.__index.changeLang(new_lang)
    end

    local data = {}
+    local headers
    local what = nil
    while true do
        local line = po:read("*l")
        if line == nil or line == "" then
-            if data.msgid and data.msgstr and data.msgstr ~= "" then
-                local unescaped_string = string.gsub(data.msgstr, "\\(.)", c_escape)
-                if data.msgctxt and data.msgctxt ~= "" then
-                    if not GetText.context[data.msgctxt] then
-                        GetText.context[data.msgctxt] = {}
+            if data.msgid and data.msgid_plural and data["msgstr[0]"] then
+                for k, v in pairs(data) do
+                    local n = tonumber(k:match("msgstr%[([0-9]+)%]"))
+                    local msgstr = v
+
+                    if n and msgstr then
+                        addTranslation(data.msgctxt, data.msgid, msgstr, n)
+                    end
+                end
+            elseif data.msgid and data.msgstr and data.msgstr ~= "" then
+                -- header
+                if not headers and data.msgid == "" then
+                    local util = require("util")
+                    headers = data.msgstr
+                    local plural_forms = data.msgstr:match("Plural%-Forms: (.*);")
+                    local nplurals = plural_forms:match("nplurals=([0-9]+);") or 2
+                    local plurals = plural_forms:match("%((.*)%)")
+
+                    if plurals:find("[^n!=%%<>&:%(%)|?0-9 ]") then
+                        -- we don't trust this input, go with default instead
+                        plurals = GetText.plural_default
+                    end
+
+                    local pl_tests = util.splitToArray(plurals, " : ")
+
+                    GetText.getPlural = getPluralFunc(pl_tests, nplurals, GetText.plural_default)
+                    if not GetText.getPlural then
+                        GetText.getPlural = getDefaultPlural
                    end
-                    GetText.context[data.msgctxt][data.msgid] = unescaped_string
-                else
-                    GetText.translation[data.msgid] = unescaped_string
                end
+
+                addTranslation(data.msgctxt, data.msgid, data.msgstr)
            end
            -- stop at EOF:
            if line == nil then break end
@ -80,7 +185,7 @@ function GetText_mt.__index.changeLang(new_lang)
            -- comment
            if not line:match("^#") then
                -- new data item (msgid, msgstr, ...
-                local w, s = line:match("^%s*(%a+)%s+\"(.*)\"%s*$")
+                local w, s = line:match("^%s*([%a_%[%]0-9]+)%s+\"(.*)\"%s*$")
                if w then
                    what = w
                else
@ -100,8 +205,30 @@ function GetText_mt.__index.changeLang(new_lang)
    GetText.current_lang = new_lang
 end

-function GetText_mt.__index.pgettext(msgctxt, msgstr)
-    return GetText.context[msgctxt] and GetText.context[msgctxt][msgstr] or msgstr
+GetText_mt.__index.getPlural = getDefaultPlural
+
+function GetText_mt.__index.ngettext(msgid, msgid_plural, n)
+    local plural = GetText.getPlural(n)
+
+    if plural == 0 then
+        return GetText.translation[msgid] and GetText.translation[msgid][plural] or msgid
+    else
+        return GetText.translation[msgid] and GetText.translation[msgid][plural] or msgid_plural
+    end
+end
+
+function GetText_mt.__index.npgettext(msgctxt, msgid, msgid_plural, n)
+    local plural = GetText.getPlural(n)
+
+    if plural == 0 then
+        return GetText.context[msgctxt] and GetText.context[msgctxt][msgid] and GetText.context[msgctxt][msgid][plural] or msgid
+    else
+        return GetText.context[msgctxt] and GetText.context[msgctxt][msgid] and GetText.context[msgctxt][msgid][plural] or msgid_plural
+    end
+end
+
+function GetText_mt.__index.pgettext(msgctxt, msgid)
+    return GetText.context[msgctxt] and GetText.context[msgctxt][msgid] or msgid
 end

 setmetatable(GetText, GetText_mt)
--- a/frontend/ui/widget/filechooser.lua
+++ b/frontend/ui/widget/filechooser.lua
@ -12,6 +12,7 @@ local ffiUtil = require("ffi/util")
 local T = ffiUtil.template
 local C = ffi.C
 local _ = require("gettext")
+local N_ = _.ngettext
 local Screen = Device.screen
 local util = require("util")
 local getFileNameSuffix = util.getFileNameSuffix
@ -210,13 +211,7 @@ function FileChooser:genItemTableFromPath(path)
        local subdir_path = self.path.."/"..dir.name
        self.list(subdir_path, sub_dirs, dir_files)
        local num_items = #sub_dirs + #dir_files
-        local istr
-        if num_items == 1 then
-            istr = _("1 item")
-        else
-            -- @translators %1 is a placeholder for a plural number. So "%1 items" will automatically show up as "2 items", "3 items", etc.
-            istr = ffiUtil.template(_("%1 items"), num_items)
-        end
+        local istr = ffiUtil.template(N_("1 item", "%1 items", num_items), num_items)
        local text
        if dir.name == ".." then
            text = up_folder_arrow
--- a/plugins/coverbrowser.koplugin/listmenu.lua
+++ b/plugins/coverbrowser.koplugin/listmenu.lua
@ -28,6 +28,7 @@ local lfs = require("libs/libkoreader-lfs")
 local logger = require("logger")
 local util = require("util")
 local _ = require("gettext")
+local N_ = _.ngettext
 local Screen = Device.screen
 local T = require("ffi/util").template

@ -381,7 +382,7 @@ function ListMenuItem:update()
                end
            else
                if pages then
-                    pages_str = T(_("%1 pages"), pages)
+                    pages_str = T(N_("1 page", "%1 pages", pages), pages)
                end
            end

--- a/plugins/evernote.koplugin/main.lua
+++ b/plugins/evernote.koplugin/main.lua
@ -12,6 +12,7 @@ local Device = require("device")
 local DEBUG = require("dbg")
 local T = require("ffi/util").template
 local _ = require("gettext")
+local N_ = _.ngettext
 local slt2 = require('slt2')
 local MyClipping = require("clip")
 local realpath = require("ffi/util").realpath
@ -409,22 +410,20 @@ function EvernoteExporter:exportClippings(clippings)
    local all_count = export_count + error_count
    if export_count > 0 and error_count == 0 then
        if all_count == 1 then
-            msg = T(_("Exported notes from the book:\n%1"), export_title)
-        else
            msg = T(
-                -- @translators %1 is the title of a book and %2 a number of 2 or higher. To track better handling of plurals please see https://github.com/koreader/koreader/issues/5249
-                _("Exported notes from the book:\n%1\nand %2 others."),
+                N_("Exported notes from the book:\n%1",
+                   "Exported notes from the book:\n%1\nand %2 others.",
+                   all_count-1),
                export_title,
                all_count-1
            )
        end
    elseif error_count > 0 then
        if all_count == 1 then
-            msg = T(_("An error occurred while trying to export notes from the book:\n%1"), error_title)
-        else
            msg = T(
-                -- @translators %1 is the title of a book and %2 a number of 2 or higher. To track better handling of plurals please see https://github.com/koreader/koreader/issues/5249
-                _("Multiple errors occurred while trying to export notes from the book:\n%1\nand %2 others."),
+                N_("An error occurred while trying to export notes from the book:\n%1",
+                   "Multiple errors occurred while trying to export notes from the book:\n%1\nand %2 others.",
+                   error_count-1),
                error_title,
                error_count-1
            )
--- a/spec/unit/gettext_spec.lua
+++ b/spec/unit/gettext_spec.lua
@ -1,9 +1,107 @@
+local test_po_part1 = [[
+# KOReader PATH/TO/FILE.PO
+# Copyright (C) 2005-2019 KOReader Development Team
+#
+# Translators:
+# Frans de Jonge <fransdejonge@gmail.com>, 2014-2019
+# Markismus <zulde.zuldemans@gmail.com>, 2014
+msgid ""
+msgstr ""
+"Project-Id-Version: KOReader\n"
+"Report-Msgid-Bugs-To: https://github.com/koreader/koreader-base/issues\n"
+"POT-Creation-Date: 2019-08-10 06:01+0000\n"
+"PO-Revision-Date: 2019-08-08 06:34+0000\n"
+"Last-Translator: Frans de Jonge <fransdejonge@gmail.com>\n"
+"Language-Team: Dutch (Netherlands) (http://www.transifex.com/houqp/koreader/language/nl_NL/)\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Language: nl_NL\n"
+]]
+
+local test_plurals_nl = [[
+"Plural-Forms: nplurals=2; plural=(n != 1);\n"
+]]
+local test_plurals_ru = [[
+"Plural-Forms: nplurals=4; plural=(n%10==1 && n%100!=11 ? 0 : n%10>=2 && n%10<=4 && (n%100<12 || n%100>14) ? 1 : n%10==0 || (n%10>=5 && n%10<=9) || (n%100>=11 && n%100<=14)? 2 : 3);\n"
+]]
+
+local test_po_part2 = [[
+
+#: frontend/ui/widget/configdialog.lua:1016
+msgid ""
+"\n"
+"message"
+msgstr "\nbericht"
+
+#: frontend/device/android/device.lua:259
+msgid "1 item"
+msgid_plural "%1 items"
+msgstr[0] "1 ding"
+msgstr[1] "%1 dingen"
+msgstr[2] "%1 dingen 2"
+
+#: frontend/ui/data/css_tweaks.lua:17
+msgctxt "Style tweaks category"
+msgid "Pages"
+msgstr "Pagina's"
+
+#: frontend/ui/data/css_tweaks.lua:20
+msgctxt "Other pages"
+msgid "Pages"
+msgstr "Pages different context"
+
+#: frontend/ui/data/css_tweaks.lua:30
+msgctxt "Context 1"
+msgid "Page"
+msgid_plural "Pages"
+msgstr[0] "Pagina"
+msgstr[1] "Pagina's"
+msgstr[2] "Pagina's plural 2"
+
+#: frontend/ui/data/css_tweaks.lua:40
+msgctxt "Context 2"
+msgid "Page"
+msgid_plural "Pages"
+msgstr[0] "Pagina context 2 plural 0"
+msgstr[1] "Pagina's context 2 plural 1"
+msgstr[2] "Pagina's context 2 plural 2"
+]]
+
 describe("GetText module", function()
    local GetText
+    local test_po_nl, test_po_ru
+
    setup(function()
        require("commonrequire")
        GetText = require("gettext")
+        GetText.dirname = "i18n-test"
+
+        local lfs = require("libs/libkoreader-lfs")
+        lfs.mkdir(GetText.dirname)
+        lfs.mkdir(GetText.dirname.."/nl_NL")
+        lfs.mkdir(GetText.dirname.."/ru")
+
+        test_po_nl = GetText.dirname.."/nl_NL/koreader.po"
+        local f = io.open(test_po_nl, "w")
+        f:write(test_po_part1, test_plurals_nl, test_po_part2)
+        f:close()
+
+        -- same file, just different plural for testing
+        test_po_ru = GetText.dirname.."/ru/koreader.po"
+        f = io.open(test_po_ru, "w")
+        f:write(test_po_part1, test_plurals_ru, test_po_part2)
+        f:close()
    end)
+
+    teardown(function()
+        os.remove(test_po_nl)
+        os.remove(test_po_ru)
+        os.remove(GetText.dirname.."/nl_NL")
+        os.remove(GetText.dirname.."/ru")
+        os.remove(GetText.dirname)
+    end)
+
    describe("changeLang", function()
        it("should return nil when passing newlang = C", function()
            assert.is_nil(GetText.changeLang("C"))
@ -22,4 +120,67 @@ describe("GetText module", function()
            assert.is_false(GetText.changeLang("more_NONSENSE"))
        end)
    end)
+
+    describe("cannot find string", function()
+        it("gettext should return input string", function()
+            assert.is_equal("bla", GetText("bla"))
+        end)
+        it("ngettext should return input string", function()
+            assert.is_equal("bla", GetText.ngettext("bla", "blabla", 1))
+            assert.is_equal("blabla", GetText.ngettext("bla", "blabla", 2))
+        end)
+        it("pgettext should return input string", function()
+            assert.is_equal("bla", GetText.pgettext("some context", "bla"))
+        end)
+        it("npgettext should return input string", function()
+            assert.is_equal("bla", GetText.npgettext("some context", "bla", "blabla", 1))
+            assert.is_equal("blabla", GetText.npgettext("some context", "bla", "blabla", 2))
+        end)
+    end)
+
+    describe("language with standard plurals", function()
+        GetText.changeLang("nl_NL")
+        it("gettext should translate multiline string", function()
+            assert.is_equal("\nbericht", GetText("\nmessage"))
+        end)
+        it("ngettext should translate plurals", function()
+            assert.is_equal("1 ding", GetText.ngettext("1 item", "%1 items", 1))
+            assert.is_equal("%1 dingen", GetText.ngettext("1 item", "%1 items", 2))
+            assert.is_equal("%1 dingen", GetText.ngettext("1 item", "%1 items", 5))
+        end)
+        it("pgettext should distinguish context", function()
+            assert.is_equal("Pagina's", GetText.pgettext("Style tweaks category", "Pages"))
+            assert.is_equal("Pages different context", GetText.pgettext("Other pages", "Pages"))
+        end)
+        it("npgettext should translate plurals and distinguish context", function()
+            assert.is_equal("Pagina", GetText.npgettext("Context 1", "Page", "Pages", 1))
+            assert.is_equal("Pagina's", GetText.npgettext("Context 1", "Page", "Pages", 2))
+            assert.is_equal("Pagina context 2 plural 0", GetText.npgettext("Context 2", "Page", "Pages", 1))
+            assert.is_equal("Pagina's context 2 plural 1", GetText.npgettext("Context 2", "Page", "Pages", 2))
+        end)
+    end)
+
+    describe("language with complex plurals", function()
+        GetText.changeLang("ru")
+        it("gettext should translate multiline string", function()
+            assert.is_equal("\nbericht", GetText("\nmessage"))
+        end)
+        it("ngettext should translate plurals", function()
+            assert.is_equal("1 ding", GetText.ngettext("1 item", "%1 items", 1))
+            assert.is_equal("%1 dingen", GetText.ngettext("1 item", "%1 items", 2))
+            assert.is_equal("%1 dingen 2", GetText.ngettext("1 item", "%1 items", 5))
+        end)
+        it("pgettext should distinguish context", function()
+            assert.is_equal("Pagina's", GetText.pgettext("Style tweaks category", "Pages"))
+            assert.is_equal("Pages different context", GetText.pgettext("Other pages", "Pages"))
+        end)
+        it("npgettext should translate plurals and distinguish context", function()
+            assert.is_equal("Pagina", GetText.npgettext("Context 1", "Page", "Pages", 1))
+            assert.is_equal("Pagina's", GetText.npgettext("Context 1", "Page", "Pages", 2))
+            assert.is_equal("Pagina's plural 2", GetText.npgettext("Context 1", "Page", "Pages", 5))
+            assert.is_equal("Pagina context 2 plural 0", GetText.npgettext("Context 2", "Page", "Pages", 1))
+            assert.is_equal("Pagina's context 2 plural 1", GetText.npgettext("Context 2", "Page", "Pages", 2))
+            assert.is_equal("Pagina's context 2 plural 2", GetText.npgettext("Context 2", "Page", "Pages", 5))
+        end)
+    end)
 end)