From 3cf3c97e876e4d949046c6be03bf5a8e0ee12e72 Mon Sep 17 00:00:00 2001 From: Dylan Garrett Date: Sat, 25 Dec 2021 01:24:48 -0800 Subject: [PATCH] [plugin] Exporter: add Readwise.io support (#8548) This extends exporter.koplugin with support for [Readwise.io](https://readwise.io), a highlight/notes aggregation service. [Readwise API documentation](https://readwise.io/api_deets) This additionally improves the highlight exporter's ability to get the correct title and author of a document, by checking actual metadata instead of inferring from filename. It also includes a modification to the plugin's highlight parsing logic to separate the highlight contents in `.text` from the notes in `.note`. This change actually fixes an existing bug in the HTML export template note.tpl, which has been missing notes because of the lack of the `.note` field. --- plugins/exporter.koplugin/ReadwiseClient.lua | 70 +++++++++++ plugins/exporter.koplugin/clip.lua | 52 +++++++- plugins/exporter.koplugin/main.lua | 121 ++++++++++++++++--- 3 files changed, 222 insertions(+), 21 deletions(-) create mode 100644 plugins/exporter.koplugin/ReadwiseClient.lua diff --git a/plugins/exporter.koplugin/ReadwiseClient.lua b/plugins/exporter.koplugin/ReadwiseClient.lua new file mode 100644 index 000000000..61d8e774e --- /dev/null +++ b/plugins/exporter.koplugin/ReadwiseClient.lua @@ -0,0 +1,70 @@ +local http = require("socket.http") +local json = require("json") +local logger = require("logger") +local ltn12 = require("ltn12") +local socket = require("socket") +local socketutil = require("socketutil") + +local ReadwiseClient = { + auth_token = "" +} + +function ReadwiseClient:new(o) + o = o or {} + self.__index = self + setmetatable(o, self) + return o +end + +function ReadwiseClient:_makeRequest(endpoint, method, request_body) + local sink = {} + local request_body_json = json.encode(request_body) + local source = ltn12.source.string(request_body_json) + socketutil:set_timeout(socketutil.LARGE_BLOCK_TIMEOUT, socketutil.LARGE_TOTAL_TIMEOUT) + local request = { + url = "https://readwise.io/api/v2/" .. endpoint, + method = method, + sink = ltn12.sink.table(sink), + source = source, + headers = { + ["Content-Length"] = #request_body_json, + ["Content-Type"] = "application/json", + ["Authorization"] = "Token " .. self.auth_token + }, + } + local code, _, status = socket.skip(1, http.request(request)) + socketutil:reset_timeout() + + if code ~= 200 then + logger.warn("ReadwiseClient: HTTP response code <> 200. Response status: ", status) + error("ReadwiseClient: HTTP response code <> 200.") + end + + local response = json.decode(sink[1]) + + return response +end + +function ReadwiseClient:createHighlights(booknotes) + local highlights = {} + for _, chapter in ipairs(booknotes) do + for _, clipping in ipairs(chapter) do + local highlight = { + text = clipping.text, + title = booknotes.title, + author = booknotes.author ~= "" and booknotes.author or nil, -- optional author + source_type = "koreader", + category = "books", + note = clipping.note, + location = clipping.page, + location_type = "page", + highlighted_at = os.date("!%Y-%m-%dT%TZ", clipping.time), + } + table.insert(highlights, highlight) + end + end + local result = self:_makeRequest("highlights", "POST", { highlights = highlights }) + logger.dbg("ReadwiseClient createHighlights result", result) +end + +return ReadwiseClient diff --git a/plugins/exporter.koplugin/clip.lua b/plugins/exporter.koplugin/clip.lua index 2d40c1270..7c30e653c 100644 --- a/plugins/exporter.koplugin/clip.lua +++ b/plugins/exporter.koplugin/clip.lua @@ -4,6 +4,8 @@ local ReadHistory = require("readhistory") local logger = require("logger") local md5 = require("ffi/sha2").md5 local util = require("util") +local _ = require("gettext") +local T = require("ffi/util").template local MyClipping = { my_clippings = "/mnt/us/documents/My Clippings.txt", @@ -98,10 +100,18 @@ local extensions = { [".doc"] = true, } +-- first attempt to parse from document metadata -- remove file extensions added by former KOReader -- extract author name in "Title(Author)" format -- extract author name in "Title - Author" format -function MyClipping:getTitle(line) +function MyClipping:getTitle(line, path) + if path then + local props = self:getProps(path) + if props and props.title ~= "" then + return props.title, props.authors or props.author + end + end + line = line:match("^%s*(.-)%s*$") or "" if extensions[line:sub(-4):lower()] then line = line:sub(1, -5) @@ -228,6 +238,15 @@ end function MyClipping:parseHighlight(highlights, bookmarks, book) --DEBUG("book", book.file) + + -- create a translated pattern that matches bookmark auto-text + -- see ReaderBookmark:getBookmarkAutoText and ReaderBookmark:getBookmarkPageString + --- @todo Remove this once we get rid of auto-text or improve the data model. + local pattern = "^" .. T(_("Page %1 %2 @ %3"), + "%[?%d*%]?%d+", + "(.*)", + "%d%d%d%d%-%d%d%-%d%d %d%d:%d%d:%d%d") .. "$" + for page, items in pairs(highlights) do for _, item in ipairs(items) do local clipping = {} @@ -238,8 +257,11 @@ function MyClipping:parseHighlight(highlights, bookmarks, book) clipping.chapter = item.chapter for _, bookmark in pairs(bookmarks) do if bookmark.datetime == item.datetime and bookmark.text then - local tmp = string.gsub(bookmark.text, "Page %d+ ", "") - clipping.text = string.gsub(tmp, " @ %d%d%d%d%-%d%d%-%d%d %d%d:%d%d:%d%d", "") + local bookmark_quote = bookmark.text:match(pattern) + if bookmark_quote ~= clipping.text and bookmark.text ~= clipping.text then + -- use modified quoted text or entire bookmark text if it's not a match + clipping.note = bookmark_quote or bookmark.text + end end end if item.text == "" and item.pos0 and item.pos1 and @@ -282,7 +304,7 @@ function MyClipping:parseHistoryFile(clippings, history_file, doc_file) return end local _, docname = util.splitFilePathName(doc_file) - local title, author = self:getTitle(util.splitFileNameSuffix(docname)) + local title, author = self:getTitle(util.splitFileNameSuffix(docname), doc_file) clippings[title] = { file = doc_file, title = title, @@ -309,11 +331,31 @@ function MyClipping:parseHistory() return clippings end +function MyClipping:getProps(file) + local document = DocumentRegistry:openDocument(file) + local book_props = nil + if document then + local loaded = true + if document.loadDocument then -- CreDocument + if not document:loadDocument(false) then -- load only metadata + -- failed loading, calling other methods would segfault + loaded = false + end + end + if loaded then + book_props = document:getProps() + end + document:close() + end + + return book_props +end + function MyClipping:parseCurrentDoc(view) local clippings = {} local path = view.document.file local _, _, docname = path:find(".*/(.*)") - local title, author = self:getTitle(docname) + local title, author = self:getTitle(docname, path) clippings[title] = { file = view.document.file, title = title, diff --git a/plugins/exporter.koplugin/main.lua b/plugins/exporter.koplugin/main.lua index 29bfc77e4..1917b4f2d 100644 --- a/plugins/exporter.koplugin/main.lua +++ b/plugins/exporter.koplugin/main.lua @@ -4,11 +4,13 @@ local InfoMessage = require("ui/widget/infomessage") local NetworkMgr = require("ui/network/manager") local DataStorage = require("datastorage") local DocSettings = require("docsettings") +local InputDialog = require("ui/widget/inputdialog") local UIManager = require("ui/uimanager") local logger = require("logger") local util = require("ffi/util") local Device = require("device") local JoplinClient = require("JoplinClient") +local ReadwiseClient = require("ReadwiseClient") local T = util.template local _ = require("gettext") local N_ = _.ngettext @@ -49,10 +51,12 @@ function Exporter:init() self.joplin_port = settings.joplin_port or 41185 self.joplin_token = settings.joplin_token -- or your token self.joplin_notebook_guid = settings.joplin_notebook_guid or nil + self.readwise_token = settings.readwise_token or nil self.html_export = settings.html_export or false self.joplin_export = settings.joplin_export or false self.txt_export = settings.txt_export or false self.json_export = settings.json_export or false + self.readwise_export = settings.readwise_export or false --- @todo Is this if block necessary? Nowhere in the code they are assigned both true. -- Do they check against external modifications to settings file? @@ -60,11 +64,14 @@ function Exporter:init() self.txt_export = false self.joplin_export = false self.json_export = false + self.readwise_export = false elseif self.txt_export then self.joplin_export = false self.json_export = false + self.readwise_export = false elseif self.json_export then self.joplin_export = false + self.readwise_export = false end self.parser = MyClipping:new{ @@ -86,7 +93,8 @@ function Exporter:readyToExport() return self.html_export ~= false or self.txt_export ~= false or self.json_export ~= false or - self.joplin_export ~= false + self.joplin_export ~= false or + self.readwise_export ~= false end function Exporter:migrateClippings() @@ -106,7 +114,6 @@ function Exporter:addToMainMenu(menu_items) { text = _("Joplin") , checked_func = function() return self.joplin_export end, - separator = true, sub_item_table ={ { text = _("Set Joplin IP and Port"), @@ -161,16 +168,10 @@ function Exporter:addToMainMenu(menu_items) text = _("Set authorization token"), keep_menu_open = true, callback = function() - local MultiInputDialog = require("ui/widget/multiinputdialog") local auth_dialog - auth_dialog = MultiInputDialog:new{ + auth_dialog = InputDialog:new{ title = _("Set authorization token for Joplin"), - fields = { - { - text = self.joplin_token, - input_type = "string" - } - }, + input = self.joplin_token, buttons = { { { @@ -182,8 +183,7 @@ function Exporter:addToMainMenu(menu_items) { text = _("Set token"), callback = function() - local auth_field = auth_dialog:getFields() - self.joplin_token = auth_field[1] + self.joplin_token = auth_dialog:getInputText() self:saveSettings() UIManager:close(auth_dialog) end @@ -204,6 +204,7 @@ function Exporter:addToMainMenu(menu_items) self.html_export = false self.txt_export = false self.json_export = false + self.readwise_export = false end self:saveSettings() end @@ -213,7 +214,7 @@ function Exporter:addToMainMenu(menu_items) keep_menu_open = true, callback = function() UIManager:show(InfoMessage:new{ - text = T(_([[You can enter your auth token on your computer by saving an empty token. Then quit KOReader, edit the evernote.joplin_token field in %1/settings.reader.lua after creating a backup, and restart KOReader once you're done. + text = T(_([[You can enter your auth token on your computer by saving an empty token. Then quit KOReader, edit the exporter.joplin_token field in %1/settings.reader.lua after creating a backup, and restart KOReader once you're done. To export to Joplin, you must forward the IP and port used by this plugin to the localhost:port on which Joplin is listening. This can be done with socat or a similar program. For example: @@ -221,7 +222,71 @@ For Windows: netsh interface portproxy add v4tov4 listenaddress=0.0.0.0 listenpo For Linux: $socat tcp-listen:41185,reuseaddr,fork tcp:localhost:41184 -For more information, please visit https://github.com/koreader/koreader/wiki/Evernote-export.]]) +For more information, please visit https://github.com/koreader/koreader/wiki/Highlight-export.]]) + , BD.dirpath(DataStorage:getDataDir())) + }) + end + } + } + }, + { + text = _("Readwise") , + checked_func = function() return self.readwise_export end, + separator = true, + sub_item_table ={ + { + text = _("Set authorization token"), + keep_menu_open = true, + callback = function() + local auth_dialog + auth_dialog = InputDialog:new{ + title = _("Set authorization token for Readwise"), + input = self.readwise_token, + buttons = { + { + { + text = _("Cancel"), + callback = function() + UIManager:close(auth_dialog) + end + }, + { + text = _("Set token"), + callback = function() + self.readwise_token = auth_dialog:getInputText() + self:saveSettings() + UIManager:close(auth_dialog) + end + } + } + } + } + UIManager:show(auth_dialog) + auth_dialog:onShowKeyboard() + end + }, + { + text = _("Export to Readwise"), + checked_func = function() return self.readwise_export end, + callback = function() + self.readwise_export = not self.readwise_export + if self.readwise_export then + self.html_export = false + self.txt_export = false + self.json_export = false + self.joplin_export = false + end + self:saveSettings() + end + }, + { + text = _("Help"), + keep_menu_open = true, + callback = function() + UIManager:show(InfoMessage:new{ + text = T(_([[You can enter your auth token on your computer by saving an empty token. Then quit KOReader, edit the exporter.readwise_token field in %1/settings.reader.lua after creating a backup, and restart KOReader once you're done. + +For more information, please visit https://github.com/koreader/koreader/wiki/Highlight-export.]]) , BD.dirpath(DataStorage:getDataDir())) }) end @@ -270,6 +335,7 @@ For more information, please visit https://github.com/koreader/koreader/wiki/Eve self.txt_export = false self.html_export = false self.joplin_export = false + self.readwise_export = false end self:saveSettings() end @@ -283,6 +349,7 @@ For more information, please visit https://github.com/koreader/koreader/wiki/Eve self.txt_export = false self.json_export = false self.joplin_export = false + self.readwise_export = false end self:saveSettings() end @@ -296,6 +363,7 @@ For more information, please visit https://github.com/koreader/koreader/wiki/Eve self.html_export = false self.json_export = false self.joplin_export = false + self.readwise_export = false end self:saveSettings() end, @@ -325,7 +393,9 @@ function Exporter:saveSettings() joplin_port = self.joplin_port, joplin_token = self.joplin_token, joplin_notebook_guid = self.joplin_notebook_guid, - joplin_export = self.joplin_export + joplin_export = self.joplin_export, + readwise_token = self.readwise_token, + readwise_export = self.readwise_export } G_reader_settings:saveSetting("exporter", settings) end @@ -413,6 +483,7 @@ end function Exporter:exportClippings(clippings) local exported_stamp local joplin_client + local readwise_client if self.html_export then exported_stamp= "html" elseif self.json_export then @@ -433,6 +504,11 @@ function Exporter:exportClippings(clippings) self.joplin_notebook_guid = joplin_client:createNotebook(self.notebook_name) self:saveSettings() end + elseif self.readwise_export then + exported_stamp = "readwise" + readwise_client = ReadwiseClient:new{ + auth_token = self.readwise_token + } else assert("an exported_stamp is expected for a new export type") end @@ -456,6 +532,8 @@ function Exporter:exportClippings(clippings) ok, err = pcall(self.exportBooknotesToJSON, self, title, booknotes) elseif self.joplin_export then ok, err = pcall(self.exportBooknotesToJoplin, self, joplin_client, title, booknotes) + elseif self.readwise_export then + ok, err = pcall(self.exportBooknotesToReadwise, self, readwise_client, title, booknotes) end -- Error reporting if not ok and err and err:find("Transport not open") then @@ -539,6 +617,9 @@ function Exporter:exportBooknotesToTXT(title, booknotes) if clipping.text then file:write(clipping.text) end + if clipping.note then + file:write("\n---\n" .. clipping.note) + end if clipping.image then file:write(_("")) end @@ -565,7 +646,11 @@ function Exporter:exportBooknotesToJoplin(client, title, booknotes) for _, clipping in ipairs(chapter) do note = note .. os.date("%Y-%m-%d %H:%M:%S \n", clipping.time) - note = note .. clipping.text .. "\n * * *\n" + note = note .. clipping.text + if clipping.note then + note = note .. "\n---\n" .. clipping.note + end + note = note .. "\n * * *\n" end end @@ -577,4 +662,8 @@ function Exporter:exportBooknotesToJoplin(client, title, booknotes) end +function Exporter:exportBooknotesToReadwise(client, title, booknotes) + client:createHighlights(booknotes) +end + return Exporter