From 1e9074ca6a3d36250ba35e4667ce371ef20b1d45 Mon Sep 17 00:00:00 2001 From: Tom Hall Date: Sun, 26 May 2019 12:46:25 +0100 Subject: [PATCH] NewsDownloader: Allow including images when using text from feed (#5040) Split the html page download out of createEpub, so that createFromDescription can pass in its own html for epub packaging (with optional image download). Still pass in a url, so that relative urls can be made absolute. --- .../epubdownloadbackend.lua | 12 +---- plugins/newsdownloader.koplugin/main.lua | 45 +++++++++++-------- 2 files changed, 28 insertions(+), 29 deletions(-) diff --git a/plugins/newsdownloader.koplugin/epubdownloadbackend.lua b/plugins/newsdownloader.koplugin/epubdownloadbackend.lua index d9a73825e..41a0f38e6 100644 --- a/plugins/newsdownloader.koplugin/epubdownloadbackend.lua +++ b/plugins/newsdownloader.koplugin/epubdownloadbackend.lua @@ -20,11 +20,6 @@ local EpubDownloadBackend = { } local max_redirects = 5; --prevent infinite redirects -function EpubDownloadBackend:download(url, path, include_images, message) - logger.dbg("EpubDownloadBackend:download") - self:createEpub(path, url, include_images, message) -end - -- Codes that getUrlContent may get from requester.request() local TIMEOUT_CODE = "timeout" -- from socket.lua local MAXTIME_CODE = "maxtime reached" -- from sink_table_with_maxtime @@ -188,16 +183,13 @@ local ext_to_mimetype = { } -- Create an epub file (with possibly images) -function EpubDownloadBackend:createEpub(epub_path, url, include_images, message) - logger.dbg("EpubDownloadBackend:createEpub(", epub_path, ",", url, ")") +function EpubDownloadBackend:createEpub(epub_path, html, url, include_images, message) + logger.dbg("EpubDownloadBackend:createEpub(", epub_path, ")") -- Use Trapper to display progress and ask questions through the UI. -- We need to have been Trapper.wrap()'ed for UI to be used, otherwise -- Trapper:info() and Trapper:confirm() will just use logger. local UI = require("ui/trapper") - UI:info(T(_("%1\n\nRetrieving article…"), message)) - local html = self:loadPage(url) - logger.dbg("Successfully retrieved article html") -- We may need to build absolute urls for non-absolute links and images urls local base_url = socket_url.parse(url) diff --git a/plugins/newsdownloader.koplugin/main.lua b/plugins/newsdownloader.koplugin/main.lua index 4740841f8..e68ec752a 100644 --- a/plugins/newsdownloader.koplugin/main.lua +++ b/plugins/newsdownloader.koplugin/main.lua @@ -276,7 +276,7 @@ function NewsDownloader:processAtom(feeds, limit, download_full_article, include if download_full_article then self:downloadFeed(feed, feed_output_dir, include_images, article_message) else - self:createFromDescription(feed, feed.context, feed_output_dir) + self:createFromDescription(feed, feed.context, feed_output_dir, include_images, article_message) end end end @@ -296,7 +296,7 @@ function NewsDownloader:processRSS(feeds, limit, download_full_article, include_ if download_full_article then self:downloadFeed(feed, feed_output_dir, include_images, article_message) else - self:createFromDescription(feed, feed.description, feed_output_dir) + self:createFromDescription(feed, feed.description, feed_output_dir, include_images, article_message) end end end @@ -322,39 +322,46 @@ local function getTitleWithDate(feed) end function NewsDownloader:downloadFeed(feed, feed_output_dir, include_images, message) - local link = getFeedLink(feed.link) local title_with_date = getTitleWithDate(feed) - local news_dl_path = ("%s%s%s"):format(feed_output_dir, - title_with_date, - file_extension) + local news_file_path = ("%s%s%s"):format(feed_output_dir, + title_with_date, + file_extension) - local file_mode = lfs.attributes(news_dl_path, "mode") + local file_mode = lfs.attributes(news_file_path, "mode") if file_mode == "file" then - logger.dbg("NewsDownloader:", news_dl_path, "already exists. Skipping") + logger.dbg("NewsDownloader:", news_file_path, "already exists. Skipping") else - logger.dbg("NewsDownloader: News file will be stored to :", news_dl_path) + logger.dbg("NewsDownloader: News file will be stored to :", news_file_path) local article_message = T(_("%1\n%2"), message, title_with_date) - DownloadBackend:download(link, news_dl_path, include_images, article_message) + local link = getFeedLink(feed.link) + local html = DownloadBackend:loadPage(link) + DownloadBackend:createEpub(news_file_path, html, link, include_images, article_message) end end -function NewsDownloader:createFromDescription(feed, context, feed_output_dir) +function NewsDownloader:createFromDescription(feed, context, feed_output_dir, include_images, message) + local title_with_date = getTitleWithDate(feed) local news_file_path = ("%s%s%s"):format(feed_output_dir, - getTitleWithDate(feed), - file_extension) - logger.dbg("NewsDownloader: News file will be created :", news_file_path) - local file = io.open(news_file_path, "w") - local footer = _("This is just description of the feed. To download full article go to News Downloader settings and change 'download_full_article' to 'true'") + title_with_date, + file_extension) + local file_mode = lfs.attributes(news_file_path, "mode") + if file_mode == "file" then + logger.dbg("NewsDownloader:", news_file_path, "already exists. Skipping") + else + logger.dbg("NewsDownloader: News file will be stored to :", news_file_path) + local article_message = T(_("%1\n%2"), message, title_with_date) + local footer = _("This is just a description of the feed. To download the full article instead, go to the News Downloader settings and change 'download_full_article' to 'true'.") - local html = string.format([[ + local html = string.format([[ %s

%s

%s

]], feed.title, feed.title, context, footer) - file:write(html) - file:close() + local link = getFeedLink(feed.link) + DownloadBackend:createEpub(news_file_path, html, link, include_images, article_message) + end end function NewsDownloader:removeNewsButKeepFeedConfig()