From 97a72e2dd4e07a4c1d7d4f60ed063e64c21f2397 Mon Sep 17 00:00:00 2001 From: mwoz123 Date: Thu, 26 Oct 2017 22:44:03 +0200 Subject: [PATCH] NewsDownloader: use feed.description as news context instead download full web page (#3426) Fixes #3425. --- .../newsdownloader.koplugin/feed_config.lua | 26 ++++++++--- plugins/newsdownloader.koplugin/main.lua | 44 +++++++++++++++---- 2 files changed, 57 insertions(+), 13 deletions(-) diff --git a/plugins/newsdownloader.koplugin/feed_config.lua b/plugins/newsdownloader.koplugin/feed_config.lua index 9033c0fdc..2b4d2a993 100644 --- a/plugins/newsdownloader.koplugin/feed_config.lua +++ b/plugins/newsdownloader.koplugin/feed_config.lua @@ -1,11 +1,27 @@ return { -- list your feeds here: - { "http://feeds.reuters.com/Reuters/worldNews?format=xml", limit = 2}, - -- set 'limit' to change number of 'news' to be downloaded from source + { "http://feeds.reuters.com/Reuters/worldNews?format=xml", limit = 2, download_full_article=false}, + + { "https://www.pcworld.com/index.rss", limit = 7 , download_full_article=true}, + + -- comment out line ("--" at line start) to stop downloading source + --{ "http://www.football.co.uk/international/rss.xml", limit = 0 , download_full_article=true}, + + + + + + + + --HELP: + -- use syntax: {"your_url", limit= max_number_of_items_to_be_created, download_full_article=true/false} + + -- set 'limit' to change number of 'news' to be created -- 'limit' equal "0" means no limit. - { "http://www.pcworld.com/index.rss", limit = 1 }, - -- comment out line to stop downloading source - --{ "http://www.football.co.uk/international/rss.xml", limit = 0 }, + -- 'download_full_article=false' - means download full article using feed link (may not always work correctly) + -- 'download_full_article=true' - means use only feed description to create feeds (usually only part of the article) + + } diff --git a/plugins/newsdownloader.koplugin/main.lua b/plugins/newsdownloader.koplugin/main.lua index 061631c01..f23390763 100644 --- a/plugins/newsdownloader.koplugin/main.lua +++ b/plugins/newsdownloader.koplugin/main.lua @@ -160,12 +160,13 @@ function NewsDownloader:loadConfigAndProcessFeeds() for idx, feed in ipairs(feed_config) do local url = feed[1] local limit = feed.limit + local download_full_article = feed.download_full_article if url and limit then info = InfoMessage:new{ text = T(_("Processing: %1"), url) } UIManager:show(info) -- processFeedSource is a blocking call, so manually force a UI refresh beforehand UIManager:forceRePaint() - self:processFeedSource(url, tonumber(limit), unsupported_feeds_urls) + self:processFeedSource(url, tonumber(limit), unsupported_feeds_urls, download_full_article) UIManager:close(info) else logger.warn('NewsDownloader: invalid feed config entry', feed) @@ -191,7 +192,7 @@ function NewsDownloader:loadConfigAndProcessFeeds() end end -function NewsDownloader:processFeedSource(url, limit, unsupported_feeds_urls) +function NewsDownloader:processFeedSource(url, limit, unsupported_feeds_urls, download_full_article) local resp_lines = {} local parsed = socket_url.parse(url) local httpRequest = parsed.scheme == 'http' and http.request or https.request @@ -207,9 +208,9 @@ function NewsDownloader:processFeedSource(url, limit, unsupported_feeds_urls) local is_atom = feeds.feed and feeds.feed.title and feeds.feed.entry[1] and feeds.feed.entry[1].title and feeds.feed.entry[1].link if is_atom then - self:processAtom(feeds, limit) + self:processAtom(feeds, limit, download_full_article) elseif is_rss then - self:processRSS(feeds, limit) + self:processRSS(feeds, limit, download_full_article) else table.insert(unsupported_feeds_urls, url) return @@ -233,7 +234,7 @@ function NewsDownloader:deserializeXMLString(xml_str) return xmlhandler.root end -function NewsDownloader:processAtom(feeds, limit) +function NewsDownloader:processAtom(feeds, limit, download_full_article) local feed_output_dir = string.format("%s%s/", news_download_dir_path, util.replaceInvalidChars(getFeedTitle(feeds.feed.title))) @@ -245,11 +246,15 @@ function NewsDownloader:processAtom(feeds, limit) if limit ~= 0 and index - 1 == limit then break end - self:downloadFeed(feed, feed_output_dir) + if download_full_article then + self:downloadFeed(feed, feed_output_dir) + else + self:createFromDescription(feed, feed.context, feed_output_dir) + end end end -function NewsDownloader:processRSS(feeds, limit) +function NewsDownloader:processRSS(feeds, limit, download_full_article) local feed_output_dir = ("%s%s/"):format( news_download_dir_path, util.replaceInvalidChars(feeds.rss.channel.title)) if not lfs.attributes(feed_output_dir, "mode") then @@ -260,7 +265,11 @@ function NewsDownloader:processRSS(feeds, limit) if limit ~= 0 and index - 1 == limit then break end - self:downloadFeed(feed, feed_output_dir) + if download_full_article then + self:downloadFeed(feed, feed_output_dir) + else + self:createFromDescription(feed, feed.description, feed_output_dir) + end end end @@ -276,6 +285,25 @@ function NewsDownloader:downloadFeed(feed, feed_output_dir) httpRequest({ url = link, sink = ltn12.sink.file(io.open(news_dl_path, 'w')), }) end +function NewsDownloader:createFromDescription(feed, context, feed_output_dir) + local news_file_path = ("%s%s%s"):format(feed_output_dir, + util.replaceInvalidChars(getFeedTitle(feed.title)), + file_extension) + logger.dbg("NewsDownloader: News file will be created :", news_file_path) + local file = io.open(news_file_path, "w") + local footer = _("This is just description of the feed. To download full article go to News Downloader settings and change 'download_full_article' to 'true'") + + local html = string.format([[ + +%s +

%s

%s
+
+ +]], feed.title, feed.title, context, footer) + file:write(html) + file:close() +end + function NewsDownloader:removeNewsButKeepFeedConfig() logger.dbg("NewsDownloader: Removing news from :", news_download_dir_path) for entry in lfs.dir(news_download_dir_path) do