@ -160,12 +160,13 @@ function NewsDownloader:loadConfigAndProcessFeeds()
for idx , feed in ipairs ( feed_config ) do
for idx , feed in ipairs ( feed_config ) do
local url = feed [ 1 ]
local url = feed [ 1 ]
local limit = feed.limit
local limit = feed.limit
local download_full_article = feed.download_full_article
if url and limit then
if url and limit then
info = InfoMessage : new { text = T ( _ ( " Processing: %1 " ) , url ) }
info = InfoMessage : new { text = T ( _ ( " Processing: %1 " ) , url ) }
UIManager : show ( info )
UIManager : show ( info )
-- processFeedSource is a blocking call, so manually force a UI refresh beforehand
-- processFeedSource is a blocking call, so manually force a UI refresh beforehand
UIManager : forceRePaint ( )
UIManager : forceRePaint ( )
self : processFeedSource ( url , tonumber ( limit ) , unsupported_feeds_urls )
self : processFeedSource ( url , tonumber ( limit ) , unsupported_feeds_urls , download_full_article )
UIManager : close ( info )
UIManager : close ( info )
else
else
logger.warn ( ' NewsDownloader: invalid feed config entry ' , feed )
logger.warn ( ' NewsDownloader: invalid feed config entry ' , feed )
@ -191,7 +192,7 @@ function NewsDownloader:loadConfigAndProcessFeeds()
end
end
end
end
function NewsDownloader : processFeedSource ( url , limit , unsupported_feeds_urls )
function NewsDownloader : processFeedSource ( url , limit , unsupported_feeds_urls , download_full_article )
local resp_lines = { }
local resp_lines = { }
local parsed = socket_url.parse ( url )
local parsed = socket_url.parse ( url )
local httpRequest = parsed.scheme == ' http ' and http.request or https.request
local httpRequest = parsed.scheme == ' http ' and http.request or https.request
@ -207,9 +208,9 @@ function NewsDownloader:processFeedSource(url, limit, unsupported_feeds_urls)
local is_atom = feeds.feed and feeds.feed . title and feeds.feed . entry [ 1 ] and feeds.feed . entry [ 1 ] . title and feeds.feed . entry [ 1 ] . link
local is_atom = feeds.feed and feeds.feed . title and feeds.feed . entry [ 1 ] and feeds.feed . entry [ 1 ] . title and feeds.feed . entry [ 1 ] . link
if is_atom then
if is_atom then
self : processAtom ( feeds , limit )
self : processAtom ( feeds , limit , download_full_article )
elseif is_rss then
elseif is_rss then
self : processRSS ( feeds , limit )
self : processRSS ( feeds , limit , download_full_article )
else
else
table.insert ( unsupported_feeds_urls , url )
table.insert ( unsupported_feeds_urls , url )
return
return
@ -233,7 +234,7 @@ function NewsDownloader:deserializeXMLString(xml_str)
return xmlhandler.root
return xmlhandler.root
end
end
function NewsDownloader : processAtom ( feeds , limit )
function NewsDownloader : processAtom ( feeds , limit , download_full_article )
local feed_output_dir = string.format ( " %s%s/ " ,
local feed_output_dir = string.format ( " %s%s/ " ,
news_download_dir_path ,
news_download_dir_path ,
util.replaceInvalidChars ( getFeedTitle ( feeds.feed . title ) ) )
util.replaceInvalidChars ( getFeedTitle ( feeds.feed . title ) ) )
@ -245,11 +246,15 @@ function NewsDownloader:processAtom(feeds, limit)
if limit ~= 0 and index - 1 == limit then
if limit ~= 0 and index - 1 == limit then
break
break
end
end
self : downloadFeed ( feed , feed_output_dir )
if download_full_article then
self : downloadFeed ( feed , feed_output_dir )
else
self : createFromDescription ( feed , feed.context , feed_output_dir )
end
end
end
end
end
function NewsDownloader : processRSS ( feeds , limit )
function NewsDownloader : processRSS ( feeds , limit , download_full_article )
local feed_output_dir = ( " %s%s/ " ) : format (
local feed_output_dir = ( " %s%s/ " ) : format (
news_download_dir_path , util.replaceInvalidChars ( feeds.rss . channel.title ) )
news_download_dir_path , util.replaceInvalidChars ( feeds.rss . channel.title ) )
if not lfs.attributes ( feed_output_dir , " mode " ) then
if not lfs.attributes ( feed_output_dir , " mode " ) then
@ -260,7 +265,11 @@ function NewsDownloader:processRSS(feeds, limit)
if limit ~= 0 and index - 1 == limit then
if limit ~= 0 and index - 1 == limit then
break
break
end
end
self : downloadFeed ( feed , feed_output_dir )
if download_full_article then
self : downloadFeed ( feed , feed_output_dir )
else
self : createFromDescription ( feed , feed.description , feed_output_dir )
end
end
end
end
end
@ -276,6 +285,25 @@ function NewsDownloader:downloadFeed(feed, feed_output_dir)
httpRequest ( { url = link , sink = ltn12.sink . file ( io.open ( news_dl_path , ' w ' ) ) , } )
httpRequest ( { url = link , sink = ltn12.sink . file ( io.open ( news_dl_path , ' w ' ) ) , } )
end
end
function NewsDownloader : createFromDescription ( feed , context , feed_output_dir )
local news_file_path = ( " %s%s%s " ) : format ( feed_output_dir ,
util.replaceInvalidChars ( getFeedTitle ( feed.title ) ) ,
file_extension )
logger.dbg ( " NewsDownloader: News file will be created : " , news_file_path )
local file = io.open ( news_file_path , " w " )
local footer = _ ( " This is just description of the feed. To download full article go to News Downloader settings and change 'download_full_article' to 'true' " )
local html = string.format ( [ [ < ! DOCTYPE html >
< html >
< head >< meta charset = ' UTF-8 ' >< title >% s </ title ></ head >
< body >< header >< h2 >% s </ h2 ></ header >< article >% s </ article >
< br >< footer >< small >% s </ small ></ footer >
</ body >
</ html > ] ] , feed.title , feed.title , context , footer )
file : write ( html )
file : close ( )
end
function NewsDownloader : removeNewsButKeepFeedConfig ( )
function NewsDownloader : removeNewsButKeepFeedConfig ( )
logger.dbg ( " NewsDownloader: Removing news from : " , news_download_dir_path )
logger.dbg ( " NewsDownloader: Removing news from : " , news_download_dir_path )
for entry in lfs.dir ( news_download_dir_path ) do
for entry in lfs.dir ( news_download_dir_path ) do