feat(core): skip files that can’t be processed + show a summary at the end

Thumbsup used to stop at the first problem processing a file.
This was a problem on large galleries where you’d need to run it again and again, fixing files as you went.
This change:
- skips problematic files and shows a summary at the end
- logs all warnings/errors to <thumbsup.log> when running the default output

Also refactor and cleanup of the logging logic.
pull/107/head
Romain 6 years ago
parent 7208642d95
commit e9beb830a2

@ -1,17 +1,98 @@
const debug = require('debug')
const fs = require('fs')
const path = require('path')
const util = require('util')
const tty = require('tty')
exports.init = (logLevel) => {
// enable particular debug() prefixes
/*
Thumbsup uses the <debug> package for logging.
It supports two main ways of logging:
1. when --log is not specified
- This ignores all detailed logging calls
- It simply renders progress using Listr on <stdout>
- Warning and error are sent to <thumbsup.log> in the output folder for troubleshooting
2. when --log is specified
- This switches Listr to text mode, e.g. "Task XYZ [completed]"
- It renders log messages to <stdout>
- The logging level can be configured between info|debug|trace
- Warnings and errors are always shown, because level is > info|debug|trace
- <thumbsup.log> is not created by default
- You can always redirect or tee the output to a file if needed
If --log is not specified, but the output doesn't support ANSI (e.g. non-TTY terminal, or file redirection)
then the mode is automatically switched to "--log info"
*/
exports.init = (logLevel, outputFolder) => {
// if the output doesn't support ANSI codes (e.g. pipe, redirect to file)
// then switch to full-text mode, because Listr's output won't make much sense
if (!logLevel && !tty.isatty(process.stdout.fd)) {
logLevel = 'info'
}
// Configure the loggers
if (!logLevel) {
configureDefaultMode(outputFolder)
} else {
configureDebugMode(logLevel)
}
}
/*
The <debug> package repeats the date + prefix for every log line, but only when using colors
This fix adds the same behaviour in non-color mode
It's important so the log files are easy to grep
*/
function overrideDebugFormat () {
debug.formatArgs = function (args) {
const prefix = new Date().toISOString() + ' ' + this.namespace + ' '
args[0] = prefix + args[0].split('\n').join('\n' + prefix)
}
}
/*
If --log is not specified, we won't show any detailed log on stdout
Instead we send all errors to a file for troubleshooting
*/
function configureDefaultMode (outputFolder) {
const logfile = path.join(outputFolder, 'thumbsup.log')
const stream = fs.createWriteStream(logfile, {flags: 'a'})
overrideDebugFormat()
debug.enable('thumbsup:error,thumbsup:warn')
debug.useColors = () => false
debug.log = function () {
const line = util.format.apply(util, arguments) + '\n'
stream.write(line)
}
}
/*
--log mode configuration
*/
function configureDebugMode (logLevel) {
// because Listr logs to stdout (not configurable), make debug() do the same
// otherwise file redirection gets very confusing
debug.log = function () {
const line = util.format.apply(util, arguments) + '\n'
process.stdout.write(line)
}
// TTY-related configuration
const supportsANSI = tty.isatty(process.stdout.fd)
debug.useColors = () => supportsANSI
if (!supportsANSI) {
overrideDebugFormat()
}
// enable the right log levels
if (logLevel === 'trace') debug.enable('*')
if (logLevel === 'debug') debug.enable('thumbsup:error,thumbsup:warn,thumbsup:info,thumbsup:debug')
if (logLevel === 'info') debug.enable('thumbsup:error,thumbsup:warn,thumbsup:info')
// when running in text-mode, make sure all console.log() calls go through debug()
// don't touch them in normal-mode, since it would affect Listr's dnyamic rendering
if (typeof logLevel === 'string') {
console.log = require('debug')('thumbsup:info')
console.info = require('debug')('thumbsup:info')
console.warn = require('debug')('thumbsup:warn')
console.error = require('debug')('thumbsup:error')
}
// capture any additional console.log() calls, including Listr task updates
console.log = require('debug')('thumbsup:info')
console.info = require('debug')('thumbsup:info')
console.warn = require('debug')('thumbsup:warn')
console.error = require('debug')('thumbsup:error')
}

@ -32,6 +32,11 @@ Gallery generated successfully!
${stats.albums} albums, ${stats.photos} photos, ${stats.videos} videos
`)
exports.PROBLEMS = (count) => chalk.yellow(`
Warning: there was an issue with ${count} file${count > 1 ? 's' : ''}.
Please check the full log for more detail.
`)
exports.GREETING = () => box(`
Thanks for using thumbsup!

@ -1,8 +1,7 @@
#!/usr/bin/env node
const fs = require('fs')
const fs = require('fs-extra')
const path = require('path')
const tty = require('tty')
const Analytics = require('./analytics')
const dependencies = require('./dependencies')
const messages = require('./messages')
@ -14,12 +13,9 @@ console.log('')
const args = process.argv.slice(2)
const opts = options.get(args)
// If stdout is non TTY, make sure there is at least some text logging on stderr
if (!opts.log && tty.isatty(process.stdout.fd) === false) {
opts.log = 'info'
}
// Only require the index after logging options have been set
require('./log').init(opts.log)
fs.mkdirpSync(opts.output)
require('./log').init(opts.log, opts.output)
const index = require('../src/index')
// If this is the first run, display a welcome message
@ -37,6 +33,7 @@ analytics.start()
// Catch all exceptions and exit gracefully
process.on('uncaughtException', handleError)
process.on('unhandledRejection', handleError)
// Check that all binary dependencies are present
dependencies.checkOptional()
@ -47,18 +44,21 @@ if (missingErrors) {
}
// Build the gallery!
index.build(opts, (err, album) => {
index.build(opts, (err, result) => {
console.log('')
if (err) {
handleError(err)
} else {
// Print any problems
result.problems.print()
// And then a summary of the gallery
const stats = {
albums: countAlbums(0, album),
photos: album.stats.photos,
videos: album.stats.videos
albums: countAlbums(0, result.album),
photos: result.album.stats.photos,
videos: result.album.stats.videos
}
analytics.finish(stats)
const message = messages.SUCCESS(stats)
console.log(`\n${message}\n`)
console.log(messages.SUCCESS(stats) + '\n')
exit(0)
}
})

@ -1,7 +1,7 @@
const fs = require('fs-extra')
const Listr = require('listr')
const steps = require('./steps/index')
const website = require('./website/website')
const Problems = require('./problems')
exports.build = function (opts, done) {
// How to render tasks
@ -11,7 +11,6 @@ exports.build = function (opts, done) {
{
title: 'Indexing folder',
task: (ctx, task) => {
fs.mkdirpSync(opts.output)
return steps.index(opts, (err, files, album) => {
if (!err) {
ctx.files = files
@ -23,7 +22,8 @@ exports.build = function (opts, done) {
{
title: 'Resizing media',
task: (ctx, task) => {
const tasks = steps.process(ctx.files, opts, task)
ctx.problems = new Problems()
const tasks = steps.process(ctx.files, ctx.problems, opts, task)
if (!opts.dryRun) {
return tasks
} else {
@ -55,7 +55,10 @@ exports.build = function (opts, done) {
})
tasks.run().then(ctx => {
done(null, ctx.album)
done(null, {
album: ctx.album,
problems: ctx.problems
})
}).catch(err => {
done(err)
})

@ -0,0 +1,26 @@
const warn = require('debug')('thumbsup:warn')
const messages = require('../bin/messages')
/*
Keeps track of which source files we failed to process
*/
module.exports = class Problems {
constructor () {
this.files = {}
}
addFile (path) {
this.files[path] = true
}
print () {
// only print the number of failed files in the standard output
const paths = Object.keys(this.files)
if (paths.length > 0) {
// print a short summary on stdout
console.warn(messages.PROBLEMS(paths.length))
// and a full log to the log file
warn('The following sources files were not processed:\n' + paths.join(','))
}
}
}

@ -1,12 +1,13 @@
const debug = require('debug')('thumbsup:debug')
const error = require('debug')('thumbsup:error')
const downsize = require('thumbsup-downsize')
const fs = require('fs-extra')
const info = require('debug')('thumbsup:info')
const ListrWorkQueue = require('listr-work-queue')
const path = require('path')
exports.run = function (files, opts, parentTask) {
const jobs = exports.create(files, opts)
exports.run = function (files, problems, opts, parentTask) {
const jobs = exports.create(files, opts, problems)
// wrap each job in a Listr task that returns a Promise
const tasks = jobs.map(job => listrTaskFromJob(job, opts.output))
const listr = new ListrWorkQueue(tasks, {
@ -22,18 +23,18 @@ exports.run = function (files, opts, parentTask) {
/*
Return a list of task to build all required outputs (new or updated)
*/
exports.create = function (files, opts) {
var tasks = {}
exports.create = function (files, opts, problems) {
const tasks = {}
const sourceFiles = new Set()
const actionMap = getActionMap(opts)
// accumulate all tasks into an object
// to remove duplicate destinations
files.forEach(f => {
Object.keys(f.output).forEach(out => {
var src = path.join(opts.input, f.path)
var dest = path.join(opts.output, f.output[out].path)
var destDate = modifiedDate(dest)
var action = actionMap[f.output[out].rel]
const src = path.join(opts.input, f.path)
const dest = path.join(opts.output, f.output[out].path)
const destDate = modifiedDate(dest)
const action = actionMap[f.output[out].rel]
// ignore output files that don't have an action (e.g. existing links)
debug(`Comparing ${f.path} (${f.date}) and ${f.output[out].path} (${destDate})`)
if (action && f.date > destDate) {
@ -45,7 +46,13 @@ exports.create = function (files, opts) {
action: (done) => {
fs.mkdirsSync(path.dirname(dest))
debug(`${f.output[out].rel} from ${src} to ${dest}`)
return action({src: src, dest: dest}, done)
return action({src: src, dest: dest}, err => {
if (err) {
error(`Error processing ${f.path} -> ${f.output[out].path}\n${err}`)
problems.addFile(f.path)
}
done()
})
}
}
}

@ -18,4 +18,9 @@ describe('messages', function () {
should(required.indexOf('bin1\n')).above(-1)
should(required.indexOf('bin2\n')).above(-1)
})
it('can print one or more problem', () => {
should(messages.PROBLEMS(1).indexOf('with 1 file.')).above(-1)
should(messages.PROBLEMS(2).indexOf('with 2 files.')).above(-1)
})
})

Loading…
Cancel
Save