mirror of https://github.com/thumbsup/thumbsup
Render progress using Listr + split the main process into "steps" which are easier to test
Romain
7 years ago
parent
179cc57644
commit
30f203af4b
@ -0,0 +1,15 @@
|
||||
# EditorConfig is awesome: http://EditorConfig.org
|
||||
|
||||
# top-most EditorConfig file
|
||||
root = true
|
||||
|
||||
# Unix-style newlines with a newline ending every file
|
||||
[*]
|
||||
end_of_line = lf
|
||||
insert_final_newline = true
|
||||
|
||||
# Indentation override for all JS under lib directory
|
||||
[**/**.js]
|
||||
charset = utf-8
|
||||
indent_style = space
|
||||
indent_size = 2
|
||||
@ -1,94 +1,61 @@
|
||||
const async = require('async')
|
||||
const fs = require('fs-extra')
|
||||
const path = require('path')
|
||||
const os = require('os')
|
||||
const cleanup = require('./output-media/cleanup')
|
||||
const database = require('./input/database')
|
||||
const Picasa = require('./input/picasa')
|
||||
const progress = require('./utils/progress')
|
||||
const hierarchy = require('./input/hierarchy.js')
|
||||
const mapper = require('./input/mapper')
|
||||
const File = require('./model/file')
|
||||
const Metadata = require('./model/metadata')
|
||||
const tasks = require('./output-media/tasks')
|
||||
const website = require('./output-website/website')
|
||||
const Listr = require('listr')
|
||||
const steps = require('./steps/index')
|
||||
const summary = require('./steps/summary')
|
||||
const website = require('./website/website')
|
||||
|
||||
exports.build = function (opts) {
|
||||
fs.mkdirpSync(opts.output)
|
||||
const databaseFile = path.join(opts.output, 'metadata.json')
|
||||
|
||||
// all files, unsorted
|
||||
var files = null
|
||||
|
||||
// root album with nested albums
|
||||
var album = null
|
||||
|
||||
async.series([
|
||||
|
||||
function updateDatabase (callback) {
|
||||
const picasaReader = new Picasa()
|
||||
database.update(opts.input, databaseFile, (err, entries) => {
|
||||
if (err) return callback(err)
|
||||
files = entries.map(entry => {
|
||||
// create standarised metadata model
|
||||
const picasa = picasaReader.file(entry.SourceFile)
|
||||
const meta = new Metadata(entry, picasa || {})
|
||||
// create a file entry for the albums
|
||||
return new File(entry, meta, opts)
|
||||
const tasks = new Listr([
|
||||
{
|
||||
title: 'Updating database',
|
||||
task: (ctx, task) => {
|
||||
// returns an observable which will complete when the database is loaded
|
||||
fs.mkdirpSync(opts.output)
|
||||
const databaseFile = path.join(opts.output, 'metadata.json')
|
||||
return steps.database(opts.input, databaseFile, (err, res) => {
|
||||
if (!err) {
|
||||
ctx.database = res.database
|
||||
}
|
||||
})
|
||||
callback()
|
||||
})
|
||||
}
|
||||
},
|
||||
|
||||
function processPhotos (callback) {
|
||||
const photos = tasks.create(opts, files, 'image')
|
||||
const bar = progress.create('Processing photos', photos.length)
|
||||
parallel(photos, bar, callback)
|
||||
{
|
||||
title: 'Creating model',
|
||||
task: (ctx) => {
|
||||
const res = steps.model(ctx.database, opts)
|
||||
ctx.files = res.files
|
||||
ctx.album = res.album
|
||||
}
|
||||
},
|
||||
|
||||
function processVideos (callback) {
|
||||
const videos = tasks.create(opts, files, 'video')
|
||||
const bar = progress.create('Processing videos', videos.length)
|
||||
parallel(videos, bar, callback)
|
||||
{
|
||||
title: 'Processing media',
|
||||
task: (ctx, task) => {
|
||||
return steps.process(ctx.files, opts, task)
|
||||
}
|
||||
},
|
||||
|
||||
function removeOldOutput (callback) {
|
||||
if (!opts.cleanup) return callback()
|
||||
cleanup.run(files, opts.output, callback)
|
||||
{
|
||||
title: 'Cleaning up',
|
||||
enabled: (ctx) => opts.cleanup,
|
||||
task: (ctx) => {
|
||||
return steps.cleanup(ctx.files, opts.output)
|
||||
}
|
||||
},
|
||||
|
||||
function createAlbums (callback) {
|
||||
const bar = progress.create('Creating albums')
|
||||
const albumMapper = mapper.create(opts)
|
||||
album = hierarchy.createAlbums(files, albumMapper, opts)
|
||||
bar.tick(1)
|
||||
callback()
|
||||
},
|
||||
|
||||
function createWebsite (callback) {
|
||||
const bar = progress.create('Building website')
|
||||
website.build(album, opts, (err) => {
|
||||
bar.tick(1)
|
||||
callback(err)
|
||||
{
|
||||
title: 'Creating website',
|
||||
task: (ctx) => new Promise((resolve, reject) => {
|
||||
website.build(ctx.album, opts, err => {
|
||||
err ? reject(err) : resolve()
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
], finish)
|
||||
}
|
||||
|
||||
function parallel (tasks, bar, callback) {
|
||||
const decorated = tasks.map(t => done => {
|
||||
t(err => {
|
||||
bar.tick(1)
|
||||
done(err)
|
||||
})
|
||||
])
|
||||
|
||||
tasks.run().then(ctx => {
|
||||
console.log('\n' + summary.create(ctx) + '\n')
|
||||
process.exit(0)
|
||||
}).catch(err => {
|
||||
console.log('\nUnexpected error', err)
|
||||
process.exit(1)
|
||||
})
|
||||
async.parallelLimit(decorated, os.cpus().length, callback)
|
||||
}
|
||||
|
||||
function finish (err) {
|
||||
console.log(err ? 'Unexpected error' : '')
|
||||
console.log(err || 'Gallery generated successfully')
|
||||
console.log()
|
||||
process.exit(err ? 1 : 0)
|
||||
}
|
||||
|
||||
@ -1,40 +0,0 @@
|
||||
/*
|
||||
--------------------------------------------------------------------------------
|
||||
Provides most metadata based on the output of <exiftool>
|
||||
Caches the resulting DB in <metadata.json> for faster re-runs
|
||||
--------------------------------------------------------------------------------
|
||||
*/
|
||||
|
||||
const debug = require('debug')('thumbsup')
|
||||
const exifdb = require('exiftool-json-db')
|
||||
const progress = require('../utils/progress')
|
||||
|
||||
exports.update = function (media, databasePath, callback) {
|
||||
var updateBar = null
|
||||
var emitter = null
|
||||
|
||||
try {
|
||||
emitter = exifdb.create({media: media, database: databasePath})
|
||||
} catch (ex) {
|
||||
const message = 'Loading database\n' + ex.toString() + '\n' +
|
||||
'If migrating from thumbsup v1, delete <metadata.json> to rebuild the database from scratch'
|
||||
callback(new Error(message))
|
||||
}
|
||||
|
||||
emitter.on('stats', (stats) => {
|
||||
debug(`Database stats: total=${stats.total}`)
|
||||
const totalBar = progress.create('Finding media', stats.total)
|
||||
totalBar.tick(stats.total)
|
||||
updateBar = progress.create('Updating database', stats.added + stats.modified)
|
||||
})
|
||||
|
||||
emitter.on('file', (file) => {
|
||||
updateBar.tick()
|
||||
})
|
||||
|
||||
emitter.on('done', (files) => {
|
||||
callback(null, files)
|
||||
})
|
||||
|
||||
emitter.on('error', callback)
|
||||
}
|
||||
@ -1,24 +0,0 @@
|
||||
const _ = require('lodash')
|
||||
const fs = require('fs')
|
||||
const path = require('path')
|
||||
const readdir = require('fs-readdir-recursive')
|
||||
const progress = require('../utils/progress')
|
||||
|
||||
exports.run = function (fileCollection, outputRoot, callback) {
|
||||
const mediaRoot = path.join(outputRoot, 'media')
|
||||
const diskFiles = readdir(mediaRoot).map(f => path.join(mediaRoot, f))
|
||||
const requiredFiles = []
|
||||
fileCollection.forEach(f => {
|
||||
Object.keys(f.output).forEach(out => {
|
||||
var dest = path.join(outputRoot, f.output[out].path)
|
||||
requiredFiles.push(dest)
|
||||
})
|
||||
})
|
||||
const useless = _.difference(diskFiles, requiredFiles)
|
||||
if (useless.length) {
|
||||
const bar = progress.create('Cleaning up', useless.length)
|
||||
useless.forEach(f => fs.unlinkSync(f))
|
||||
bar.tick(useless.length)
|
||||
}
|
||||
callback()
|
||||
}
|
||||
@ -0,0 +1,5 @@
|
||||
exports.database = require('./step-database').run
|
||||
exports.model = require('./step-model').run
|
||||
exports.process = require('./step-process').run
|
||||
exports.cleanup = require('./step-cleanup').run
|
||||
// exports.website = require('./website')
|
||||
@ -0,0 +1,28 @@
|
||||
const _ = require('lodash')
|
||||
const fs = require('fs')
|
||||
const Observable = require('zen-observable')
|
||||
const path = require('path')
|
||||
const readdir = require('fs-readdir-recursive')
|
||||
|
||||
exports.run = function (fileCollection, outputRoot) {
|
||||
return new Observable(observer => {
|
||||
const mediaRoot = path.join(outputRoot, 'media')
|
||||
const diskFiles = readdir(mediaRoot).map(f => path.join(mediaRoot, f))
|
||||
const requiredFiles = []
|
||||
fileCollection.forEach(f => {
|
||||
Object.keys(f.output).forEach(out => {
|
||||
var dest = path.join(outputRoot, f.output[out].path)
|
||||
requiredFiles.push(dest)
|
||||
})
|
||||
})
|
||||
const useless = _.difference(diskFiles, requiredFiles)
|
||||
if (useless.length) {
|
||||
// const bar = progress.create('Cleaning up', useless.length)
|
||||
useless.forEach(f => {
|
||||
observer.next(path.relative(outputRoot, f))
|
||||
fs.unlinkSync(f)
|
||||
})
|
||||
}
|
||||
observer.complete()
|
||||
})
|
||||
}
|
||||
@ -0,0 +1,57 @@
|
||||
/*
|
||||
--------------------------------------------------------------------------------
|
||||
Provides most metadata based on the output of <exiftool>
|
||||
Caches the resulting DB in <metadata.json> for faster re-runs
|
||||
--------------------------------------------------------------------------------
|
||||
*/
|
||||
|
||||
const debug = require('debug')('thumbsup')
|
||||
const exifdb = require('exiftool-json-db')
|
||||
const Observable = require('zen-observable')
|
||||
|
||||
exports.run = function (media, databasePath, callback) {
|
||||
return new Observable(observer => {
|
||||
var count = 0
|
||||
var total = 0
|
||||
var emitter = null
|
||||
|
||||
try {
|
||||
emitter = exifdb.create({media: media, database: databasePath})
|
||||
} catch (ex) {
|
||||
const message = [
|
||||
'Loading database',
|
||||
ex.toString(),
|
||||
'If migrating from thumbsup v1, delete <metadata.json> to rebuild the database from scratch'
|
||||
]
|
||||
observer.error(new Error(message.join('\n')))
|
||||
}
|
||||
|
||||
// once we know how many files need to be read
|
||||
emitter.on('stats', (stats) => {
|
||||
debug(`Database stats: total=${stats.total}`)
|
||||
total = stats.added + stats.modified
|
||||
reportProgress()
|
||||
})
|
||||
|
||||
// after every file is read
|
||||
emitter.on('file', file => {
|
||||
++count
|
||||
reportProgress()
|
||||
})
|
||||
|
||||
// when finished
|
||||
emitter.on('done', files => {
|
||||
callback(null, {database: files})
|
||||
observer.complete()
|
||||
})
|
||||
|
||||
// on error
|
||||
emitter.on('error', err => observer.error(err))
|
||||
|
||||
function reportProgress () {
|
||||
if (total === 0) return
|
||||
const percent = count * 100 / total
|
||||
observer.next(`Updated ${count}/${total} files (${percent}%)`)
|
||||
}
|
||||
})
|
||||
}
|
||||
@ -0,0 +1,22 @@
|
||||
const Picasa = require('../input/picasa')
|
||||
const mapper = require('../input/mapper')
|
||||
const hierarchy = require('../input/hierarchy.js')
|
||||
const File = require('../model/file')
|
||||
const Metadata = require('../model/metadata')
|
||||
|
||||
exports.run = function (database, opts, callback) {
|
||||
const picasaReader = new Picasa()
|
||||
// create a flat array of files
|
||||
const files = database.map(entry => {
|
||||
// create standarised metadata model
|
||||
const picasa = picasaReader.file(entry.SourceFile)
|
||||
const meta = new Metadata(entry, picasa || {})
|
||||
// create a file entry for the albums
|
||||
return new File(entry, meta, opts)
|
||||
})
|
||||
// create the full album hierarchy
|
||||
const albumMapper = mapper.create(opts)
|
||||
const album = hierarchy.createAlbums(files, albumMapper, opts)
|
||||
// return the results
|
||||
return {files, album}
|
||||
}
|
||||
@ -0,0 +1,21 @@
|
||||
|
||||
exports.create = function (ctx) {
|
||||
const stats = contextStats(ctx)
|
||||
const messages = [
|
||||
' Gallery generated successfully',
|
||||
` ${stats.albums} albums, ${stats.photos} photos, ${stats.videos} videos`
|
||||
]
|
||||
return messages.join('\n')
|
||||
}
|
||||
|
||||
function contextStats (ctx) {
|
||||
return {
|
||||
albums: countAlbums(0, ctx.album) - 1,
|
||||
photos: ctx.files.filter(f => f.type === 'image').length,
|
||||
videos: ctx.files.filter(f => f.type === 'video').length
|
||||
}
|
||||
}
|
||||
|
||||
function countAlbums (total, album) {
|
||||
return 1 + album.albums.reduce(countAlbums, total)
|
||||
}
|
||||
@ -1,63 +0,0 @@
|
||||
const pad = require('pad')
|
||||
const ProgressBar = require('progress')
|
||||
const util = require('util')
|
||||
|
||||
exports.create = function (message, count) {
|
||||
var format = ''
|
||||
if (typeof count === 'undefined') {
|
||||
format = pad(message, 20) + '[:bar] :eta'
|
||||
return new BetterProgressBar(format, 1)
|
||||
}
|
||||
if (Array.isArray(count)) count = count.length
|
||||
if (count > 0) {
|
||||
format = pad(message, 20) + '[:bar] :current/:total :eta'
|
||||
return new BetterProgressBar(format, count)
|
||||
} else {
|
||||
format = pad(message, 20) + '[:bar] up to date'
|
||||
var bar = new BetterProgressBar(format, 1)
|
||||
bar.tick(1)
|
||||
return bar
|
||||
}
|
||||
}
|
||||
|
||||
function BetterProgressBar (format, count) {
|
||||
ProgressBar.call(this, format, { total: count, width: 25 })
|
||||
this.tick(0)
|
||||
}
|
||||
|
||||
util.inherits(BetterProgressBar, ProgressBar)
|
||||
|
||||
BetterProgressBar.prototype.eta = function () {
|
||||
var ratio = this.curr / this.total
|
||||
ratio = Math.min(Math.max(ratio, 0), 1)
|
||||
var percent = ratio * 100
|
||||
var elapsed = new Date() - this.start
|
||||
return (percent === 100) ? 0 : elapsed * ((this.total / this.curr) - 1)
|
||||
}
|
||||
|
||||
BetterProgressBar.prototype.render = function (tokens) {
|
||||
const str = formatEta(this.eta())
|
||||
const actualFormat = this.fmt
|
||||
// improve display of ETA
|
||||
this.fmt = this.fmt.replace(':eta', str)
|
||||
ProgressBar.prototype.render.call(this, tokens)
|
||||
this.fmt = actualFormat
|
||||
}
|
||||
|
||||
function formatEta (ms) {
|
||||
var min = 0
|
||||
var sec = 0
|
||||
if (isNaN(ms) || !isFinite(ms)) return ''
|
||||
if (ms > 60 * 1000) {
|
||||
min = Math.floor(ms / 60 / 1000)
|
||||
return `(${min.toFixed(0)}min left)`
|
||||
} else if (ms > 10 * 1000) {
|
||||
sec = Math.floor(ms / 10000) * 10
|
||||
return `(${sec.toFixed(0)}s left)`
|
||||
} else if (ms > 0) {
|
||||
sec = ms / 1000
|
||||
return `(a few seconds left)`
|
||||
} else {
|
||||
return 'done'
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue