diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..182d70a --- /dev/null +++ b/.editorconfig @@ -0,0 +1,15 @@ +# EditorConfig is awesome: http://EditorConfig.org + +# top-most EditorConfig file +root = true + +# Unix-style newlines with a newline ending every file +[*] +end_of_line = lf +insert_final_newline = true + +# Indentation override for all JS under lib directory +[**/**.js] +charset = utf-8 +indent_style = space +indent_size = 2 diff --git a/README.md b/README.md index 2c6875c..4a05dca 100644 --- a/README.md +++ b/README.md @@ -26,11 +26,14 @@ npm install -g thumbsup thumbsup --input ./media --output ./website ``` +![Screen recording](demo.gif) + There are many more command line arguments to customise the output. See the website for the full documentation: https://thumbsup.github.io. -*Requirements* +## Requirements +Thumbsup requires the following dependencies: - [Node.js](http://nodejs.org/): `brew install Node` - [exiftool](http://www.sno.phy.queensu.ca/~phil/exiftool/): `brew install exiftool` - [GraphicsMagick](http://www.graphicsmagick.org/): `brew install graphicsmagick` @@ -107,11 +110,18 @@ The optional JSON config should contain a single object with one key per argumen We welcome all [issues](https://github.com/thumbsup/thumbsup/issues) and [pull requests](https://github.com/thumbsup/thumbsup/pulls)! -If you are facing any issues or getting crashes, you can run `thumbsup` in debug mode -to get a verbose troubleshooting log of all operations: +If you are facing any issues or getting crashes, please try the following options to help troubleshoot: ```bash -DEBUG="*" thumbsup [options] +thumbsup [options] | tee +# [16:04:56] media/thumbs/photo-1446822622709-e1c7ad6e82d52.jpg [started] +# [16:04:57] media/thumbs/photo-1446822622709-e1c7ad6e82d52.jpg [completed] + +DEBUG="*" thumbsup [options] | tee +# [16:04:56] media/thumbs/photo-1446822622709-e1c7ad6e82d52.jpg [started] +# gm "identify" "-ping" "-format" "%[EXIF:Orientation]" [...] +# gm "convert" "-quality" "90" "-resize" "x400>" "+profile" [...] +# [16:04:57] media/thumbs/photo-1446822622709-e1c7ad6e82d52.jpg [completed] ``` Please make sure the tests are passing when submitting a code change: diff --git a/demo.gif b/demo.gif new file mode 100644 index 0000000..1425aa2 Binary files /dev/null and b/demo.gif differ diff --git a/package.json b/package.json index ccfcb03..cd91850 100644 --- a/package.json +++ b/package.json @@ -35,14 +35,15 @@ "ini": "^1.3.4", "less": "^2.7.1", "lightgallery": "1.2.14", + "listr": "^0.12.0", + "listr-work-queue": "https://github.com/thumbsup/listr-work-queue.git", "lodash": "^4.16.6", "moment": "^2.16.0", - "pad": "^2.0.1", - "progress": "^2.0.0", "thumbsup-downsize": "^1.0.0", "url-join": "^2.0.2", "video.js": "^6.2.8", - "yargs": "^9.0.1" + "yargs": "^9.0.1", + "zen-observable": "^0.6.0" }, "devDependencies": { "injectmd": "^1.0.0", diff --git a/src/index.js b/src/index.js index 9a5cb2c..ed898ef 100644 --- a/src/index.js +++ b/src/index.js @@ -1,94 +1,61 @@ -const async = require('async') const fs = require('fs-extra') const path = require('path') -const os = require('os') -const cleanup = require('./output-media/cleanup') -const database = require('./input/database') -const Picasa = require('./input/picasa') -const progress = require('./utils/progress') -const hierarchy = require('./input/hierarchy.js') -const mapper = require('./input/mapper') -const File = require('./model/file') -const Metadata = require('./model/metadata') -const tasks = require('./output-media/tasks') -const website = require('./output-website/website') +const Listr = require('listr') +const steps = require('./steps/index') +const summary = require('./steps/summary') +const website = require('./website/website') exports.build = function (opts) { - fs.mkdirpSync(opts.output) - const databaseFile = path.join(opts.output, 'metadata.json') - - // all files, unsorted - var files = null - - // root album with nested albums - var album = null - - async.series([ - - function updateDatabase (callback) { - const picasaReader = new Picasa() - database.update(opts.input, databaseFile, (err, entries) => { - if (err) return callback(err) - files = entries.map(entry => { - // create standarised metadata model - const picasa = picasaReader.file(entry.SourceFile) - const meta = new Metadata(entry, picasa || {}) - // create a file entry for the albums - return new File(entry, meta, opts) + const tasks = new Listr([ + { + title: 'Updating database', + task: (ctx, task) => { + // returns an observable which will complete when the database is loaded + fs.mkdirpSync(opts.output) + const databaseFile = path.join(opts.output, 'metadata.json') + return steps.database(opts.input, databaseFile, (err, res) => { + if (!err) { + ctx.database = res.database + } }) - callback() - }) + } }, - - function processPhotos (callback) { - const photos = tasks.create(opts, files, 'image') - const bar = progress.create('Processing photos', photos.length) - parallel(photos, bar, callback) + { + title: 'Creating model', + task: (ctx) => { + const res = steps.model(ctx.database, opts) + ctx.files = res.files + ctx.album = res.album + } }, - - function processVideos (callback) { - const videos = tasks.create(opts, files, 'video') - const bar = progress.create('Processing videos', videos.length) - parallel(videos, bar, callback) + { + title: 'Processing media', + task: (ctx, task) => { + return steps.process(ctx.files, opts, task) + } }, - - function removeOldOutput (callback) { - if (!opts.cleanup) return callback() - cleanup.run(files, opts.output, callback) + { + title: 'Cleaning up', + enabled: (ctx) => opts.cleanup, + task: (ctx) => { + return steps.cleanup(ctx.files, opts.output) + } }, - - function createAlbums (callback) { - const bar = progress.create('Creating albums') - const albumMapper = mapper.create(opts) - album = hierarchy.createAlbums(files, albumMapper, opts) - bar.tick(1) - callback() - }, - - function createWebsite (callback) { - const bar = progress.create('Building website') - website.build(album, opts, (err) => { - bar.tick(1) - callback(err) + { + title: 'Creating website', + task: (ctx) => new Promise((resolve, reject) => { + website.build(ctx.album, opts, err => { + err ? reject(err) : resolve() + }) }) } - - ], finish) -} - -function parallel (tasks, bar, callback) { - const decorated = tasks.map(t => done => { - t(err => { - bar.tick(1) - done(err) - }) + ]) + + tasks.run().then(ctx => { + console.log('\n' + summary.create(ctx) + '\n') + process.exit(0) + }).catch(err => { + console.log('\nUnexpected error', err) + process.exit(1) }) - async.parallelLimit(decorated, os.cpus().length, callback) -} - -function finish (err) { - console.log(err ? 'Unexpected error' : '') - console.log(err || 'Gallery generated successfully') - console.log() - process.exit(err ? 1 : 0) } diff --git a/src/input/database.js b/src/input/database.js deleted file mode 100644 index ac452ad..0000000 --- a/src/input/database.js +++ /dev/null @@ -1,40 +0,0 @@ -/* --------------------------------------------------------------------------------- -Provides most metadata based on the output of -Caches the resulting DB in for faster re-runs --------------------------------------------------------------------------------- -*/ - -const debug = require('debug')('thumbsup') -const exifdb = require('exiftool-json-db') -const progress = require('../utils/progress') - -exports.update = function (media, databasePath, callback) { - var updateBar = null - var emitter = null - - try { - emitter = exifdb.create({media: media, database: databasePath}) - } catch (ex) { - const message = 'Loading database\n' + ex.toString() + '\n' + - 'If migrating from thumbsup v1, delete to rebuild the database from scratch' - callback(new Error(message)) - } - - emitter.on('stats', (stats) => { - debug(`Database stats: total=${stats.total}`) - const totalBar = progress.create('Finding media', stats.total) - totalBar.tick(stats.total) - updateBar = progress.create('Updating database', stats.added + stats.modified) - }) - - emitter.on('file', (file) => { - updateBar.tick() - }) - - emitter.on('done', (files) => { - callback(null, files) - }) - - emitter.on('error', callback) -} diff --git a/src/output-media/cleanup.js b/src/output-media/cleanup.js deleted file mode 100644 index d2c8fca..0000000 --- a/src/output-media/cleanup.js +++ /dev/null @@ -1,24 +0,0 @@ -const _ = require('lodash') -const fs = require('fs') -const path = require('path') -const readdir = require('fs-readdir-recursive') -const progress = require('../utils/progress') - -exports.run = function (fileCollection, outputRoot, callback) { - const mediaRoot = path.join(outputRoot, 'media') - const diskFiles = readdir(mediaRoot).map(f => path.join(mediaRoot, f)) - const requiredFiles = [] - fileCollection.forEach(f => { - Object.keys(f.output).forEach(out => { - var dest = path.join(outputRoot, f.output[out].path) - requiredFiles.push(dest) - }) - }) - const useless = _.difference(diskFiles, requiredFiles) - if (useless.length) { - const bar = progress.create('Cleaning up', useless.length) - useless.forEach(f => fs.unlinkSync(f)) - bar.tick(useless.length) - } - callback() -} diff --git a/src/steps/index.js b/src/steps/index.js new file mode 100644 index 0000000..81b0039 --- /dev/null +++ b/src/steps/index.js @@ -0,0 +1,5 @@ +exports.database = require('./step-database').run +exports.model = require('./step-model').run +exports.process = require('./step-process').run +exports.cleanup = require('./step-cleanup').run +// exports.website = require('./website') diff --git a/src/steps/step-cleanup.js b/src/steps/step-cleanup.js new file mode 100644 index 0000000..5e2a950 --- /dev/null +++ b/src/steps/step-cleanup.js @@ -0,0 +1,28 @@ +const _ = require('lodash') +const fs = require('fs') +const Observable = require('zen-observable') +const path = require('path') +const readdir = require('fs-readdir-recursive') + +exports.run = function (fileCollection, outputRoot) { + return new Observable(observer => { + const mediaRoot = path.join(outputRoot, 'media') + const diskFiles = readdir(mediaRoot).map(f => path.join(mediaRoot, f)) + const requiredFiles = [] + fileCollection.forEach(f => { + Object.keys(f.output).forEach(out => { + var dest = path.join(outputRoot, f.output[out].path) + requiredFiles.push(dest) + }) + }) + const useless = _.difference(diskFiles, requiredFiles) + if (useless.length) { + // const bar = progress.create('Cleaning up', useless.length) + useless.forEach(f => { + observer.next(path.relative(outputRoot, f)) + fs.unlinkSync(f) + }) + } + observer.complete() + }) +} diff --git a/src/steps/step-database.js b/src/steps/step-database.js new file mode 100644 index 0000000..329c042 --- /dev/null +++ b/src/steps/step-database.js @@ -0,0 +1,57 @@ +/* +-------------------------------------------------------------------------------- +Provides most metadata based on the output of +Caches the resulting DB in for faster re-runs +-------------------------------------------------------------------------------- +*/ + +const debug = require('debug')('thumbsup') +const exifdb = require('exiftool-json-db') +const Observable = require('zen-observable') + +exports.run = function (media, databasePath, callback) { + return new Observable(observer => { + var count = 0 + var total = 0 + var emitter = null + + try { + emitter = exifdb.create({media: media, database: databasePath}) + } catch (ex) { + const message = [ + 'Loading database', + ex.toString(), + 'If migrating from thumbsup v1, delete to rebuild the database from scratch' + ] + observer.error(new Error(message.join('\n'))) + } + + // once we know how many files need to be read + emitter.on('stats', (stats) => { + debug(`Database stats: total=${stats.total}`) + total = stats.added + stats.modified + reportProgress() + }) + + // after every file is read + emitter.on('file', file => { + ++count + reportProgress() + }) + + // when finished + emitter.on('done', files => { + callback(null, {database: files}) + observer.complete() + }) + + // on error + emitter.on('error', err => observer.error(err)) + + function reportProgress () { + if (total === 0) return + const percent = count * 100 / total + observer.next(`Updated ${count}/${total} files (${percent}%)`) + } + }) +} diff --git a/src/steps/step-model.js b/src/steps/step-model.js new file mode 100644 index 0000000..a7efe36 --- /dev/null +++ b/src/steps/step-model.js @@ -0,0 +1,22 @@ +const Picasa = require('../input/picasa') +const mapper = require('../input/mapper') +const hierarchy = require('../input/hierarchy.js') +const File = require('../model/file') +const Metadata = require('../model/metadata') + +exports.run = function (database, opts, callback) { + const picasaReader = new Picasa() + // create a flat array of files + const files = database.map(entry => { + // create standarised metadata model + const picasa = picasaReader.file(entry.SourceFile) + const meta = new Metadata(entry, picasa || {}) + // create a file entry for the albums + return new File(entry, meta, opts) + }) + // create the full album hierarchy + const albumMapper = mapper.create(opts) + const album = hierarchy.createAlbums(files, albumMapper, opts) + // return the results + return {files, album} +} diff --git a/src/output-media/tasks.js b/src/steps/step-process.js similarity index 53% rename from src/output-media/tasks.js rename to src/steps/step-process.js index b72a01e..a65473b 100644 --- a/src/output-media/tasks.js +++ b/src/steps/step-process.js @@ -1,18 +1,33 @@ -const fs = require('fs-extra') -const path = require('path') const debug = require('debug')('thumbsup') const downsize = require('thumbsup-downsize') +const os = require('os') +const fs = require('fs-extra') +const ListrWorkQueue = require('listr-work-queue') +const path = require('path') + +exports.run = function (files, opts, parentTask) { + const jobs = exports.create(files, opts) + // wrap each job in a Listr task that returns a Promise + const tasks = jobs.map(job => listrTaskFromJob(job, opts.output)) + const listr = new ListrWorkQueue(tasks, { + concurrent: os.cpus().length, + update: (done, total) => { + const progress = done === total ? '' : `(${done}/${total})` + parentTask.title = `Processing media ${progress}` + } + }) + return listr +} /* Return a list of task to build all required outputs (new or updated) - Can be filtered by type (image/video) to give more accurate ETAs */ -exports.create = function (opts, files, filterType) { +exports.create = function (files, opts) { var tasks = {} const actionMap = getActionMap(opts) // accumulate all tasks into an object // to remove duplicate destinations - files.filter(f => f.type === filterType).forEach(f => { + files.forEach(f => { debug(`Tasks for ${f.path}, ${JSON.stringify(f.output)}`) Object.keys(f.output).forEach(out => { var src = path.join(opts.input, f.path) @@ -21,20 +36,45 @@ exports.create = function (opts, files, filterType) { var action = actionMap[f.output[out].rel] // ignore output files that don't have an action (e.g. existing links) if (action && f.date > destDate) { - tasks[dest] = (done) => { - fs.mkdirsSync(path.dirname(dest)) - debug(`${f.output[out].rel} from ${src} to ${dest}`) - action({src: src, dest: dest}, done) + tasks[dest] = { + file: f, + dest: dest, + rel: f.output[out].rel, + action: (done) => { + fs.mkdirsSync(path.dirname(dest)) + debug(`${f.output[out].rel} from ${src} to ${dest}`) + return action({src: src, dest: dest}, done) + } } } }) }) // back into an array - const list = Object.keys(tasks).map(t => tasks[t]) - debug(`Created ${list.length} ${filterType} tasks`) + const list = Object.keys(tasks).map(dest => tasks[dest]) + debug(`Created ${list.length} tasks`) return list } +function listrTaskFromJob (job, outputRoot) { + const relative = path.relative(outputRoot, job.dest) + return { + title: relative, + task: (ctx, task) => { + return new Promise((resolve, reject) => { + var progressEmitter = job.action(err => { + err ? reject(err) : resolve() + }) + // render progress percentage for videos + if (progressEmitter) { + progressEmitter.on('progress', (percent) => { + task.title = `${relative} (${percent}%)` + }) + } + }) + } + } +} + function modifiedDate (filepath) { try { return fs.statSync(filepath).mtime.getTime() diff --git a/src/steps/summary.js b/src/steps/summary.js new file mode 100644 index 0000000..67d1390 --- /dev/null +++ b/src/steps/summary.js @@ -0,0 +1,21 @@ + +exports.create = function (ctx) { + const stats = contextStats(ctx) + const messages = [ + ' Gallery generated successfully', + ` ${stats.albums} albums, ${stats.photos} photos, ${stats.videos} videos` + ] + return messages.join('\n') +} + +function contextStats (ctx) { + return { + albums: countAlbums(0, ctx.album) - 1, + photos: ctx.files.filter(f => f.type === 'image').length, + videos: ctx.files.filter(f => f.type === 'video').length + } +} + +function countAlbums (total, album) { + return 1 + album.albums.reduce(countAlbums, total) +} diff --git a/src/utils/progress.js b/src/utils/progress.js deleted file mode 100644 index 24a3394..0000000 --- a/src/utils/progress.js +++ /dev/null @@ -1,63 +0,0 @@ -const pad = require('pad') -const ProgressBar = require('progress') -const util = require('util') - -exports.create = function (message, count) { - var format = '' - if (typeof count === 'undefined') { - format = pad(message, 20) + '[:bar] :eta' - return new BetterProgressBar(format, 1) - } - if (Array.isArray(count)) count = count.length - if (count > 0) { - format = pad(message, 20) + '[:bar] :current/:total :eta' - return new BetterProgressBar(format, count) - } else { - format = pad(message, 20) + '[:bar] up to date' - var bar = new BetterProgressBar(format, 1) - bar.tick(1) - return bar - } -} - -function BetterProgressBar (format, count) { - ProgressBar.call(this, format, { total: count, width: 25 }) - this.tick(0) -} - -util.inherits(BetterProgressBar, ProgressBar) - -BetterProgressBar.prototype.eta = function () { - var ratio = this.curr / this.total - ratio = Math.min(Math.max(ratio, 0), 1) - var percent = ratio * 100 - var elapsed = new Date() - this.start - return (percent === 100) ? 0 : elapsed * ((this.total / this.curr) - 1) -} - -BetterProgressBar.prototype.render = function (tokens) { - const str = formatEta(this.eta()) - const actualFormat = this.fmt - // improve display of ETA - this.fmt = this.fmt.replace(':eta', str) - ProgressBar.prototype.render.call(this, tokens) - this.fmt = actualFormat -} - -function formatEta (ms) { - var min = 0 - var sec = 0 - if (isNaN(ms) || !isFinite(ms)) return '' - if (ms > 60 * 1000) { - min = Math.floor(ms / 60 / 1000) - return `(${min.toFixed(0)}min left)` - } else if (ms > 10 * 1000) { - sec = Math.floor(ms / 10000) * 10 - return `(${sec.toFixed(0)}s left)` - } else if (ms > 0) { - sec = ms / 1000 - return `(a few seconds left)` - } else { - return 'done' - } -} diff --git a/src/output-website/template.js b/src/website/template.js similarity index 100% rename from src/output-website/template.js rename to src/website/template.js diff --git a/src/output-website/website.js b/src/website/website.js similarity index 100% rename from src/output-website/website.js rename to src/website/website.js