diff --git a/src/cli/options.js b/src/cli/options.js index 2ee301d..0c56e8f 100644 --- a/src/cli/options.js +++ b/src/cli/options.js @@ -29,6 +29,12 @@ const OPTIONS = { // ------------------------------------ // Input options // ------------------------------------ + 'scan-mode': { + group: 'Input options:', + description: 'How files are indexed', + choices: ['full', 'partial', 'incremental'], + 'default': 'full' + }, 'include-photos': { group: 'Input options:', description: 'Include photos in the gallery', diff --git a/src/components/index/delta.js b/src/components/index/delta.js index faa7116..d29cb79 100644 --- a/src/components/index/delta.js +++ b/src/components/index/delta.js @@ -1,28 +1,44 @@ /* eslint-disable no-prototype-builtins */ const _ = require('lodash') +const GlobPattern = require('./pattern') /* Calculate the difference between files on disk and already indexed - databaseMap = hashmap of {path, timestamp} - diskMap = hashmap of {path, timestamp} */ -exports.calculate = (databaseMap, diskMap) => { +exports.calculate = (databaseMap, diskMap, { scanMode = 'full', include, exclude }) => { const delta = { unchanged: [], added: [], modified: [], - deleted: [] + deleted: [], + skipped: [] } + // TODO: the glob pattern should be passed in + // It should be identical to the one used by the Glob object that scans the disk + // For now, partial scans only uses the include/exclude filter + // If we pass it it, other filters would apply as well (e.g. photo/video/raw...) + const pattern = new GlobPattern({ include, exclude, extensions: [] }) _.each(databaseMap, (dbTime, dbPath) => { - if (diskMap.hasOwnProperty(dbPath)) { - const modified = Math.abs(dbTime - diskMap[dbPath]) > 1000 - if (modified) { - delta.modified.push(dbPath) + const shouldProcessDBEntry = (scanMode === 'full') ? true : pattern.match(dbPath) + if (shouldProcessDBEntry) { + if (diskMap.hasOwnProperty(dbPath)) { + const modified = Math.abs(dbTime - diskMap[dbPath]) > 1000 + if (modified) { + delta.modified.push(dbPath) + } else { + delta.unchanged.push(dbPath) + } } else { - delta.unchanged.push(dbPath) + if (scanMode === 'incremental') { + delta.skipped.push(dbPath) + } else { + delta.deleted.push(dbPath) + } } } else { - delta.deleted.push(dbPath) + delta.skipped.push(dbPath) } }) _.each(diskMap, (diskTime, diskPath) => { diff --git a/src/components/index/index.js b/src/components/index/index.js index c3b7fc8..b7f1bca 100644 --- a/src/components/index/index.js +++ b/src/components/index/index.js @@ -57,12 +57,13 @@ class Index { if (err) return console.error('error', err) // calculate the difference: which files have been added, modified, etc - const deltaFiles = delta.calculate(databaseMap, diskMap) + const deltaFiles = delta.calculate(databaseMap, diskMap, options) emitter.emit('stats', { unchanged: deltaFiles.unchanged.length, added: deltaFiles.added.length, modified: deltaFiles.modified.length, deleted: deltaFiles.deleted.length, + skipped: deltaFiles.skipped.length, total: Object.keys(diskMap).length }) diff --git a/src/components/index/pattern.js b/src/components/index/pattern.js index f59f33d..ead17d4 100644 --- a/src/components/index/pattern.js +++ b/src/components/index/pattern.js @@ -4,10 +4,10 @@ const micromatch = require('micromatch') class GlobPattern { constructor ({ include, exclude, extensions }) { - this.includeList = include - this.excludeList = exclude + this.includeList = (include && include.length > 0) ? include : ['**/**'] + this.excludeList = exclude || [] this.includeFolders = _.uniq(_.flatMap(this.includeList, this.subFolders)) - this.directoryExcludeList = exclude.concat(['**/@eaDir/**', '#recycle/**']) + this.directoryExcludeList = this.excludeList.concat(['**/@eaDir/**', '#recycle/**']) this.extensions = extPattern(extensions) } @@ -43,7 +43,9 @@ class GlobPattern { } function extPattern (extensions) { - if (extensions.length === 1) { + if (extensions.length === 0) { + return '**/*' + } else if (extensions.length === 1) { return '**/*.' + extensions[0] } else { return '**/*.{' + extensions.join(',') + '}' diff --git a/test/components/index/delta.spec.js b/test/components/index/delta.spec.js index 1b50331..c033f59 100644 --- a/test/components/index/delta.spec.js +++ b/test/components/index/delta.spec.js @@ -2,113 +2,213 @@ const delta = require('../../../src/components/index/delta') const should = require('should/as-function') describe('Index: delta', () => { - it('no changes', () => { - const database = { - IMG_0001: 1410000000000, - IMG_0002: 1420000000000 - } - const disk = { - IMG_0001: 1410000000000, - IMG_0002: 1420000000000 - } - const res = delta.calculate(database, disk) - should(res).eql({ - unchanged: ['IMG_0001', 'IMG_0002'], - added: [], - modified: [], - deleted: [] + describe('Scan mode: full', () => { + it('no changes', () => { + const database = { + IMG_0001: 1410000000000, + IMG_0002: 1420000000000 + } + const disk = { + IMG_0001: 1410000000000, + IMG_0002: 1420000000000 + } + const res = delta.calculate(database, disk, {}) + should(res).eql({ + unchanged: ['IMG_0001', 'IMG_0002'], + added: [], + modified: [], + deleted: [], + skipped: [] + }) }) - }) - it('no changes within a second', () => { - const database = { - IMG_0001: 1410000001000, - IMG_0002: 1420000001000 - } - const disk = { - IMG_0001: 1410000001500, // 500ms later - IMG_0002: 1420000000500 // 500ms earlier - } - const res = delta.calculate(database, disk) - should(res).eql({ - unchanged: ['IMG_0001', 'IMG_0002'], - added: [], - modified: [], - deleted: [] + it('no changes within a second', () => { + const database = { + IMG_0001: 1410000001000, + IMG_0002: 1420000001000 + } + const disk = { + IMG_0001: 1410000001500, // 500ms later + IMG_0002: 1420000000500 // 500ms earlier + } + const res = delta.calculate(database, disk, {}) + should(res).eql({ + unchanged: ['IMG_0001', 'IMG_0002'], + added: [], + modified: [], + deleted: [], + skipped: [] + }) }) - }) - it('new files', () => { - const database = { - IMG_0001: 1410000000000, - IMG_0002: 1420000000000 - } - const disk = { - IMG_0001: 1410000000000, - IMG_0002: 1420000000000, - IMG_0003: 1430000000000 - } - const res = delta.calculate(database, disk) - should(res).eql({ - unchanged: ['IMG_0001', 'IMG_0002'], - added: ['IMG_0003'], - modified: [], - deleted: [] + it('new files', () => { + const database = { + IMG_0001: 1410000000000, + IMG_0002: 1420000000000 + } + const disk = { + IMG_0001: 1410000000000, + IMG_0002: 1420000000000, + IMG_0003: 1430000000000 + } + const res = delta.calculate(database, disk, {}) + should(res).eql({ + unchanged: ['IMG_0001', 'IMG_0002'], + added: ['IMG_0003'], + modified: [], + deleted: [], + skipped: [] + }) }) - }) - it('deleted files', () => { - const database = { - IMG_0001: 1410000000000, - IMG_0002: 1420000000000 - } - const disk = { - IMG_0001: 1410000000000 - } - const res = delta.calculate(database, disk) - should(res).eql({ - unchanged: ['IMG_0001'], - added: [], - modified: [], - deleted: ['IMG_0002'] + it('deleted files', () => { + const database = { + IMG_0001: 1410000000000, + IMG_0002: 1420000000000 + } + const disk = { + IMG_0001: 1410000000000 + } + const res = delta.calculate(database, disk, {}) + should(res).eql({ + unchanged: ['IMG_0001'], + added: [], + modified: [], + deleted: ['IMG_0002'], + skipped: [] + }) + }) + + it('modified files', () => { + const database = { + IMG_0001: 1410000000000, + IMG_0002: 1420000000000 + } + const disk = { + IMG_0001: 1410000000000, + IMG_0002: 1420000002000 + } + const res = delta.calculate(database, disk, {}) + should(res).eql({ + unchanged: ['IMG_0001'], + added: [], + modified: ['IMG_0002'], + deleted: [], + skipped: [] + }) + }) + + it('all cases', () => { + const database = { + IMG_0001: 1410000000000, + IMG_0002: 1420000000000, + IMG_0003: 1430000000000 + } + const disk = { + IMG_0001: 1410000000000, + IMG_0002: 1420000002000, + IMG_0004: 1445000000000 + } + const res = delta.calculate(database, disk, {}) + should(res).eql({ + unchanged: ['IMG_0001'], + added: ['IMG_0004'], + modified: ['IMG_0002'], + deleted: ['IMG_0003'], + skipped: [] + }) }) }) - it('modified files', () => { - const database = { - IMG_0001: 1410000000000, - IMG_0002: 1420000000000 - } - const disk = { - IMG_0001: 1410000000000, - IMG_0002: 1420000002000 - } - const res = delta.calculate(database, disk) - should(res).eql({ - unchanged: ['IMG_0001'], - added: [], - modified: ['IMG_0002'], - deleted: [] + describe('Scan mode: partial', () => { + it('considers deleted files outside the inclusion pattern as skipped', () => { + const database = { + 'London/IMG_0001': 1410000000000, + 'Tokyo/IMG_0002': 1420000000000 + } + const disk = { + 'London/IMG_0001': 1410000000000 + } + const res = delta.calculate(database, disk, { + scanMode: 'incremental', + include: ['London/**'], + exclude: [] + }) + should(res).eql({ + unchanged: ['London/IMG_0001'], + added: [], + modified: [], + deleted: [], + skipped: ['Tokyo/IMG_0002'] + }) + }) + + it('considers deleted files matching an exclusion pattern as skipped', () => { + const database = { + 'London/IMG_0001': 1410000000000, + 'Tokyo/IMG_0002': 1420000000000 + } + const disk = { + 'London/IMG_0001': 1410000000000 + } + const res = delta.calculate(database, disk, { + scanMode: 'incremental', + include: [], + exclude: ['Tokyo/**'] + }) + should(res).eql({ + unchanged: ['London/IMG_0001'], + added: [], + modified: [], + deleted: [], + skipped: ['Tokyo/IMG_0002'] + }) + }) + + it('considers files inside the inclusion pattern as deleted', () => { + const database = { + 'London/IMG_0001': 1410000000000, + 'Tokyo/IMG_0002': 1420000000000 + } + const disk = { + 'London/IMG_0001': 1410000000000 + } + const res = delta.calculate(database, disk, { + scanMode: 'partial', + include: ['**/**'], + exclude: [] + }) + should(res).eql({ + unchanged: ['London/IMG_0001'], + added: [], + modified: [], + deleted: ['Tokyo/IMG_0002'], + skipped: [] + }) }) }) - it('all cases', () => { - const database = { - IMG_0001: 1410000000000, - IMG_0002: 1420000000000, - IMG_0003: 1430000000000 - } - const disk = { - IMG_0001: 1410000000000, - IMG_0002: 1420000002000, - IMG_0004: 1445000000000 - } - const res = delta.calculate(database, disk) - should(res).eql({ - unchanged: ['IMG_0001'], - added: ['IMG_0004'], - modified: ['IMG_0002'], - deleted: ['IMG_0003'] + describe('Scan mode: incremental', () => { + it('considers files inside the inclusion pattern as skipped', () => { + const database = { + 'London/IMG_0001': 1410000000000, + 'Tokyo/IMG_0002': 1420000000000 + } + const disk = { + 'London/IMG_0001': 1410000000000 + } + const res = delta.calculate(database, disk, { + scanMode: 'incremental', + include: [], + exclude: [] + }) + should(res).eql({ + unchanged: ['London/IMG_0001'], + added: [], + modified: [], + deleted: [], + skipped: ['Tokyo/IMG_0002'] + }) }) }) }) diff --git a/test/components/index/index.spec.js b/test/components/index/index.spec.js index 0727c12..19c1854 100644 --- a/test/components/index/index.spec.js +++ b/test/components/index/index.spec.js @@ -1,4 +1,3 @@ -const fs = require('fs') const path = require('path') const should = require('should/as-function') const Index = require('../../../src/components/index/index') @@ -9,18 +8,18 @@ describe('Index', function () { this.timeout(1000) let tmpdir = null + const image = fixtures.fromDisk('photo.jpg') - before(() => { - const image = fixtures.fromDisk('photo.jpg') + beforeEach(() => { tmpdir = fixtures.createTempStructure({ 'input/london/IMG_0001.jpg': image, 'input/newyork/IMG_0002.jpg': image }) }) - it('indexes a folder', (done) => { + function runIndex (options, done) { const index = new Index(path.join(tmpdir, 'thumbsup.db')) - const emitter = index.update(path.join(tmpdir, 'input')) + const emitter = index.update(path.join(tmpdir, 'input'), options) const emitted = [] let processed = 0 let stats = null @@ -28,62 +27,74 @@ describe('Index', function () { emitter.on('file', meta => emitted.push(meta)) emitter.on('stats', s => { stats = s }) emitter.on('done', result => { - // check stats - should(result.count).eql(2) - should(stats).eql({ unchanged: 0, added: 2, modified: 0, deleted: 0, total: 2 }) + done({ result, stats, emitted, processed }) + }) + } + + it('indexes a folder', (done) => { + runIndex({}, data => { + should(data.result.count).eql(2) + should(data.stats).eql({ unchanged: 0, added: 2, modified: 0, deleted: 0, skipped: 0, total: 2 }) // check all files were indexed - const paths = emitted.map(e => e.path).sort() + const paths = data.emitted.map(e => e.path).sort() should(paths).eql([ 'london/IMG_0001.jpg', 'newyork/IMG_0002.jpg' ]) // check all files were sent to exiftool - should(processed).eql(2) + should(data.processed).eql(2) // check the path matches the SourceFile property - const sourceFiles = emitted.map(e => e.metadata.SourceFile).sort() + const sourceFiles = data.emitted.map(e => e.metadata.SourceFile).sort() should(paths).eql(sourceFiles) done() }) }) it('can re-index with no changes', (done) => { - const index = new Index(path.join(tmpdir, 'thumbsup.db')) - const emitter = index.update(path.join(tmpdir, 'input')) - let emitted = 0 - let processed = 0 - let stats = null - emitter.on('progress', () => ++processed) - emitter.on('file', () => ++emitted) - emitter.on('stats', s => { stats = s }) - emitter.on('done', result => { - // check stats - should(result.count).eql(2) - should(stats).eql({ unchanged: 2, added: 0, modified: 0, deleted: 0, total: 2 }) - // all files are emitted, but they were not processed again - should(emitted).eql(2) - should(processed).eql(0) - done() + runIndex({}, () => { + // then do a second run + runIndex({}, data => { + should(data.result.count).eql(2) + should(data.stats).eql({ unchanged: 2, added: 0, modified: 0, deleted: 0, skipped: 0, total: 2 }) + // all files are emitted, but they were not processed again + should(data.emitted).has.length(2) + should(data.processed).eql(0) + done() + }) }) }) it('can un-index a deleted file', (done) => { - fs.unlinkSync(path.join(tmpdir, 'input/newyork/IMG_0002.jpg')) - const index = new Index(path.join(tmpdir, 'thumbsup.db')) - const emitter = index.update(path.join(tmpdir, 'input')) - let emitted = 0 - let processed = 0 - let stats = null - emitter.on('progress', () => ++processed) - emitter.on('file', () => ++emitted) - emitter.on('stats', s => { stats = s }) - emitter.on('done', result => { - // check stats - should(result.count).eql(1) - should(stats).eql({ unchanged: 1, added: 0, modified: 0, deleted: 1, total: 1 }) - // the remaining file was emitted - should(emitted).eql(1) - should(processed).eql(0) - done() + runIndex({}, () => { + // then do a second run + fixtures.deleteTempFile(tmpdir, 'input/newyork/IMG_0002.jpg') + runIndex({}, data => { + should(data.result.count).eql(1) + should(data.stats).eql({ unchanged: 1, added: 0, modified: 0, deleted: 1, skipped: 0, total: 1 }) + // the remaining file was emitted + should(data.emitted).has.length(1) + should(data.processed).eql(0) + done() + }) + }) + }) + + describe('scan modes', () => { + it('partial ignores changes outside the include pattern', (done) => { + runIndex({}, () => { + // then do a second run + fixtures.deleteTempFile(tmpdir, 'input/newyork/IMG_0002.jpg') + const options = { scanMode: 'partial', include: ['london/**'] } + runIndex(options, data => { + should(data.result.count).eql(2) + // note: total is 1 because it scanned 1 file on disk + should(data.stats).eql({ unchanged: 1, added: 0, modified: 0, deleted: 0, skipped: 1, total: 1 }) + // but it still emitted 2 files + should(data.emitted).has.length(2) + should(data.processed).eql(0) + done() + }) + }) }) }) diff --git a/test/fixtures.js b/test/fixtures.js index 8ee140e..38da22f 100644 --- a/test/fixtures.js +++ b/test/fixtures.js @@ -72,6 +72,10 @@ exports.createTempStructure = function (files) { return tmpdir } +exports.deleteTempFile = function (tmpdir, filepath) { + fs.unlinkSync(path.join(tmpdir, filepath)) +} + // convert to OS-dependent style paths for testing exports.ospath = function (filepath) { return filepath.replace(/\//g, path.sep) diff --git a/test/integration/integration-scan-modes.js b/test/integration/integration-scan-modes.js new file mode 100644 index 0000000..bd59f4a --- /dev/null +++ b/test/integration/integration-scan-modes.js @@ -0,0 +1,100 @@ +const should = require('should/as-function') +const IntegrationTest = require('./integration-test') +const fixtures = require('../fixtures') + +describe('Integration: scan modes', function () { + this.slow(5000) + this.timeout(5000) + + const image = fixtures.fromDisk('photo.jpg') + + beforeEach(IntegrationTest.before) + afterEach(IntegrationTest.after) + + function newIntegrationTest () { + return new IntegrationTest({ + 'input/london/IMG_0001.jpg': image, + 'input/london/IMG_0002.jpg': image, + 'input/newyork/day 1/IMG_0003.jpg': image, + 'input/newyork/day 2/IMG_0004.jpg': image + }) + } + + describe('Full', () => { + it('removes files that no longer exist in the source', function (done) { + const integration = newIntegrationTest() + integration.run(['--scan-mode', 'full'], () => { + const london1 = integration.parseYaml('london.html') + should(london1.files).have.length(2) + // delete a file and run again + integration.deleteInputFile('input/london/IMG_0002.jpg') + integration.run(['--scan-mode', 'full', '--cleanup', 'true'], () => { + const london2 = integration.parseYaml('london.html') + // the deleted file was removed + should(london2.files).have.length(1) + integration.assertNotExist(['media/thumbs/london/IMG_0002.jpg']) + done() + }) + }) + }) + + it("removes files that don't match the include filter", function (done) { + const integration = newIntegrationTest() + integration.run(['--scan-mode', 'full'], () => { + // first run, there's 2 albums (London + New York) + const index1 = integration.parseYaml('index.html') + should(index1.albums).have.length(2) + // run again, only including New York + integration.run(['--scan-mode', 'full', '--include', 'newyork/**', '--cleanup', 'true'], () => { + const index2 = integration.parseYaml('index.html') + // the London album is no longer there + should(index2.albums).have.length(1) + should(index2.albums[0].title).eql('newyork') + integration.assertNotExist(['media/thumbs/london/IMG_0001.jpg']) + done() + }) + }) + }) + }) + + describe('Partial', () => { + it('ignores changes outside the include pattern', function (done) { + const integration = newIntegrationTest() + integration.run(['--scan-mode', 'full'], () => { + const london1 = integration.parseYaml('london.html') + should(london1.files).have.length(2) + integration.deleteInputFile('input/london/IMG_0002.jpg') + // run again, with only processing New York + integration.run(['--scan-mode', 'partial', '--include', 'newyork/**', '--cleanup', 'true'], () => { + // the London album still exists + const index2 = integration.parseYaml('index.html') + should(index2.albums).have.length(2) + // and it still has 2 files + const london2 = integration.parseYaml('london.html') + should(london2.files).have.length(2) + // and the excluded thumbnails are not deleted + integration.assertExist(['media/thumbs/london/IMG_0002.jpg']) + done() + }) + }) + }) + }) + + describe('Incremental', () => { + it('does not remove deleted files', function (done) { + const integration = newIntegrationTest() + integration.run(['--scan-mode', 'full'], () => { + const london1 = integration.parseYaml('london.html') + should(london1.files).have.length(2) + // run again after deleting a file + integration.deleteInputFile('input/london/IMG_0002.jpg') + integration.run(['--scan-mode', 'incremental'], () => { + const london2 = integration.parseYaml('london.html') + should(london2.files).have.length(2) + integration.assertExist(['media/thumbs/london/IMG_0002.jpg']) + done() + }) + }) + }) + }) +}) diff --git a/test/integration/integration-test.js b/test/integration/integration-test.js index a92e111..87b7bb2 100644 --- a/test/integration/integration-test.js +++ b/test/integration/integration-test.js @@ -39,11 +39,20 @@ class IntegrationTest { }) } + deleteInputFile (filepath) { + fixtures.deleteTempFile(this.tmpdir, filepath) + } + assertExist (expected) { const missing = expected.filter(f => this.actualFiles.indexOf(f) === -1) should(missing).eql([]) } + assertNotExist (expected) { + const present = expected.filter(f => this.actualFiles.indexOf(f) !== -1) + should(present).eql([]) + } + parse (filepath) { const fullpath = path.join(this.output, filepath) return fs.readFileSync(fullpath, { encoding: 'utf8' })