feat: support multiple scan modes for partial gallery generation

src/cli/options.js

@@ -29,6 +29,12 @@ const OPTIONS = {
   // ------------------------------------
   // Input options
   // ------------------------------------
+  'scan-mode': {
+    group: 'Input options:',
+    description: 'How files are indexed',
+    choices: ['full', 'partial', 'incremental'],
+    'default': 'full'
+  },
   'include-photos': {
     group: 'Input options:',
     description: 'Include photos in the gallery',

src/components/index/delta.js

@@ -1,28 +1,44 @@
 /* eslint-disable no-prototype-builtins */
 const _ = require('lodash')
+const GlobPattern = require('./pattern')
 
 /*
   Calculate the difference between files on disk and already indexed
   - databaseMap = hashmap of {path, timestamp}
   - diskMap = hashmap of {path, timestamp}
 */
-exports.calculate = (databaseMap, diskMap) => {
+exports.calculate = (databaseMap, diskMap, { scanMode = 'full', include, exclude }) => {
   const delta = {
     unchanged: [],
     added: [],
     modified: [],
-    deleted: []
+    deleted: [],
+    skipped: []
   }
+  // TODO: the glob pattern should be passed in
+  // It should be identical to the one used by the Glob object that scans the disk
+  // For now, partial scans only uses the include/exclude filter
+  // If we pass it it, other filters would apply as well (e.g. photo/video/raw...)
+  const pattern = new GlobPattern({ include, exclude, extensions: [] })
   _.each(databaseMap, (dbTime, dbPath) => {
-    if (diskMap.hasOwnProperty(dbPath)) {
+    const shouldProcessDBEntry = (scanMode === 'full') ? true : pattern.match(dbPath)
-      const modified = Math.abs(dbTime - diskMap[dbPath]) > 1000
+    if (shouldProcessDBEntry) {
-      if (modified) {
+      if (diskMap.hasOwnProperty(dbPath)) {
-        delta.modified.push(dbPath)
+        const modified = Math.abs(dbTime - diskMap[dbPath]) > 1000
+        if (modified) {
+          delta.modified.push(dbPath)
+        } else {
+          delta.unchanged.push(dbPath)
+        }
       } else {
-        delta.unchanged.push(dbPath)
+        if (scanMode === 'incremental') {
+          delta.skipped.push(dbPath)
+        } else {
+          delta.deleted.push(dbPath)
+        }
       }
     } else {
-      delta.deleted.push(dbPath)
+      delta.skipped.push(dbPath)
     }
   })
   _.each(diskMap, (diskTime, diskPath) => {

src/components/index/index.js

@@ -57,12 +57,13 @@ class Index {
       if (err) return console.error('error', err)
 
       // calculate the difference: which files have been added, modified, etc
-      const deltaFiles = delta.calculate(databaseMap, diskMap)
+      const deltaFiles = delta.calculate(databaseMap, diskMap, options)
       emitter.emit('stats', {
         unchanged: deltaFiles.unchanged.length,
         added: deltaFiles.added.length,
         modified: deltaFiles.modified.length,
         deleted: deltaFiles.deleted.length,
+        skipped: deltaFiles.skipped.length,
         total: Object.keys(diskMap).length
       })
 

src/components/index/pattern.js

@@ -4,10 +4,10 @@ const micromatch = require('micromatch')
 
 class GlobPattern {
   constructor ({ include, exclude, extensions }) {
-    this.includeList = include
+    this.includeList = (include && include.length > 0) ? include : ['**/**']
-    this.excludeList = exclude
+    this.excludeList = exclude || []
     this.includeFolders = _.uniq(_.flatMap(this.includeList, this.subFolders))
-    this.directoryExcludeList = exclude.concat(['**/@eaDir/**', '#recycle/**'])
+    this.directoryExcludeList = this.excludeList.concat(['**/@eaDir/**', '#recycle/**'])
     this.extensions = extPattern(extensions)
   }
 
@@ -43,7 +43,9 @@ class GlobPattern {
 }
 
 function extPattern (extensions) {
-  if (extensions.length === 1) {
+  if (extensions.length === 0) {
+    return '**/*'
+  } else if (extensions.length === 1) {
     return '**/*.' + extensions[0]
   } else {
     return '**/*.{' + extensions.join(',') + '}'

test/components/index/delta.spec.js

@@ -2,113 +2,213 @@ const delta = require('../../../src/components/index/delta')
 const should = require('should/as-function')
 
 describe('Index: delta', () => {
-  it('no changes', () => {
+  describe('Scan mode: full', () => {
-    const database = {
+    it('no changes', () => {
-      IMG_0001: 1410000000000,
+      const database = {
-      IMG_0002: 1420000000000
+        IMG_0001: 1410000000000,
-    }
+        IMG_0002: 1420000000000
-    const disk = {
+      }
-      IMG_0001: 1410000000000,
+      const disk = {
-      IMG_0002: 1420000000000
+        IMG_0001: 1410000000000,
-    }
+        IMG_0002: 1420000000000
-    const res = delta.calculate(database, disk)
+      }
-    should(res).eql({
+      const res = delta.calculate(database, disk, {})
-      unchanged: ['IMG_0001', 'IMG_0002'],
+      should(res).eql({
-      added: [],
+        unchanged: ['IMG_0001', 'IMG_0002'],
-      modified: [],
+        added: [],
-      deleted: []
+        modified: [],
+        deleted: [],
+        skipped: []
+      })
     })
-  })
 
-  it('no changes within a second', () => {
+    it('no changes within a second', () => {
-    const database = {
+      const database = {
-      IMG_0001: 1410000001000,
+        IMG_0001: 1410000001000,
-      IMG_0002: 1420000001000
+        IMG_0002: 1420000001000
-    }
+      }
-    const disk = {
+      const disk = {
-      IMG_0001: 1410000001500, // 500ms later
+        IMG_0001: 1410000001500, // 500ms later
-      IMG_0002: 1420000000500 // 500ms earlier
+        IMG_0002: 1420000000500 // 500ms earlier
-    }
+      }
-    const res = delta.calculate(database, disk)
+      const res = delta.calculate(database, disk, {})
-    should(res).eql({
+      should(res).eql({
-      unchanged: ['IMG_0001', 'IMG_0002'],
+        unchanged: ['IMG_0001', 'IMG_0002'],
-      added: [],
+        added: [],
-      modified: [],
+        modified: [],
-      deleted: []
+        deleted: [],
+        skipped: []
+      })
     })
-  })
 
-  it('new files', () => {
+    it('new files', () => {
-    const database = {
+      const database = {
-      IMG_0001: 1410000000000,
+        IMG_0001: 1410000000000,
-      IMG_0002: 1420000000000
+        IMG_0002: 1420000000000
-    }
+      }
-    const disk = {
+      const disk = {
-      IMG_0001: 1410000000000,
+        IMG_0001: 1410000000000,
-      IMG_0002: 1420000000000,
+        IMG_0002: 1420000000000,
-      IMG_0003: 1430000000000
+        IMG_0003: 1430000000000
-    }
+      }
-    const res = delta.calculate(database, disk)
+      const res = delta.calculate(database, disk, {})
-    should(res).eql({
+      should(res).eql({
-      unchanged: ['IMG_0001', 'IMG_0002'],
+        unchanged: ['IMG_0001', 'IMG_0002'],
-      added: ['IMG_0003'],
+        added: ['IMG_0003'],
-      modified: [],
+        modified: [],
-      deleted: []
+        deleted: [],
+        skipped: []
+      })
     })
-  })
 
-  it('deleted files', () => {
+    it('deleted files', () => {
-    const database = {
+      const database = {
-      IMG_0001: 1410000000000,
+        IMG_0001: 1410000000000,
-      IMG_0002: 1420000000000
+        IMG_0002: 1420000000000
-    }
+      }
-    const disk = {
+      const disk = {
-      IMG_0001: 1410000000000
+        IMG_0001: 1410000000000
-    }
+      }
-    const res = delta.calculate(database, disk)
+      const res = delta.calculate(database, disk, {})
-    should(res).eql({
+      should(res).eql({
-      unchanged: ['IMG_0001'],
+        unchanged: ['IMG_0001'],
-      added: [],
+        added: [],
-      modified: [],
+        modified: [],
-      deleted: ['IMG_0002']
+        deleted: ['IMG_0002'],
+        skipped: []
+      })
+    })
+
+    it('modified files', () => {
+      const database = {
+        IMG_0001: 1410000000000,
+        IMG_0002: 1420000000000
+      }
+      const disk = {
+        IMG_0001: 1410000000000,
+        IMG_0002: 1420000002000
+      }
+      const res = delta.calculate(database, disk, {})
+      should(res).eql({
+        unchanged: ['IMG_0001'],
+        added: [],
+        modified: ['IMG_0002'],
+        deleted: [],
+        skipped: []
+      })
+    })
+
+    it('all cases', () => {
+      const database = {
+        IMG_0001: 1410000000000,
+        IMG_0002: 1420000000000,
+        IMG_0003: 1430000000000
+      }
+      const disk = {
+        IMG_0001: 1410000000000,
+        IMG_0002: 1420000002000,
+        IMG_0004: 1445000000000
+      }
+      const res = delta.calculate(database, disk, {})
+      should(res).eql({
+        unchanged: ['IMG_0001'],
+        added: ['IMG_0004'],
+        modified: ['IMG_0002'],
+        deleted: ['IMG_0003'],
+        skipped: []
+      })
     })
   })
 
-  it('modified files', () => {
+  describe('Scan mode: partial', () => {
-    const database = {
+    it('considers deleted files outside the inclusion pattern as skipped', () => {
-      IMG_0001: 1410000000000,
+      const database = {
-      IMG_0002: 1420000000000
+        'London/IMG_0001': 1410000000000,
-    }
+        'Tokyo/IMG_0002': 1420000000000
-    const disk = {
+      }
-      IMG_0001: 1410000000000,
+      const disk = {
-      IMG_0002: 1420000002000
+        'London/IMG_0001': 1410000000000
-    }
+      }
-    const res = delta.calculate(database, disk)
+      const res = delta.calculate(database, disk, {
-    should(res).eql({
+        scanMode: 'incremental',
-      unchanged: ['IMG_0001'],
+        include: ['London/**'],
-      added: [],
+        exclude: []
-      modified: ['IMG_0002'],
+      })
-      deleted: []
+      should(res).eql({
+        unchanged: ['London/IMG_0001'],
+        added: [],
+        modified: [],
+        deleted: [],
+        skipped: ['Tokyo/IMG_0002']
+      })
+    })
+
+    it('considers deleted files matching an exclusion pattern as skipped', () => {
+      const database = {
+        'London/IMG_0001': 1410000000000,
+        'Tokyo/IMG_0002': 1420000000000
+      }
+      const disk = {
+        'London/IMG_0001': 1410000000000
+      }
+      const res = delta.calculate(database, disk, {
+        scanMode: 'incremental',
+        include: [],
+        exclude: ['Tokyo/**']
+      })
+      should(res).eql({
+        unchanged: ['London/IMG_0001'],
+        added: [],
+        modified: [],
+        deleted: [],
+        skipped: ['Tokyo/IMG_0002']
+      })
+    })
+
+    it('considers files inside the inclusion pattern as deleted', () => {
+      const database = {
+        'London/IMG_0001': 1410000000000,
+        'Tokyo/IMG_0002': 1420000000000
+      }
+      const disk = {
+        'London/IMG_0001': 1410000000000
+      }
+      const res = delta.calculate(database, disk, {
+        scanMode: 'partial',
+        include: ['**/**'],
+        exclude: []
+      })
+      should(res).eql({
+        unchanged: ['London/IMG_0001'],
+        added: [],
+        modified: [],
+        deleted: ['Tokyo/IMG_0002'],
+        skipped: []
+      })
     })
   })
 
-  it('all cases', () => {
+  describe('Scan mode: incremental', () => {
-    const database = {
+    it('considers files inside the inclusion pattern as skipped', () => {
-      IMG_0001: 1410000000000,
+      const database = {
-      IMG_0002: 1420000000000,
+        'London/IMG_0001': 1410000000000,
-      IMG_0003: 1430000000000
+        'Tokyo/IMG_0002': 1420000000000
-    }
+      }
-    const disk = {
+      const disk = {
-      IMG_0001: 1410000000000,
+        'London/IMG_0001': 1410000000000
-      IMG_0002: 1420000002000,
+      }
-      IMG_0004: 1445000000000
+      const res = delta.calculate(database, disk, {
-    }
+        scanMode: 'incremental',
-    const res = delta.calculate(database, disk)
+        include: [],
-    should(res).eql({
+        exclude: []
-      unchanged: ['IMG_0001'],
+      })
-      added: ['IMG_0004'],
+      should(res).eql({
-      modified: ['IMG_0002'],
+        unchanged: ['London/IMG_0001'],
-      deleted: ['IMG_0003']
+        added: [],
+        modified: [],
+        deleted: [],
+        skipped: ['Tokyo/IMG_0002']
+      })
     })
   })
 })

test/components/index/index.spec.js

@@ -1,4 +1,3 @@
-const fs = require('fs')
 const path = require('path')
 const should = require('should/as-function')
 const Index = require('../../../src/components/index/index')
@@ -9,18 +8,18 @@ describe('Index', function () {
   this.timeout(1000)
 
   let tmpdir = null
+  const image = fixtures.fromDisk('photo.jpg')
 
-  before(() => {
+  beforeEach(() => {
-    const image = fixtures.fromDisk('photo.jpg')
     tmpdir = fixtures.createTempStructure({
       'input/london/IMG_0001.jpg': image,
       'input/newyork/IMG_0002.jpg': image
     })
   })
 
-  it('indexes a folder', (done) => {
+  function runIndex (options, done) {
     const index = new Index(path.join(tmpdir, 'thumbsup.db'))
-    const emitter = index.update(path.join(tmpdir, 'input'))
+    const emitter = index.update(path.join(tmpdir, 'input'), options)
     const emitted = []
     let processed = 0
     let stats = null
@@ -28,62 +27,74 @@ describe('Index', function () {
     emitter.on('file', meta => emitted.push(meta))
     emitter.on('stats', s => { stats = s })
     emitter.on('done', result => {
-      // check stats
+      done({ result, stats, emitted, processed })
-      should(result.count).eql(2)
+    })
-      should(stats).eql({ unchanged: 0, added: 2, modified: 0, deleted: 0, total: 2 })
+  }
+
+  it('indexes a folder', (done) => {
+    runIndex({}, data => {
+      should(data.result.count).eql(2)
+      should(data.stats).eql({ unchanged: 0, added: 2, modified: 0, deleted: 0, skipped: 0, total: 2 })
       // check all files were indexed
-      const paths = emitted.map(e => e.path).sort()
+      const paths = data.emitted.map(e => e.path).sort()
       should(paths).eql([
         'london/IMG_0001.jpg',
         'newyork/IMG_0002.jpg'
       ])
       // check all files were sent to exiftool
-      should(processed).eql(2)
+      should(data.processed).eql(2)
       // check the path matches the SourceFile property
-      const sourceFiles = emitted.map(e => e.metadata.SourceFile).sort()
+      const sourceFiles = data.emitted.map(e => e.metadata.SourceFile).sort()
       should(paths).eql(sourceFiles)
       done()
     })
   })
 
   it('can re-index with no changes', (done) => {
-    const index = new Index(path.join(tmpdir, 'thumbsup.db'))
+    runIndex({}, () => {
-    const emitter = index.update(path.join(tmpdir, 'input'))
+      // then do a second run
-    let emitted = 0
+      runIndex({}, data => {
-    let processed = 0
+        should(data.result.count).eql(2)
-    let stats = null
+        should(data.stats).eql({ unchanged: 2, added: 0, modified: 0, deleted: 0, skipped: 0, total: 2 })
-    emitter.on('progress', () => ++processed)
+        // all files are emitted, but they were not processed again
-    emitter.on('file', () => ++emitted)
+        should(data.emitted).has.length(2)
-    emitter.on('stats', s => { stats = s })
+        should(data.processed).eql(0)
-    emitter.on('done', result => {
+        done()
-      // check stats
+      })
-      should(result.count).eql(2)
-      should(stats).eql({ unchanged: 2, added: 0, modified: 0, deleted: 0, total: 2 })
-      // all files are emitted, but they were not processed again
-      should(emitted).eql(2)
-      should(processed).eql(0)
-      done()
     })
   })
 
   it('can un-index a deleted file', (done) => {
-    fs.unlinkSync(path.join(tmpdir, 'input/newyork/IMG_0002.jpg'))
+    runIndex({}, () => {
-    const index = new Index(path.join(tmpdir, 'thumbsup.db'))
+      // then do a second run
-    const emitter = index.update(path.join(tmpdir, 'input'))
+      fixtures.deleteTempFile(tmpdir, 'input/newyork/IMG_0002.jpg')
-    let emitted = 0
+      runIndex({}, data => {
-    let processed = 0
+        should(data.result.count).eql(1)
-    let stats = null
+        should(data.stats).eql({ unchanged: 1, added: 0, modified: 0, deleted: 1, skipped: 0, total: 1 })
-    emitter.on('progress', () => ++processed)
+        // the remaining file was emitted
-    emitter.on('file', () => ++emitted)
+        should(data.emitted).has.length(1)
-    emitter.on('stats', s => { stats = s })
+        should(data.processed).eql(0)
-    emitter.on('done', result => {
+        done()
-      // check stats
+      })
-      should(result.count).eql(1)
+    })
-      should(stats).eql({ unchanged: 1, added: 0, modified: 0, deleted: 1, total: 1 })
+  })
-      // the remaining file was emitted
+
-      should(emitted).eql(1)
+  describe('scan modes', () => {
-      should(processed).eql(0)
+    it('partial ignores changes outside the include pattern', (done) => {
-      done()
+      runIndex({}, () => {
+        // then do a second run
+        fixtures.deleteTempFile(tmpdir, 'input/newyork/IMG_0002.jpg')
+        const options = { scanMode: 'partial', include: ['london/**'] }
+        runIndex(options, data => {
+          should(data.result.count).eql(2)
+          // note: total is 1 because it scanned 1 file on disk
+          should(data.stats).eql({ unchanged: 1, added: 0, modified: 0, deleted: 0, skipped: 1, total: 1 })
+          // but it still emitted 2 files
+          should(data.emitted).has.length(2)
+          should(data.processed).eql(0)
+          done()
+        })
+      })
     })
   })
 

test/fixtures.js

@@ -72,6 +72,10 @@ exports.createTempStructure = function (files) {
   return tmpdir
 }
 
+exports.deleteTempFile = function (tmpdir, filepath) {
+  fs.unlinkSync(path.join(tmpdir, filepath))
+}
+
 // convert to OS-dependent style paths for testing
 exports.ospath = function (filepath) {
   return filepath.replace(/\//g, path.sep)

test/integration/integration-scan-modes.js

@@ -0,0 +1,100 @@
+const should = require('should/as-function')
+const IntegrationTest = require('./integration-test')
+const fixtures = require('../fixtures')
+
+describe('Integration: scan modes', function () {
+  this.slow(5000)
+  this.timeout(5000)
+
+  const image = fixtures.fromDisk('photo.jpg')
+
+  beforeEach(IntegrationTest.before)
+  afterEach(IntegrationTest.after)
+
+  function newIntegrationTest () {
+    return new IntegrationTest({
+      'input/london/IMG_0001.jpg': image,
+      'input/london/IMG_0002.jpg': image,
+      'input/newyork/day 1/IMG_0003.jpg': image,
+      'input/newyork/day 2/IMG_0004.jpg': image
+    })
+  }
+
+  describe('Full', () => {
+    it('removes files that no longer exist in the source', function (done) {
+      const integration = newIntegrationTest()
+      integration.run(['--scan-mode', 'full'], () => {
+        const london1 = integration.parseYaml('london.html')
+        should(london1.files).have.length(2)
+        // delete a file and run again
+        integration.deleteInputFile('input/london/IMG_0002.jpg')
+        integration.run(['--scan-mode', 'full', '--cleanup', 'true'], () => {
+          const london2 = integration.parseYaml('london.html')
+          // the deleted file was removed
+          should(london2.files).have.length(1)
+          integration.assertNotExist(['media/thumbs/london/IMG_0002.jpg'])
+          done()
+        })
+      })
+    })
+
+    it("removes files that don't match the include filter", function (done) {
+      const integration = newIntegrationTest()
+      integration.run(['--scan-mode', 'full'], () => {
+        // first run, there's 2 albums (London + New York)
+        const index1 = integration.parseYaml('index.html')
+        should(index1.albums).have.length(2)
+        // run again, only including New York
+        integration.run(['--scan-mode', 'full', '--include', 'newyork/**', '--cleanup', 'true'], () => {
+          const index2 = integration.parseYaml('index.html')
+          // the London album is no longer there
+          should(index2.albums).have.length(1)
+          should(index2.albums[0].title).eql('newyork')
+          integration.assertNotExist(['media/thumbs/london/IMG_0001.jpg'])
+          done()
+        })
+      })
+    })
+  })
+
+  describe('Partial', () => {
+    it('ignores changes outside the include pattern', function (done) {
+      const integration = newIntegrationTest()
+      integration.run(['--scan-mode', 'full'], () => {
+        const london1 = integration.parseYaml('london.html')
+        should(london1.files).have.length(2)
+        integration.deleteInputFile('input/london/IMG_0002.jpg')
+        // run again, with only processing New York
+        integration.run(['--scan-mode', 'partial', '--include', 'newyork/**', '--cleanup', 'true'], () => {
+          // the London album still exists
+          const index2 = integration.parseYaml('index.html')
+          should(index2.albums).have.length(2)
+          // and it still has 2 files
+          const london2 = integration.parseYaml('london.html')
+          should(london2.files).have.length(2)
+          // and the excluded thumbnails are not deleted
+          integration.assertExist(['media/thumbs/london/IMG_0002.jpg'])
+          done()
+        })
+      })
+    })
+  })
+
+  describe('Incremental', () => {
+    it('does not remove deleted files', function (done) {
+      const integration = newIntegrationTest()
+      integration.run(['--scan-mode', 'full'], () => {
+        const london1 = integration.parseYaml('london.html')
+        should(london1.files).have.length(2)
+        // run again after deleting a file
+        integration.deleteInputFile('input/london/IMG_0002.jpg')
+        integration.run(['--scan-mode', 'incremental'], () => {
+          const london2 = integration.parseYaml('london.html')
+          should(london2.files).have.length(2)
+          integration.assertExist(['media/thumbs/london/IMG_0002.jpg'])
+          done()
+        })
+      })
+    })
+  })
+})

test/integration/integration-test.js

@@ -39,11 +39,20 @@ class IntegrationTest {
     })
   }
 
+  deleteInputFile (filepath) {
+    fixtures.deleteTempFile(this.tmpdir, filepath)
+  }
+
   assertExist (expected) {
     const missing = expected.filter(f => this.actualFiles.indexOf(f) === -1)
     should(missing).eql([])
   }
 
+  assertNotExist (expected) {
+    const present = expected.filter(f => this.actualFiles.indexOf(f) !== -1)
+    should(present).eql([])
+  }
+
   parse (filepath) {
     const fullpath = path.join(this.output, filepath)
     return fs.readFileSync(fullpath, { encoding: 'utf8' })