Archives
/
thumbsup
mirror of https://github.com/thumbsup/thumbsup
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

feat: support multiple scan modes for partial gallery generation

Browse Source
pull/336/head
Romain 1 year ago
parent c77654d0fe
commit 8a43cebc72
9 changed files with 401 additions and 152 deletions
Show Stats Download Patch File Download Diff File

6
src/cli/options.js
Unescape Escape View File

@ -29,6 +29,12 @@ const OPTIONS = {
// ------------------------------------
// Input options
// ------------------------------------
'scan-mode': {
group: 'Input options:',
description: 'How files are indexed',
choices: ['full', 'partial', 'incremental'],
'default': 'full'
},
'include-photos': {
group: 'Input options:',
description: 'Include photos in the gallery',

32
src/components/index/delta.js
Unescape Escape View File

@ -1,28 +1,44 @@
/* eslint-disable no-prototype-builtins */
const _ = require('lodash')
const GlobPattern = require('./pattern')
/*
Calculate the difference between files on disk and already indexed
- databaseMap = hashmap of {path, timestamp}
- diskMap = hashmap of {path, timestamp}
*/
exports.calculate = (databaseMap, diskMap) => {
exports.calculate = (databaseMap, diskMap, { scanMode = 'full', include, exclude }) => {
const delta = {
unchanged: [],
added: [],
modified: [],
deleted: []
deleted: [],
skipped: []
}
// TODO: the glob pattern should be passed in
// It should be identical to the one used by the Glob object that scans the disk
// For now, partial scans only uses the include/exclude filter
// If we pass it it, other filters would apply as well (e.g. photo/video/raw...)
const pattern = new GlobPattern({ include, exclude, extensions: [] })
_.each(databaseMap, (dbTime, dbPath) => {
if (diskMap.hasOwnProperty(dbPath)) {
const modified = Math.abs(dbTime - diskMap[dbPath]) > 1000
if (modified) {
delta.modified.push(dbPath)
const shouldProcessDBEntry = (scanMode === 'full') ? true : pattern.match(dbPath)
if (shouldProcessDBEntry) {
if (diskMap.hasOwnProperty(dbPath)) {
const modified = Math.abs(dbTime - diskMap[dbPath]) > 1000
if (modified) {
delta.modified.push(dbPath)
} else {
delta.unchanged.push(dbPath)
}
} else {
delta.unchanged.push(dbPath)
if (scanMode === 'incremental') {
delta.skipped.push(dbPath)
} else {
delta.deleted.push(dbPath)
}
}
} else {
delta.deleted.push(dbPath)
delta.skipped.push(dbPath)
}
})
_.each(diskMap, (diskTime, diskPath) => {

3
src/components/index/index.js
Unescape Escape View File

@ -57,12 +57,13 @@ class Index {
if (err) return console.error('error', err)
// calculate the difference: which files have been added, modified, etc
const deltaFiles = delta.calculate(databaseMap, diskMap)
const deltaFiles = delta.calculate(databaseMap, diskMap, options)
emitter.emit('stats', {
unchanged: deltaFiles.unchanged.length,
added: deltaFiles.added.length,
modified: deltaFiles.modified.length,
deleted: deltaFiles.deleted.length,
skipped: deltaFiles.skipped.length,
total: Object.keys(diskMap).length
})

10
src/components/index/pattern.js
Unescape Escape View File

@ -4,10 +4,10 @@ const micromatch = require('micromatch')
class GlobPattern {
constructor ({ include, exclude, extensions }) {
this.includeList = include
this.excludeList = exclude
this.includeList = (include && include.length > 0) ? include : ['**/**']
this.excludeList = exclude || []
this.includeFolders = _.uniq(_.flatMap(this.includeList, this.subFolders))
this.directoryExcludeList = exclude.concat(['**/@eaDir/**', '#recycle/**'])
this.directoryExcludeList = this.excludeList.concat(['**/@eaDir/**', '#recycle/**'])
this.extensions = extPattern(extensions)
}
@ -43,7 +43,9 @@ class GlobPattern {
}
function extPattern (extensions) {
if (extensions.length === 1) {
if (extensions.length === 0) {
return '**/*'
} else if (extensions.length === 1) {
return '**/*.' + extensions[0]
} else {
return '**/*.{' + extensions.join(',') + '}'

290
test/components/index/delta.spec.js
Unescape Escape View File

@ -2,113 +2,213 @@ const delta = require('../../../src/components/index/delta')
const should = require('should/as-function')
describe('Index: delta', () => {
it('no changes', () => {
const database = {
IMG_0001: 1410000000000,
IMG_0002: 1420000000000
}
const disk = {
IMG_0001: 1410000000000,
IMG_0002: 1420000000000
}
const res = delta.calculate(database, disk)
should(res).eql({
unchanged: ['IMG_0001', 'IMG_0002'],
added: [],
modified: [],
deleted: []
describe('Scan mode: full', () => {
it('no changes', () => {
const database = {
IMG_0001: 1410000000000,
IMG_0002: 1420000000000
}
const disk = {
IMG_0001: 1410000000000,
IMG_0002: 1420000000000
}
const res = delta.calculate(database, disk, {})
should(res).eql({
unchanged: ['IMG_0001', 'IMG_0002'],
added: [],
modified: [],
deleted: [],
skipped: []
})
})
})
it('no changes within a second', () => {
const database = {
IMG_0001: 1410000001000,
IMG_0002: 1420000001000
}
const disk = {
IMG_0001: 1410000001500, // 500ms later
IMG_0002: 1420000000500 // 500ms earlier
}
const res = delta.calculate(database, disk)
should(res).eql({
unchanged: ['IMG_0001', 'IMG_0002'],
added: [],
modified: [],
deleted: []
it('no changes within a second', () => {
const database = {
IMG_0001: 1410000001000,
IMG_0002: 1420000001000
}
const disk = {
IMG_0001: 1410000001500, // 500ms later
IMG_0002: 1420000000500 // 500ms earlier
}
const res = delta.calculate(database, disk, {})
should(res).eql({
unchanged: ['IMG_0001', 'IMG_0002'],
added: [],
modified: [],
deleted: [],
skipped: []
})
})
})
it('new files', () => {
const database = {
IMG_0001: 1410000000000,
IMG_0002: 1420000000000
}
const disk = {
IMG_0001: 1410000000000,
IMG_0002: 1420000000000,
IMG_0003: 1430000000000
}
const res = delta.calculate(database, disk)
should(res).eql({
unchanged: ['IMG_0001', 'IMG_0002'],
added: ['IMG_0003'],
modified: [],
deleted: []
it('new files', () => {
const database = {
IMG_0001: 1410000000000,
IMG_0002: 1420000000000
}
const disk = {
IMG_0001: 1410000000000,
IMG_0002: 1420000000000,
IMG_0003: 1430000000000
}
const res = delta.calculate(database, disk, {})
should(res).eql({
unchanged: ['IMG_0001', 'IMG_0002'],
added: ['IMG_0003'],
modified: [],
deleted: [],
skipped: []
})
})
})
it('deleted files', () => {
const database = {
IMG_0001: 1410000000000,
IMG_0002: 1420000000000
}
const disk = {
IMG_0001: 1410000000000
}
const res = delta.calculate(database, disk)
should(res).eql({
unchanged: ['IMG_0001'],
added: [],
modified: [],
deleted: ['IMG_0002']
it('deleted files', () => {
const database = {
IMG_0001: 1410000000000,
IMG_0002: 1420000000000
}
const disk = {
IMG_0001: 1410000000000
}
const res = delta.calculate(database, disk, {})
should(res).eql({
unchanged: ['IMG_0001'],
added: [],
modified: [],
deleted: ['IMG_0002'],
skipped: []
})
})
it('modified files', () => {
const database = {
IMG_0001: 1410000000000,
IMG_0002: 1420000000000
}
const disk = {
IMG_0001: 1410000000000,
IMG_0002: 1420000002000
}
const res = delta.calculate(database, disk, {})
should(res).eql({
unchanged: ['IMG_0001'],
added: [],
modified: ['IMG_0002'],
deleted: [],
skipped: []
})
})
it('all cases', () => {
const database = {
IMG_0001: 1410000000000,
IMG_0002: 1420000000000,
IMG_0003: 1430000000000
}
const disk = {
IMG_0001: 1410000000000,
IMG_0002: 1420000002000,
IMG_0004: 1445000000000
}
const res = delta.calculate(database, disk, {})
should(res).eql({
unchanged: ['IMG_0001'],
added: ['IMG_0004'],
modified: ['IMG_0002'],
deleted: ['IMG_0003'],
skipped: []
})
})
})
it('modified files', () => {
const database = {
IMG_0001: 1410000000000,
IMG_0002: 1420000000000
}
const disk = {
IMG_0001: 1410000000000,
IMG_0002: 1420000002000
}
const res = delta.calculate(database, disk)
should(res).eql({
unchanged: ['IMG_0001'],
added: [],
modified: ['IMG_0002'],
deleted: []
describe('Scan mode: partial', () => {
it('considers deleted files outside the inclusion pattern as skipped', () => {
const database = {
'London/IMG_0001': 1410000000000,
'Tokyo/IMG_0002': 1420000000000
}
const disk = {
'London/IMG_0001': 1410000000000
}
const res = delta.calculate(database, disk, {
scanMode: 'incremental',
include: ['London/**'],
exclude: []
})
should(res).eql({
unchanged: ['London/IMG_0001'],
added: [],
modified: [],
deleted: [],
skipped: ['Tokyo/IMG_0002']
})
})
it('considers deleted files matching an exclusion pattern as skipped', () => {
const database = {
'London/IMG_0001': 1410000000000,
'Tokyo/IMG_0002': 1420000000000
}
const disk = {
'London/IMG_0001': 1410000000000
}
const res = delta.calculate(database, disk, {
scanMode: 'incremental',
include: [],
exclude: ['Tokyo/**']
})
should(res).eql({
unchanged: ['London/IMG_0001'],
added: [],
modified: [],
deleted: [],
skipped: ['Tokyo/IMG_0002']
})
})
it('considers files inside the inclusion pattern as deleted', () => {
const database = {
'London/IMG_0001': 1410000000000,
'Tokyo/IMG_0002': 1420000000000
}
const disk = {
'London/IMG_0001': 1410000000000
}
const res = delta.calculate(database, disk, {
scanMode: 'partial',
include: ['**/**'],
exclude: []
})
should(res).eql({
unchanged: ['London/IMG_0001'],
added: [],
modified: [],
deleted: ['Tokyo/IMG_0002'],
skipped: []
})
})
})
it('all cases', () => {
const database = {
IMG_0001: 1410000000000,
IMG_0002: 1420000000000,
IMG_0003: 1430000000000
}
const disk = {
IMG_0001: 1410000000000,
IMG_0002: 1420000002000,
IMG_0004: 1445000000000
}
const res = delta.calculate(database, disk)
should(res).eql({
unchanged: ['IMG_0001'],
added: ['IMG_0004'],
modified: ['IMG_0002'],
deleted: ['IMG_0003']
describe('Scan mode: incremental', () => {
it('considers files inside the inclusion pattern as skipped', () => {
const database = {
'London/IMG_0001': 1410000000000,
'Tokyo/IMG_0002': 1420000000000
}
const disk = {
'London/IMG_0001': 1410000000000
}
const res = delta.calculate(database, disk, {
scanMode: 'incremental',
include: [],
exclude: []
})
should(res).eql({
unchanged: ['London/IMG_0001'],
added: [],
modified: [],
deleted: [],
skipped: ['Tokyo/IMG_0002']
})
})
})
})

99
test/components/index/index.spec.js
Unescape Escape View File

@ -1,4 +1,3 @@
const fs = require('fs')
const path = require('path')
const should = require('should/as-function')
const Index = require('../../../src/components/index/index')
@ -9,18 +8,18 @@ describe('Index', function () {
this.timeout(1000)
let tmpdir = null
const image = fixtures.fromDisk('photo.jpg')
before(() => {
const image = fixtures.fromDisk('photo.jpg')
beforeEach(() => {
tmpdir = fixtures.createTempStructure({
'input/london/IMG_0001.jpg': image,
'input/newyork/IMG_0002.jpg': image
})
})
it('indexes a folder', (done) => {
function runIndex (options, done) {
const index = new Index(path.join(tmpdir, 'thumbsup.db'))
const emitter = index.update(path.join(tmpdir, 'input'))
const emitter = index.update(path.join(tmpdir, 'input'), options)
const emitted = []
let processed = 0
let stats = null
@ -28,62 +27,74 @@ describe('Index', function () {
emitter.on('file', meta => emitted.push(meta))
emitter.on('stats', s => { stats = s })
emitter.on('done', result => {
// check stats
should(result.count).eql(2)
should(stats).eql({ unchanged: 0, added: 2, modified: 0, deleted: 0, total: 2 })
done({ result, stats, emitted, processed })
})
}
it('indexes a folder', (done) => {
runIndex({}, data => {
should(data.result.count).eql(2)
should(data.stats).eql({ unchanged: 0, added: 2, modified: 0, deleted: 0, skipped: 0, total: 2 })
// check all files were indexed
const paths = emitted.map(e => e.path).sort()
const paths = data.emitted.map(e => e.path).sort()
should(paths).eql([
'london/IMG_0001.jpg',
'newyork/IMG_0002.jpg'
])
// check all files were sent to exiftool
should(processed).eql(2)
should(data.processed).eql(2)
// check the path matches the SourceFile property
const sourceFiles = emitted.map(e => e.metadata.SourceFile).sort()
const sourceFiles = data.emitted.map(e => e.metadata.SourceFile).sort()
should(paths).eql(sourceFiles)
done()
})
})
it('can re-index with no changes', (done) => {
const index = new Index(path.join(tmpdir, 'thumbsup.db'))
const emitter = index.update(path.join(tmpdir, 'input'))
let emitted = 0
let processed = 0
let stats = null
emitter.on('progress', () => ++processed)
emitter.on('file', () => ++emitted)
emitter.on('stats', s => { stats = s })
emitter.on('done', result => {
// check stats
should(result.count).eql(2)
should(stats).eql({ unchanged: 2, added: 0, modified: 0, deleted: 0, total: 2 })
// all files are emitted, but they were not processed again
should(emitted).eql(2)
should(processed).eql(0)
done()
runIndex({}, () => {
// then do a second run
runIndex({}, data => {
should(data.result.count).eql(2)
should(data.stats).eql({ unchanged: 2, added: 0, modified: 0, deleted: 0, skipped: 0, total: 2 })
// all files are emitted, but they were not processed again
should(data.emitted).has.length(2)
should(data.processed).eql(0)
done()
})
})
})
it('can un-index a deleted file', (done) => {
fs.unlinkSync(path.join(tmpdir, 'input/newyork/IMG_0002.jpg'))
const index = new Index(path.join(tmpdir, 'thumbsup.db'))
const emitter = index.update(path.join(tmpdir, 'input'))
let emitted = 0
let processed = 0
let stats = null
emitter.on('progress', () => ++processed)
emitter.on('file', () => ++emitted)
emitter.on('stats', s => { stats = s })
emitter.on('done', result => {
// check stats
should(result.count).eql(1)
should(stats).eql({ unchanged: 1, added: 0, modified: 0, deleted: 1, total: 1 })
// the remaining file was emitted
should(emitted).eql(1)
should(processed).eql(0)
done()
runIndex({}, () => {
// then do a second run
fixtures.deleteTempFile(tmpdir, 'input/newyork/IMG_0002.jpg')
runIndex({}, data => {
should(data.result.count).eql(1)
should(data.stats).eql({ unchanged: 1, added: 0, modified: 0, deleted: 1, skipped: 0, total: 1 })
// the remaining file was emitted
should(data.emitted).has.length(1)
should(data.processed).eql(0)
done()
})
})
})
describe('scan modes', () => {
it('partial ignores changes outside the include pattern', (done) => {
runIndex({}, () => {
// then do a second run
fixtures.deleteTempFile(tmpdir, 'input/newyork/IMG_0002.jpg')
const options = { scanMode: 'partial', include: ['london/**'] }
runIndex(options, data => {
should(data.result.count).eql(2)
// note: total is 1 because it scanned 1 file on disk
should(data.stats).eql({ unchanged: 1, added: 0, modified: 0, deleted: 0, skipped: 1, total: 1 })
// but it still emitted 2 files
should(data.emitted).has.length(2)
should(data.processed).eql(0)
done()
})
})
})
})

4
test/fixtures.js
Unescape Escape View File

@ -72,6 +72,10 @@ exports.createTempStructure = function (files) {
return tmpdir
}
exports.deleteTempFile = function (tmpdir, filepath) {
fs.unlinkSync(path.join(tmpdir, filepath))
}
// convert to OS-dependent style paths for testing
exports.ospath = function (filepath) {
return filepath.replace(/\//g, path.sep)

100
test/integration/integration-scan-modes.js
Unescape Escape View File

@ -0,0 +1,100 @@
const should = require('should/as-function')
const IntegrationTest = require('./integration-test')
const fixtures = require('../fixtures')
describe('Integration: scan modes', function () {
this.slow(5000)
this.timeout(5000)
const image = fixtures.fromDisk('photo.jpg')
beforeEach(IntegrationTest.before)
afterEach(IntegrationTest.after)
function newIntegrationTest () {
return new IntegrationTest({
'input/london/IMG_0001.jpg': image,
'input/london/IMG_0002.jpg': image,
'input/newyork/day 1/IMG_0003.jpg': image,
'input/newyork/day 2/IMG_0004.jpg': image
})
}
describe('Full', () => {
it('removes files that no longer exist in the source', function (done) {
const integration = newIntegrationTest()
integration.run(['--scan-mode', 'full'], () => {
const london1 = integration.parseYaml('london.html')
should(london1.files).have.length(2)
// delete a file and run again
integration.deleteInputFile('input/london/IMG_0002.jpg')
integration.run(['--scan-mode', 'full', '--cleanup', 'true'], () => {
const london2 = integration.parseYaml('london.html')
// the deleted file was removed
should(london2.files).have.length(1)
integration.assertNotExist(['media/thumbs/london/IMG_0002.jpg'])
done()
})
})
})
it("removes files that don't match the include filter", function (done) {
const integration = newIntegrationTest()
integration.run(['--scan-mode', 'full'], () => {
// first run, there's 2 albums (London + New York)
const index1 = integration.parseYaml('index.html')
should(index1.albums).have.length(2)
// run again, only including New York
integration.run(['--scan-mode', 'full', '--include', 'newyork/**', '--cleanup', 'true'], () => {
const index2 = integration.parseYaml('index.html')
// the London album is no longer there
should(index2.albums).have.length(1)
should(index2.albums[0].title).eql('newyork')
integration.assertNotExist(['media/thumbs/london/IMG_0001.jpg'])
done()
})
})
})
})
describe('Partial', () => {
it('ignores changes outside the include pattern', function (done) {
const integration = newIntegrationTest()
integration.run(['--scan-mode', 'full'], () => {
const london1 = integration.parseYaml('london.html')
should(london1.files).have.length(2)
integration.deleteInputFile('input/london/IMG_0002.jpg')
// run again, with only processing New York
integration.run(['--scan-mode', 'partial', '--include', 'newyork/**', '--cleanup', 'true'], () => {
// the London album still exists
const index2 = integration.parseYaml('index.html')
should(index2.albums).have.length(2)
// and it still has 2 files
const london2 = integration.parseYaml('london.html')
should(london2.files).have.length(2)
// and the excluded thumbnails are not deleted
integration.assertExist(['media/thumbs/london/IMG_0002.jpg'])
done()
})
})
})
})
describe('Incremental', () => {
it('does not remove deleted files', function (done) {
const integration = newIntegrationTest()
integration.run(['--scan-mode', 'full'], () => {
const london1 = integration.parseYaml('london.html')
should(london1.files).have.length(2)
// run again after deleting a file
integration.deleteInputFile('input/london/IMG_0002.jpg')
integration.run(['--scan-mode', 'incremental'], () => {
const london2 = integration.parseYaml('london.html')
should(london2.files).have.length(2)
integration.assertExist(['media/thumbs/london/IMG_0002.jpg'])
done()
})
})
})
})
})

9
test/integration/integration-test.js
Unescape Escape View File

@ -39,11 +39,20 @@ class IntegrationTest {
})
}
deleteInputFile (filepath) {
fixtures.deleteTempFile(this.tmpdir, filepath)
}
assertExist (expected) {
const missing = expected.filter(f => this.actualFiles.indexOf(f) === -1)
should(missing).eql([])
}
assertNotExist (expected) {
const present = expected.filter(f => this.actualFiles.indexOf(f) !== -1)
should(present).eql([])
}
parse (filepath) {
const fullpath = path.join(this.output, filepath)
return fs.readFileSync(fullpath, { encoding: 'utf8' })

Write Preview
Loading…
Cancel
Save