fastgallery/cmd/fastgallery/main.go

package main

import (
	"embed"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/cheggaaa/pb/v3"
	"github.com/davidbyttow/govips/v2/vips"
	"github.com/kr/pretty"

	"github.com/alexflint/go-arg"
)

// Embed all static assets
//go:embed assets
var assets embed.FS

// Define global exit function, so unit tests can override this
var exit = os.Exit

// configuration state is stored in this struct
type configuration struct {
	files struct {
		originalDir    string
		fullsizeDir    string
		thumbnailDir   string
		directoryMode  os.FileMode
		fileMode       os.FileMode
		imageExtension string
		videoExtension string
	}
	media struct {
		thumbnailWidth    int
		thumbnailHeight   int
		fullsizeMaxWidth  int
		fullsizeMaxHeight int
		videoMaxSize      int
	}
	concurrency int
}

// initialize the configuration with hardcoded defaults
func initializeConfig() (config configuration) {
	config.files.originalDir = "_original"
	config.files.fullsizeDir = "_fullsize"
	config.files.thumbnailDir = "_thumbnail"
	config.files.directoryMode = 0755
	config.files.fileMode = 0644
	config.files.imageExtension = ".jpg"
	config.files.videoExtension = ".mp4"

	config.media.thumbnailWidth = 280
	config.media.thumbnailHeight = 210
	config.media.fullsizeMaxWidth = 1920
	config.media.fullsizeMaxHeight = 1080
	config.media.videoMaxSize = 640

	config.concurrency = 8

	return config
}

// file struct represents an individual media file
// relPath is the relative path to from source/gallery root directory.
// For source files, exists marks whether it exists in the gallery and doesn't need to be copied.
// In this case, gallery has all three transformed files (original, full-size and thumbnail) and
// the thumbnail's modification date isn't before the original source file's.
// For gallery files, exists marks whether all three gallery files are in place (original, full-size
// and thumbnail) and there's a corresponding source file.
type file struct {
	name    string
	relPath string
	absPath string
	modTime time.Time
	exists  bool
}

// directory struct is one directory, which contains files and subdirectories
// relPath is the relative path from source/gallery root directory
// For source directories, exists reflects whether the directory exists in the gallery
// For gallery directories, exists reflects whether there's a corresponding source directory
type directory struct {
	name           string
	relPath        string
	absPath        string
	modTime        time.Time
	files          []file
	subdirectories []directory
	exists         bool
}

// exists checks whether given file, directory or symlink exists
func exists(filepath string) bool {
	if _, err := os.Stat(filepath); os.IsNotExist(err) {
		return false
	}
	return true
}

// isDirectory checks whether provided path is a directory or symlink to one
// resolves symlinks only one level deep
func isDirectory(directory string) bool {
	filestat, err := os.Stat(directory)
	if os.IsNotExist(err) {
		return false
	}

	if filestat.IsDir() {
		return true
	}

	if filestat.Mode()&os.ModeSymlink != 0 {
		realDirectory, err := filepath.EvalSymlinks(directory)
		if err != nil {
			fmt.Fprintf(os.Stderr, "error: %s\n", err.Error())
			return false
		}

		realFilestat, err := os.Stat(realDirectory)
		if err != nil {
			fmt.Fprintf(os.Stderr, "error: %s\n", err.Error())
			return false
		}

		if realFilestat.IsDir() {
			return true
		}
	}

	return false
}

// Validate that source and gallery directories given as parameters
// are valid directories. Return absolue path of source and gallery
func validateSourceAndGallery(source string, gallery string) (string, string) {
	var err error

	source, err = filepath.Abs(source)
	if err != nil {
		fmt.Fprintf(os.Stderr, "error: %s\n", err.Error())
		exit(1)
	}

	if !isDirectory(source) {
		fmt.Fprintf(os.Stderr, "Source directory doesn't exist: %s\n", source)
		exit(1)
	}

	gallery, err = filepath.Abs(gallery)
	if err != nil {
		fmt.Fprintf(os.Stderr, "error: %s\n", err.Error())
		exit(1)
	}

	if !isDirectory(gallery) {
		// Ok, gallery isn't a directory but check whether the parent directory is
		// and we're supposed to create gallery there during runtime
		galleryParent, err := filepath.Abs(gallery + "/../")
		if err != nil {
			fmt.Fprintf(os.Stderr, "error: %s\n", err.Error())
			exit(1)
		}

		if !isDirectory(galleryParent) {
			fmt.Fprintf(os.Stderr, "Neither gallery directory or it's parent directory exist: %s\n", gallery)
			exit(1)
		}
	}

	return source, gallery
}

// Checks whether directory has media files, or subdirectories with media files.
// If there's a subdirectory that's empty or that has directories or files which
// aren't media files, we leave that out of the directory tree.
func dirHasMediafiles(directory string) (isEmpty bool) {
	list, err := os.ReadDir(directory)
	if err != nil {
		// If we can't read the directory contents, it doesn't have media files in it
		return false
	}

	if len(list) == 0 {
		// If it's empty, it doesn't have media files
		return false
	}

	for _, entry := range list {
		entryAbsPath := filepath.Join(directory, entry.Name())
		if entry.IsDir() {
			// Recursion to subdirectories
			if dirHasMediafiles(entryAbsPath) {
				return true
			}
		} else if isMediaFile(entryAbsPath) {
			// We found at least one media file, return true
			return true
		}
	}

	// Didn't find at least one media file
	return false
}

// Check whether given path is a video file
func isVideoFile(filename string) bool {
	switch filepath.Ext(strings.ToLower(filename)) {
	case ".mp4", ".mov", ".3gp", ".avi", ".mts", ".m4v", ".mpg":
		return true
	default:
		return false
	}
}

// Check whether given path is an image file
func isImageFile(filename string) bool {
	switch filepath.Ext(strings.ToLower(filename)) {
	case ".jpg", ".jpeg", ".heic", ".png", ".gif", ".tif", ".tiff":
		return true
	case ".cr2", ".raw", ".arw":
		return true
	default:
		return false
	}
}

// Check whether given absolute path is a media file
func isMediaFile(filename string) bool {
	if isImageFile(filename) {
		return true
	}

	// TODO optIgnoreVideos
	if isVideoFile(filename) {
		return true
	}

	return false
}

// Create a recursive directory struct by traversing the directory absoluteDirectory.
// The function calls itself recursively, carrying state in the relativeDirectory parameter.
func createDirectoryTree(absoluteDirectory string, parentDirectory string) (tree directory) {
	// In case the target directory doesn't exist, it's the gallery directory
	// which hasn't been created yet. We'll just create a dummy tree and return it.
	if !exists(absoluteDirectory) && parentDirectory == "" {
		tree.name = filepath.Base(absoluteDirectory)
		tree.relPath = parentDirectory
		tree.absPath, _ = filepath.Abs(absoluteDirectory)
		return
	}

	// Fill in the directory name and other basic info
	tree.name = filepath.Base(absoluteDirectory)
	tree.absPath, _ = filepath.Abs(absoluteDirectory)
	tree.relPath = parentDirectory
	absoluteDirectoryStat, _ := os.Stat(absoluteDirectory)
	tree.modTime = absoluteDirectoryStat.ModTime()

	// List directory contents
	list, err := os.ReadDir(absoluteDirectory)
	if err != nil {
		log.Fatal("Couldn't list directory contents:", absoluteDirectory)
	}

	// If it's a directory and it has media files somewhere, add it to directories
	// If it's a media file, add it to the files
	for _, entry := range list {
		entryAbsPath := filepath.Join(absoluteDirectory, entry.Name())
		entryRelPath := filepath.Join(parentDirectory, entry.Name())
		if entry.IsDir() {
			if dirHasMediafiles(entryAbsPath) {
				entrySubTree := createDirectoryTree(entryAbsPath, entryRelPath)
				tree.subdirectories = append(tree.subdirectories, entrySubTree)
			}
		} else if isMediaFile(entryAbsPath) {
			entryFileInfo, err := entry.Info()
			if err != nil {
				log.Fatal("Couldn't stat file information for media file:", entry.Name())
			}
			entryFile := file{
				name:    entry.Name(),
				relPath: entryRelPath,
				absPath: entryAbsPath,
				modTime: entryFileInfo.ModTime(),
				exists:  false,
			}
			tree.files = append(tree.files, entryFile)
		}
	}
	return
}

// stripExtension strips the filename extension and returns the basename
func stripExtension(filename string) string {
	extension := filepath.Ext(filename)
	return filename[0 : len(filename)-len(extension)]
}

func reservedDirectory(path string, config configuration) bool {
	if path == config.files.thumbnailDir {
		return true
	}

	if path == config.files.fullsizeDir {
		return true
	}

	if path == config.files.originalDir {
		return true
	}

	return false
}

// hasDirectoryChanged checks whether the gallery directory has changed and thus
// the HTML file needs to be updated. Could be due to:
// At least one non-existent source file or directory (will be created in gallery)
// We're doing a cleanup, and at least one non-existent gallery file or directory (will be removed from gallery HTML)
func hasDirectoryChanged(source directory, gallery directory, cleanUp bool) bool {
	for _, sourceFile := range source.files {
		if !sourceFile.exists {
			return true
		}
	}

	for _, sourceDir := range source.subdirectories {
		if !sourceDir.exists {
			return true
		}
	}

	if cleanUp {
		for _, galleryFile := range gallery.files {
			if !galleryFile.exists {
				return true
			}
		}

		for _, galleryDir := range gallery.subdirectories {
			if !galleryDir.exists {
				return true
			}
		}
	}

	return false
}

// compareDirectoryTrees compares two directory trees (source and gallery) and marks
// each file that exists in both
func compareDirectoryTrees(source *directory, gallery *directory, config configuration) {
	// If we are comparing two directories, we know they both exist so we can set the
	// directory struct exists boolean
	source.exists = true
	gallery.exists = true

	// Iterate over each file in source directory to see whether it exists in gallery
	for i, sourceFile := range source.files {
		sourceFileBasename := stripExtension(sourceFile.name)

		var thumbnailFile, fullsizeFile, originalFile *file

		// Go through all subdirectories, and check the ones that match
		// the thumbnail, full-size or original subdirectories
		for _, subDir := range gallery.subdirectories {
			if subDir.name == config.files.thumbnailDir {
				for _, outputFile := range subDir.files {
					outputFileBasename := stripExtension(outputFile.name)
					if sourceFileBasename == outputFileBasename {
						thumbnailFile = &outputFile
						thumbnailFile.exists = true
					}
				}
			}

			if subDir.name == config.files.fullsizeDir {
				for _, outputFile := range subDir.files {
					outputFileBasename := stripExtension(outputFile.name)
					if sourceFileBasename == outputFileBasename {
						fullsizeFile = &outputFile
						fullsizeFile.exists = true
					}
				}
			}

			if subDir.name == config.files.originalDir {
				for _, outputFile := range subDir.files {
					outputFileBasename := stripExtension(outputFile.name)
					if sourceFileBasename == outputFileBasename {
						originalFile = &outputFile
						originalFile.exists = true
					}
				}
			}
		}

		// If all of thumbnail, full-size and original files exist in gallery, and they're
		// modified after the source file, the source file exists and is up to date.
		// Otherwise we overwrite gallery files in case source file's been updated since the thumbnail
		// was created.
		if thumbnailFile != nil && fullsizeFile != nil && originalFile != nil {
			if !thumbnailFile.modTime.Before(sourceFile.modTime) {
				source.files[i].exists = true
			}
		}
	}

	// After checking all the files in this directory, recurse into each subdirectory and do the same
	for k, inputDir := range source.subdirectories {
		if !reservedDirectory(inputDir.name, config) {
			for l, outputDir := range gallery.subdirectories {
				if inputDir.name == outputDir.name {
					compareDirectoryTrees(&(source.subdirectories[k]), &(gallery.subdirectories[l]), config)
				}
			}
		}
	}
}

func countChanges(source directory) (outputChanges int) {
	outputChanges = 0
	for _, file := range source.files {
		if !file.exists {
			outputChanges++
		}
	}

	for _, dir := range source.subdirectories {
		outputChanges = outputChanges + countChanges(dir)
	}

	return outputChanges
}

func createDirectory(destination string, dryRun bool, dirMode os.FileMode) {
	if _, err := os.Stat(destination); os.IsNotExist(err) {
		if dryRun {
			fmt.Println("Would create directory:", destination)
		} else {
			err := os.Mkdir(destination, dirMode)
			if err != nil {
				fmt.Fprintf(os.Stderr, "couldn't create directory %s: %s\n", destination, err.Error())
				exit(1)
			}
		}
	}
}

func symlinkFile(sourceDir string, destDir string, filename string, dryRun bool) {
	// TODO functionality
}

// TODO deprecate copyFile() function or use for originals
func copyFile(sourceDir string, destDir string, filename string, dryRun bool) {
	sourceFilename := filepath.Join(sourceDir, filename)
	destFilename := filepath.Join(destDir, filename)

	if !dryRun {
		_, err := os.Stat(sourceFilename)
		if err != nil {
			fmt.Fprintf(os.Stderr, "couldn't copy source file %s: %s\n", sourceFilename, err.Error())
			exit(1)
		}

		sourceHandle, err := os.Open(sourceFilename)
		if err != nil {
			fmt.Fprintf(os.Stderr, "couldn't open source file for copy %s: %s\n", sourceFilename, err.Error())
			exit(1)
		}
		defer sourceHandle.Close()

		destHandle, err := os.Create(destFilename)
		if err != nil {
			fmt.Fprintf(os.Stderr, "couldn't create dest file %s: %s\n", destFilename, err.Error())
			exit(1)
		}
		defer destHandle.Close()

		_, err = io.Copy(destHandle, sourceHandle)
		if err != nil {
			fmt.Fprintf(os.Stderr, "couldn't copy file %s -> %s: %s\n", sourceFilename, destFilename, err.Error())
			exit(1)
		}
	} else {
		fmt.Println("would copy", sourceFilename, "to", destFilename)
	}
}

// copyRootAssets copies all the embedded assets to the root directory of the gallery
func copyRootAssets(gallery directory, dryRun bool, fileMode os.FileMode) {
	assetDirectoryListing, err := assets.ReadDir("assets")
	if err != nil {
		log.Fatal("couldn't open embedded assets:", err.Error())
	}

	// Iterate through all the embedded assets
	for _, entry := range assetDirectoryListing {
		if !entry.IsDir() {
			switch filepath.Ext(strings.ToLower(entry.Name())) {
			// Copy all javascript and CSS files
			case ".js", ".css":
				if !dryRun {
					filebuffer, err := assets.ReadFile("assets/" + entry.Name())
					if err != nil {
						log.Fatal("couldn't open embedded asset:", entry.Name(), ":", err.Error())
					}
					err = os.WriteFile(gallery.absPath+"/"+entry.Name(), filebuffer, fileMode)
					if err != nil {
						log.Fatal("couldn't write embedded asset:", gallery.absPath+"/"+entry.Name(), ":", err.Error())
					}
				} else {
					fmt.Println("Would copy JS/CSS file", entry.Name(), "to", gallery.absPath)
				}
			}

			switch entry.Name() {
			// Copy back.png and folder.png
			case "back.png", "folder.png":
				if !dryRun {
					filebuffer, err := assets.ReadFile("assets/" + entry.Name())
					if err != nil {
						log.Fatal("couldn't open embedded asset:", entry.Name(), ":", err.Error())
					}
					err = os.WriteFile(gallery.absPath+"/"+entry.Name(), filebuffer, fileMode)
					if err != nil {
						log.Fatal("couldn't write embedded asset:", gallery.absPath+"/"+entry.Name(), ":", err.Error())
					}
				} else {
					fmt.Println("Would copy icon", entry.Name(), "to", gallery.absPath)
				}
			}
		}
	}
}

func createHTML(depth int, source directory, dryRun bool) {
	// TODO functionality
	// TODO dry-run
}

func createMedia(depth int, source directory, gallery directory, dryRun bool, config configuration, progressBar *pb.ProgressBar) {
	// TODO concurrency
	for _, file := range source.files {
		if !file.exists {
			// TODO functionality
			fmt.Println("converting:", file.name, file.relPath, file.absPath)
		}
	}
}

// Clean gallery directory of any directories or files which don't exist in source
func cleanDirectory(gallery directory, dryRun bool) {
	for _, file := range gallery.files {
		if !file.exists {
			// TODO
		}
	}

	for _, dir := range gallery.subdirectories {
		if !dir.exists {
			// TODO
			// What about reserved directories for thumbnails, pictures and originals?
		}
	}
}

func createGallery(depth int, source directory, gallery directory, dryRun bool, cleanUp bool, config configuration, progressBar *pb.ProgressBar) {

	if hasDirectoryChanged(source, gallery, cleanUp) {
		createMedia(depth, source, gallery, dryRun, config, progressBar)
		createHTML(depth, source, dryRun)
		if cleanUp {
			cleanDirectory(gallery, dryRun)
		}
	}

	for _, subdir := range source.subdirectories {
		fmt.Println("recursing to:", subdir.name, subdir.relPath, subdir.absPath)
		createGallery(depth+1, subdir, gallery, dryRun, cleanUp, config, progressBar)
	}
}

func main() {
	// Define command-line arguments
	var args struct {
		Source  string `arg:"positional,required" help:"Source directory for images/videos"`
		Gallery string `arg:"positional,required" help:"Destination directory to create gallery in"`
		Verbose bool   `arg:"-v,--verbose" help:"verbosity level"`
		DryRun  bool   `arg:"--dry-run" help:"dry run; don't change anything, just print what would be done"`
		CleanUp bool   `arg:"-c,--cleanup" help:"cleanup, delete files and directories in gallery which don't exist in source"`
	}

	// Parse command-line arguments
	arg.MustParse(&args)

	// Validate source and gallery arguments, make paths absolute
	args.Source, args.Gallery = validateSourceAndGallery(args.Source, args.Gallery)

	// Initialize configuration (assets, directories, file types)
	config := initializeConfig()

	fmt.Println("Creating gallery...")
	fmt.Println("Source:", args.Source)
	fmt.Println("Gallery:", args.Gallery)
	fmt.Println()
	fmt.Println("Finding all media files...")

	// Creating a directory struct of both source as well as gallery directories
	source := createDirectoryTree(args.Source, "")
	gallery := createDirectoryTree(args.Gallery, "")

	// Check which source media exists in gallery
	compareDirectoryTrees(&source, &gallery, config)

	// Count number of source files which don't exist in gallery
	changes := countChanges(source)
	if changes > 0 {
		fmt.Println(changes, "files to update")
		if !exists(gallery.absPath) {
			createDirectory(gallery.absPath, args.DryRun, config.files.directoryMode)
		}

		var progressBar *pb.ProgressBar
		if !args.DryRun {
			progressBar = pb.StartNew(changes)
			if args.Verbose {
				vips.LoggingSettings(nil, vips.LogLevelDebug)
				vips.Startup(&vips.Config{
					CacheTrace:   false,
					CollectStats: false,
					ReportLeaks:  true})
			} else {
				vips.LoggingSettings(nil, vips.LogLevelError)
				vips.Startup(nil)
			}
			defer vips.Shutdown()
		}

		copyRootAssets(gallery, args.DryRun, config.files.fileMode)
		createGallery(0, source, gallery, args.DryRun, args.CleanUp, config, progressBar)

		if !args.DryRun {
			progressBar.Finish()
		}

		fmt.Println("Gallery updated!")
	} else {
		fmt.Println("Gallery already up to date!")
	}

	fmt.Println("source:")
	pretty.Print(source)

	fmt.Println("gallery:")
	pretty.Print(gallery)
}