store big images in a lower resolution

There are common sizes in which images are
likely to be displayed in.
(smaller or equal to the screen size)
If images reach a specific resolution the
principle of spatial locality doesn't apply anymore
to resize operations to common sizes
(e.g. 6000x6000 to 300x300).
So to reduce cache misses and therefore avoid
a worse performance images are stored by default up to a size
which is about the same as the screen size.
Images will be fully loaded if it's required by their
image scaler or if they should be displayed with a size
which is bigger than the screen size.

ueberzug/action.py

@@ -120,40 +120,148 @@ class AddImageAction(ImageAction):
     def get_action_name():
         return 'add'
 
+    def __attrs_post_init__(self):
+        self.width = self.max_width or self.width
+        self.height = self.max_height or self.height
+        # attrs doesn't support overriding the init method
+        # pylint: disable=attribute-defined-outside-init
+        self.__scaler_class = None
+        self.__last_modified = None
+
+    @property
+    def scaler_class(self):
+        """scaling.ImageScaler: the used scaler class of this placement"""
+        if self.__scaler_class is None:
+            self.__scaler_class = \
+                scaling.ScalerOption(self.scaler).scaler_class
+        return self.__scaler_class
+
+    @property
+    def last_modified(self):
+        """float: the last modified time of the image"""
+        if self.__last_modified is None:
+            self.__last_modified = os.path.getmtime(self.path)
+        return self.__last_modified
+
+    def is_same_image(self, old_placement):
+        """Determines whether the placement contains the same image
+        after applying the changes of this command.
+
+        Args:
+            old_placement (ui.OverlayWindow.Placement):
+                the old data of the placement
+
+        Returns:
+            bool: True if it's the same file
+        """
+        return old_placement and not (
+            old_placement.last_modified < self.last_modified
+            or self.path != old_placement.path)
+
+    def is_full_reload_required(self, old_placement,
+                                screen_columns, screen_rows):
+        """Determines whether it's required to fully reload
+        the image of the placement to properly render the placement.
+
+        Args:
+            old_placement (ui.OverlayWindow.Placement):
+                the old data of the placement
+            screen_columns (float):
+                the maximum amount of columns the screen can display
+            screen_rows (float):
+                the maximum amount of rows the screen can display
+
+        Returns:
+            bool: True if the image should be reloaded
+        """
+        return old_placement and (
+            (not self.scaler_class.is_indulgent_resizing()
+             and old_placement.scaler.is_indulgent_resizing())
+            or (old_placement.width <= screen_columns < self.width)
+            or (old_placement.height <= screen_rows < self.height))
+
+    def is_partly_reload_required(self, old_placement,
+                                  screen_columns, screen_rows):
+        """Determines whether it's required to partly reload
+        the image of the placement to render the placement more quickly.
+
+        Args:
+            old_placement (ui.OverlayWindow.Placement):
+                the old data of the placement
+            screen_columns (float):
+                the maximum amount of columns the screen can display
+            screen_rows (float):
+                the maximum amount of rows the screen can display
+
+        Returns:
+            bool: True if the image should be reloaded
+        """
+        return old_placement and (
+            (self.scaler_class.is_indulgent_resizing()
+             and not old_placement.scaler.is_indulgent_resizing())
+            or (self.width <= screen_columns < old_placement.width)
+            or (self.height <= screen_rows < old_placement.height))
+
     async def apply(self, windows, view, tools):
         try:
             import ueberzug.ui as ui
+            import ueberzug.loading as loading
             old_placement = view.media.pop(self.identifier, None)
             cache = old_placement and old_placement.cache
             image = old_placement and old_placement.image
-            last_modified = old_placement and old_placement.last_modified
-            current_last_modified = os.path.getmtime(self.path)
-            width = self.max_width or self.width
-            height = self.max_height or self.height
-            scaler_class = scaling.ScalerOption(self.scaler).scaler_class
-
-            if (not image
-                    or last_modified < current_last_modified
-                    or self.path != old_placement.path):
-                last_modified = current_last_modified
+
+            max_font_width = max(map(
+                lambda i: i or 0, windows.parent_info.font_width or [0]))
+            max_font_height = max(map(
+                lambda i: i or 0, windows.parent_info.font_height or [0]))
+            font_size_available = max_font_width and max_font_height
+            screen_columns = (font_size_available and
+                              view.screen_width / max_font_width)
+            screen_rows = (font_size_available and
+                           view.screen_height / max_font_height)
+
+            # By default images are only stored up to a resolution which
+            # is about as big as the screen resolution.
+            # (loading.CoverPostLoadImageProcessor)
+            # The principle of spatial locality does not apply to
+            # resize operations of images with big resolutions
+            # which is why those operations should be applied
+            # to a resized version of those images.
+            # Sometimes we still need all pixels e.g.
+            # if the image scaler crop is used.
+            # So sometimes it's required to fully load them
+            # and sometimes it's not required anymore which is
+            # why they should be partly reloaded
+            # (to speed up the resize operations again).
+            if (not self.is_same_image(old_placement)
+                    or (font_size_available and self.is_full_reload_required(
+                        old_placement, screen_columns, screen_rows))
+                    or (font_size_available and self.is_partly_reload_required(
+                        old_placement, screen_columns, screen_rows))):
                 upper_bound_size = None
-                max_font_width = max(map(
-                    lambda i: i or 0, windows.parent_info.font_width))
-                max_font_height = max(map(
-                    lambda i: i or 0, windows.parent_info.font_height))
-                if (scaler_class != scaling.CropImageScaler and
-                        max_font_width and max_font_height):
+                image_post_load_processor = None
+                if (self.scaler_class != scaling.CropImageScaler and
+                        font_size_available):
                     upper_bound_size = (
-                        max_font_width * width, max_font_height * height)
-                image = tools.loader.load(self.path, upper_bound_size)
+                        max_font_width * self.width,
+                        max_font_height * self.height)
+                if (self.scaler_class != scaling.CropImageScaler
+                        and font_size_available
+                        and self.width <= screen_columns
+                        and self.height <= screen_rows):
+                    image_post_load_processor = \
+                        loading.CoverPostLoadImageProcessor(
+                            view.screen_width, view.screen_height)
+                image = tools.loader.load(
+                    self.path, upper_bound_size, image_post_load_processor)
                 cache = None
 
             view.media[self.identifier] = ui.OverlayWindow.Placement(
-                self.x, self.y, width, height,
+                self.x, self.y, self.width, self.height,
                 geometry.Point(self.scaling_position_x,
                                self.scaling_position_y),
-                scaler_class(),
-                self.path, image, last_modified, cache)
+                self.scaler_class(),
+                self.path, image, self.last_modified, cache)
         finally:
             await super().apply(windows, view, tools)
 

ueberzug/layer.py

@@ -173,6 +173,8 @@ class View:
     def __init__(self):
         self.offset = geometry.Distance()
         self.media = {}
+        self.screen_width = 0
+        self.screen_height = 0
 
 
 class Tools:
@@ -185,6 +187,7 @@ class Tools:
 
 def main(options):
     display = xutil.get_display()
+    screen = display.screen()
     window_infos = xutil.get_parent_window_infos()
     loop = asyncio.get_event_loop()
     executor = thread.DaemonThreadPoolExecutor(max_workers=2)
@@ -198,6 +201,8 @@ def main(options):
     window_factory = ui.OverlayWindow.Factory(display, view)
     windows = batch.BatchList(window_factory.create(*window_infos))
     image_loader.register_error_handler(error_handler)
+    view.screen_width = screen.width_in_pixels
+    view.screen_height = screen.height_in_pixels
 
     if tmux_util.is_used():
         atexit.register(setup_tmux_hooks())

ueberzug/loading.py

@@ -89,6 +89,49 @@ class ImageHolder:
         return self.image
 
 
+class PostLoadImageProcessor(metaclass=abc.ABCMeta):
+    """Describes the structure used to define callbacks which
+    will be invoked after loading an image.
+    """
+    @abc.abstractmethod
+    def on_loaded(self, image):
+        """Postprocessor of an loaded image.
+        The returned image will be assigned to the image holder.
+
+        Args:
+            image (PIL.Image): the loaded image
+
+        Returns:
+            PIL.Image:
+                the image which will be assigned
+                to the image holder of this loading process
+        """
+        raise NotImplementedError()
+
+
+class CoverPostLoadImageProcessor(PostLoadImageProcessor):
+    """Implementation of PostLoadImageProcessor
+    which resizes an image (if possible -> needs to be bigger)
+    such that it covers only just a given resolution.
+    """
+    def __init__(self, width, height):
+        self.width = width
+        self.height = height
+
+    def on_loaded(self, image):
+        import PIL.Image
+        resize_ratio = max(min(1, self.width / image.width),
+                           min(1, self.height / image.height))
+
+        if resize_ratio != 1:
+            image = image.resize(
+                (int(resize_ratio * image.width),
+                 int(resize_ratio * image.height)),
+                PIL.Image.ANTIALIAS)
+
+        return image
+
+
 class ImageLoader(metaclass=abc.ABCMeta):
     """Describes the structure used to define image loading strategies.
 
@@ -113,7 +156,7 @@ class ImageLoader(metaclass=abc.ABCMeta):
         self.error_handler = None
 
     @abc.abstractmethod
-    def load(self, path, upper_bound_size):
+    def load(self, path, upper_bound_size, post_load_processor=None):
         """Starts the image loading procedure for the passed path.
         How and when an image get's loaded depends on the implementation
         of the used ImageLoader class.
@@ -122,6 +165,8 @@ class ImageLoader(metaclass=abc.ABCMeta):
             path (str): the path to the image which should be loaded
             upper_bound_size (tuple of (width: int, height: int)):
                 the maximal size to load data for
+            post_load_processor (PostLoadImageProcessor):
+                allows to apply changes to the recently loaded image
 
         Returns:
             ImageHolder: which the image will be assigned to
@@ -162,7 +207,7 @@ class SynchronousImageLoader(ImageLoader):
     def get_loader_name():
         return "synchronous"
 
-    def load(self, path, upper_bound_size):
+    def load(self, path, upper_bound_size, post_load_processor=None):
         image = None
 
         try:
@@ -170,6 +215,9 @@ class SynchronousImageLoader(ImageLoader):
         except OSError as exception:
             self.process_error(exception)
 
+        if image and post_load_processor:
+            image = post_load_processor.on_loaded(image)
+
         return ImageHolder(path, image or self.PLACEHOLDER)
 
 
@@ -192,7 +240,7 @@ class AsynchronousImageLoader(ImageLoader):
         self.__queue_low_priority = queue.Queue()
         self.__waiter_low_priority = threading.Condition()
 
-    def _enqueue(self, queue, image_holder, upper_bound_size):
+    def _enqueue(self, queue, image_holder, upper_bound_size, post_load_processor):
         """Enqueues the image holder weakly referenced.
 
         Args:
@@ -201,8 +249,11 @@ class AsynchronousImageLoader(ImageLoader):
                 the image holder for which an image should be loaded
             upper_bound_size (tuple of (width: int, height: int)):
                 the maximal size to load data for
+            post_load_processor (PostLoadImageProcessor):
+                allows to apply changes to the recently loaded image
         """
-        queue.put((weakref.ref(image_holder), upper_bound_size))
+        queue.put((
+            weakref.ref(image_holder), upper_bound_size, post_load_processor))
 
     def _dequeue(self, queue):
         """Removes queue entries till an alive reference was found.
@@ -214,21 +265,25 @@ class AsynchronousImageLoader(ImageLoader):
             queue (queue.Queue): the queue to operate on
 
         Returns:
-            tuple of (ImageHolder, tuple of (width: int, height: int)):
-                an queued image holder or None, upper bound size or None
+            tuple of (ImageHolder, tuple of (width: int, height: int),
+                      PostLoadImageProcessor):
+                an queued image holder or None, upper bound size or None,
+                the post load image processor or None
         """
         holder_reference = None
         image_holder = None
         upper_bound_size = None
+        post_load_processor = None
 
         while not queue.empty():
-            holder_reference, upper_bound_size = queue.get_nowait()
+            holder_reference, upper_bound_size, post_load_processor = \
+                queue.get_nowait()
             image_holder = holder_reference and holder_reference()
             if (holder_reference is None or
                     image_holder is not None):
                 break
 
-        return image_holder, upper_bound_size
+        return image_holder, upper_bound_size, post_load_processor
 
     @abc.abstractmethod
     def _schedule(self, function, priority):
@@ -243,16 +298,22 @@ class AsynchronousImageLoader(ImageLoader):
         """
         raise NotImplementedError()
 
-    def _load_image(self, path, upper_bound_size):
+    def _load_image(self, path, upper_bound_size, post_load_processor):
         """Wrapper for calling load_image.
         Behaves like calling it directly,
         but allows e.g. executing the function in other processes.
         """
-        return load_image(path, upper_bound_size)
+        image, *other_data = load_image(path, upper_bound_size)
+
+        if image and post_load_processor:
+            image = post_load_processor.on_loaded(image)
+
+        return (image, *other_data)
 
-    def load(self, path, upper_bound_size):
+    def load(self, path, upper_bound_size, post_load_processor=None):
         holder = ImageHolder(path)
-        self._enqueue(self.__queue, holder, upper_bound_size)
+        self._enqueue(
+            self.__queue, holder, upper_bound_size, post_load_processor)
         self._schedule(self.__process_high_priority_entry,
                        self.Priority.HIGH)
         return holder
@@ -290,15 +351,18 @@ class AsynchronousImageLoader(ImageLoader):
             queue (queue.Queue): the queue to operate on
         """
         image = None
-        image_holder, upper_bound_size = self._dequeue(queue)
+        image_holder, upper_bound_size, post_load_processor = \
+            self._dequeue(queue)
         if image_holder is None:
             return
 
         try:
             image, downscaled = self._load_image(
-                image_holder.path, upper_bound_size)
+                image_holder.path, upper_bound_size, post_load_processor)
             if upper_bound_size and downscaled:
-                self._enqueue(self.__queue_low_priority, image_holder, None)
+                self._enqueue(
+                    self.__queue_low_priority,
+                    image_holder, None, post_load_processor)
                 self._schedule(self.__process_low_priority_entry,
                                self.Priority.LOW)
         except OSError as exception:
@@ -366,20 +430,25 @@ class ProcessImageLoader(ThreadImageLoader):
             .result()
 
     @staticmethod
-    def _load_image_extern(path, upper_bound_size):
+    def _load_image_extern(path, upper_bound_size, post_load_processor):
         """This function is a wrapper for the image loading function
         as sometimes pillow restores decoded images
         received from other processes wrongly.
         E.g. a PNG is reported as webp (-> crash on using an image function)
         So this function is a workaround which prevents these crashs to happen.
         """
-        image, downscaled = load_image(path, upper_bound_size)
-        return image.mode, image.size, image.tobytes(), downscaled
+        image, *other_data = load_image(path, upper_bound_size)
+
+        if image and post_load_processor:
+            image = post_load_processor.on_loaded(image)
+
+        return (image.mode, image.size, image.tobytes(), *other_data)
 
-    def _load_image(self, path, upper_bound_size):
+    def _load_image(self, path, upper_bound_size, post_load_processor=None):
         import PIL.Image
         future = self.__executor_loader.submit(
-            ProcessImageLoader._load_image_extern, path, upper_bound_size)
+            ProcessImageLoader._load_image_extern,
+            path, upper_bound_size, post_load_processor)
         mode, size, data, downscaled = future.result()
         return PIL.Image.frombytes(mode, size, data), downscaled
 

ueberzug/scaling.py

@@ -17,7 +17,22 @@ class ImageScaler(metaclass=abc.ABCMeta):
     @staticmethod
     @abc.abstractmethod
     def get_scaler_name():
-        """Returns the constant name which is associated to this scaler."""
+        """Returns:
+            str: the constant name which is associated to this scaler.
+        """
+        raise NotImplementedError()
+
+    @staticmethod
+    @abc.abstractmethod
+    def is_indulgent_resizing():
+        """This method specifies whether the
+        algorithm returns noticeable different results for
+        the same image with different sizes (bigger than the
+        maximum size which is passed to the scale method).
+
+        Returns:
+            bool: False if the results differ
+        """
         raise NotImplementedError()
 
     @abc.abstractmethod
@@ -98,6 +113,10 @@ class CropImageScaler(MinSizeImageScaler, OffsetImageScaler):
     def get_scaler_name():
         return "crop"
 
+    @staticmethod
+    def is_indulgent_resizing():
+        return False
+
     def scale(self, image, position: geometry.Point,
               width: int, height: int):
         width, height = self.calculate_resolution(image, width, height)
@@ -118,6 +137,10 @@ class DistortImageScaler(ImageScaler):
     def get_scaler_name():
         return "distort"
 
+    @staticmethod
+    def is_indulgent_resizing():
+        return True
+
     def calculate_resolution(self, image, width: int, height: int):
         return width, height
 
@@ -140,6 +163,10 @@ class FitContainImageScaler(DistortImageScaler):
     def get_scaler_name():
         return "fit_contain"
 
+    @staticmethod
+    def is_indulgent_resizing():
+        return True
+
     def calculate_resolution(self, image, width: int, height: int):
         factor = min(width / image.width, height / image.height)
         return int(image.width * factor), int(image.height * factor)
@@ -155,6 +182,10 @@ class ContainImageScaler(FitContainImageScaler):
     def get_scaler_name():
         return "contain"
 
+    @staticmethod
+    def is_indulgent_resizing():
+        return True
+
     def calculate_resolution(self, image, width: int, height: int):
         return super().calculate_resolution(
             image, min(width, image.width), min(height, image.height))
@@ -174,6 +205,10 @@ class ForcedCoverImageScaler(DistortImageScaler, OffsetImageScaler):
     def get_scaler_name():
         return "forced_cover"
 
+    @staticmethod
+    def is_indulgent_resizing():
+        return True
+
     def scale(self, image, position: geometry.Point,
               width: int, height: int):
         import PIL.Image
@@ -203,6 +238,10 @@ class CoverImageScaler(MinSizeImageScaler, ForcedCoverImageScaler):
     def get_scaler_name():
         return "cover"
 
+    @staticmethod
+    def is_indulgent_resizing():
+        return True
+
 
 @enum.unique
 class ScalerOption(str, enum.Enum):