From ea6d514a9d5f71ff6ca3e034cf98a7d101069735 Mon Sep 17 00:00:00 2001 From: seebye Date: Tue, 24 Dec 2019 18:05:45 +0100 Subject: [PATCH] store big images in a lower resolution There are common sizes in which images are likely to be displayed in. (smaller or equal to the screen size) If images reach a specific resolution the principle of spatial locality doesn't apply anymore to resize operations to common sizes (e.g. 6000x6000 to 300x300). So to reduce cache misses and therefore avoid a worse performance images are stored by default up to a size which is about the same as the screen size. Images will be fully loaded if it's required by their image scaler or if they should be displayed with a size which is bigger than the screen size. --- ueberzug/action.py | 150 +++++++++++++++++++++++++++++++++++++------- ueberzug/layer.py | 5 ++ ueberzug/loading.py | 109 ++++++++++++++++++++++++++------ ueberzug/scaling.py | 41 +++++++++++- 4 files changed, 263 insertions(+), 42 deletions(-) diff --git a/ueberzug/action.py b/ueberzug/action.py index 716e583..e7ca057 100644 --- a/ueberzug/action.py +++ b/ueberzug/action.py @@ -120,40 +120,148 @@ class AddImageAction(ImageAction): def get_action_name(): return 'add' + def __attrs_post_init__(self): + self.width = self.max_width or self.width + self.height = self.max_height or self.height + # attrs doesn't support overriding the init method + # pylint: disable=attribute-defined-outside-init + self.__scaler_class = None + self.__last_modified = None + + @property + def scaler_class(self): + """scaling.ImageScaler: the used scaler class of this placement""" + if self.__scaler_class is None: + self.__scaler_class = \ + scaling.ScalerOption(self.scaler).scaler_class + return self.__scaler_class + + @property + def last_modified(self): + """float: the last modified time of the image""" + if self.__last_modified is None: + self.__last_modified = os.path.getmtime(self.path) + return self.__last_modified + + def is_same_image(self, old_placement): + """Determines whether the placement contains the same image + after applying the changes of this command. + + Args: + old_placement (ui.OverlayWindow.Placement): + the old data of the placement + + Returns: + bool: True if it's the same file + """ + return old_placement and not ( + old_placement.last_modified < self.last_modified + or self.path != old_placement.path) + + def is_full_reload_required(self, old_placement, + screen_columns, screen_rows): + """Determines whether it's required to fully reload + the image of the placement to properly render the placement. + + Args: + old_placement (ui.OverlayWindow.Placement): + the old data of the placement + screen_columns (float): + the maximum amount of columns the screen can display + screen_rows (float): + the maximum amount of rows the screen can display + + Returns: + bool: True if the image should be reloaded + """ + return old_placement and ( + (not self.scaler_class.is_indulgent_resizing() + and old_placement.scaler.is_indulgent_resizing()) + or (old_placement.width <= screen_columns < self.width) + or (old_placement.height <= screen_rows < self.height)) + + def is_partly_reload_required(self, old_placement, + screen_columns, screen_rows): + """Determines whether it's required to partly reload + the image of the placement to render the placement more quickly. + + Args: + old_placement (ui.OverlayWindow.Placement): + the old data of the placement + screen_columns (float): + the maximum amount of columns the screen can display + screen_rows (float): + the maximum amount of rows the screen can display + + Returns: + bool: True if the image should be reloaded + """ + return old_placement and ( + (self.scaler_class.is_indulgent_resizing() + and not old_placement.scaler.is_indulgent_resizing()) + or (self.width <= screen_columns < old_placement.width) + or (self.height <= screen_rows < old_placement.height)) + async def apply(self, windows, view, tools): try: import ueberzug.ui as ui + import ueberzug.loading as loading old_placement = view.media.pop(self.identifier, None) cache = old_placement and old_placement.cache image = old_placement and old_placement.image - last_modified = old_placement and old_placement.last_modified - current_last_modified = os.path.getmtime(self.path) - width = self.max_width or self.width - height = self.max_height or self.height - scaler_class = scaling.ScalerOption(self.scaler).scaler_class - - if (not image - or last_modified < current_last_modified - or self.path != old_placement.path): - last_modified = current_last_modified + + max_font_width = max(map( + lambda i: i or 0, windows.parent_info.font_width or [0])) + max_font_height = max(map( + lambda i: i or 0, windows.parent_info.font_height or [0])) + font_size_available = max_font_width and max_font_height + screen_columns = (font_size_available and + view.screen_width / max_font_width) + screen_rows = (font_size_available and + view.screen_height / max_font_height) + + # By default images are only stored up to a resolution which + # is about as big as the screen resolution. + # (loading.CoverPostLoadImageProcessor) + # The principle of spatial locality does not apply to + # resize operations of images with big resolutions + # which is why those operations should be applied + # to a resized version of those images. + # Sometimes we still need all pixels e.g. + # if the image scaler crop is used. + # So sometimes it's required to fully load them + # and sometimes it's not required anymore which is + # why they should be partly reloaded + # (to speed up the resize operations again). + if (not self.is_same_image(old_placement) + or (font_size_available and self.is_full_reload_required( + old_placement, screen_columns, screen_rows)) + or (font_size_available and self.is_partly_reload_required( + old_placement, screen_columns, screen_rows))): upper_bound_size = None - max_font_width = max(map( - lambda i: i or 0, windows.parent_info.font_width)) - max_font_height = max(map( - lambda i: i or 0, windows.parent_info.font_height)) - if (scaler_class != scaling.CropImageScaler and - max_font_width and max_font_height): + image_post_load_processor = None + if (self.scaler_class != scaling.CropImageScaler and + font_size_available): upper_bound_size = ( - max_font_width * width, max_font_height * height) - image = tools.loader.load(self.path, upper_bound_size) + max_font_width * self.width, + max_font_height * self.height) + if (self.scaler_class != scaling.CropImageScaler + and font_size_available + and self.width <= screen_columns + and self.height <= screen_rows): + image_post_load_processor = \ + loading.CoverPostLoadImageProcessor( + view.screen_width, view.screen_height) + image = tools.loader.load( + self.path, upper_bound_size, image_post_load_processor) cache = None view.media[self.identifier] = ui.OverlayWindow.Placement( - self.x, self.y, width, height, + self.x, self.y, self.width, self.height, geometry.Point(self.scaling_position_x, self.scaling_position_y), - scaler_class(), - self.path, image, last_modified, cache) + self.scaler_class(), + self.path, image, self.last_modified, cache) finally: await super().apply(windows, view, tools) diff --git a/ueberzug/layer.py b/ueberzug/layer.py index c253d86..fe51cf2 100644 --- a/ueberzug/layer.py +++ b/ueberzug/layer.py @@ -173,6 +173,8 @@ class View: def __init__(self): self.offset = geometry.Distance() self.media = {} + self.screen_width = 0 + self.screen_height = 0 class Tools: @@ -185,6 +187,7 @@ class Tools: def main(options): display = xutil.get_display() + screen = display.screen() window_infos = xutil.get_parent_window_infos() loop = asyncio.get_event_loop() executor = thread.DaemonThreadPoolExecutor(max_workers=2) @@ -198,6 +201,8 @@ def main(options): window_factory = ui.OverlayWindow.Factory(display, view) windows = batch.BatchList(window_factory.create(*window_infos)) image_loader.register_error_handler(error_handler) + view.screen_width = screen.width_in_pixels + view.screen_height = screen.height_in_pixels if tmux_util.is_used(): atexit.register(setup_tmux_hooks()) diff --git a/ueberzug/loading.py b/ueberzug/loading.py index e1eb8ee..9eed228 100644 --- a/ueberzug/loading.py +++ b/ueberzug/loading.py @@ -89,6 +89,49 @@ class ImageHolder: return self.image +class PostLoadImageProcessor(metaclass=abc.ABCMeta): + """Describes the structure used to define callbacks which + will be invoked after loading an image. + """ + @abc.abstractmethod + def on_loaded(self, image): + """Postprocessor of an loaded image. + The returned image will be assigned to the image holder. + + Args: + image (PIL.Image): the loaded image + + Returns: + PIL.Image: + the image which will be assigned + to the image holder of this loading process + """ + raise NotImplementedError() + + +class CoverPostLoadImageProcessor(PostLoadImageProcessor): + """Implementation of PostLoadImageProcessor + which resizes an image (if possible -> needs to be bigger) + such that it covers only just a given resolution. + """ + def __init__(self, width, height): + self.width = width + self.height = height + + def on_loaded(self, image): + import PIL.Image + resize_ratio = max(min(1, self.width / image.width), + min(1, self.height / image.height)) + + if resize_ratio != 1: + image = image.resize( + (int(resize_ratio * image.width), + int(resize_ratio * image.height)), + PIL.Image.ANTIALIAS) + + return image + + class ImageLoader(metaclass=abc.ABCMeta): """Describes the structure used to define image loading strategies. @@ -113,7 +156,7 @@ class ImageLoader(metaclass=abc.ABCMeta): self.error_handler = None @abc.abstractmethod - def load(self, path, upper_bound_size): + def load(self, path, upper_bound_size, post_load_processor=None): """Starts the image loading procedure for the passed path. How and when an image get's loaded depends on the implementation of the used ImageLoader class. @@ -122,6 +165,8 @@ class ImageLoader(metaclass=abc.ABCMeta): path (str): the path to the image which should be loaded upper_bound_size (tuple of (width: int, height: int)): the maximal size to load data for + post_load_processor (PostLoadImageProcessor): + allows to apply changes to the recently loaded image Returns: ImageHolder: which the image will be assigned to @@ -162,7 +207,7 @@ class SynchronousImageLoader(ImageLoader): def get_loader_name(): return "synchronous" - def load(self, path, upper_bound_size): + def load(self, path, upper_bound_size, post_load_processor=None): image = None try: @@ -170,6 +215,9 @@ class SynchronousImageLoader(ImageLoader): except OSError as exception: self.process_error(exception) + if image and post_load_processor: + image = post_load_processor.on_loaded(image) + return ImageHolder(path, image or self.PLACEHOLDER) @@ -192,7 +240,7 @@ class AsynchronousImageLoader(ImageLoader): self.__queue_low_priority = queue.Queue() self.__waiter_low_priority = threading.Condition() - def _enqueue(self, queue, image_holder, upper_bound_size): + def _enqueue(self, queue, image_holder, upper_bound_size, post_load_processor): """Enqueues the image holder weakly referenced. Args: @@ -201,8 +249,11 @@ class AsynchronousImageLoader(ImageLoader): the image holder for which an image should be loaded upper_bound_size (tuple of (width: int, height: int)): the maximal size to load data for + post_load_processor (PostLoadImageProcessor): + allows to apply changes to the recently loaded image """ - queue.put((weakref.ref(image_holder), upper_bound_size)) + queue.put(( + weakref.ref(image_holder), upper_bound_size, post_load_processor)) def _dequeue(self, queue): """Removes queue entries till an alive reference was found. @@ -214,21 +265,25 @@ class AsynchronousImageLoader(ImageLoader): queue (queue.Queue): the queue to operate on Returns: - tuple of (ImageHolder, tuple of (width: int, height: int)): - an queued image holder or None, upper bound size or None + tuple of (ImageHolder, tuple of (width: int, height: int), + PostLoadImageProcessor): + an queued image holder or None, upper bound size or None, + the post load image processor or None """ holder_reference = None image_holder = None upper_bound_size = None + post_load_processor = None while not queue.empty(): - holder_reference, upper_bound_size = queue.get_nowait() + holder_reference, upper_bound_size, post_load_processor = \ + queue.get_nowait() image_holder = holder_reference and holder_reference() if (holder_reference is None or image_holder is not None): break - return image_holder, upper_bound_size + return image_holder, upper_bound_size, post_load_processor @abc.abstractmethod def _schedule(self, function, priority): @@ -243,16 +298,22 @@ class AsynchronousImageLoader(ImageLoader): """ raise NotImplementedError() - def _load_image(self, path, upper_bound_size): + def _load_image(self, path, upper_bound_size, post_load_processor): """Wrapper for calling load_image. Behaves like calling it directly, but allows e.g. executing the function in other processes. """ - return load_image(path, upper_bound_size) + image, *other_data = load_image(path, upper_bound_size) + + if image and post_load_processor: + image = post_load_processor.on_loaded(image) + + return (image, *other_data) - def load(self, path, upper_bound_size): + def load(self, path, upper_bound_size, post_load_processor=None): holder = ImageHolder(path) - self._enqueue(self.__queue, holder, upper_bound_size) + self._enqueue( + self.__queue, holder, upper_bound_size, post_load_processor) self._schedule(self.__process_high_priority_entry, self.Priority.HIGH) return holder @@ -290,15 +351,18 @@ class AsynchronousImageLoader(ImageLoader): queue (queue.Queue): the queue to operate on """ image = None - image_holder, upper_bound_size = self._dequeue(queue) + image_holder, upper_bound_size, post_load_processor = \ + self._dequeue(queue) if image_holder is None: return try: image, downscaled = self._load_image( - image_holder.path, upper_bound_size) + image_holder.path, upper_bound_size, post_load_processor) if upper_bound_size and downscaled: - self._enqueue(self.__queue_low_priority, image_holder, None) + self._enqueue( + self.__queue_low_priority, + image_holder, None, post_load_processor) self._schedule(self.__process_low_priority_entry, self.Priority.LOW) except OSError as exception: @@ -366,20 +430,25 @@ class ProcessImageLoader(ThreadImageLoader): .result() @staticmethod - def _load_image_extern(path, upper_bound_size): + def _load_image_extern(path, upper_bound_size, post_load_processor): """This function is a wrapper for the image loading function as sometimes pillow restores decoded images received from other processes wrongly. E.g. a PNG is reported as webp (-> crash on using an image function) So this function is a workaround which prevents these crashs to happen. """ - image, downscaled = load_image(path, upper_bound_size) - return image.mode, image.size, image.tobytes(), downscaled + image, *other_data = load_image(path, upper_bound_size) + + if image and post_load_processor: + image = post_load_processor.on_loaded(image) + + return (image.mode, image.size, image.tobytes(), *other_data) - def _load_image(self, path, upper_bound_size): + def _load_image(self, path, upper_bound_size, post_load_processor=None): import PIL.Image future = self.__executor_loader.submit( - ProcessImageLoader._load_image_extern, path, upper_bound_size) + ProcessImageLoader._load_image_extern, + path, upper_bound_size, post_load_processor) mode, size, data, downscaled = future.result() return PIL.Image.frombytes(mode, size, data), downscaled diff --git a/ueberzug/scaling.py b/ueberzug/scaling.py index 4eec61a..947cb98 100644 --- a/ueberzug/scaling.py +++ b/ueberzug/scaling.py @@ -17,7 +17,22 @@ class ImageScaler(metaclass=abc.ABCMeta): @staticmethod @abc.abstractmethod def get_scaler_name(): - """Returns the constant name which is associated to this scaler.""" + """Returns: + str: the constant name which is associated to this scaler. + """ + raise NotImplementedError() + + @staticmethod + @abc.abstractmethod + def is_indulgent_resizing(): + """This method specifies whether the + algorithm returns noticeable different results for + the same image with different sizes (bigger than the + maximum size which is passed to the scale method). + + Returns: + bool: False if the results differ + """ raise NotImplementedError() @abc.abstractmethod @@ -98,6 +113,10 @@ class CropImageScaler(MinSizeImageScaler, OffsetImageScaler): def get_scaler_name(): return "crop" + @staticmethod + def is_indulgent_resizing(): + return False + def scale(self, image, position: geometry.Point, width: int, height: int): width, height = self.calculate_resolution(image, width, height) @@ -118,6 +137,10 @@ class DistortImageScaler(ImageScaler): def get_scaler_name(): return "distort" + @staticmethod + def is_indulgent_resizing(): + return True + def calculate_resolution(self, image, width: int, height: int): return width, height @@ -140,6 +163,10 @@ class FitContainImageScaler(DistortImageScaler): def get_scaler_name(): return "fit_contain" + @staticmethod + def is_indulgent_resizing(): + return True + def calculate_resolution(self, image, width: int, height: int): factor = min(width / image.width, height / image.height) return int(image.width * factor), int(image.height * factor) @@ -155,6 +182,10 @@ class ContainImageScaler(FitContainImageScaler): def get_scaler_name(): return "contain" + @staticmethod + def is_indulgent_resizing(): + return True + def calculate_resolution(self, image, width: int, height: int): return super().calculate_resolution( image, min(width, image.width), min(height, image.height)) @@ -174,6 +205,10 @@ class ForcedCoverImageScaler(DistortImageScaler, OffsetImageScaler): def get_scaler_name(): return "forced_cover" + @staticmethod + def is_indulgent_resizing(): + return True + def scale(self, image, position: geometry.Point, width: int, height: int): import PIL.Image @@ -203,6 +238,10 @@ class CoverImageScaler(MinSizeImageScaler, ForcedCoverImageScaler): def get_scaler_name(): return "cover" + @staticmethod + def is_indulgent_resizing(): + return True + @enum.unique class ScalerOption(str, enum.Enum):