From 5e9ec706c5554e0128b958c196c35d4da6708a12 Mon Sep 17 00:00:00 2001 From: Thore Schillmann Date: Sun, 19 Jun 2022 22:59:54 +0000 Subject: [PATCH 01/13] first draft of embedding metadata on conversion --- cps/tasks/convert.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/cps/tasks/convert.py b/cps/tasks/convert.py index 3062850d..9d2ee636 100644 --- a/cps/tasks/convert.py +++ b/cps/tasks/convert.py @@ -19,8 +19,12 @@ import os import re from glob import glob -from shutil import copyfile +from shutil import copyfile, copyfileobj from markupsafe import escape +#donrar +from tempfile import gettempdir +from time import time +#enddonrar from sqlalchemy.exc import SQLAlchemyError from flask_babel import lazy_gettext as N_ @@ -39,6 +43,7 @@ from cps import gdriveutils log = logger.create() +current_milli_time = lambda: int(round(time() * 1000)) class TaskConvert(CalibreTask): def __init__(self, file_path, book_id, task_message, settings, kindle_mail, user=None): @@ -223,6 +228,7 @@ class TaskConvert(CalibreTask): return check, None def _convert_calibre(self, file_path, format_old_ext, format_new_ext): + book_id = self.book_id try: # Linux py2.7 encode as list without quotes no empty element for parameters # linux py3.x no encode and as list without quotes no empty element for parameters @@ -230,8 +236,18 @@ class TaskConvert(CalibreTask): # windows py 3.x no encode and as string with quotes empty element for parameters is okay # separate handling for windows and linux quotes = [1, 2] + + # TODO: Clean up and make cli work with windows. Also, implement changing covers. + tmp_dir = os.path.join(gettempdir(), 'calibre_web') + path_calibrecli = os.path.join(os.path.dirname(config.config_converterpath), "calibredb") + opf_command = [path_calibrecli, 'show_metadata', '--as-opf', str(book_id), '--with-library', config.config_calibre_dir] + p = process_open(opf_command) + path_tmp_opf = os.path.join(tmp_dir, "metadata_" + str(current_milli_time()) + ".opf") + with open(path_tmp_opf, 'w') as fd: + copyfileobj(p.stdout, fd) + command = [config.config_converterpath, (file_path + format_old_ext), - (file_path + format_new_ext)] + (file_path + format_new_ext), '--from-opf', path_tmp_opf] quotes_index = 3 if config.config_calibre: parameters = config.config_calibre.split(" ") From 8c781ad4a438450ca1a1d79ac1d84f31ac3e7596 Mon Sep 17 00:00:00 2001 From: Thore Schillmann Date: Sun, 19 Jun 2022 23:20:59 +0000 Subject: [PATCH 02/13] code cleanup and implements cover change --- cps/tasks/convert.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cps/tasks/convert.py b/cps/tasks/convert.py index 9d2ee636..6d723ac8 100644 --- a/cps/tasks/convert.py +++ b/cps/tasks/convert.py @@ -21,10 +21,8 @@ import re from glob import glob from shutil import copyfile, copyfileobj from markupsafe import escape -#donrar from tempfile import gettempdir from time import time -#enddonrar from sqlalchemy.exc import SQLAlchemyError from flask_babel import lazy_gettext as N_ @@ -237,7 +235,7 @@ class TaskConvert(CalibreTask): # separate handling for windows and linux quotes = [1, 2] - # TODO: Clean up and make cli work with windows. Also, implement changing covers. + # TODO: Clean up and make cli work with windows. tmp_dir = os.path.join(gettempdir(), 'calibre_web') path_calibrecli = os.path.join(os.path.dirname(config.config_converterpath), "calibredb") opf_command = [path_calibrecli, 'show_metadata', '--as-opf', str(book_id), '--with-library', config.config_calibre_dir] @@ -247,7 +245,8 @@ class TaskConvert(CalibreTask): copyfileobj(p.stdout, fd) command = [config.config_converterpath, (file_path + format_old_ext), - (file_path + format_new_ext), '--from-opf', path_tmp_opf] + (file_path + format_new_ext), '--from-opf', path_tmp_opf, + '--cover', os.path.join(os.path.dirname(file_path), 'cover.jpg')] quotes_index = 3 if config.config_calibre: parameters = config.config_calibre.split(" ") From 3c4330ba510aa31bb60e58c00a24226225576a73 Mon Sep 17 00:00:00 2001 From: Thore Schillmann Date: Tue, 21 Jun 2022 17:04:44 +0000 Subject: [PATCH 03/13] refactoring of calibre binary detection --- cps/admin.py | 1 + cps/config_sql.py | 55 +++++++++++++++++++++++++++------- cps/constants.py | 3 ++ cps/tasks/convert.py | 7 +++-- cps/templates/config_edit.html | 7 +++++ 5 files changed, 60 insertions(+), 13 deletions(-) diff --git a/cps/admin.py b/cps/admin.py index 76275922..43103eed 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -1721,6 +1721,7 @@ def _configuration_update_helper(): constants.EXTENSIONS_UPLOAD = config.config_upload_formats.split(',') _config_string(to_save, "config_calibre") + _config_string(to_save, "config_binariesdir") _config_string(to_save, "config_converterpath") _config_string(to_save, "config_kepubifypath") diff --git a/cps/config_sql.py b/cps/config_sql.py index 743b2ce7..9e3472bd 100644 --- a/cps/config_sql.py +++ b/cps/config_sql.py @@ -131,6 +131,7 @@ class _Settings(_Base): config_kepubifypath = Column(String, default=None) config_converterpath = Column(String, default=None) + config_binariesdir = Column(String, default=None) config_calibre = Column(String) config_rarfile_location = Column(String, default=None) config_upload_formats = Column(String, default=','.join(constants.EXTENSIONS_UPLOAD)) @@ -166,12 +167,16 @@ class _ConfigSQL(object): self.cli = cli change = False + + if self.config_binariesdir == None: # pylint: disable=access-member-before-definition + change = True + self.config_binariesdir = autodetect_calibre_binaries() + if self.config_converterpath == None: # pylint: disable=access-member-before-definition change = True - self.config_converterpath = autodetect_calibre_binary() + self.config_converterpath = autodetect_converter_binary() if self.config_kepubifypath == None: # pylint: disable=access-member-before-definition - change = True self.config_kepubifypath = autodetect_kepubify_binary() @@ -266,6 +271,21 @@ class _ConfigSQL(object): def get_scheduled_task_settings(self): return {k:v for k, v in self.__dict__.items() if k.startswith('schedule_')} + def get_calibre_binarypath(self, binary): + binariesdir = self.config_binariesdir + if binariesdir: + # TODO: Need to make sure that all supported calibre binaries are actually in the specified directory when set via UI + if sys.platform == "win32": + extension = ".exe" + else: + extension = "" + if binary in constants.SUPPORTED_CALIBRE_BINARIES: + return os.path.join(binariesdir, binary + extension) + else: + # TODO: Error handling + pass + return "" + def set_from_dictionary(self, dictionary, field, convertor=None, default=None, encode=None): """Possibly updates a field of this object. The new value, if present, is grabbed from the given dictionary, and optionally passed through a convertor. @@ -407,17 +427,31 @@ def _migrate_table(session, orm_class): session.rollback() -def autodetect_calibre_binary(): +def autodetect_calibre_binaries(): if sys.platform == "win32": - calibre_path = ["C:\\program files\\calibre\\ebook-convert.exe", - "C:\\program files(x86)\\calibre\\ebook-convert.exe", - "C:\\program files(x86)\\calibre2\\ebook-convert.exe", - "C:\\program files\\calibre2\\ebook-convert.exe"] + extension = ".exe" + calibre_path = ["C:\\program files\\calibre\\", + "C:\\program files(x86)\\calibre\\", + "C:\\program files(x86)\\calibre2\\", + "C:\\program files\\calibre2\\"] else: - calibre_path = ["/opt/calibre/ebook-convert"] + extension = "" + calibre_path = ["/opt/calibre/"] for element in calibre_path: - if os.path.isfile(element) and os.access(element, os.X_OK): - return element + supported_binary_paths = [os.path.join(element, binary + extension) for binary in constants.SUPPORTED_CALIBRE_BINARIES] + if all(os.path.isfile(binary_path) and os.access(binary_path, os.X_OK) for binary_path in supported_binary_paths): + return element + return "" + + +def autodetect_converter_binary(): + calibre_path = autodetect_calibre_binaries() + if sys.platform == "win32": + converter_path = os.path.join(calibre_path, "ebook-convert.exe") + else: + converter_path = os.path.join(calibre_path, "ebook-convert") + if os.path.isfile(converter_path) and os.access(converter_path, os.X_OK): + return converter_path return "" @@ -462,6 +496,7 @@ def load_configuration(conf, session, cli): conf.init_config(session, cli) # return conf + def get_flask_session_key(_session): flask_settings = _session.query(_Flask_Settings).one_or_none() if flask_settings == None: diff --git a/cps/constants.py b/cps/constants.py index 0f3b2530..c684453c 100644 --- a/cps/constants.py +++ b/cps/constants.py @@ -152,6 +152,9 @@ EXTENSIONS_UPLOAD = {'txt', 'pdf', 'epub', 'kepub', 'mobi', 'azw', 'azw3', 'cbr' 'opus', 'wav', 'flac', 'm4a', 'm4b'} +SUPPORTED_CALIBRE_BINARIES = ["ebook-convert", "calibredb"] + + def has_flag(value, bit_flag): return bit_flag == (bit_flag & (value or 0)) diff --git a/cps/tasks/convert.py b/cps/tasks/convert.py index 6d723ac8..74455837 100644 --- a/cps/tasks/convert.py +++ b/cps/tasks/convert.py @@ -235,10 +235,11 @@ class TaskConvert(CalibreTask): # separate handling for windows and linux quotes = [1, 2] - # TODO: Clean up and make cli work with windows. + # TODO: Clean up. + # TODO: Maybe delete/clean-up tmp files directly. tmp_dir = os.path.join(gettempdir(), 'calibre_web') - path_calibrecli = os.path.join(os.path.dirname(config.config_converterpath), "calibredb") - opf_command = [path_calibrecli, 'show_metadata', '--as-opf', str(book_id), '--with-library', config.config_calibre_dir] + calibredb_binarypath = config.get_calibre_binarypath("calibredb") + opf_command = [calibredb_binarypath, 'show_metadata', '--as-opf', str(book_id), '--with-library', config.config_calibre_dir] p = process_open(opf_command) path_tmp_opf = os.path.join(tmp_dir, "metadata_" + str(current_milli_time()) + ".opf") with open(path_tmp_opf, 'w') as fd: diff --git a/cps/templates/config_edit.html b/cps/templates/config_edit.html index f61ca9a5..725fb88d 100644 --- a/cps/templates/config_edit.html +++ b/cps/templates/config_edit.html @@ -323,6 +323,13 @@
+ +
+ + + + +
From 03359599edcc91dd9dffa410265640253a6141b6 Mon Sep 17 00:00:00 2001 From: Thore Schillmann Date: Thu, 23 Jun 2022 20:02:54 +0000 Subject: [PATCH 04/13] input validation for calibre binary directory --- cps/admin.py | 4 ++++ cps/config_sql.py | 18 ++++++++---------- cps/constants.py | 6 ++++-- cps/helper.py | 29 ++++++++++++++++++++++++++++- 4 files changed, 44 insertions(+), 13 deletions(-) diff --git a/cps/admin.py b/cps/admin.py index 43103eed..3b24e518 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -1724,6 +1724,10 @@ def _configuration_update_helper(): _config_string(to_save, "config_binariesdir") _config_string(to_save, "config_converterpath") _config_string(to_save, "config_kepubifypath") + if "config_binariesdir" in to_save: + calibre_status = helper.check_calibre(config.config_binariesdir) + if calibre_status: + return _configuration_result(calibre_status) reboot_required |= _config_int(to_save, "config_login_type") diff --git a/cps/config_sql.py b/cps/config_sql.py index 9e3472bd..a3798765 100644 --- a/cps/config_sql.py +++ b/cps/config_sql.py @@ -30,6 +30,7 @@ except ImportError: from sqlalchemy.ext.declarative import declarative_base from . import constants, logger +from .subproc_wrapper import process_wait log = logger.create() @@ -274,13 +275,8 @@ class _ConfigSQL(object): def get_calibre_binarypath(self, binary): binariesdir = self.config_binariesdir if binariesdir: - # TODO: Need to make sure that all supported calibre binaries are actually in the specified directory when set via UI - if sys.platform == "win32": - extension = ".exe" - else: - extension = "" if binary in constants.SUPPORTED_CALIBRE_BINARIES: - return os.path.join(binariesdir, binary + extension) + return os.path.join(binariesdir, binary) else: # TODO: Error handling pass @@ -429,18 +425,20 @@ def _migrate_table(session, orm_class): def autodetect_calibre_binaries(): if sys.platform == "win32": - extension = ".exe" calibre_path = ["C:\\program files\\calibre\\", "C:\\program files(x86)\\calibre\\", "C:\\program files(x86)\\calibre2\\", "C:\\program files\\calibre2\\"] else: - extension = "" calibre_path = ["/opt/calibre/"] for element in calibre_path: - supported_binary_paths = [os.path.join(element, binary + extension) for binary in constants.SUPPORTED_CALIBRE_BINARIES] + supported_binary_paths = [os.path.join(element, binary) for binary in constants.SUPPORTED_CALIBRE_BINARIES] if all(os.path.isfile(binary_path) and os.access(binary_path, os.X_OK) for binary_path in supported_binary_paths): - return element + values = [process_wait([binary_path, "--version"], pattern='\(calibre (.*)\)') for binary_path in supported_binary_paths] + if all(values): + version = values[0].group(1) + log.debug("calibre version %s", version) + return element return "" diff --git a/cps/constants.py b/cps/constants.py index c684453c..849f9562 100644 --- a/cps/constants.py +++ b/cps/constants.py @@ -151,8 +151,10 @@ EXTENSIONS_UPLOAD = {'txt', 'pdf', 'epub', 'kepub', 'mobi', 'azw', 'azw3', 'cbr' 'prc', 'doc', 'docx', 'fb2', 'html', 'rtf', 'lit', 'odt', 'mp3', 'mp4', 'ogg', 'opus', 'wav', 'flac', 'm4a', 'm4b'} - -SUPPORTED_CALIBRE_BINARIES = ["ebook-convert", "calibredb"] +_extension = "" +if sys.platform == "win32": + _extension = ".exe" +SUPPORTED_CALIBRE_BINARIES = [binary + _extension for binary in ["ebook-convert", "calibredb"]] def has_flag(value, bit_flag): diff --git a/cps/helper.py b/cps/helper.py index ed11e1c0..aba18dcc 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -53,7 +53,7 @@ from . import calibre_db, cli_param from .tasks.convert import TaskConvert from . import logger, config, db, ub, fs from . import gdriveutils as gd -from .constants import STATIC_DIR as _STATIC_DIR, CACHE_TYPE_THUMBNAILS, THUMBNAIL_TYPE_COVER, THUMBNAIL_TYPE_SERIES +from .constants import STATIC_DIR as _STATIC_DIR, CACHE_TYPE_THUMBNAILS, THUMBNAIL_TYPE_COVER, THUMBNAIL_TYPE_SERIES, SUPPORTED_CALIBRE_BINARIES from .subproc_wrapper import process_wait from .services.worker import WorkerThread from .tasks.mail import TaskEmail @@ -940,6 +940,33 @@ def check_unrar(unrar_location): return _('Error excecuting UnRar') +def check_calibre(calibre_location): + if not calibre_location: + return + + if not os.path.exists(calibre_location): + return _('Could not find the specified directory') + + if not os.path.isdir(calibre_location): + return _('Please specify a directory, not a file') + + try: + supported_binary_paths = [os.path.join(calibre_location, binary) for binary in SUPPORTED_CALIBRE_BINARIES] + if all(os.path.isfile(binary_path) and os.access(binary_path, os.X_OK) for binary_path in supported_binary_paths): + values = [process_wait([binary_path, "--version"], pattern='\(calibre (.*)\)') for binary_path in supported_binary_paths] + if all(values): + version = values[0].group(1) + log.debug("calibre version %s", version) + else: + return _('Calibre binaries not viable') + else: + return _('Missing calibre binaries in the specified directory') + + except (OSError, UnicodeDecodeError) as err: + log.error_or_exception(err) + return _('Error excecuting Calibre') + + def json_serial(obj): """JSON serializer for objects not serializable by default json code""" From e39c6130c34a1bc4bb93cffa77a55865a9c21e36 Mon Sep 17 00:00:00 2001 From: Thore Schillmann Date: Wed, 29 Jun 2022 19:54:53 +0000 Subject: [PATCH 05/13] cleanup and better error handling --- cps/config_sql.py | 3 +-- cps/helper.py | 6 ++++-- cps/tasks/convert.py | 11 ++++++----- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/cps/config_sql.py b/cps/config_sql.py index a3798765..9dc0abdd 100644 --- a/cps/config_sql.py +++ b/cps/config_sql.py @@ -278,8 +278,7 @@ class _ConfigSQL(object): if binary in constants.SUPPORTED_CALIBRE_BINARIES: return os.path.join(binariesdir, binary) else: - # TODO: Error handling - pass + raise ValueError("'{}' is not a supported Calibre binary".format(binary)) return "" def set_from_dictionary(self, dictionary, field, convertor=None, default=None, encode=None): diff --git a/cps/helper.py b/cps/helper.py index aba18dcc..31b15eb9 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -952,7 +952,8 @@ def check_calibre(calibre_location): try: supported_binary_paths = [os.path.join(calibre_location, binary) for binary in SUPPORTED_CALIBRE_BINARIES] - if all(os.path.isfile(binary_path) and os.access(binary_path, os.X_OK) for binary_path in supported_binary_paths): + binaries_available=[os.path.isfile(binary_path) and os.access(binary_path, os.X_OK) for binary_path in supported_binary_paths] + if all(binaries_available): values = [process_wait([binary_path, "--version"], pattern='\(calibre (.*)\)') for binary_path in supported_binary_paths] if all(values): version = values[0].group(1) @@ -960,7 +961,8 @@ def check_calibre(calibre_location): else: return _('Calibre binaries not viable') else: - return _('Missing calibre binaries in the specified directory') + missing_binaries=[path for path, available in zip(SUPPORTED_CALIBRE_BINARIES, binaries_available) if not available] + return _('Missing calibre binaries: %(missing)s', missing=", ".join(missing_binaries)) except (OSError, UnicodeDecodeError) as err: log.error_or_exception(err) diff --git a/cps/tasks/convert.py b/cps/tasks/convert.py index 74455837..1c9975e1 100644 --- a/cps/tasks/convert.py +++ b/cps/tasks/convert.py @@ -233,18 +233,19 @@ class TaskConvert(CalibreTask): # windows py2.7 encode as string with quotes empty element for parameters is okay # windows py 3.x no encode and as string with quotes empty element for parameters is okay # separate handling for windows and linux - quotes = [1, 2] - # TODO: Clean up. - # TODO: Maybe delete/clean-up tmp files directly. + quotes = [3, 5] tmp_dir = os.path.join(gettempdir(), 'calibre_web') + if not os.path.isdir(tmp_dir): + os.mkdir(tmp_dir) calibredb_binarypath = config.get_calibre_binarypath("calibredb") opf_command = [calibredb_binarypath, 'show_metadata', '--as-opf', str(book_id), '--with-library', config.config_calibre_dir] - p = process_open(opf_command) + p = process_open(opf_command, quotes) path_tmp_opf = os.path.join(tmp_dir, "metadata_" + str(current_milli_time()) + ".opf") with open(path_tmp_opf, 'w') as fd: copyfileobj(p.stdout, fd) + quotes = [1, 2, 4, 6] command = [config.config_converterpath, (file_path + format_old_ext), (file_path + format_new_ext), '--from-opf', path_tmp_opf, '--cover', os.path.join(os.path.dirname(file_path), 'cover.jpg')] @@ -257,7 +258,7 @@ class TaskConvert(CalibreTask): quotes_index += 1 p = process_open(command, quotes, newlines=False) - except OSError as e: + except (ValueError, OSError) as e: return 1, N_(u"Ebook-converter failed: %(error)s", error=e) while p.poll() is None: From a77aef83c67f292c6ddac7e6c57d77cb372a996c Mon Sep 17 00:00:00 2001 From: Thore Schillmann Date: Thu, 30 Jun 2022 13:05:36 +0000 Subject: [PATCH 06/13] automatically set `config_converterpath` --- cps/admin.py | 4 +++- cps/templates/config_edit.html | 7 ------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/cps/admin.py b/cps/admin.py index 3b24e518..149d4aa4 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -1722,12 +1722,14 @@ def _configuration_update_helper(): _config_string(to_save, "config_calibre") _config_string(to_save, "config_binariesdir") - _config_string(to_save, "config_converterpath") _config_string(to_save, "config_kepubifypath") if "config_binariesdir" in to_save: calibre_status = helper.check_calibre(config.config_binariesdir) if calibre_status: return _configuration_result(calibre_status) + # ToDo: Remove this and 'self.config_converterpath' and replace with 'config.get_calibre_binarypath("ebook-convert")' everywhere + to_save["config_converterpath"] = config.get_calibre_binarypath("ebook-convert") + _config_string(to_save, "config_converterpath") reboot_required |= _config_int(to_save, "config_login_type") diff --git a/cps/templates/config_edit.html b/cps/templates/config_edit.html index 725fb88d..c9cf9a29 100644 --- a/cps/templates/config_edit.html +++ b/cps/templates/config_edit.html @@ -330,13 +330,6 @@
- -
- - - - -
From 0d34f41a48e3056c0cd3c8bb0cc5882878481642 Mon Sep 17 00:00:00 2001 From: Thore Schillmann Date: Fri, 1 Jul 2022 12:06:33 +0000 Subject: [PATCH 07/13] cleanup of `autodetect_converter_binary` --- cps/config_sql.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/cps/config_sql.py b/cps/config_sql.py index 9dc0abdd..7867b6f3 100644 --- a/cps/config_sql.py +++ b/cps/config_sql.py @@ -172,10 +172,7 @@ class _ConfigSQL(object): if self.config_binariesdir == None: # pylint: disable=access-member-before-definition change = True self.config_binariesdir = autodetect_calibre_binaries() - - if self.config_converterpath == None: # pylint: disable=access-member-before-definition - change = True - self.config_converterpath = autodetect_converter_binary() + self.config_converterpath = autodetect_converter_binary(self.config_binariesdir) if self.config_kepubifypath == None: # pylint: disable=access-member-before-definition change = True @@ -441,13 +438,12 @@ def autodetect_calibre_binaries(): return "" -def autodetect_converter_binary(): - calibre_path = autodetect_calibre_binaries() +def autodetect_converter_binary(calibre_path): if sys.platform == "win32": converter_path = os.path.join(calibre_path, "ebook-convert.exe") else: converter_path = os.path.join(calibre_path, "ebook-convert") - if os.path.isfile(converter_path) and os.access(converter_path, os.X_OK): + if calibre_path and os.path.isfile(converter_path) and os.access(converter_path, os.X_OK): return converter_path return "" From c5c3874243c45436975dd780c96198c3ee5199b6 Mon Sep 17 00:00:00 2001 From: Thore Schillmann Date: Fri, 1 Jul 2022 16:04:25 +0000 Subject: [PATCH 08/13] first implementation --- cps/helper.py | 25 +++++++++++++++++++++++-- cps/tasks/convert.py | 1 + 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/cps/helper.py b/cps/helper.py index 31b15eb9..885ffba5 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -54,7 +54,7 @@ from .tasks.convert import TaskConvert from . import logger, config, db, ub, fs from . import gdriveutils as gd from .constants import STATIC_DIR as _STATIC_DIR, CACHE_TYPE_THUMBNAILS, THUMBNAIL_TYPE_COVER, THUMBNAIL_TYPE_SERIES, SUPPORTED_CALIBRE_BINARIES -from .subproc_wrapper import process_wait +from .subproc_wrapper import process_wait, process_open from .services.worker import WorkerThread from .tasks.mail import TaskEmail from .tasks.thumbnail import TaskClearCoverThumbnailCache, TaskGenerateCoverThumbnails @@ -911,7 +911,28 @@ def do_download_file(book, book_format, client, data, headers): if client == "kobo" and book_format == "kepub": headers["Content-Disposition"] = headers["Content-Disposition"].replace(".kepub", ".kepub.epub") - response = make_response(send_from_directory(filename, data.name + "." + book_format)) + if config.config_binariesdir: + try: + quotes = [3, 5, 7, 9] + tmp_dir = os.path.join(gettempdir(), 'calibre_web') + if not os.path.isdir(tmp_dir): + os.mkdir(tmp_dir) + calibredb_binarypath = config.get_calibre_binarypath("calibredb") + opf_command = [calibredb_binarypath, 'export', '--dont-write-opf', str(book.id), + '--with-library', config.config_calibre_dir, '--to-dir', tmp_dir, + '--formats', book_format, "--template", "{} - {{authors}}".format(book.title)] + file_name = book.title + if len(book.authors) > 0: + file_name = file_name + ' - ' + book.authors[0].name + p = process_open(opf_command, quotes) + _, err = p.communicate() + if err: + log.error('Metadata embedder encountered an error: %s', err) + except (ValueError, OSError) as e: + # ToDo real error handling + log.error_or_exception(e) + + response = make_response(send_from_directory(tmp_dir, file_name + "." + book_format)) # ToDo Check headers parameter for element in headers: response.headers[element[0]] = element[1] diff --git a/cps/tasks/convert.py b/cps/tasks/convert.py index 1c9975e1..594f0c38 100644 --- a/cps/tasks/convert.py +++ b/cps/tasks/convert.py @@ -241,6 +241,7 @@ class TaskConvert(CalibreTask): calibredb_binarypath = config.get_calibre_binarypath("calibredb") opf_command = [calibredb_binarypath, 'show_metadata', '--as-opf', str(book_id), '--with-library', config.config_calibre_dir] p = process_open(opf_command, quotes) + p.wait() path_tmp_opf = os.path.join(tmp_dir, "metadata_" + str(current_milli_time()) + ".opf") with open(path_tmp_opf, 'w') as fd: copyfileobj(p.stdout, fd) From fc004f4f0c9d4bbb0e9817f6983f82bce76dd2ac Mon Sep 17 00:00:00 2001 From: Thore Schillmann Date: Thu, 7 Jul 2022 11:41:51 +0000 Subject: [PATCH 09/13] moved `get_calibre_binarypath()` to `helper.py` --- cps/admin.py | 5 ++--- cps/config_sql.py | 11 +---------- cps/constants.py | 2 +- cps/helper.py | 15 +++++++++++++-- cps/tasks/convert.py | 6 +++--- 5 files changed, 20 insertions(+), 19 deletions(-) diff --git a/cps/admin.py b/cps/admin.py index 149d4aa4..e487b5ca 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -43,7 +43,7 @@ from . import constants, logger, helper, services, cli_param from . import db, calibre_db, ub, web_server, config, updater_thread, gdriveutils, \ kobo_sync_status, schedule from .helper import check_valid_domain, send_test_mail, reset_password, generate_password_hash, check_email, \ - valid_email, check_username + valid_email, check_username, get_calibre_binarypath from .gdriveutils import is_gdrive_ready, gdrive_support from .render_template import render_title_template, get_sidebar_config from .services.worker import WorkerThread @@ -1727,8 +1727,7 @@ def _configuration_update_helper(): calibre_status = helper.check_calibre(config.config_binariesdir) if calibre_status: return _configuration_result(calibre_status) - # ToDo: Remove this and 'self.config_converterpath' and replace with 'config.get_calibre_binarypath("ebook-convert")' everywhere - to_save["config_converterpath"] = config.get_calibre_binarypath("ebook-convert") + to_save["config_converterpath"] = get_calibre_binarypath("ebook-convert") _config_string(to_save, "config_converterpath") reboot_required |= _config_int(to_save, "config_login_type") diff --git a/cps/config_sql.py b/cps/config_sql.py index 7867b6f3..9b8c29fa 100644 --- a/cps/config_sql.py +++ b/cps/config_sql.py @@ -269,15 +269,6 @@ class _ConfigSQL(object): def get_scheduled_task_settings(self): return {k:v for k, v in self.__dict__.items() if k.startswith('schedule_')} - def get_calibre_binarypath(self, binary): - binariesdir = self.config_binariesdir - if binariesdir: - if binary in constants.SUPPORTED_CALIBRE_BINARIES: - return os.path.join(binariesdir, binary) - else: - raise ValueError("'{}' is not a supported Calibre binary".format(binary)) - return "" - def set_from_dictionary(self, dictionary, field, convertor=None, default=None, encode=None): """Possibly updates a field of this object. The new value, if present, is grabbed from the given dictionary, and optionally passed through a convertor. @@ -428,7 +419,7 @@ def autodetect_calibre_binaries(): else: calibre_path = ["/opt/calibre/"] for element in calibre_path: - supported_binary_paths = [os.path.join(element, binary) for binary in constants.SUPPORTED_CALIBRE_BINARIES] + supported_binary_paths = [os.path.join(element, binary) for binary in constants.SUPPORTED_CALIBRE_BINARIES.values()] if all(os.path.isfile(binary_path) and os.access(binary_path, os.X_OK) for binary_path in supported_binary_paths): values = [process_wait([binary_path, "--version"], pattern='\(calibre (.*)\)') for binary_path in supported_binary_paths] if all(values): diff --git a/cps/constants.py b/cps/constants.py index 849f9562..b613d0aa 100644 --- a/cps/constants.py +++ b/cps/constants.py @@ -154,7 +154,7 @@ EXTENSIONS_UPLOAD = {'txt', 'pdf', 'epub', 'kepub', 'mobi', 'azw', 'azw3', 'cbr' _extension = "" if sys.platform == "win32": _extension = ".exe" -SUPPORTED_CALIBRE_BINARIES = [binary + _extension for binary in ["ebook-convert", "calibredb"]] +SUPPORTED_CALIBRE_BINARIES = {binary:binary + _extension for binary in ["ebook-convert", "calibredb"]} def has_flag(value, bit_flag): diff --git a/cps/helper.py b/cps/helper.py index 31b15eb9..a16245fa 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -951,7 +951,7 @@ def check_calibre(calibre_location): return _('Please specify a directory, not a file') try: - supported_binary_paths = [os.path.join(calibre_location, binary) for binary in SUPPORTED_CALIBRE_BINARIES] + supported_binary_paths = [os.path.join(calibre_location, binary) for binary in SUPPORTED_CALIBRE_BINARIES.values()] binaries_available=[os.path.isfile(binary_path) and os.access(binary_path, os.X_OK) for binary_path in supported_binary_paths] if all(binaries_available): values = [process_wait([binary_path, "--version"], pattern='\(calibre (.*)\)') for binary_path in supported_binary_paths] @@ -961,7 +961,7 @@ def check_calibre(calibre_location): else: return _('Calibre binaries not viable') else: - missing_binaries=[path for path, available in zip(SUPPORTED_CALIBRE_BINARIES, binaries_available) if not available] + missing_binaries=[path for path, available in zip(SUPPORTED_CALIBRE_BINARIES.values(), binaries_available) if not available] return _('Missing calibre binaries: %(missing)s', missing=", ".join(missing_binaries)) except (OSError, UnicodeDecodeError) as err: @@ -1032,6 +1032,17 @@ def get_download_link(book_id, book_format, client): abort(404) +def get_calibre_binarypath(binary): + binariesdir = config.config_binariesdir + if binariesdir: + try: + return os.path.join(binariesdir, SUPPORTED_CALIBRE_BINARIES[binary]) + except KeyError as ex: + log.error("Binary not supported by Calibre-Web: %s", SUPPORTED_CALIBRE_BINARIES[binary]) + pass + return "" + + def clear_cover_thumbnail_cache(book_id): if config.schedule_generate_book_covers: WorkerThread.add(None, TaskClearCoverThumbnailCache(book_id), hidden=True) diff --git a/cps/tasks/convert.py b/cps/tasks/convert.py index 1c9975e1..25cf562d 100644 --- a/cps/tasks/convert.py +++ b/cps/tasks/convert.py @@ -37,7 +37,7 @@ from cps.ub import init_db_thread from cps.tasks.mail import TaskEmail from cps import gdriveutils - +from cps.constants import SUPPORTED_CALIBRE_BINARIES log = logger.create() @@ -238,7 +238,7 @@ class TaskConvert(CalibreTask): tmp_dir = os.path.join(gettempdir(), 'calibre_web') if not os.path.isdir(tmp_dir): os.mkdir(tmp_dir) - calibredb_binarypath = config.get_calibre_binarypath("calibredb") + calibredb_binarypath = os.path.join(config.config_binariesdir, SUPPORTED_CALIBRE_BINARIES["calibredb"]) opf_command = [calibredb_binarypath, 'show_metadata', '--as-opf', str(book_id), '--with-library', config.config_calibre_dir] p = process_open(opf_command, quotes) path_tmp_opf = os.path.join(tmp_dir, "metadata_" + str(current_milli_time()) + ".opf") @@ -258,7 +258,7 @@ class TaskConvert(CalibreTask): quotes_index += 1 p = process_open(command, quotes, newlines=False) - except (ValueError, OSError) as e: + except OSError as e: return 1, N_(u"Ebook-converter failed: %(error)s", error=e) while p.poll() is None: From 4913673e8fe4f394b48926252402b66c95a7a1fd Mon Sep 17 00:00:00 2001 From: Thore Schillmann Date: Thu, 7 Jul 2022 11:47:10 +0000 Subject: [PATCH 10/13] added `subprocess.wait()` when getting metadata --- cps/tasks/convert.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cps/tasks/convert.py b/cps/tasks/convert.py index 25cf562d..79d7ddbd 100644 --- a/cps/tasks/convert.py +++ b/cps/tasks/convert.py @@ -241,6 +241,7 @@ class TaskConvert(CalibreTask): calibredb_binarypath = os.path.join(config.config_binariesdir, SUPPORTED_CALIBRE_BINARIES["calibredb"]) opf_command = [calibredb_binarypath, 'show_metadata', '--as-opf', str(book_id), '--with-library', config.config_calibre_dir] p = process_open(opf_command, quotes) + p.wait() path_tmp_opf = os.path.join(tmp_dir, "metadata_" + str(current_milli_time()) + ".opf") with open(path_tmp_opf, 'w') as fd: copyfileobj(p.stdout, fd) From fc7ce8da2d2dee4f2c235ebce8c3ab59be832778 Mon Sep 17 00:00:00 2001 From: Thore Schillmann Date: Wed, 13 Jul 2022 15:39:01 +0000 Subject: [PATCH 11/13] cleanup --- cps/helper.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cps/helper.py b/cps/helper.py index 77323d0b..52174b02 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -917,7 +917,7 @@ def do_download_file(book, book_format, client, data, headers): tmp_dir = os.path.join(gettempdir(), 'calibre_web') if not os.path.isdir(tmp_dir): os.mkdir(tmp_dir) - calibredb_binarypath = config.get_calibre_binarypath("calibredb") + calibredb_binarypath = get_calibre_binarypath("calibredb") opf_command = [calibredb_binarypath, 'export', '--dont-write-opf', str(book.id), '--with-library', config.config_calibre_dir, '--to-dir', tmp_dir, '--formats', book_format, "--template", "{} - {{authors}}".format(book.title)] @@ -928,9 +928,9 @@ def do_download_file(book, book_format, client, data, headers): _, err = p.communicate() if err: log.error('Metadata embedder encountered an error: %s', err) - except (ValueError, OSError) as e: + except OSError as ex: # ToDo real error handling - log.error_or_exception(e) + log.error_or_exception(ex) response = make_response(send_from_directory(tmp_dir, file_name + "." + book_format)) # ToDo Check headers parameter From 0b4731913eadcd511246aa286867e7b9850d52f9 Mon Sep 17 00:00:00 2001 From: Thore Schillmann Date: Thu, 14 Jul 2022 09:25:37 +0000 Subject: [PATCH 12/13] created `do_calibre_export` function --- cps/helper.py | 55 +++++++++++++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 24 deletions(-) diff --git a/cps/helper.py b/cps/helper.py index 52174b02..282a22d3 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -894,9 +894,10 @@ def save_cover(img, book_path): def do_download_file(book, book_format, client, data, headers): + book_name = data.name if config.config_use_google_drive: # startTime = time.time() - df = gd.getFileFromEbooksFolder(book.path, data.name + "." + book_format) + df = gd.getFileFromEbooksFolder(book.path, book_name + "." + book_format) # log.debug('%s', time.time() - startTime) if df: return gd.do_gdrive_download(df, headers) @@ -904,41 +905,47 @@ def do_download_file(book, book_format, client, data, headers): abort(404) else: filename = os.path.join(config.config_calibre_dir, book.path) - if not os.path.isfile(os.path.join(filename, data.name + "." + book_format)): + if not os.path.isfile(os.path.join(filename, book_name + "." + book_format)): # ToDo: improve error handling - log.error('File not found: %s', os.path.join(filename, data.name + "." + book_format)) + log.error('File not found: %s', os.path.join(filename, book_name + "." + book_format)) if client == "kobo" and book_format == "kepub": headers["Content-Disposition"] = headers["Content-Disposition"].replace(".kepub", ".kepub.epub") if config.config_binariesdir: - try: - quotes = [3, 5, 7, 9] - tmp_dir = os.path.join(gettempdir(), 'calibre_web') - if not os.path.isdir(tmp_dir): - os.mkdir(tmp_dir) - calibredb_binarypath = get_calibre_binarypath("calibredb") - opf_command = [calibredb_binarypath, 'export', '--dont-write-opf', str(book.id), - '--with-library', config.config_calibre_dir, '--to-dir', tmp_dir, - '--formats', book_format, "--template", "{} - {{authors}}".format(book.title)] - file_name = book.title - if len(book.authors) > 0: - file_name = file_name + ' - ' + book.authors[0].name - p = process_open(opf_command, quotes) - _, err = p.communicate() - if err: - log.error('Metadata embedder encountered an error: %s', err) - except OSError as ex: - # ToDo real error handling - log.error_or_exception(ex) + filename, book_name = do_calibre_export(book, book_format) - response = make_response(send_from_directory(tmp_dir, file_name + "." + book_format)) + response = make_response(send_from_directory(filename, book_name + "." + book_format)) # ToDo Check headers parameter for element in headers: response.headers[element[0]] = element[1] - log.info('Downloading file: {}'.format(os.path.join(filename, data.name + "." + book_format))) + log.info('Downloading file: {}'.format(os.path.join(filename, book_name + "." + book_format))) return response + +def do_calibre_export(book, book_format): + try: + quotes = [3, 5, 7, 9] + tmp_dir = os.path.join(gettempdir(), 'calibre_web') + if not os.path.isdir(tmp_dir): + os.mkdir(tmp_dir) + calibredb_binarypath = get_calibre_binarypath("calibredb") + opf_command = [calibredb_binarypath, 'export', '--dont-write-opf', str(book.id), + '--with-library', config.config_calibre_dir, '--to-dir', tmp_dir, + '--formats', book_format, "--template", "{} - {{authors}}".format(book.title)] + file_name = book.title + if len(book.authors) > 0: + file_name = file_name + ' - ' + book.authors[0].name + p = process_open(opf_command, quotes) + _, err = p.communicate() + if err: + log.error('Metadata embedder encountered an error: %s', err) + return tmp_dir, file_name + except OSError as ex: + # ToDo real error handling + log.error_or_exception(ex) + + ################################## From 80b0e8865079a076939f3547405b556b485abb59 Mon Sep 17 00:00:00 2001 From: Thore Schillmann Date: Thu, 14 Jul 2022 13:34:42 +0000 Subject: [PATCH 13/13] fix for GDrive integration --- cps/tasks/convert.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cps/tasks/convert.py b/cps/tasks/convert.py index 79d7ddbd..ac0d3a2a 100644 --- a/cps/tasks/convert.py +++ b/cps/tasks/convert.py @@ -64,15 +64,20 @@ class TaskConvert(CalibreTask): data = worker_db.get_book_format(self.book_id, self.settings['old_book_format']) df = gdriveutils.getFileFromEbooksFolder(cur_book.path, data.name + "." + self.settings['old_book_format'].lower()) - if df: + df_cover = gdriveutils.getFileFromEbooksFolder(cur_book.path, "cover.jpg") + if df and df_cover: datafile = os.path.join(config.config_calibre_dir, cur_book.path, data.name + u"." + self.settings['old_book_format'].lower()) + datafile_cover = os.path.join(config.config_calibre_dir, + cur_book.path, "cover.jpg") if not os.path.exists(os.path.join(config.config_calibre_dir, cur_book.path)): os.makedirs(os.path.join(config.config_calibre_dir, cur_book.path)) df.GetContentFile(datafile) + df_cover.GetContentFile(datafile_cover) worker_db.session.close() else: + # ToDo Include cover in error handling error_message = _(u"%(format)s not found on Google Drive: %(fn)s", format=self.settings['old_book_format'], fn=data.name + "." + self.settings['old_book_format'].lower()) @@ -82,6 +87,7 @@ class TaskConvert(CalibreTask): filename = self._convert_ebook_format() if config.config_use_google_drive: os.remove(self.file_path + u'.' + self.settings['old_book_format'].lower()) + os.remove(os.path.join(config.config_calibre_dir, cur_book.path, "cover.jpg")) if filename: if config.config_use_google_drive: