diff --git a/cps/about.py b/cps/about.py index 7b6cc71a..1d081fe2 100644 --- a/cps/about.py +++ b/cps/about.py @@ -49,9 +49,9 @@ sorted_modules = OrderedDict((sorted(modules.items(), key=lambda x: x[0].casefol def collect_stats(): if constants.NIGHTLY_VERSION[0] == "$Format:%H$": - calibre_web_version = constants.STABLE_VERSION['version'] + calibre_web_version = constants.STABLE_VERSION['version'].replace("b", " Beta") else: - calibre_web_version = (constants.STABLE_VERSION['version'] + ' - ' + calibre_web_version = (constants.STABLE_VERSION['version'].replace("b", " Beta") + ' - ' + constants.NIGHTLY_VERSION[0].replace('%', '%%') + ' - ' + constants.NIGHTLY_VERSION[1].replace('%', '%%')) diff --git a/cps/admin.py b/cps/admin.py index 20e901e3..fa29759e 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -47,7 +47,7 @@ from . import constants, logger, helper, services, cli_param from . import db, calibre_db, ub, web_server, config, updater_thread, gdriveutils, \ kobo_sync_status, schedule from .helper import check_valid_domain, send_test_mail, reset_password, generate_password_hash, check_email, \ - valid_email, check_username + valid_email, check_username, get_calibre_binarypath from .gdriveutils import is_gdrive_ready, gdrive_support from .render_template import render_title_template, get_sidebar_config from .services.worker import WorkerThread @@ -217,7 +217,7 @@ def admin(): form_date += timedelta(hours=int(commit[20:22]), minutes=int(commit[23:])) commit = format_datetime(form_date - tz, format='short') else: - commit = version['version'] + commit = version['version'].replace("b", " Beta") all_user = ub.session.query(ub.User).all() # email_settings = mail_config.get_mail_settings() @@ -1705,7 +1705,7 @@ def _db_configuration_update_helper(): return _db_configuration_result('{}'.format(ex), gdrive_error) if db_change or not db_valid or not config.db_configured \ - or config.config_calibre_dir != to_save["config_calibre_dir"]: + or config.config_calibre_dir != to_save["config_calibre_dir"]: if not os.path.exists(metadata_db) or not to_save['config_calibre_dir']: return _db_configuration_result(_('DB Location is not Valid, Please Enter Correct Path'), gdrive_error) else: @@ -1728,6 +1728,9 @@ def _db_configuration_update_helper(): calibre_db.update_config(config) if not os.access(os.path.join(config.config_calibre_dir, "metadata.db"), os.W_OK): flash(_("DB is not Writeable"), category="warning") + _config_string(to_save, "config_calibre_split_dir") + config.config_calibre_split = to_save.get('config_calibre_split', 0) == "on" + calibre_db.update_config(config) config.save() return _db_configuration_result(None, gdrive_error) @@ -1748,6 +1751,7 @@ def _configuration_update_helper(): _config_checkbox_int(to_save, "config_uploading") _config_checkbox_int(to_save, "config_unicode_filename") + _config_checkbox_int(to_save, "config_embed_metadata") # Reboot on config_anonbrowse with enabled ldap, as decoraters are changed in this case reboot_required |= (_config_checkbox_int(to_save, "config_anonbrowse") and config.config_login_type == constants.LOGIN_LDAP) @@ -1764,8 +1768,14 @@ def _configuration_update_helper(): constants.EXTENSIONS_UPLOAD = config.config_upload_formats.split(',') _config_string(to_save, "config_calibre") - _config_string(to_save, "config_converterpath") + _config_string(to_save, "config_binariesdir") _config_string(to_save, "config_kepubifypath") + if "config_binariesdir" in to_save: + calibre_status = helper.check_calibre(config.config_binariesdir) + if calibre_status: + return _configuration_result(calibre_status) + to_save["config_converterpath"] = get_calibre_binarypath("ebook-convert") + _config_string(to_save, "config_converterpath") reboot_required |= _config_int(to_save, "config_login_type") diff --git a/cps/cli.py b/cps/cli.py index e9b97b9d..855ad899 100644 --- a/cps/cli.py +++ b/cps/cli.py @@ -29,8 +29,8 @@ from .constants import DEFAULT_SETTINGS_FILE, DEFAULT_GDRIVE_FILE def version_info(): if _NIGHTLY_VERSION[1].startswith('$Format'): - return "Calibre-Web version: %s - unknown git-clone" % _STABLE_VERSION['version'] - return "Calibre-Web version: %s -%s" % (_STABLE_VERSION['version'], _NIGHTLY_VERSION[1]) + return "Calibre-Web version: %s - unknown git-clone" % _STABLE_VERSION['version'].replace("b", " Beta") + return "Calibre-Web version: %s -%s" % (_STABLE_VERSION['version'].replace("b", " Beta"), _NIGHTLY_VERSION[1]) class CliParameter(object): diff --git a/cps/config_sql.py b/cps/config_sql.py index 21644ccd..c4f94b4e 100644 --- a/cps/config_sql.py +++ b/cps/config_sql.py @@ -34,6 +34,7 @@ except ImportError: from sqlalchemy.ext.declarative import declarative_base from . import constants, logger +from .subproc_wrapper import process_wait log = logger.create() @@ -69,6 +70,8 @@ class _Settings(_Base): config_calibre_dir = Column(String) config_calibre_uuid = Column(String) + config_calibre_split = Column(Boolean, default=False) + config_calibre_split_dir = Column(String) config_port = Column(Integer, default=constants.DEFAULT_PORT) config_external_port = Column(Integer, default=constants.DEFAULT_PORT) config_certfile = Column(String) @@ -138,10 +141,12 @@ class _Settings(_Base): config_kepubifypath = Column(String, default=None) config_converterpath = Column(String, default=None) + config_binariesdir = Column(String, default=None) config_calibre = Column(String) config_rarfile_location = Column(String, default=None) config_upload_formats = Column(String, default=','.join(constants.EXTENSIONS_UPLOAD)) config_unicode_filename = Column(Boolean, default=False) + config_embed_metadata = Column(Boolean, default=True) config_updatechannel = Column(Integer, default=constants.UPDATE_STABLE) @@ -184,9 +189,11 @@ class ConfigSQL(object): self.load() change = False - if self.config_converterpath == None: # pylint: disable=access-member-before-definition + + if self.config_binariesdir == None: # pylint: disable=access-member-before-definition change = True - self.config_converterpath = autodetect_calibre_binary() + self.config_binariesdir = autodetect_calibre_binaries() + self.config_converterpath = autodetect_converter_binary(self.config_binariesdir) if self.config_kepubifypath == None: # pylint: disable=access-member-before-definition change = True @@ -389,6 +396,9 @@ class ConfigSQL(object): self.db_configured = False self.save() + def get_book_path(self): + return self.config_calibre_split_dir if self.config_calibre_split_dir else self.config_calibre_dir + def store_calibre_uuid(self, calibre_db, Library_table): try: calibre_uuid = calibre_db.session.query(Library_table).one_or_none() @@ -469,17 +479,32 @@ def _migrate_table(session, orm_class, secret_key=None): session.rollback() -def autodetect_calibre_binary(): +def autodetect_calibre_binaries(): if sys.platform == "win32": - calibre_path = ["C:\\program files\\calibre\\ebook-convert.exe", - "C:\\program files(x86)\\calibre\\ebook-convert.exe", - "C:\\program files(x86)\\calibre2\\ebook-convert.exe", - "C:\\program files\\calibre2\\ebook-convert.exe"] + calibre_path = ["C:\\program files\\calibre\\", + "C:\\program files(x86)\\calibre\\", + "C:\\program files(x86)\\calibre2\\", + "C:\\program files\\calibre2\\"] else: - calibre_path = ["/opt/calibre/ebook-convert"] + calibre_path = ["/opt/calibre/"] for element in calibre_path: - if os.path.isfile(element) and os.access(element, os.X_OK): - return element + supported_binary_paths = [os.path.join(element, binary) for binary in constants.SUPPORTED_CALIBRE_BINARIES.values()] + if all(os.path.isfile(binary_path) and os.access(binary_path, os.X_OK) for binary_path in supported_binary_paths): + values = [process_wait([binary_path, "--version"], pattern='\(calibre (.*)\)') for binary_path in supported_binary_paths] + if all(values): + version = values[0].group(1) + log.debug("calibre version %s", version) + return element + return "" + + +def autodetect_converter_binary(calibre_path): + if sys.platform == "win32": + converter_path = os.path.join(calibre_path, "ebook-convert.exe") + else: + converter_path = os.path.join(calibre_path, "ebook-convert") + if calibre_path and os.path.isfile(converter_path) and os.access(converter_path, os.X_OK): + return converter_path return "" @@ -521,6 +546,7 @@ def load_configuration(session, secret_key): session.commit() + def get_flask_session_key(_session): flask_settings = _session.query(_Flask_Settings).one_or_none() if flask_settings == None: diff --git a/cps/constants.py b/cps/constants.py index 87ce6f59..ef207e02 100644 --- a/cps/constants.py +++ b/cps/constants.py @@ -156,6 +156,11 @@ EXTENSIONS_UPLOAD = {'txt', 'pdf', 'epub', 'kepub', 'mobi', 'azw', 'azw3', 'cbr' 'prc', 'doc', 'docx', 'fb2', 'html', 'rtf', 'lit', 'odt', 'mp3', 'mp4', 'ogg', 'opus', 'wav', 'flac', 'm4a', 'm4b'} +_extension = "" +if sys.platform == "win32": + _extension = ".exe" +SUPPORTED_CALIBRE_BINARIES = {binary:binary + _extension for binary in ["ebook-convert", "calibredb"]} + def has_flag(value, bit_flag): return bit_flag == (bit_flag & (value or 0)) @@ -169,13 +174,11 @@ BookMeta = namedtuple('BookMeta', 'file_path, extension, title, author, cover, d 'series_id, languages, publisher, pubdate, identifiers') # python build process likes to have x.y.zbw -> b for beta and w a counting number -STABLE_VERSION = {'version': '0.6.22 Beta'} +STABLE_VERSION = {'version': '0.6.22b'} NIGHTLY_VERSION = dict() NIGHTLY_VERSION[0] = '$Format:%H$' NIGHTLY_VERSION[1] = '$Format:%cI$' -# NIGHTLY_VERSION[0] = 'bb7d2c6273ae4560e83950d36d64533343623a57' -# NIGHTLY_VERSION[1] = '2018-09-09T10:13:08+02:00' # CACHE CACHE_TYPE_THUMBNAILS = 'thumbnails' diff --git a/cps/editbooks.py b/cps/editbooks.py index 4b33f15e..4d195eb7 100644 --- a/cps/editbooks.py +++ b/cps/editbooks.py @@ -137,7 +137,7 @@ def edit_book(book_id): edited_books_id = book.id modify_date = True title_author_error = helper.update_dir_structure(edited_books_id, - config.config_calibre_dir, + config.get_book_path(), input_authors[0], renamed_author=renamed) if title_author_error: @@ -282,7 +282,7 @@ def upload(): meta.extension.lower()) else: error = helper.update_dir_structure(book_id, - config.config_calibre_dir, + config.get_book_path(), input_authors[0], meta.file_path, title_dir + meta.extension.lower(), @@ -332,7 +332,7 @@ def convert_bookformat(book_id): return redirect(url_for('edit-book.show_edit_book', book_id=book_id)) log.info('converting: book id: %s from: %s to: %s', book_id, book_format_from, book_format_to) - rtn = helper.convert_book_format(book_id, config.config_calibre_dir, book_format_from.upper(), + rtn = helper.convert_book_format(book_id, config.get_book_path(), book_format_from.upper(), book_format_to.upper(), current_user.name) if rtn is None: @@ -402,7 +402,7 @@ def edit_list_book(param): elif param == 'title': sort_param = book.sort if handle_title_on_edit(book, vals.get('value', "")): - rename_error = helper.update_dir_structure(book.id, config.config_calibre_dir) + rename_error = helper.update_dir_structure(book.id, config.get_book_path()) if not rename_error: ret = Response(json.dumps({'success': True, 'newValue': book.title}), mimetype='application/json') @@ -420,7 +420,7 @@ def edit_list_book(param): mimetype='application/json') elif param == 'authors': input_authors, __, renamed = handle_author_on_edit(book, vals['value'], vals.get('checkA', None) == "true") - rename_error = helper.update_dir_structure(book.id, config.config_calibre_dir, input_authors[0], + rename_error = helper.update_dir_structure(book.id, config.get_book_path(), input_authors[0], renamed_author=renamed) if not rename_error: ret = Response(json.dumps({ @@ -524,10 +524,10 @@ def merge_list_book(): for element in from_book.data: if element.format not in to_file: # create new data entry with: book_id, book_format, uncompressed_size, name - filepath_new = os.path.normpath(os.path.join(config.config_calibre_dir, + filepath_new = os.path.normpath(os.path.join(config.get_book_path(), to_book.path, to_name + "." + element.format.lower())) - filepath_old = os.path.normpath(os.path.join(config.config_calibre_dir, + filepath_old = os.path.normpath(os.path.join(config.get_book_path(), from_book.path, element.name + "." + element.format.lower())) copyfile(filepath_old, filepath_new) @@ -567,7 +567,7 @@ def table_xchange_author_title(): if edited_books_id: # toDo: Handle error - edit_error = helper.update_dir_structure(edited_books_id, config.config_calibre_dir, input_authors[0], + edit_error = helper.update_dir_structure(edited_books_id, config.get_book_path(), input_authors[0], renamed_author=renamed) if modify_date: book.last_modified = datetime.utcnow() @@ -764,7 +764,7 @@ def move_coverfile(meta, db_book): cover_file = meta.cover else: cover_file = os.path.join(constants.STATIC_DIR, 'generic_cover.jpg') - new_cover_path = os.path.join(config.config_calibre_dir, db_book.path) + new_cover_path = os.path.join(config.get_book_path(), db_book.path) try: os.makedirs(new_cover_path, exist_ok=True) copyfile(cover_file, os.path.join(new_cover_path, "cover.jpg")) @@ -850,7 +850,7 @@ def delete_book_from_table(book_id, book_format, json_response): book = calibre_db.get_book(book_id) if book: try: - result, error = helper.delete_book(book, config.config_calibre_dir, book_format=book_format.upper()) + result, error = helper.delete_book(book, config.get_book_path(), book_format=book_format.upper()) if not result: if json_response: return json.dumps([{"location": url_for("edit-book.show_edit_book", book_id=book_id), @@ -1190,7 +1190,7 @@ def upload_single_file(file_request, book, book_id): return False file_name = book.path.rsplit('/', 1)[-1] - filepath = os.path.normpath(os.path.join(config.config_calibre_dir, book.path)) + filepath = os.path.normpath(os.path.join(config.get_book_path(), book.path)) saved_filename = os.path.join(filepath, file_name + '.' + file_ext) # check if file path exists, otherwise create it, copy file to calibre path and delete temp file diff --git a/cps/epub.py b/cps/epub.py index ca6820b1..a45fb926 100644 --- a/cps/epub.py +++ b/cps/epub.py @@ -23,10 +23,12 @@ from lxml import etree from . import isoLanguages, cover from . import config, logger from .helper import split_authors +from .epub_helper import get_content_opf, default_ns from .constants import BookMeta log = logger.create() + def _extract_cover(zip_file, cover_file, cover_path, tmp_file_name): if cover_file is None: return None @@ -44,23 +46,14 @@ def _extract_cover(zip_file, cover_file, cover_path, tmp_file_name): return cover.cover_processing(tmp_file_name, cf, extension) def get_epub_layout(book, book_data): - ns = { - 'n': 'urn:oasis:names:tc:opendocument:xmlns:container', - 'pkg': 'http://www.idpf.org/2007/opf', - } - file_path = os.path.normpath(os.path.join(config.config_calibre_dir, book.path, book_data.name + "." + book_data.format.lower())) + file_path = os.path.normpath(os.path.join(config.get_book_path(), + book.path, book_data.name + "." + book_data.format.lower())) try: - epubZip = zipfile.ZipFile(file_path) - txt = epubZip.read('META-INF/container.xml') - tree = etree.fromstring(txt) - cfname = tree.xpath('n:rootfiles/n:rootfile/@full-path', namespaces=ns)[0] - cf = epubZip.read(cfname) + tree, __ = get_content_opf(file_path, default_ns) + p = tree.xpath('/pkg:package/pkg:metadata', namespaces=default_ns)[0] - tree = etree.fromstring(cf) - p = tree.xpath('/pkg:package/pkg:metadata', namespaces=ns)[0] - - layout = p.xpath('pkg:meta[@property="rendition:layout"]/text()', namespaces=ns) + layout = p.xpath('pkg:meta[@property="rendition:layout"]/text()', namespaces=default_ns) except (etree.XMLSyntaxError, KeyError, IndexError) as e: log.error("Could not parse epub metadata of book {} during kobo sync: {}".format(book.id, e)) layout = [] @@ -78,13 +71,7 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension): 'dc': 'http://purl.org/dc/elements/1.1/' } - epub_zip = zipfile.ZipFile(tmp_file_path) - - txt = epub_zip.read('META-INF/container.xml') - tree = etree.fromstring(txt) - cf_name = tree.xpath('n:rootfiles/n:rootfile/@full-path', namespaces=ns)[0] - cf = epub_zip.read(cf_name) - tree = etree.fromstring(cf) + tree, cf_name = get_content_opf(tmp_file_path, ns) cover_path = os.path.dirname(cf_name) @@ -127,6 +114,7 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension): epub_metadata = parse_epub_series(ns, tree, epub_metadata) + epub_zip = zipfile.ZipFile(tmp_file_path) cover_file = parse_epub_cover(ns, tree, epub_zip, cover_path, tmp_file_path) identifiers = [] diff --git a/cps/epub_helper.py b/cps/epub_helper.py new file mode 100644 index 00000000..603ccc3d --- /dev/null +++ b/cps/epub_helper.py @@ -0,0 +1,162 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2018 lemmsh, Kennyl, Kyosfonica, matthazinski +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import zipfile +from lxml import etree + +from . import isoLanguages + +default_ns = { + 'n': 'urn:oasis:names:tc:opendocument:xmlns:container', + 'pkg': 'http://www.idpf.org/2007/opf', +} + +OPF_NAMESPACE = "http://www.idpf.org/2007/opf" +PURL_NAMESPACE = "http://purl.org/dc/elements/1.1/" + +OPF = "{%s}" % OPF_NAMESPACE +PURL = "{%s}" % PURL_NAMESPACE + +etree.register_namespace("opf", OPF_NAMESPACE) +etree.register_namespace("dc", PURL_NAMESPACE) + +OPF_NS = {None: OPF_NAMESPACE} # the default namespace (no prefix) +NSMAP = {'dc': PURL_NAMESPACE, 'opf': OPF_NAMESPACE} + + +def updateEpub(src, dest, filename, data, ): + # create a temp copy of the archive without filename + with zipfile.ZipFile(src, 'r') as zin: + with zipfile.ZipFile(dest, 'w') as zout: + zout.comment = zin.comment # preserve the comment + for item in zin.infolist(): + if item.filename != filename: + zout.writestr(item, zin.read(item.filename)) + + # now add filename with its new data + with zipfile.ZipFile(dest, mode='a', compression=zipfile.ZIP_DEFLATED) as zf: + zf.writestr(filename, data) + + +def get_content_opf(file_path, ns=default_ns): + epubZip = zipfile.ZipFile(file_path) + txt = epubZip.read('META-INF/container.xml') + tree = etree.fromstring(txt) + cf_name = tree.xpath('n:rootfiles/n:rootfile/@full-path', namespaces=ns)[0] + cf = epubZip.read(cf_name) + + return etree.fromstring(cf), cf_name + + +def create_new_metadata_backup(book, custom_columns, export_language, translated_cover_name, lang_type=3): + # generate root package element + package = etree.Element(OPF + "package", nsmap=OPF_NS) + package.set("unique-identifier", "uuid_id") + package.set("version", "2.0") + + # generate metadata element and all sub elements of it + metadata = etree.SubElement(package, "metadata", nsmap=NSMAP) + identifier = etree.SubElement(metadata, PURL + "identifier", id="calibre_id", nsmap=NSMAP) + identifier.set(OPF + "scheme", "calibre") + identifier.text = str(book.id) + identifier2 = etree.SubElement(metadata, PURL + "identifier", id="uuid_id", nsmap=NSMAP) + identifier2.set(OPF + "scheme", "uuid") + identifier2.text = book.uuid + title = etree.SubElement(metadata, PURL + "title", nsmap=NSMAP) + title.text = book.title + for author in book.authors: + creator = etree.SubElement(metadata, PURL + "creator", nsmap=NSMAP) + creator.text = str(author.name) + creator.set(OPF + "file-as", book.author_sort) # ToDo Check + creator.set(OPF + "role", "aut") + contributor = etree.SubElement(metadata, PURL + "contributor", nsmap=NSMAP) + contributor.text = "calibre (5.7.2) [https://calibre-ebook.com]" + contributor.set(OPF + "file-as", "calibre") # ToDo Check + contributor.set(OPF + "role", "bkp") + + date = etree.SubElement(metadata, PURL + "date", nsmap=NSMAP) + date.text = '{d.year:04}-{d.month:02}-{d.day:02}T{d.hour:02}:{d.minute:02}:{d.second:02}'.format(d=book.pubdate) + if book.comments and book.comments[0].text: + for b in book.comments: + description = etree.SubElement(metadata, PURL + "description", nsmap=NSMAP) + description.text = b.text + for b in book.publishers: + publisher = etree.SubElement(metadata, PURL + "publisher", nsmap=NSMAP) + publisher.text = str(b.name) + if not book.languages: + language = etree.SubElement(metadata, PURL + "language", nsmap=NSMAP) + language.text = export_language + else: + for b in book.languages: + language = etree.SubElement(metadata, PURL + "language", nsmap=NSMAP) + language.text = str(b.lang_code) if lang_type == 3 else isoLanguages.get(part3=b.lang_code).part1 + for b in book.tags: + subject = etree.SubElement(metadata, PURL + "subject", nsmap=NSMAP) + subject.text = str(b.name) + etree.SubElement(metadata, "meta", name="calibre:author_link_map", + content="{" + ", ".join(['"' + str(a.name) + '": ""' for a in book.authors]) + "}", + nsmap=NSMAP) + for b in book.series: + etree.SubElement(metadata, "meta", name="calibre:series", + content=str(str(b.name)), + nsmap=NSMAP) + if book.series: + etree.SubElement(metadata, "meta", name="calibre:series_index", + content=str(book.series_index), + nsmap=NSMAP) + if len(book.ratings) and book.ratings[0].rating > 0: + etree.SubElement(metadata, "meta", name="calibre:rating", + content=str(book.ratings[0].rating), + nsmap=NSMAP) + etree.SubElement(metadata, "meta", name="calibre:timestamp", + content='{d.year:04}-{d.month:02}-{d.day:02}T{d.hour:02}:{d.minute:02}:{d.second:02}'.format( + d=book.timestamp), + nsmap=NSMAP) + etree.SubElement(metadata, "meta", name="calibre:title_sort", + content=book.sort, + nsmap=NSMAP) + sequence = 0 + for cc in custom_columns: + value = None + extra = None + cc_entry = getattr(book, "custom_column_" + str(cc.id)) + if cc_entry.__len__(): + value = [c.value for c in cc_entry] if cc.is_multiple else cc_entry[0].value + extra = cc_entry[0].extra if hasattr(cc_entry[0], "extra") else None + etree.SubElement(metadata, "meta", name="calibre:user_metadata:#{}".format(cc.label), + content=cc.to_json(value, extra, sequence), + nsmap=NSMAP) + sequence += 1 + + # generate guide element and all sub elements of it + # Title is translated from default export language + guide = etree.SubElement(package, "guide") + etree.SubElement(guide, "reference", type="cover", title=translated_cover_name, href="cover.jpg") + + return package + +def replace_metadata(tree, package): + rep_element = tree.xpath('/pkg:package/pkg:metadata', namespaces=default_ns)[0] + new_element = package.xpath('//metadata', namespaces=default_ns)[0] + tree.replace(rep_element, new_element) + return etree.tostring(tree, + xml_declaration=True, + encoding='utf-8', + pretty_print=True).decode('utf-8') + + diff --git a/cps/file_helper.py b/cps/file_helper.py new file mode 100644 index 00000000..7c3e5291 --- /dev/null +++ b/cps/file_helper.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2023 OzzieIsaacs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from tempfile import gettempdir +import os +import shutil + +def get_temp_dir(): + tmp_dir = os.path.join(gettempdir(), 'calibre_web') + if not os.path.isdir(tmp_dir): + os.mkdir(tmp_dir) + return tmp_dir + + +def del_temp_dir(): + tmp_dir = os.path.join(gettempdir(), 'calibre_web') + shutil.rmtree(tmp_dir) diff --git a/cps/gdrive.py b/cps/gdrive.py index 832350e1..4d110f83 100644 --- a/cps/gdrive.py +++ b/cps/gdrive.py @@ -23,7 +23,6 @@ import os import hashlib import json -import tempfile from uuid import uuid4 from time import time from shutil import move, copyfile @@ -34,6 +33,7 @@ from flask_login import login_required from . import logger, gdriveutils, config, ub, calibre_db, csrf from .admin import admin_required +from .file_helper import get_temp_dir gdrive = Blueprint('gdrive', __name__, url_prefix='/gdrive') log = logger.create() @@ -139,9 +139,7 @@ try: dbpath = os.path.join(config.config_calibre_dir, "metadata.db").encode() if not response['deleted'] and response['file']['title'] == 'metadata.db' \ and response['file']['md5Checksum'] != hashlib.md5(dbpath): # nosec - tmp_dir = os.path.join(tempfile.gettempdir(), 'calibre_web') - if not os.path.isdir(tmp_dir): - os.mkdir(tmp_dir) + tmp_dir = get_temp_dir() log.info('Database file updated') copyfile(dbpath, os.path.join(tmp_dir, "metadata.db_" + str(current_milli_time()))) diff --git a/cps/gdriveutils.py b/cps/gdriveutils.py index 08ead47d..b1d30596 100644 --- a/cps/gdriveutils.py +++ b/cps/gdriveutils.py @@ -34,7 +34,6 @@ except ImportError: from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.exc import OperationalError, InvalidRequestError, IntegrityError from sqlalchemy.orm.exc import StaleDataError -from sqlalchemy.sql.expression import text try: from httplib2 import __version__ as httplib2_version diff --git a/cps/helper.py b/cps/helper.py index 0c526d01..975a2523 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -25,9 +25,10 @@ import re import shutil import socket from datetime import datetime, timedelta -from tempfile import gettempdir import requests import unidecode +from uuid import uuid4 +from lxml import etree from flask import send_from_directory, make_response, redirect, abort, url_for from flask_babel import gettext as _ @@ -54,12 +55,14 @@ from . import calibre_db, cli_param from .tasks.convert import TaskConvert from . import logger, config, db, ub, fs from . import gdriveutils as gd -from .constants import STATIC_DIR as _STATIC_DIR, CACHE_TYPE_THUMBNAILS, THUMBNAIL_TYPE_COVER, THUMBNAIL_TYPE_SERIES -from .subproc_wrapper import process_wait +from .constants import STATIC_DIR as _STATIC_DIR, CACHE_TYPE_THUMBNAILS, THUMBNAIL_TYPE_COVER, THUMBNAIL_TYPE_SERIES, SUPPORTED_CALIBRE_BINARIES +from .subproc_wrapper import process_wait, process_open from .services.worker import WorkerThread from .tasks.mail import TaskEmail from .tasks.thumbnail import TaskClearCoverThumbnailCache, TaskGenerateCoverThumbnails from .tasks.metadata_backup import TaskBackupMetadata +from .file_helper import get_temp_dir +from .epub_helper import get_content_opf, create_new_metadata_backup, updateEpub, replace_metadata log = logger.create() @@ -781,7 +784,7 @@ def get_book_cover_internal(book, resolution=None): # Send the book cover from the Calibre directory else: - cover_file_path = os.path.join(config.config_calibre_dir, book.path) + cover_file_path = os.path.join(config.get_book_path(), book.path) if os.path.isfile(os.path.join(cover_file_path, "cover.jpg")): return send_from_directory(cover_file_path, "cover.jpg") else: @@ -921,10 +924,7 @@ def save_cover(img, book_path): return False, _("Only jpg/jpeg files are supported as coverfile") if config.config_use_google_drive: - tmp_dir = os.path.join(gettempdir(), 'calibre_web') - - if not os.path.isdir(tmp_dir): - os.mkdir(tmp_dir) + tmp_dir = get_temp_dir() ret, message = save_cover_from_filestorage(tmp_dir, "uploaded_cover.jpg", img) if ret is True: gd.uploadFileToEbooksFolder(os.path.join(book_path, 'cover.jpg').replace("\\", "/"), @@ -934,33 +934,92 @@ def save_cover(img, book_path): else: return False, message else: - return save_cover_from_filestorage(os.path.join(config.config_calibre_dir, book_path), "cover.jpg", img) + return save_cover_from_filestorage(os.path.join(config.get_book_path(), book_path), "cover.jpg", img) def do_download_file(book, book_format, client, data, headers): + book_name = data.name if config.config_use_google_drive: # startTime = time.time() - df = gd.getFileFromEbooksFolder(book.path, data.name + "." + book_format) + df = gd.getFileFromEbooksFolder(book.path, book_name + "." + book_format) # log.debug('%s', time.time() - startTime) if df: - return gd.do_gdrive_download(df, headers) + if config.config_embed_metadata and ( + (book_format == "kepub" and config.config_kepubifypath ) or + (book_format != "kepub" and config.config_binariesdir)): + output_path = os.path.join(config.config_calibre_dir, book.path) + if not os.path.exists(output_path): + os.makedirs(output_path) + output = os.path.join(config.config_calibre_dir, book.path, book_name + "." + book_format) + gd.downloadFile(book.path, book_name + "." + book_format, output) + if book_format == "kepub" and config.config_kepubifypath: + filename, download_name = do_kepubify_metadata_replace(book, output) + elif book_format != "kepub" and config.config_binariesdir: + filename, download_name = do_calibre_export(book.id, book_format) + else: + return gd.do_gdrive_download(df, headers) else: abort(404) else: - filename = os.path.join(config.config_calibre_dir, book.path) - if not os.path.isfile(os.path.join(filename, data.name + "." + book_format)): + filename = os.path.join(config.get_book_path(), book.path) + if not os.path.isfile(os.path.join(filename, book_name + "." + book_format)): # ToDo: improve error handling - log.error('File not found: %s', os.path.join(filename, data.name + "." + book_format)) + log.error('File not found: %s', os.path.join(filename, book_name + "." + book_format)) if client == "kobo" and book_format == "kepub": headers["Content-Disposition"] = headers["Content-Disposition"].replace(".kepub", ".kepub.epub") - response = make_response(send_from_directory(filename, data.name + "." + book_format)) - # ToDo Check headers parameter - for element in headers: - response.headers[element[0]] = element[1] - log.info('Downloading file: {}'.format(os.path.join(filename, data.name + "." + book_format))) - return response + if book_format == "kepub" and config.config_kepubifypath and config.config_embed_metadata: + filename, download_name = do_kepubify_metadata_replace(book, os.path.join(filename, + book_name + "." + book_format)) + elif book_format != "kepub" and config.config_binariesdir and config.config_embed_metadata: + filename, download_name = do_calibre_export(book.id, book_format) + else: + download_name = book_name + + response = make_response(send_from_directory(filename, download_name + "." + book_format)) + # ToDo Check headers parameter + for element in headers: + response.headers[element[0]] = element[1] + log.info('Downloading file: {}'.format(os.path.join(filename, book_name + "." + book_format))) + return response + + +def do_kepubify_metadata_replace(book, file_path): + custom_columns = (calibre_db.session.query(db.CustomColumns) + .filter(db.CustomColumns.mark_for_delete == 0) + .filter(db.CustomColumns.datatype.notin_(db.cc_exceptions)) + .order_by(db.CustomColumns.label).all()) + + tree, cf_name = get_content_opf(file_path) + package = create_new_metadata_backup(book, custom_columns, current_user.locale, _("Cover"), lang_type=2) + content = replace_metadata(tree, package) + tmp_dir = get_temp_dir() + temp_file_name = str(uuid4()) + # open zipfile and replace metadata block in content.opf + updateEpub(file_path, os.path.join(tmp_dir, temp_file_name + ".kepub"), cf_name, content) + return tmp_dir, temp_file_name + + +def do_calibre_export(book_id, book_format, ): + try: + quotes = [3, 5, 7, 9] + tmp_dir = get_temp_dir() + calibredb_binarypath = get_calibre_binarypath("calibredb") + temp_file_name = str(uuid4()) + opf_command = [calibredb_binarypath, 'export', '--dont-write-opf', '--with-library', config.config_calibre_dir, + '--to-dir', tmp_dir, '--formats', book_format, "--template", "{}".format(temp_file_name), + str(book_id)] + p = process_open(opf_command, quotes) + _, err = p.communicate() + if err: + log.error('Metadata embedder encountered an error: %s', err) + return tmp_dir, temp_file_name + except OSError as ex: + # ToDo real error handling + log.error_or_exception(ex) + return None, None + ################################## @@ -984,6 +1043,47 @@ def check_unrar(unrar_location): return _('Error executing UnRar') +def check_calibre(calibre_location): + if not calibre_location: + return + + if not os.path.exists(calibre_location): + return _('Could not find the specified directory') + + if not os.path.isdir(calibre_location): + return _('Please specify a directory, not a file') + + try: + supported_binary_paths = [os.path.join(calibre_location, binary) + for binary in SUPPORTED_CALIBRE_BINARIES.values()] + binaries_available = [os.path.isfile(binary_path) for binary_path in supported_binary_paths] + binaries_executable = [os.access(binary_path, os.X_OK) for binary_path in supported_binary_paths] + if all(binaries_available) and all(binaries_executable): + values = [process_wait([binary_path, "--version"], pattern='\(calibre (.*)\)') + for binary_path in supported_binary_paths] + if all(values): + version = values[0].group(1) + log.debug("calibre version %s", version) + else: + return _('Calibre binaries not viable') + else: + ret_val = [] + missing_binaries=[path for path, available in + zip(SUPPORTED_CALIBRE_BINARIES.values(), binaries_available) if not available] + + missing_perms=[path for path, available in + zip(SUPPORTED_CALIBRE_BINARIES.values(), binaries_executable) if not available] + if missing_binaries: + ret_val.append(_('Missing calibre binaries: %(missing)s', missing=", ".join(missing_binaries))) + if missing_perms: + ret_val.append(_('Missing executable permissions: %(missing)s', missing=", ".join(missing_perms))) + return ", ".join(ret_val) + + except (OSError, UnicodeDecodeError) as err: + log.error_or_exception(err) + return _('Error excecuting Calibre') + + def json_serial(obj): """JSON serializer for objects not serializable by default json code""" @@ -1008,43 +1108,49 @@ def tags_filters(): # checks if domain is in database (including wildcards) -# example SELECT * FROM @TABLE WHERE 'abcdefg' LIKE Name; +# example SELECT * FROM @TABLE WHERE 'abcdefg' LIKE Name; # from https://code.luasoftware.com/tutorials/flask/execute-raw-sql-in-flask-sqlalchemy/ # in all calls the email address is checked for validity def check_valid_domain(domain_text): sql = "SELECT * FROM registration WHERE (:domain LIKE domain and allow = 1);" - result = ub.session.query(ub.Registration).from_statement(text(sql)).params(domain=domain_text).all() - if not len(result): + if not len(ub.session.query(ub.Registration).from_statement(text(sql)).params(domain=domain_text).all()): return False sql = "SELECT * FROM registration WHERE (:domain LIKE domain and allow = 0);" - result = ub.session.query(ub.Registration).from_statement(text(sql)).params(domain=domain_text).all() - return not len(result) + return not len(ub.session.query(ub.Registration).from_statement(text(sql)).params(domain=domain_text).all()) def get_download_link(book_id, book_format, client): book_format = book_format.split(".")[0] book = calibre_db.get_filtered_book(book_id, allow_show_archived=True) - data1= "" if book: data1 = calibre_db.get_book_format(book.id, book_format.upper()) + if data1: + # collect downloaded books only for registered user and not for anonymous user + if current_user.is_authenticated: + ub.update_download(book_id, int(current_user.id)) + file_name = book.title + if len(book.authors) > 0: + file_name = file_name + ' - ' + book.authors[0].name + file_name = get_valid_filename(file_name, replace_whitespace=False) + headers = Headers() + headers["Content-Type"] = mimetypes.types_map.get('.' + book_format, "application/octet-stream") + headers["Content-Disposition"] = "attachment; filename=%s.%s; filename*=UTF-8''%s.%s" % ( + quote(file_name), book_format, quote(file_name), book_format) + return do_download_file(book, book_format, client, data1, headers) else: log.error("Book id {} not found for downloading".format(book_id)) - abort(404) - if data1: - # collect downloaded books only for registered user and not for anonymous user - if current_user.is_authenticated: - ub.update_download(book_id, int(current_user.id)) - file_name = book.title - if len(book.authors) > 0: - file_name = file_name + ' - ' + book.authors[0].name - file_name = get_valid_filename(file_name, replace_whitespace=False) - headers = Headers() - headers["Content-Type"] = mimetypes.types_map.get('.' + book_format, "application/octet-stream") - headers["Content-Disposition"] = "attachment; filename=%s.%s; filename*=UTF-8''%s.%s" % ( - quote(file_name), book_format, quote(file_name), book_format) - return do_download_file(book, book_format, client, data1, headers) - else: - abort(404) + abort(404) + + +def get_calibre_binarypath(binary): + binariesdir = config.config_binariesdir + if binariesdir: + try: + return os.path.join(binariesdir, SUPPORTED_CALIBRE_BINARIES[binary]) + except KeyError as ex: + log.error("Binary not supported by Calibre-Web: %s", SUPPORTED_CALIBRE_BINARIES[binary]) + pass + return "" def clear_cover_thumbnail_cache(book_id): diff --git a/cps/kobo.py b/cps/kobo.py index 5f8c72e1..00e40b49 100644 --- a/cps/kobo.py +++ b/cps/kobo.py @@ -208,7 +208,7 @@ def HandleSyncRequest(): for book in books: formats = [data.format for data in book.Books.data] if 'KEPUB' not in formats and config.config_kepubifypath and 'EPUB' in formats: - helper.convert_book_format(book.Books.id, config.config_calibre_dir, 'EPUB', 'KEPUB', current_user.name) + helper.convert_book_format(book.Books.id, config.get_book_path(), 'EPUB', 'KEPUB', current_user.name) kobo_reading_state = get_or_create_reading_state(book.Books.id) entitlement = { diff --git a/cps/opds.py b/cps/opds.py index 4067712f..b13b0570 100644 --- a/cps/opds.py +++ b/cps/opds.py @@ -502,7 +502,7 @@ def render_element_index(database_column, linked_table, folder): entries = entries.join(linked_table).join(db.Books) entries = entries.filter(calibre_db.common_filters()).group_by(func.upper(func.substr(database_column, 1, 1))).all() elements = [] - if off == 0: + if off == 0 and entries: elements.append({'id': "00", 'name': _("All")}) shift = 1 for entry in entries[ diff --git a/cps/schedule.py b/cps/schedule.py index 05367e99..bf622b36 100644 --- a/cps/schedule.py +++ b/cps/schedule.py @@ -21,6 +21,7 @@ import datetime from . import config, constants from .services.background_scheduler import BackgroundScheduler, CronTrigger, use_APScheduler from .tasks.database import TaskReconnectDatabase +from .tasks.tempFolder import TaskDeleteTempFolder from .tasks.thumbnail import TaskGenerateCoverThumbnails, TaskGenerateSeriesThumbnails, TaskClearCoverThumbnailCache from .services.worker import WorkerThread from .tasks.metadata_backup import TaskBackupMetadata @@ -31,6 +32,9 @@ def get_scheduled_tasks(reconnect=True): if reconnect: tasks.append([lambda: TaskReconnectDatabase(), 'reconnect', False]) + # Delete temp folder + tasks.append([lambda: TaskDeleteTempFolder(), 'delete temp', True]) + # Generate metadata.opf file for each changed book if config.schedule_metadata_backup: tasks.append([lambda: TaskBackupMetadata("en"), 'backup metadata', False]) @@ -86,6 +90,8 @@ def register_startup_tasks(): # Ignore tasks that should currently be running, as these will be added when registering scheduled tasks if constants.APP_MODE in ['development', 'test'] and not should_task_be_running(start, duration): scheduler.schedule_tasks_immediately(tasks=get_scheduled_tasks(False)) + else: + scheduler.schedule_tasks_immediately(tasks=[[lambda: TaskDeleteTempFolder(), 'delete temp', True]]) def should_task_be_running(start, duration): diff --git a/cps/static/js/kthoom.js b/cps/static/js/kthoom.js index 67b18fc1..b7ed6c8b 100644 --- a/cps/static/js/kthoom.js +++ b/cps/static/js/kthoom.js @@ -179,8 +179,9 @@ kthoom.ImageFile = function(file) { }; function updateDirectionButtons(){ - var left, right = 1; - if (currentImage == 0 ) { + var left = 1; + var right = 1; + if (currentImage <= 0 ) { if (settings.direction === 0) { left = 0; } else { diff --git a/cps/tasks/convert.py b/cps/tasks/convert.py old mode 100755 new mode 100644 index df6ae104..8cb29197 --- a/cps/tasks/convert.py +++ b/cps/tasks/convert.py @@ -19,8 +19,10 @@ import os import re from glob import glob -from shutil import copyfile +from shutil import copyfile, copyfileobj from markupsafe import escape +from time import time +from uuid import uuid4 from sqlalchemy.exc import SQLAlchemyError from flask_babel import lazy_gettext as N_ @@ -32,13 +34,15 @@ from cps.subproc_wrapper import process_open from flask_babel import gettext as _ from cps.kobo_sync_status import remove_synced_book from cps.ub import init_db_thread +from cps.file_helper import get_temp_dir from cps.tasks.mail import TaskEmail -from cps import gdriveutils - +from cps import gdriveutils, helper +from cps.constants import SUPPORTED_CALIBRE_BINARIES log = logger.create() +current_milli_time = lambda: int(round(time() * 1000)) class TaskConvert(CalibreTask): def __init__(self, file_path, book_id, task_message, settings, ereader_mail, user=None): @@ -61,24 +65,33 @@ class TaskConvert(CalibreTask): data = worker_db.get_book_format(self.book_id, self.settings['old_book_format']) df = gdriveutils.getFileFromEbooksFolder(cur_book.path, data.name + "." + self.settings['old_book_format'].lower()) + df_cover = gdriveutils.getFileFromEbooksFolder(cur_book.path, "cover.jpg") if df: - datafile = os.path.join(config.config_calibre_dir, + datafile = os.path.join(config.get_book_path(), cur_book.path, data.name + "." + self.settings['old_book_format'].lower()) - if not os.path.exists(os.path.join(config.config_calibre_dir, cur_book.path)): - os.makedirs(os.path.join(config.config_calibre_dir, cur_book.path)) + if df_cover: + datafile_cover = os.path.join(config.get_book_path(), + cur_book.path, "cover.jpg") + if not os.path.exists(os.path.join(config.get_book_path(), cur_book.path)): + os.makedirs(os.path.join(config.get_book_path(), cur_book.path)) df.GetContentFile(datafile) + if df_cover: + df_cover.GetContentFile(datafile_cover) worker_db.session.close() else: + # ToDo Include cover in error handling error_message = _("%(format)s not found on Google Drive: %(fn)s", format=self.settings['old_book_format'], fn=data.name + "." + self.settings['old_book_format'].lower()) worker_db.session.close() - return error_message + return self._handleError(self, error_message) filename = self._convert_ebook_format() if config.config_use_google_drive: os.remove(self.file_path + '.' + self.settings['old_book_format'].lower()) + if df_cover: + os.remove(os.path.join(config.config_calibre_dir, cur_book.path, "cover.jpg")) if filename: if config.config_use_google_drive: @@ -112,7 +125,7 @@ class TaskConvert(CalibreTask): # check to see if destination format already exists - or if book is in database # if it does - mark the conversion task as complete and return a success - # this will allow send to E-Reader workflow to continue to work + # this will allow to send to E-Reader workflow to continue to work if os.path.isfile(file_path + format_new_ext) or\ local_db.get_book_format(self.book_id, self.settings['new_book_format']): log.info("Book id %d already converted to %s", book_id, format_new_ext) @@ -152,7 +165,8 @@ class TaskConvert(CalibreTask): if not os.path.exists(config.config_converterpath): self._handleError(N_("Calibre ebook-convert %(tool)s not found", tool=config.config_converterpath)) return - check, error_message = self._convert_calibre(file_path, format_old_ext, format_new_ext) + has_cover = local_db.get_book(book_id).has_cover + check, error_message = self._convert_calibre(file_path, format_old_ext, format_new_ext, has_cover) if check == 0: cur_book = local_db.get_book(book_id) @@ -194,8 +208,15 @@ class TaskConvert(CalibreTask): return def _convert_kepubify(self, file_path, format_old_ext, format_new_ext): + if config.config_embed_metadata and config.config_binariesdir: + tmp_dir, temp_file_name = helper.do_calibre_export(self.book_id, format_old_ext[1:]) + filename = os.path.join(tmp_dir, temp_file_name + format_old_ext) + temp_file_path = tmp_dir + else: + filename = file_path + format_old_ext + temp_file_path = os.path.dirname(file_path) quotes = [1, 3] - command = [config.config_kepubifypath, (file_path + format_old_ext), '-o', os.path.dirname(file_path)] + command = [config.config_kepubifypath, filename, '-o', temp_file_path, '-i'] try: p = process_open(command, quotes) except OSError as e: @@ -209,13 +230,12 @@ class TaskConvert(CalibreTask): if p.poll() is not None: break - # ToD Handle # process returncode check = p.returncode # move file if check == 0: - converted_file = glob(os.path.join(os.path.dirname(file_path), "*.kepub.epub")) + converted_file = glob(os.path.splitext(filename)[0] + "*.kepub.epub") if len(converted_file) == 1: copyfile(converted_file[0], (file_path + format_new_ext)) os.unlink(converted_file[0]) @@ -224,16 +244,28 @@ class TaskConvert(CalibreTask): folder=os.path.dirname(file_path)) return check, None - def _convert_calibre(self, file_path, format_old_ext, format_new_ext): + def _convert_calibre(self, file_path, format_old_ext, format_new_ext, has_cover): try: - # Linux py2.7 encode as list without quotes no empty element for parameters - # linux py3.x no encode and as list without quotes no empty element for parameters - # windows py2.7 encode as string with quotes empty element for parameters is okay - # windows py 3.x no encode and as string with quotes empty element for parameters is okay - # separate handling for windows and linux - quotes = [1, 2] + # path_tmp_opf = self._embed_metadata() + if config.config_embed_metadata: + quotes = [3, 5] + tmp_dir = get_temp_dir() + calibredb_binarypath = os.path.join(config.config_binariesdir, SUPPORTED_CALIBRE_BINARIES["calibredb"]) + opf_command = [calibredb_binarypath, 'show_metadata', '--as-opf', str(self.book_id), + '--with-library', config.config_calibre_dir] + p = process_open(opf_command, quotes) + p.wait() + path_tmp_opf = os.path.join(tmp_dir, "metadata_" + str(uuid4()) + ".opf") + with open(path_tmp_opf, 'w') as fd: + copyfileobj(p.stdout, fd) + + quotes = [1, 2, 4, 6] command = [config.config_converterpath, (file_path + format_old_ext), (file_path + format_new_ext)] + if config.config_embed_metadata: + command.extend(['--from-opf', path_tmp_opf]) + if has_cover: + command.extend(['--cover', os.path.join(os.path.dirname(file_path), 'cover.jpg')]) quotes_index = 3 if config.config_calibre: parameters = config.config_calibre.split(" ") diff --git a/cps/tasks/mail.py b/cps/tasks/mail.py old mode 100755 new mode 100644 index a305b623..36133ccf --- a/cps/tasks/mail.py +++ b/cps/tasks/mail.py @@ -239,7 +239,7 @@ class TaskEmail(CalibreTask): @classmethod def _get_attachment(cls, book_path, filename): """Get file as MIMEBase message""" - calibre_path = config.config_calibre_dir + calibre_path = config.get_book_path() if config.config_use_google_drive: df = gdriveutils.getFileFromEbooksFolder(book_path, filename) if df: diff --git a/cps/tasks/metadata_backup.py b/cps/tasks/metadata_backup.py index 1751feeb..2f402448 100644 --- a/cps/tasks/metadata_backup.py +++ b/cps/tasks/metadata_backup.py @@ -17,26 +17,13 @@ # along with this program. If not, see . import os -from urllib.request import urlopen from lxml import etree - from cps import config, db, gdriveutils, logger from cps.services.worker import CalibreTask from flask_babel import lazy_gettext as N_ -OPF_NAMESPACE = "http://www.idpf.org/2007/opf" -PURL_NAMESPACE = "http://purl.org/dc/elements/1.1/" - -OPF = "{%s}" % OPF_NAMESPACE -PURL = "{%s}" % PURL_NAMESPACE - -etree.register_namespace("opf", OPF_NAMESPACE) -etree.register_namespace("dc", PURL_NAMESPACE) - -OPF_NS = {None: OPF_NAMESPACE} # the default namespace (no prefix) -NSMAP = {'dc': PURL_NAMESPACE, 'opf': OPF_NAMESPACE} - +from ..epub_helper import create_new_metadata_backup class TaskBackupMetadata(CalibreTask): @@ -101,7 +88,8 @@ class TaskBackupMetadata(CalibreTask): self.calibre_db.session.close() def open_metadata(self, book, custom_columns): - package = self.create_new_metadata_backup(book, custom_columns) + # package = self.create_new_metadata_backup(book, custom_columns) + package = create_new_metadata_backup(book, custom_columns, self.export_language, self.translated_title) if config.config_use_google_drive: if not gdriveutils.is_gdrive_ready(): raise Exception('Google Drive is configured but not ready') @@ -114,7 +102,7 @@ class TaskBackupMetadata(CalibreTask): True) else: # ToDo: Handle book folder not found or not readable - book_metadata_filepath = os.path.join(config.config_calibre_dir, book.path, 'metadata.opf') + book_metadata_filepath = os.path.join(config.get_book_path(), book.path, 'metadata.opf') # prepare finalize everything and output doc = etree.ElementTree(package) try: @@ -123,7 +111,7 @@ class TaskBackupMetadata(CalibreTask): except Exception as ex: raise Exception('Writing Metadata failed with error: {} '.format(ex)) - def create_new_metadata_backup(self, book, custom_columns): + '''def create_new_metadata_backup(self, book, custom_columns): # generate root package element package = etree.Element(OPF + "package", nsmap=OPF_NS) package.set("unique-identifier", "uuid_id") @@ -208,7 +196,7 @@ class TaskBackupMetadata(CalibreTask): guide = etree.SubElement(package, "guide") etree.SubElement(guide, "reference", type="cover", title=self.translated_title, href="cover.jpg") - return package + return package''' @property def name(self): diff --git a/cps/tasks/tempFolder.py b/cps/tasks/tempFolder.py new file mode 100644 index 00000000..e740cd1e --- /dev/null +++ b/cps/tasks/tempFolder.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2023 OzzieIsaacs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from urllib.request import urlopen + +from flask_babel import lazy_gettext as N_ + +from cps import logger, file_helper +from cps.services.worker import CalibreTask + + +class TaskDeleteTempFolder(CalibreTask): + def __init__(self, task_message=N_('Delete temp folder contents')): + super(TaskDeleteTempFolder, self).__init__(task_message) + self.log = logger.create() + + def run(self, worker_thread): + try: + file_helper.del_temp_dir() + except FileNotFoundError: + pass + except (PermissionError, OSError) as e: + self.log.error("Error deleting temp folder: {}".format(e)) + self._handleSuccess() + + @property + def name(self): + return "Delete Temp Folder" + + @property + def is_cancellable(self): + return False diff --git a/cps/tasks/thumbnail.py b/cps/tasks/thumbnail.py index 6d11fe97..dd9ee1e0 100644 --- a/cps/tasks/thumbnail.py +++ b/cps/tasks/thumbnail.py @@ -209,7 +209,7 @@ class TaskGenerateCoverThumbnails(CalibreTask): if stream is not None: stream.close() else: - book_cover_filepath = os.path.join(config.config_calibre_dir, book.path, 'cover.jpg') + book_cover_filepath = os.path.join(config.get_book_path(), book.path, 'cover.jpg') if not os.path.isfile(book_cover_filepath): raise Exception('Book cover file not found') @@ -404,7 +404,7 @@ class TaskGenerateSeriesThumbnails(CalibreTask): if stream is not None: stream.close() - book_cover_filepath = os.path.join(config.config_calibre_dir, book.path, 'cover.jpg') + book_cover_filepath = os.path.join(config.get_book_path(), book.path, 'cover.jpg') if not os.path.isfile(book_cover_filepath): raise Exception('Book cover file not found') diff --git a/cps/templates/config_db.html b/cps/templates/config_db.html index 0090bd95..e79bb630 100644 --- a/cps/templates/config_db.html +++ b/cps/templates/config_db.html @@ -16,6 +16,18 @@ +
+ + +
+
+
+ + + + +
+
{% if feature_support['gdrive'] %}
diff --git a/cps/templates/config_edit.html b/cps/templates/config_edit.html index d101f960..d83831db 100644 --- a/cps/templates/config_edit.html +++ b/cps/templates/config_edit.html @@ -103,6 +103,10 @@
+
+ + +
@@ -323,12 +327,12 @@
- +
- - - - + + + +
diff --git a/cps/templates/shelfdown.html b/cps/templates/shelfdown.html index e42652b2..19a42b01 100644 --- a/cps/templates/shelfdown.html +++ b/cps/templates/shelfdown.html @@ -19,13 +19,6 @@ {% endif %} - - - - {% block header %}{% endblock %} diff --git a/cps/updater.py b/cps/updater.py index 6d6e408f..67b3653f 100644 --- a/cps/updater.py +++ b/cps/updater.py @@ -25,13 +25,13 @@ import threading import time import zipfile from io import BytesIO -from tempfile import gettempdir - import requests + from flask_babel import format_datetime from flask_babel import gettext as _ from . import constants, logger # config, web_server +from .file_helper import get_temp_dir log = logger.create() @@ -85,7 +85,7 @@ class Updater(threading.Thread): z = zipfile.ZipFile(BytesIO(r.content)) self.status = 3 log.debug('Extracting zipfile') - tmp_dir = gettempdir() + tmp_dir = get_temp_dir() z.extractall(tmp_dir) folder_name = os.path.join(tmp_dir, z.namelist()[0])[:-1] if not os.path.isdir(folder_name): @@ -566,7 +566,7 @@ class Updater(threading.Thread): try: current_version[2] = int(current_version[2]) except ValueError: - current_version[2] = int(current_version[2].split(' ')[0])-1 + current_version[2] = int(current_version[2].replace("b", "").split(' ')[0])-1 # Check if major versions are identical search for newest non-equal commit and update to this one if major_version_update == current_version[0]: diff --git a/cps/uploader.py b/cps/uploader.py index 23dfc4a6..8f20762f 100644 --- a/cps/uploader.py +++ b/cps/uploader.py @@ -18,12 +18,12 @@ import os import hashlib -from tempfile import gettempdir from flask_babel import gettext as _ from . import logger, comic, isoLanguages from .constants import BookMeta from .helper import split_authors +from .file_helper import get_temp_dir log = logger.create() @@ -249,10 +249,7 @@ def get_magick_version(): def upload(uploadfile, rar_excecutable): - tmp_dir = os.path.join(gettempdir(), 'calibre_web') - - if not os.path.isdir(tmp_dir): - os.mkdir(tmp_dir) + tmp_dir = get_temp_dir() filename = uploadfile.filename filename_root, file_extension = os.path.splitext(filename) diff --git a/cps/web.py b/cps/web.py old mode 100755 new mode 100644 index 67d22ec7..24c5cacd --- a/cps/web.py +++ b/cps/web.py @@ -1192,7 +1192,7 @@ def serve_book(book_id, book_format, anyname): if book_format.upper() == 'TXT': log.info('Serving book: %s', data.name) try: - rawdata = open(os.path.join(config.config_calibre_dir, book.path, data.name + "." + book_format), + rawdata = open(os.path.join(config.get_book_path(), book.path, data.name + "." + book_format), "rb").read() result = chardet.detect(rawdata) try: @@ -1209,7 +1209,7 @@ def serve_book(book_id, book_format, anyname): return "File Not Found" # enable byte range read of pdf response = make_response( - send_from_directory(os.path.join(config.config_calibre_dir, book.path), data.name + "." + book_format)) + send_from_directory(os.path.join(config.get_book_path(), book.path), data.name + "." + book_format)) if not range_header: log.info('Serving book: %s', data.name) response.headers['Accept-Ranges'] = 'bytes' @@ -1233,7 +1233,7 @@ def send_to_ereader(book_id, book_format, convert): response = [{'type': "danger", 'message': _("Please configure the SMTP mail settings first...")}] return Response(json.dumps(response), mimetype='application/json') elif current_user.kindle_mail: - result = send_mail(book_id, book_format, convert, current_user.kindle_mail, config.config_calibre_dir, + result = send_mail(book_id, book_format, convert, current_user.kindle_mail, config.get_book_path(), current_user.name) if result is None: ub.update_download(book_id, int(current_user.id)) @@ -1354,7 +1354,7 @@ def login(): @limiter.limit("3/minute", key_func=lambda: request.form.get('username', "").strip().lower()) def login_post(): form = request.form.to_dict() - username = form.get('username', "").strip().lower().replace("\n","\\n").replace("\r","") + username = form.get('username', "").strip().lower().replace("\n","").replace("\r","") try: limiter.check() except RateLimitExceeded: diff --git a/requirements.txt b/requirements.txt index f1e5b712..c28f2019 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ iso-639>=0.4.5,<0.5.0 PyPDF>=3.0.0,<3.16.0 pytz>=2016.10 requests>=2.28.0,<2.32.0 -SQLAlchemy>=1.3.0,<2.0.0 +SQLAlchemy>=1.3.0,<2.1.0 tornado>=6.3,<6.4 Wand>=0.4.4,<0.7.0 unidecode>=0.04.19,<1.4.0 diff --git a/setup.cfg b/setup.cfg index 4bcd1a11..eb73462d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -41,7 +41,7 @@ install_requires = Werkzeug<3.0.0 APScheduler>=3.6.3,<3.11.0 Babel>=1.3,<3.0 - Flask-Babel>=0.11.1,<3.2.0 + Flask-Babel>=0.11.1,<4.1.0 Flask-Login>=0.3.2,<0.6.3 Flask-Principal>=0.3.2,<0.5.1 Flask>=1.0.2,<2.4.0 @@ -49,15 +49,15 @@ install_requires = PyPDF>=3.0.0,<3.16.0 pytz>=2016.10 requests>=2.28.0,<2.32.0 - SQLAlchemy>=1.3.0,<2.0.0 + SQLAlchemy>=1.3.0,<2.1.0 tornado>=6.3,<6.4 Wand>=0.4.4,<0.7.0 unidecode>=0.04.19,<1.4.0 lxml>=3.8.0,<5.0.0 - flask-wtf>=0.14.2,<1.2.0 + flask-wtf>=0.14.2,<1.3.0 chardet>=3.0.0,<4.1.0 advocate>=1.0.0,<1.1.0 - Flask-Limiter>=2.3.0,<3.5.0 + Flask-Limiter>=2.3.0,<3.6.0 [options.packages.find] @@ -66,7 +66,7 @@ include = cps/services* [options.extras_require] gdrive = - google-api-python-client>=1.7.11,<2.98.0 + google-api-python-client>=1.7.11,<2.108.0 gevent>20.6.0,<24.0.0 greenlet>=0.4.17,<2.1.0 httplib2>=0.9.2,<0.23.0 @@ -79,7 +79,7 @@ gdrive = rsa>=3.4.2,<4.10.0 gmail = google-auth-oauthlib>=0.4.3,<1.1.0 - google-api-python-client>=1.7.11,<2.98.0 + google-api-python-client>=1.7.11,<2.108.0 goodreads = goodreads>=0.3.2,<0.4.0 python-Levenshtein>=0.12.0,<0.22.0 diff --git a/test/Calibre-Web TestSummary_Linux.html b/test/Calibre-Web TestSummary_Linux.html index 7ca3dad5..6bafbd11 100644 --- a/test/Calibre-Web TestSummary_Linux.html +++ b/test/Calibre-Web TestSummary_Linux.html @@ -37,20 +37,20 @@
-

Start Time: 2023-10-16 19:38:22

+

Start Time: 2024-01-17 20:30:42

-

Stop Time: 2023-10-17 02:18:49

+

Stop Time: 2024-01-18 03:25:01

-

Duration: 5h 37 min

+

Duration: 5h 47 min

@@ -234,15 +234,15 @@ - + TestBackupMetadata - 22 - 22 - 0 + 21 + 20 + 1 0 0 - Detail + Detail @@ -320,11 +320,35 @@ - +
TestBackupMetadata - test_backup_change_book_series_index
- PASS + +
+ FAIL +
+ + + + @@ -430,42 +454,225 @@ -
TestBackupMetadata - test_gdrive
+
TestBackupMetadata - test_upload_book
PASS + + + + TestBackupMetadataGdrive + 1 + 0 + 0 + 1 + 0 + + Detail + + + + - + -
TestBackupMetadata - test_upload_book
+
TestBackupMetadataGdrive - test_backup_gdrive
+ + +
+ ERROR +
+ + + - PASS - - TestBackupMetadataGdrive - 1 + + _ErrorHolder 1 0 0 + 1 0 - Detail + Detail - + -
TestBackupMetadataGdrive - test_backup_gdrive
+
tearDownClass (test_backup_metadata_gdrive)
+ + +
+ ERROR +
+ + + - PASS @@ -479,13 +686,13 @@ 0 0 - Detail + Detail - +
TestCli - test_already_started
@@ -494,7 +701,7 @@ - +
TestCli - test_bind_to_single_interface
@@ -503,7 +710,7 @@ - +
TestCli - test_change_password
@@ -512,7 +719,7 @@ - +
TestCli - test_cli_SSL_files
@@ -521,7 +728,7 @@ - +
TestCli - test_cli_different_folder
@@ -530,7 +737,7 @@ - +
TestCli - test_cli_different_settings_database
@@ -539,7 +746,7 @@ - +
TestCli - test_dryrun_update
@@ -548,7 +755,7 @@ - +
TestCli - test_enable_reconnect
@@ -557,7 +764,7 @@ - +
TestCli - test_environ_port_setting
@@ -566,7 +773,7 @@ - +
TestCli - test_logfile
@@ -575,7 +782,7 @@ - +
TestCli - test_no_database
@@ -584,7 +791,7 @@ - +
TestCli - test_settingsdb_not_writeable
@@ -593,7 +800,7 @@ - +
TestCli - test_writeonly_static_files
@@ -611,13 +818,13 @@ 0 0 - Detail + Detail - +
TestCliGdrivedb - test_cli_gdrive_folder
@@ -626,7 +833,7 @@ - +
TestCliGdrivedb - test_cli_gdrive_location
@@ -635,7 +842,7 @@ - +
TestCliGdrivedb - test_gdrive_db_nonwrite
@@ -644,7 +851,7 @@ - +
TestCliGdrivedb - test_no_database
@@ -662,13 +869,13 @@ 0 0 - Detail + Detail - +
TestCoverEditBooks - test_invalid_jpg_hdd
@@ -677,7 +884,7 @@ - +
TestCoverEditBooks - test_upload_jpg
@@ -695,13 +902,13 @@ 0 0 - Detail + Detail - +
TestDeleteDatabase - test_delete_books_in_database
@@ -719,13 +926,13 @@ 0 0 - Detail + Detail - +
TestEbookConvertCalibre - test_calibre_log
@@ -734,7 +941,7 @@ - +
TestEbookConvertCalibre - test_convert_deactivate
@@ -743,7 +950,7 @@ - +
TestEbookConvertCalibre - test_convert_email
@@ -752,7 +959,7 @@ - +
TestEbookConvertCalibre - test_convert_failed_and_email
@@ -761,7 +968,7 @@ - +
TestEbookConvertCalibre - test_convert_only
@@ -770,7 +977,7 @@ - +
TestEbookConvertCalibre - test_convert_options
@@ -779,7 +986,7 @@ - +
TestEbookConvertCalibre - test_convert_parameter
@@ -788,7 +995,7 @@ - +
TestEbookConvertCalibre - test_convert_wrong_excecutable
@@ -797,7 +1004,7 @@ - +
TestEbookConvertCalibre - test_convert_xss
@@ -806,7 +1013,7 @@ - +
TestEbookConvertCalibre - test_email_failed
@@ -815,7 +1022,7 @@ - +
TestEbookConvertCalibre - test_email_only
@@ -824,7 +1031,7 @@ - +
TestEbookConvertCalibre - test_kindle_send_not_configured
@@ -833,7 +1040,7 @@ - +
TestEbookConvertCalibre - test_ssl_smtp_setup_error
@@ -842,7 +1049,7 @@ - +
TestEbookConvertCalibre - test_starttls_smtp_setup_error
@@ -851,7 +1058,7 @@ - +
TestEbookConvertCalibre - test_user_convert_xss
@@ -869,13 +1076,13 @@ 0 0 - Detail + Detail - +
TestEbookConvertCalibreGDrive - test_convert_email
@@ -884,7 +1091,7 @@ - +
TestEbookConvertCalibreGDrive - test_convert_failed_and_email
@@ -893,7 +1100,7 @@ - +
TestEbookConvertCalibreGDrive - test_convert_only
@@ -902,7 +1109,7 @@ - +
TestEbookConvertCalibreGDrive - test_convert_parameter
@@ -911,7 +1118,7 @@ - +
TestEbookConvertCalibreGDrive - test_email_failed
@@ -920,7 +1127,7 @@ - +
TestEbookConvertCalibreGDrive - test_email_only
@@ -938,13 +1145,13 @@ 0 0 - Detail + Detail - +
TestEbookConvertKepubify - test_convert_deactivate
@@ -953,7 +1160,7 @@ - +
TestEbookConvertKepubify - test_convert_only
@@ -962,7 +1169,7 @@ - +
TestEbookConvertKepubify - test_convert_wrong_excecutable
@@ -980,13 +1187,13 @@ 0 0 - Detail + Detail - +
TestEbookConvertGDriveKepubify - test_convert_deactivate
@@ -995,7 +1202,7 @@ - +
TestEbookConvertGDriveKepubify - test_convert_only
@@ -1004,7 +1211,7 @@ - +
TestEbookConvertGDriveKepubify - test_convert_wrong_excecutable
@@ -1022,13 +1229,13 @@ 0 2 - Detail + Detail - +
TestEditAdditionalBooks - test_cbz_comicinfo
@@ -1037,7 +1244,7 @@ - +
TestEditAdditionalBooks - test_change_upload_formats
@@ -1046,7 +1253,7 @@ - +
TestEditAdditionalBooks - test_delete_book
@@ -1055,7 +1262,7 @@ - +
TestEditAdditionalBooks - test_delete_role
@@ -1064,7 +1271,7 @@ - +
TestEditAdditionalBooks - test_details_popup
@@ -1073,7 +1280,7 @@ - +
TestEditAdditionalBooks - test_edit_book_identifier
@@ -1082,7 +1289,7 @@ - +
TestEditAdditionalBooks - test_edit_book_identifier_capital
@@ -1091,7 +1298,7 @@ - +
TestEditAdditionalBooks - test_edit_book_identifier_standard
@@ -1100,7 +1307,7 @@ - +
TestEditAdditionalBooks - test_edit_special_book_identifier
@@ -1109,7 +1316,7 @@ - +
TestEditAdditionalBooks - test_title_sort
@@ -1118,7 +1325,7 @@ - +
TestEditAdditionalBooks - test_upload_cbz_coverformats
@@ -1127,7 +1334,7 @@ - +
TestEditAdditionalBooks - test_upload_edit_role
@@ -1136,7 +1343,7 @@ - +
TestEditAdditionalBooks - test_upload_metadata_cb7
@@ -1145,7 +1352,7 @@ - +
TestEditAdditionalBooks - test_upload_metadata_cbr
@@ -1154,7 +1361,7 @@ - +
TestEditAdditionalBooks - test_upload_metadata_cbt
@@ -1163,19 +1370,19 @@ - +
TestEditAdditionalBooks - test_writeonly_calibre_database
- SKIP + SKIP
-