From ba23ada1fe63ed79326ca4a8f3ff220dc09739b0 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sat, 12 Feb 2022 12:32:35 +0100 Subject: [PATCH 01/24] Reenable showing of academic cover in case no cover was found from scholary --- cps/metadata_provider/scholar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cps/metadata_provider/scholar.py b/cps/metadata_provider/scholar.py index bbf50fb3..f94a65f4 100644 --- a/cps/metadata_provider/scholar.py +++ b/cps/metadata_provider/scholar.py @@ -47,7 +47,7 @@ class scholar(Metadata): scholar_gen = itertools.islice(scholarly.search_pubs(query), 10) for result in scholar_gen: match = self._parse_search_result( - result=result, generic_cover=generic_cover, locale=locale + result=result, generic_cover="", locale=locale ) val.append(match) return val From 9c5970bbfc0f16e71eeb8641104c09f75d7f16bd Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sat, 12 Feb 2022 12:34:54 +0100 Subject: [PATCH 02/24] Bugfix for empty search results from google --- cps/metadata_provider/google.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cps/metadata_provider/google.py b/cps/metadata_provider/google.py index 11e86aa1..fbb68965 100644 --- a/cps/metadata_provider/google.py +++ b/cps/metadata_provider/google.py @@ -46,7 +46,7 @@ class Google(Metadata): tokens = [quote(t.encode("utf-8")) for t in title_tokens] query = "+".join(tokens) results = requests.get(Google.SEARCH_URL + query) - for result in results.json()["items"]: + for result in results.json().get("items", []): val.append( self._parse_search_result( result=result, generic_cover=generic_cover, locale=locale From 7bb3cac7fbab8e370287483d6213873566d99038 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sat, 12 Feb 2022 12:41:29 +0100 Subject: [PATCH 03/24] Avoid problems with percent encoded utf-8 abstracts on certain chinese papers --- cps/metadata_provider/scholar.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cps/metadata_provider/scholar.py b/cps/metadata_provider/scholar.py index f94a65f4..b0c10b66 100644 --- a/cps/metadata_provider/scholar.py +++ b/cps/metadata_provider/scholar.py @@ -17,7 +17,7 @@ # along with this program. If not, see . import itertools from typing import Dict, List, Optional -from urllib.parse import quote +from urllib.parse import quote, unquote try: from fake_useragent.errors import FakeUserAgentError @@ -66,7 +66,7 @@ class scholar(Metadata): ) match.cover = result.get("image", {}).get("original_url", generic_cover) - match.description = result["bib"].get("abstract", "") + match.description = unquote(result["bib"].get("abstract", "")) match.publisher = result["bib"].get("venue", "") match.publishedDate = result["bib"].get("pub_year") + "-01-01" match.identifiers = {"scholar": match.id} From e9b674f46e3fc08e9e83f3d1b6968725e294c9cf Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Tue, 8 Feb 2022 19:35:00 +0100 Subject: [PATCH 04/24] Fix a problem with sending emails from custom domain name server (#2301) --- cps/tasks/convert.py | 2 ++ cps/tasks/mail.py | 54 +++++++++++++++++--------------------------- 2 files changed, 23 insertions(+), 33 deletions(-) diff --git a/cps/tasks/convert.py b/cps/tasks/convert.py index e610b14b..98cd7b48 100644 --- a/cps/tasks/convert.py +++ b/cps/tasks/convert.py @@ -35,6 +35,8 @@ from cps.ub import init_db_thread from cps.tasks.mail import TaskEmail from cps import gdriveutils + + log = logger.create() diff --git a/cps/tasks/mail.py b/cps/tasks/mail.py index 03526c8b..1e652b02 100644 --- a/cps/tasks/mail.py +++ b/cps/tasks/mail.py @@ -23,16 +23,8 @@ import threading import socket import mimetypes -try: - from StringIO import StringIO - from email.MIMEBase import MIMEBase - from email.MIMEMultipart import MIMEMultipart - from email.MIMEText import MIMEText -except ImportError: - from io import StringIO - from email.mime.base import MIMEBase - from email.mime.multipart import MIMEMultipart - from email.mime.text import MIMEText +from io import StringIO +from email.message import EmailMessage @@ -131,19 +123,24 @@ class TaskEmail(CalibreTask): self.results = dict() def prepare_message(self): - message = MIMEMultipart() + message = EmailMessage() + # message = MIMEMultipart() message['to'] = self.recipent message['from'] = self.settings["mail_from"] message['subject'] = self.subject message['Message-Id'] = make_msgid('calibre-web') message['Date'] = formatdate(localtime=True) - text = self.text - msg = MIMEText(text.encode('UTF-8'), 'plain', 'UTF-8') - message.attach(msg) + # text = self.text + message.set_content(self.text.encode('UTF-8'), "text", "plain") if self.attachment: - result = self._get_attachment(self.filepath, self.attachment) - if result: - message.attach(result) + data = self._get_attachment(self.filepath, self.attachment) + if data: + # Set mimetype + content_type, encoding = mimetypes.guess_type(self.attachment) + if content_type is None or encoding is not None: + content_type = 'application/octet-stream' + main_type, sub_type = content_type.split('/', 1) + message.add_attachment(data, maintype=main_type, subtype=sub_type, filename=self.attachment) else: self._handleError(u"Attachment not found") return @@ -226,15 +223,15 @@ class TaskEmail(CalibreTask): self._progress = x @classmethod - def _get_attachment(cls, bookpath, filename): + def _get_attachment(cls, book_path, filename): """Get file as MIMEBase message""" calibre_path = config.config_calibre_dir if config.config_use_google_drive: - df = gdriveutils.getFileFromEbooksFolder(bookpath, filename) + df = gdriveutils.getFileFromEbooksFolder(book_path, filename) if df: - datafile = os.path.join(calibre_path, bookpath, filename) - if not os.path.exists(os.path.join(calibre_path, bookpath)): - os.makedirs(os.path.join(calibre_path, bookpath)) + datafile = os.path.join(calibre_path, book_path, filename) + if not os.path.exists(os.path.join(calibre_path, book_path)): + os.makedirs(os.path.join(calibre_path, book_path)) df.GetContentFile(datafile) else: return None @@ -244,23 +241,14 @@ class TaskEmail(CalibreTask): os.remove(datafile) else: try: - file_ = open(os.path.join(calibre_path, bookpath, filename), 'rb') + file_ = open(os.path.join(calibre_path, book_path, filename), 'rb') data = file_.read() file_.close() except IOError as e: log.debug_or_exception(e, stacklevel=3) log.error(u'The requested file could not be read. Maybe wrong permissions?') return None - # Set mimetype - content_type, encoding = mimetypes.guess_type(filename) - if content_type is None or encoding is not None: - content_type = 'application/octet-stream' - main_type, sub_type = content_type.split('/', 1) - attachment = MIMEBase(main_type, sub_type) - attachment.set_payload(data) - encoders.encode_base64(attachment) - attachment.add_header('Content-Disposition', 'attachment', filename=filename) - return attachment + return data @property def name(self): From ef7c6731bcb2035320fce731114d22b209869928 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Mon, 14 Feb 2022 17:35:20 +0100 Subject: [PATCH 05/24] Possible fix for #2301 (sending emails from custom domain name server) --- cps/tasks/mail.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) mode change 100644 => 100755 cps/tasks/mail.py diff --git a/cps/tasks/mail.py b/cps/tasks/mail.py old mode 100644 new mode 100755 index 1e652b02..80adbbcc --- a/cps/tasks/mail.py +++ b/cps/tasks/mail.py @@ -16,7 +16,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import sys import os import smtplib import threading @@ -25,7 +24,7 @@ import mimetypes from io import StringIO from email.message import EmailMessage - +from email.utils import parseaddr from email import encoders @@ -37,6 +36,7 @@ from cps.services import gmail from cps import logger, config from cps import gdriveutils +import uuid log = logger.create() @@ -122,15 +122,27 @@ class TaskEmail(CalibreTask): self.asyncSMTP = None self.results = dict() + # from calibre code: + # https://github.com/kovidgoyal/calibre/blob/731ccd92a99868de3e2738f65949f19768d9104c/src/calibre/utils/smtp.py#L60 + def get_msgid_domain(self): + try: + # Parse out the address from the From line, and then the domain from that + from_email = parseaddr(self.settings["mail_from"])[1] + msgid_domain = from_email.partition('@')[2].strip() + # This can sometimes sneak through parseaddr if the input is malformed + msgid_domain = msgid_domain.rstrip('>').strip() + except Exception: + msgid_domain = '' + return msgid_domain or 'calibre-web.com' + def prepare_message(self): message = EmailMessage() # message = MIMEMultipart() - message['to'] = self.recipent - message['from'] = self.settings["mail_from"] - message['subject'] = self.subject - message['Message-Id'] = make_msgid('calibre-web') + message['From'] = self.settings["mail_from"] + message['To'] = self.recipent + message['Subject'] = self.subject message['Date'] = formatdate(localtime=True) - # text = self.text + message['Message-Id'] = "{}@{}".format(uuid.uuid4(), self.get_msgid_domain()) # f"<{uuid.uuid4()}@{get_msgid_domain(from_)}>" # make_msgid('calibre-web') message.set_content(self.text.encode('UTF-8'), "text", "plain") if self.attachment: data = self._get_attachment(self.filepath, self.attachment) From 0aac961cde83c22abd4985f720679aa588f79e05 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sat, 19 Feb 2022 09:41:10 +0100 Subject: [PATCH 06/24] Update readme Bugfix debug logging during update unrar-free is now also recognized for displaying unrar version in about section, removed unused not configured string --- README.md | 14 +++++--------- cps/converter.py | 11 ++++++----- cps/updater.py | 4 ++-- 3 files changed, 13 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index ea9d9ba0..d9c50f7f 100644 --- a/README.md +++ b/README.md @@ -40,16 +40,12 @@ Calibre-Web is a web app providing a clean interface for browsing, reading and d ## Installation #### Installation via pip (recommended) -1. Install calibre web via pip with the command `pip install calibreweb` (Depending on your OS and or distro the command could also be `pip3`). -2. Optional features can also be installed via pip, please refer to [this page](https://github.com/janeczku/calibre-web/wiki/Dependencies-in-Calibre-Web-Linux-Windows) for details -3. Calibre-Web can be started afterwards by typing `cps` or `python3 -m cps` +1. To avoid problems with already installed python dependencies, it's recommended to create a virtual environment for Calibre-Web +2. Install Calibre-Web via pip with the command `pip install calibreweb` (Depending on your OS and or distro the command could also be `pip3`). +3. Optional features can also be installed via pip, please refer to [this page](https://github.com/janeczku/calibre-web/wiki/Dependencies-in-Calibre-Web-Linux-Windows) for details +4. Calibre-Web can be started afterwards by typing `cps` or `python3 -m cps` -#### Manual installation -1. Install dependencies by running `pip3 install --target vendor -r requirements.txt` (python3.x). Alternativly set up a python virtual environment. -2. Execute the command: `python3 cps.py` (or `nohup python3 cps.py` - recommended if you want to exit the terminal window) - -Issues with Ubuntu: -Please note that running the above install command can fail on some versions of Ubuntu, saying `"can't combine user with prefix"`. This is a [known bug](https://github.com/pypa/pip/issues/3826) and can be remedied by using the command `pip install --system --target vendor -r requirements.txt` instead. +In the Wiki there are also examples for a [manual installation](https://github.com/janeczku/calibre-web/wiki/Manual-installation) and for installation on [Linux Mint](https://github.com/janeczku/calibre-web/wiki/How-To:Install-Calibre-Web-in-Linux-Mint-19-or-20) ## Quick start diff --git a/cps/converter.py b/cps/converter.py index fcbabbfc..bb197467 100644 --- a/cps/converter.py +++ b/cps/converter.py @@ -27,7 +27,6 @@ from .subproc_wrapper import process_wait log = logger.create() # _() necessary to make babel aware of string for translation -_NOT_CONFIGURED = _('not configured') _NOT_INSTALLED = _('not installed') _EXECUTION_ERROR = _('Execution permissions missing') @@ -48,14 +47,16 @@ def _get_command_version(path, pattern, argument=None): def get_calibre_version(): - return _get_command_version(config.config_converterpath, r'ebook-convert.*\(calibre', '--version') \ - or _NOT_CONFIGURED + return _get_command_version(config.config_converterpath, r'ebook-convert.*\(calibre', '--version') def get_unrar_version(): - return _get_command_version(config.config_rarfile_location, r'UNRAR.*\d') or _NOT_CONFIGURED + unrar_version = _get_command_version(config.config_rarfile_location, r'UNRAR.*\d') + if unrar_version == "not installed": + unrar_version = _get_command_version(config.config_rarfile_location, r'unrar.*\d','-V') + return unrar_version def get_kepubify_version(): - return _get_command_version(config.config_kepubifypath, r'kepubify\s','--version') or _NOT_CONFIGURED + return _get_command_version(config.config_kepubifypath, r'kepubify\s','--version') diff --git a/cps/updater.py b/cps/updater.py index 2166b334..181c4374 100644 --- a/cps/updater.py +++ b/cps/updater.py @@ -214,7 +214,7 @@ class Updater(threading.Thread): if not os.path.exists(dst_dir): try: os.makedirs(dst_dir) - log.debug('Create directory: {}', dst_dir) + log.debug('Create directory: {}'.format(dst_dir)) except OSError as e: log.error('Failed creating folder: {} with error {}'.format(dst_dir, e)) if change_permissions: @@ -233,7 +233,7 @@ class Updater(threading.Thread): permission = os.stat(dst_file) try: os.remove(dst_file) - log.debug('Remove file before copy: %s', dst_file) + log.debug('Remove file before copy: {}'.format(dst_file)) except OSError as e: log.error('Failed removing file: {} with error {}'.format(dst_file, e)) else: From 8007e450b3178f517b83b0989744c6df38867932 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sat, 19 Feb 2022 10:04:21 +0100 Subject: [PATCH 07/24] Bugfix for cbr support without comicapi --- cps/comic.py | 2 +- optional-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cps/comic.py b/cps/comic.py index 64711460..9e7f4f8f 100644 --- a/cps/comic.py +++ b/cps/comic.py @@ -97,7 +97,7 @@ def _extract_Cover_from_archive(original_file_extension, tmp_file_name, rarExecu try: rarfile.UNRAR_TOOL = rarExecutable cf = rarfile.RarFile(tmp_file_name) - for name in cf.getnames(): + for name in cf.namelist(): ext = os.path.splitext(name) if len(ext) > 1: extension = ext[1].lower() diff --git a/optional-requirements.txt b/optional-requirements.txt index f7c7b572..04f7bb0c 100644 --- a/optional-requirements.txt +++ b/optional-requirements.txt @@ -28,7 +28,7 @@ Flask-Dance>=2.0.0,<5.2.0 SQLAlchemy-Utils>=0.33.5,<0.39.0 # metadata extraction -rarfile>=2.7 +rarfile>=3.2 scholarly>=1.2.0,<1.6 markdown2>=2.0.0,<2.5.0 html2text>=2020.1.16,<2022.1.1 From 965352c8d96c9eae7a6867ff76b0db137d04b0b8 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sat, 26 Feb 2022 08:05:35 +0100 Subject: [PATCH 08/24] Don't allow redirects on cover uploads, catch more addresses which resolve to localhost --- cps/helper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cps/helper.py b/cps/helper.py index b5495930..c162f7ee 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -734,10 +734,10 @@ def save_cover_from_url(url, book_path): if not cli.allow_localhost: # 127.0.x.x, localhost, [::1], [::ffff:7f00:1] ip = socket.getaddrinfo(urlparse(url).hostname, 0)[0][4][0] - if ip.startswith("127.") or ip.startswith('::ffff:7f') or ip == "::1": + if ip.startswith("127.") or ip.startswith('::ffff:7f') or ip == "::1" or ip == "0.0.0.0" or ip == "::": log.error("Localhost was accessed for cover upload") return False, _("You are not allowed to access localhost for cover uploads") - img = requests.get(url, timeout=(10, 200)) # ToDo: Error Handling + img = requests.get(url, timeout=(10, 200), allow_redirects=False) # ToDo: Error Handling img.raise_for_status() return save_cover(img, book_path) except (socket.gaierror, From 598618e4285d4450c74c9920941eef3242db880e Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Tue, 1 Mar 2022 16:36:46 +0100 Subject: [PATCH 09/24] Fix rename_files_on_change return handling Updated test result --- cps/helper.py | 11 +- test/Calibre-Web TestSummary_Linux.html | 3905 ++++++----------------- 2 files changed, 1057 insertions(+), 2859 deletions(-) diff --git a/cps/helper.py b/cps/helper.py index c162f7ee..03a2d03b 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -493,12 +493,10 @@ def upload_new_file_gdrive(book_id, first_author, renamed_author, title, title_d title_dir + " (" + str(book_id) + ")") book.path = gdrive_path.replace("\\", "/") gd.uploadFileToEbooksFolder(os.path.join(gdrive_path, file_name).replace("\\", "/"), original_filepath) - error |= rename_files_on_change(first_author, renamed_author, localbook=book, gdrive=True) - return error + return rename_files_on_change(first_author, renamed_author, localbook=book, gdrive=True) def update_dir_structure_gdrive(book_id, first_author, renamed_author): - error = False book = calibre_db.get_book(book_id) authordir = book.path.split('/')[0] @@ -513,7 +511,7 @@ def update_dir_structure_gdrive(book_id, first_author, renamed_author): book.path = book.path.split('/')[0] + u'/' + new_titledir gd.updateDatabaseOnEdit(gFile['id'], book.path) # only child folder affected else: - error = _(u'File %(file)s not found on Google Drive', file=book.path) # file not found + return _(u'File %(file)s not found on Google Drive', file=book.path) # file not found if authordir != new_authordir and authordir not in renamed_author: gFile = gd.getFileFromEbooksFolder(os.path.dirname(book.path), new_titledir) @@ -522,12 +520,11 @@ def update_dir_structure_gdrive(book_id, first_author, renamed_author): book.path = new_authordir + u'/' + book.path.split('/')[1] gd.updateDatabaseOnEdit(gFile['id'], book.path) else: - error = _(u'File %(file)s not found on Google Drive', file=authordir) # file not found + return _(u'File %(file)s not found on Google Drive', file=authordir) # file not found # change location in database to new author/title path book.path = os.path.join(new_authordir, new_titledir).replace('\\', '/') - error |= rename_files_on_change(first_author, renamed_author, book, gdrive=True) - return error + return rename_files_on_change(first_author, renamed_author, book, gdrive=True) def move_files_on_change(calibre_path, new_authordir, new_titledir, localbook, db_filename, original_filepath, path): diff --git a/test/Calibre-Web TestSummary_Linux.html b/test/Calibre-Web TestSummary_Linux.html index af8cfcd9..c569a45d 100644 --- a/test/Calibre-Web TestSummary_Linux.html +++ b/test/Calibre-Web TestSummary_Linux.html @@ -37,20 +37,20 @@
-

Start Time: 2022-02-05 20:19:09

+

Start Time: 2022-02-28 21:38:15

-

Stop Time: 2022-02-06 00:20:42

+

Stop Time: 2022-03-01 01:36:56

-

Duration: 3h 15 min

+

Duration: 3h 14 min

@@ -330,12 +330,12 @@ - + TestCliGdrivedb 2 + 2 0 0 - 2 0 Detail @@ -344,89 +344,20 @@ - +
TestCliGdrivedb - test_cli_gdrive_location
- -
- ERROR -
- - - - + PASS - +
TestCliGdrivedb - test_gdrive_db_nonwrite
- -
- ERROR -
- - - - + PASS @@ -1496,9 +1427,9 @@ receiveMessage@chrome://remote/content/marionette/actors/MarionetteCommandsChild 
Traceback (most recent call last):
-  File "/home/ozzie/Development/calibre-web-test/test/test_edit_books_author_gdrive.py", line 544, in test_rename_capital_on_upload
-    self.check_element_on_page((By.ID, 'edit_cancel')).click()
-AttributeError: 'bool' object has no attribute 'click'
+ File "/home/ozzie/Development/calibre-web-test/test/test_edit_books_author_gdrive.py", line 579, in test_rename_capital_on_upload + self.assertEqual('Useless', details['title']) +KeyError: 'title'
@@ -1713,9 +1644,9 @@ AttributeError: 'bool' object has no attribute 'click' TestEditBooksOnGdrive 20 - 0 - 0 - 20 + 15 + 3 + 2 0 Detail @@ -1724,31 +1655,11 @@ AttributeError: 'bool' object has no attribute 'click' - +
TestEditBooksOnGdrive - test_download_book
- -
- ERROR -
- - - - + PASS @@ -1770,77 +1681,9 @@ TypeError: object of type 'bool' has no len() 
Traceback (most recent call last):
-  File "/home/ozzie/Development/calibre-web-test/venv/lib/python3.8/site-packages/urllib3/connection.py", line 174, in _new_conn
-    conn = connection.create_connection(
-  File "/home/ozzie/Development/calibre-web-test/venv/lib/python3.8/site-packages/urllib3/util/connection.py", line 95, in create_connection
-    raise err
-  File "/home/ozzie/Development/calibre-web-test/venv/lib/python3.8/site-packages/urllib3/util/connection.py", line 85, in create_connection
-    sock.connect(sa)
-ConnectionRefusedError: [Errno 111] Connection refused
-
-During handling of the above exception, another exception occurred:
-
-Traceback (most recent call last):
-  File "/home/ozzie/Development/calibre-web-test/venv/lib/python3.8/site-packages/urllib3/connectionpool.py", line 703, in urlopen
-    httplib_response = self._make_request(
-  File "/home/ozzie/Development/calibre-web-test/venv/lib/python3.8/site-packages/urllib3/connectionpool.py", line 398, in _make_request
-    conn.request(method, url, **httplib_request_kw)
-  File "/home/ozzie/Development/calibre-web-test/venv/lib/python3.8/site-packages/urllib3/connection.py", line 239, in request
-    super(HTTPConnection, self).request(method, url, body=body, headers=headers)
-  File "/usr/lib/python3.8/http/client.py", line 1256, in request
-    self._send_request(method, url, body, headers, encode_chunked)
-  File "/usr/lib/python3.8/http/client.py", line 1302, in _send_request
-    self.endheaders(body, encode_chunked=encode_chunked)
-  File "/usr/lib/python3.8/http/client.py", line 1251, in endheaders
-    self._send_output(message_body, encode_chunked=encode_chunked)
-  File "/usr/lib/python3.8/http/client.py", line 1011, in _send_output
-    self.send(msg)
-  File "/usr/lib/python3.8/http/client.py", line 951, in send
-    self.connect()
-  File "/home/ozzie/Development/calibre-web-test/venv/lib/python3.8/site-packages/urllib3/connection.py", line 205, in connect
-    conn = self._new_conn()
-  File "/home/ozzie/Development/calibre-web-test/venv/lib/python3.8/site-packages/urllib3/connection.py", line 186, in _new_conn
-    raise NewConnectionError(
-urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPConnection object at 0x7f7cee201a90>: Failed to establish a new connection: [Errno 111] Connection refused
-
-During handling of the above exception, another exception occurred:
-
-Traceback (most recent call last):
-  File "/home/ozzie/Development/calibre-web-test/test/test_edit_ebooks_gdrive.py", line 291, in test_edit_author
-    self.fill_basic_config({"config_unicode_filename": 1})
-  File "/home/ozzie/Development/calibre-web-test/test/helper_ui.py", line 358, in fill_basic_config
-    cls._fill_basic_config(elements)
-  File "/home/ozzie/Development/calibre-web-test/test/helper_ui.py", line 268, in _fill_basic_config
-    WebDriverWait(cls.driver, 5).until(EC.presence_of_element_located((By.ID, "config_port")))
-  File "/home/ozzie/Development/calibre-web-test/venv/lib/python3.8/site-packages/selenium/webdriver/support/wait.py", line 78, in until
-    value = method(self._driver)
-  File "/home/ozzie/Development/calibre-web-test/venv/lib/python3.8/site-packages/selenium/webdriver/support/expected_conditions.py", line 64, in _predicate
-    return driver.find_element(*locator)
-  File "/home/ozzie/Development/calibre-web-test/venv/lib/python3.8/site-packages/selenium/webdriver/remote/webdriver.py", line 1244, in find_element
-    return self.execute(Command.FIND_ELEMENT, {
-  File "/home/ozzie/Development/calibre-web-test/venv/lib/python3.8/site-packages/selenium/webdriver/remote/webdriver.py", line 422, in execute
-    response = self.command_executor.execute(driver_command, params)
-  File "/home/ozzie/Development/calibre-web-test/venv/lib/python3.8/site-packages/selenium/webdriver/remote/remote_connection.py", line 421, in execute
-    return self._request(command_info[0], url, body=data)
-  File "/home/ozzie/Development/calibre-web-test/venv/lib/python3.8/site-packages/selenium/webdriver/remote/remote_connection.py", line 443, in _request
-    resp = self._conn.request(method, url, body=body, headers=headers)
-  File "/home/ozzie/Development/calibre-web-test/venv/lib/python3.8/site-packages/urllib3/request.py", line 78, in request
-    return self.request_encode_body(
-  File "/home/ozzie/Development/calibre-web-test/venv/lib/python3.8/site-packages/urllib3/request.py", line 170, in request_encode_body
-    return self.urlopen(method, url, **extra_kw)
-  File "/home/ozzie/Development/calibre-web-test/venv/lib/python3.8/site-packages/urllib3/poolmanager.py", line 375, in urlopen
-    response = conn.urlopen(method, u.request_uri, **kw)
-  File "/home/ozzie/Development/calibre-web-test/venv/lib/python3.8/site-packages/urllib3/connectionpool.py", line 813, in urlopen
-    return self.urlopen(
-  File "/home/ozzie/Development/calibre-web-test/venv/lib/python3.8/site-packages/urllib3/connectionpool.py", line 813, in urlopen
-    return self.urlopen(
-  File "/home/ozzie/Development/calibre-web-test/venv/lib/python3.8/site-packages/urllib3/connectionpool.py", line 813, in urlopen
-    return self.urlopen(
-  File "/home/ozzie/Development/calibre-web-test/venv/lib/python3.8/site-packages/urllib3/connectionpool.py", line 785, in urlopen
-    retries = retries.increment(
-  File "/home/ozzie/Development/calibre-web-test/venv/lib/python3.8/site-packages/urllib3/util/retry.py", line 592, in increment
-    raise MaxRetryError(_pool, url, error or ResponseError(cause))
-urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='localhost', port=51341): Max retries exceeded with url: /session/2d4e9139-cf96-4ca3-bca3-9646475259b3/element (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7f7cee201a90>: Failed to establish a new connection: [Errno 111] Connection refused'))
+ File "/home/ozzie/Development/calibre-web-test/test/test_edit_ebooks_gdrive.py", line 392, in test_edit_author + self.assertEqual(values['author'][0], 'Pipo, Pipe') +IndexError: list index out of range
@@ -1850,295 +1693,143 @@ urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='localhost', p - +
TestEditBooksOnGdrive - test_edit_category
- -
- ERROR -
- - - - + PASS - +
TestEditBooksOnGdrive - test_edit_comments
- -
- ERROR -
- - - - + PASS - +
TestEditBooksOnGdrive - test_edit_custom_bool
- -
- ERROR -
- - - - + PASS - +
TestEditBooksOnGdrive - test_edit_custom_categories
- -
- ERROR -
- - - - + PASS - +
TestEditBooksOnGdrive - test_edit_custom_float
- -
- ERROR -
- - - - + PASS - +
TestEditBooksOnGdrive - test_edit_custom_int
+ PASS + + + + + + +
TestEditBooksOnGdrive - test_edit_custom_rating
+ + PASS + + + + + + +
TestEditBooksOnGdrive - test_edit_custom_single_select
+ + PASS + + + + + + +
TestEditBooksOnGdrive - test_edit_custom_text
+ + PASS + + + + + + +
TestEditBooksOnGdrive - test_edit_language
+ + PASS + + + + + + +
TestEditBooksOnGdrive - test_edit_publisher
+ + PASS + + + + + + +
TestEditBooksOnGdrive - test_edit_rating
+ + PASS + + + + + + +
TestEditBooksOnGdrive - test_edit_series
+ + PASS + + + + + + +
TestEditBooksOnGdrive - test_edit_title
+
- ERROR + ERROR
- From 753319c8b63df34db9dfdf2e1078950070d410f3 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sun, 6 Mar 2022 16:30:50 +0100 Subject: [PATCH 12/24] Version bump --- cps/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cps/constants.py b/cps/constants.py index a96f614b..cb5348d5 100644 --- a/cps/constants.py +++ b/cps/constants.py @@ -154,7 +154,7 @@ def selected_roles(dictionary): BookMeta = namedtuple('BookMeta', 'file_path, extension, title, author, cover, description, tags, series, ' 'series_id, languages, publisher') -STABLE_VERSION = {'version': '0.6.17'} +STABLE_VERSION = {'version': '0.6.18 Beta'} NIGHTLY_VERSION = dict() NIGHTLY_VERSION[0] = '$Format:%H$' From 34478079d83755ce843ba200638717929ecc3010 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Wed, 9 Mar 2022 14:45:39 +0100 Subject: [PATCH 13/24] Prevent local variable 'from_book' referenced before assignment during merge of books Merge books source book: Each book in own row Merge books, sources are deleted before dialog shows up again --- cps/editbooks.py | 2 +- cps/static/js/table.js | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cps/editbooks.py b/cps/editbooks.py index fcf043a5..483e8a7e 100755 --- a/cps/editbooks.py +++ b/cps/editbooks.py @@ -1263,8 +1263,8 @@ def simulate_merge_list_book(): to_book = calibre_db.get_book(vals[0]).title vals.pop(0) if to_book: + from_book = [] for book_id in vals: - from_book = [] from_book.append(calibre_db.get_book(book_id).title) return json.dumps({'to': to_book, 'from': from_book}) return "" diff --git a/cps/static/js/table.js b/cps/static/js/table.js index dce1d06c..ae8c591b 100644 --- a/cps/static/js/table.js +++ b/cps/static/js/table.js @@ -107,8 +107,9 @@ $(function() { url: window.location.pathname + "/../ajax/simulatemerge", data: JSON.stringify({"Merge_books":selections}), success: function success(booTitles) { + $('#merge_from').empty(); $.each(booTitles.from, function(i, item) { - $("- " + item + "").appendTo("#merge_from"); + $("- " + item + "

").appendTo("#merge_from"); }); $("#merge_to").text("- " + booTitles.to); From 49692b4a454f762edbefc6642dfb0f50b75418e6 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sat, 12 Mar 2022 08:27:18 +0100 Subject: [PATCH 14/24] Update catch errors for load metadata from amazon (#2333) --- cps/metadata_provider/amazon.py | 62 +++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 27 deletions(-) diff --git a/cps/metadata_provider/amazon.py b/cps/metadata_provider/amazon.py index 558edebc..acea7e07 100644 --- a/cps/metadata_provider/amazon.py +++ b/cps/metadata_provider/amazon.py @@ -25,8 +25,11 @@ try: except ImportError: pass from cps.services.Metadata import MetaRecord, MetaSourceInfo, Metadata +import cps.logger as logger + #from time import time from operator import itemgetter +log = logger.create() class Amazon(Metadata): __name__ = "Amazon" @@ -48,15 +51,15 @@ class Amazon(Metadata): self, query: str, generic_cover: str = "", locale: str = "en" ): #timer=time() - def inner(link,index)->[dict,int]: - with self.session as session: - r = session.get(f"https://www.amazon.com/{link}") - r.raise_for_status() - long_soup = BS(r.text, "lxml") #~4sec :/ - soup2 = long_soup.find("div", attrs={"cel_widget_id": "dpx-books-ppd_csm_instrumentation_wrapper"}) - if soup2 is None: - return - try: + def inner(link, index) -> [dict, int]: + try: + with self.session as session: + r = session.get(f"https://www.amazon.com{link}") + r.raise_for_status() + long_soup = BS(r.text, "lxml") #~4sec :/ + soup2 = long_soup.find("div", attrs={"cel_widget_id": "dpx-books-ppd_csm_instrumentation_wrapper"}) + if soup2 is None: + return match = MetaRecord( title = "", authors = "", @@ -65,7 +68,7 @@ class Amazon(Metadata): description="Amazon Books", link="https://amazon.com/" ), - url = f"https://www.amazon.com/{link}", + url = f"https://www.amazon.com{link}", #the more searches the slower, these are too hard to find in reasonable time or might not even exist publisher= "", # very unreliable publishedDate= "", # very unreliable @@ -101,22 +104,27 @@ class Amazon(Metadata): except (AttributeError, TypeError): match.cover = "" return match, index - except Exception as e: - print(e) - return + except Exception as e: + log.debug_or_exception(e) + return val = list() - if self.active: - results = self.session.get( - f"https://www.amazon.com/s?k={query.replace(' ', '+')}&i=digital-text&sprefix={query.replace(' ', '+')}" - f"%2Cdigital-text&ref=nb_sb_noss", - headers=self.headers) - results.raise_for_status() - soup = BS(results.text, 'html.parser') - links_list = [next(filter(lambda i: "digital-text" in i["href"], x.findAll("a")))["href"] for x in - soup.findAll("div", attrs={"data-component-type": "s-search-result"})] - with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: - fut = {executor.submit(inner, link, index) for index, link in enumerate(links_list[:5])} - val=list(map(lambda x : x.result() ,concurrent.futures.as_completed(fut))) - result=list(filter(lambda x: x, val)) - return [x[0] for x in sorted(result, key=itemgetter(1))] #sort by amazons listing order for best relevance + try: + if self.active: + results = self.session.get( + f"https://www.amazon.com/s?k={query.replace(' ', '+')}" + f"&i=digital-text&sprefix={query.replace(' ', '+')}" + f"%2Cdigital-text&ref=nb_sb_noss", + headers=self.headers) + results.raise_for_status() + soup = BS(results.text, 'html.parser') + links_list = [next(filter(lambda i: "digital-text" in i["href"], x.findAll("a")))["href"] for x in + soup.findAll("div", attrs={"data-component-type": "s-search-result"})] + with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: + fut = {executor.submit(inner, link, index) for index, link in enumerate(links_list[:5])} + val = list(map(lambda x: x.result(), concurrent.futures.as_completed(fut))) + result = list(filter(lambda x: x, val)) + return [x[0] for x in sorted(result, key=itemgetter(1))] #sort by amazons listing order for best relevance + except requests.exceptions.HTTPError as e: + log.debug_or_exception(e) + return [] From d80297e1a87523adc2b5731695cd91f06b265ae6 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sat, 12 Mar 2022 10:00:38 +0100 Subject: [PATCH 15/24] Bugfix sorting user table --- cps/admin.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cps/admin.py b/cps/admin.py index 4876d421..18bd87ec 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -298,10 +298,13 @@ def list_users(): limit = int(request.args.get("limit") or 10) search = request.args.get("search") sort = request.args.get("sort", "id") - order = request.args.get("order", "").lower() state = None if sort == "state": state = json.loads(request.args.get("state", "[]")) + else: + if sort not in ub.User.__table__.columns.keys(): + sort = "id" + order = request.args.get("order", "").lower() if sort != "state" and order: order = text(sort + " " + order) From 547ea93dc9b8cace6383374d66da1b51de64342d Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sat, 12 Mar 2022 10:19:21 +0100 Subject: [PATCH 16/24] First fix for #2325 (edit book table with readonly database) --- cps/editbooks.py | 148 ++++++++++++++++++++++++----------------------- 1 file changed, 76 insertions(+), 72 deletions(-) diff --git a/cps/editbooks.py b/cps/editbooks.py index 483e8a7e..7e251696 100755 --- a/cps/editbooks.py +++ b/cps/editbooks.py @@ -1152,80 +1152,81 @@ def edit_list_book(param): vals = request.form.to_dict() book = calibre_db.get_book(vals['pk']) # ret = "" - if param == 'series_index': - edit_book_series_index(vals['value'], book) - ret = Response(json.dumps({'success': True, 'newValue': book.series_index}), mimetype='application/json') - elif param == 'tags': - edit_book_tags(vals['value'], book) - ret = Response(json.dumps({'success': True, 'newValue': ', '.join([tag.name for tag in book.tags])}), - mimetype='application/json') - elif param == 'series': - edit_book_series(vals['value'], book) - ret = Response(json.dumps({'success': True, 'newValue': ', '.join([serie.name for serie in book.series])}), - mimetype='application/json') - elif param == 'publishers': - edit_book_publisher(vals['value'], book) - ret = Response(json.dumps({'success': True, - 'newValue': ', '.join([publisher.name for publisher in book.publishers])}), - mimetype='application/json') - elif param == 'languages': - invalid = list() - edit_book_languages(vals['value'], book, invalid=invalid) - if invalid: - ret = Response(json.dumps({'success': False, - 'msg': 'Invalid languages in request: {}'.format(','.join(invalid))}), + try: + if param == 'series_index': + edit_book_series_index(vals['value'], book) + ret = Response(json.dumps({'success': True, 'newValue': book.series_index}), mimetype='application/json') + elif param == 'tags': + edit_book_tags(vals['value'], book) + ret = Response(json.dumps({'success': True, 'newValue': ', '.join([tag.name for tag in book.tags])}), mimetype='application/json') - else: - lang_names = list() - for lang in book.languages: - lang_names.append(isoLanguages.get_language_name(get_locale(), lang.lang_code)) - ret = Response(json.dumps({'success': True, 'newValue': ', '.join(lang_names)}), - mimetype='application/json') - elif param == 'author_sort': - book.author_sort = vals['value'] - ret = Response(json.dumps({'success': True, 'newValue': book.author_sort}), - mimetype='application/json') - elif param == 'title': - sort = book.sort - handle_title_on_edit(book, vals.get('value', "")) - helper.update_dir_structure(book.id, config.config_calibre_dir) - ret = Response(json.dumps({'success': True, 'newValue': book.title}), - mimetype='application/json') - elif param == 'sort': - book.sort = vals['value'] - ret = Response(json.dumps({'success': True, 'newValue': book.sort}), - mimetype='application/json') - elif param == 'comments': - edit_book_comments(vals['value'], book) - ret = Response(json.dumps({'success': True, 'newValue': book.comments[0].text}), - mimetype='application/json') - elif param == 'authors': - input_authors, __, renamed = handle_author_on_edit(book, vals['value'], vals.get('checkA', None) == "true") - helper.update_dir_structure(book.id, config.config_calibre_dir, input_authors[0], renamed_author=renamed) - ret = Response(json.dumps({'success': True, - 'newValue': ' & '.join([author.replace('|',',') for author in input_authors])}), - mimetype='application/json') - elif param == 'is_archived': - change_archived_books(book.id, vals['value'] == "True") - ret = "" - elif param == 'read_status': - ret = helper.edit_book_read_status(book.id, vals['value'] == "True") - if ret: - return ret, 400 - elif param.startswith("custom_column_"): - new_val = dict() - new_val[param] = vals['value'] - edit_single_cc_data(book.id, book, param[14:], new_val) - # ToDo: Very hacky find better solution - if vals['value'] in ["True", "False"]: + elif param == 'series': + edit_book_series(vals['value'], book) + ret = Response(json.dumps({'success': True, 'newValue': ', '.join([serie.name for serie in book.series])}), + mimetype='application/json') + elif param == 'publishers': + edit_book_publisher(vals['value'], book) + ret = Response(json.dumps({'success': True, + 'newValue': ', '.join([publisher.name for publisher in book.publishers])}), + mimetype='application/json') + elif param == 'languages': + invalid = list() + edit_book_languages(vals['value'], book, invalid=invalid) + if invalid: + ret = Response(json.dumps({'success': False, + 'msg': 'Invalid languages in request: {}'.format(','.join(invalid))}), + mimetype='application/json') + else: + lang_names = list() + for lang in book.languages: + lang_names.append(isoLanguages.get_language_name(get_locale(), lang.lang_code)) + ret = Response(json.dumps({'success': True, 'newValue': ', '.join(lang_names)}), + mimetype='application/json') + elif param == 'author_sort': + book.author_sort = vals['value'] + ret = Response(json.dumps({'success': True, 'newValue': book.author_sort}), + mimetype='application/json') + elif param == 'title': + sort = book.sort + handle_title_on_edit(book, vals.get('value', "")) + helper.update_dir_structure(book.id, config.config_calibre_dir) + ret = Response(json.dumps({'success': True, 'newValue': book.title}), + mimetype='application/json') + elif param == 'sort': + book.sort = vals['value'] + ret = Response(json.dumps({'success': True, 'newValue': book.sort}), + mimetype='application/json') + elif param == 'comments': + edit_book_comments(vals['value'], book) + ret = Response(json.dumps({'success': True, 'newValue': book.comments[0].text}), + mimetype='application/json') + elif param == 'authors': + input_authors, __, renamed = handle_author_on_edit(book, vals['value'], vals.get('checkA', None) == "true") + helper.update_dir_structure(book.id, config.config_calibre_dir, input_authors[0], renamed_author=renamed) + ret = Response(json.dumps({'success': True, + 'newValue': ' & '.join([author.replace('|',',') for author in input_authors])}), + mimetype='application/json') + elif param == 'is_archived': + change_archived_books(book.id, vals['value'] == "True") ret = "" + elif param == 'read_status': + ret = helper.edit_book_read_status(book.id, vals['value'] == "True") + if ret: + return ret, 400 + elif param.startswith("custom_column_"): + new_val = dict() + new_val[param] = vals['value'] + edit_single_cc_data(book.id, book, param[14:], new_val) + # ToDo: Very hacky find better solution + if vals['value'] in ["True", "False"]: + ret = "" + else: + ret = Response(json.dumps({'success': True, 'newValue': vals['value']}), + mimetype='application/json') else: - ret = Response(json.dumps({'success': True, 'newValue': vals['value']}), - mimetype='application/json') - else: - return _("Parameter not found"), 400 - book.last_modified = datetime.utcnow() - try: + return _("Parameter not found"), 400 + book.last_modified = datetime.utcnow() + calibre_db.session.commit() # revert change for sort if automatic fields link is deactivated if param == 'title' and vals.get('checkT') == "false": @@ -1233,7 +1234,10 @@ def edit_list_book(param): calibre_db.session.commit() except (OperationalError, IntegrityError) as e: calibre_db.session.rollback() - log.error("Database error: %s", e) + log.error("Database error: {}".format(e)) + ret = Response(json.dumps({'success': False, + 'msg': 'Database error: {}'.format(e.orig)}), + mimetype='application/json') return ret From 3a0dacc6a69d3d7edf4abc362d1052f55b0330a3 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sat, 12 Mar 2022 14:27:41 +0100 Subject: [PATCH 17/24] log message on not found author --- cps/db.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cps/db.py b/cps/db.py index a3b93d89..caf0ee6e 100644 --- a/cps/db.py +++ b/cps/db.py @@ -788,8 +788,9 @@ class CalibreDB(): error = False for auth in sort_authors: results = self.session.query(Authors).filter(Authors.sort == auth.lstrip().strip()).all() - # ToDo: How to handle not found authorname + # ToDo: How to handle not found author name if not len(results): + log.error("Author {} not found to display name in right order".format(auth)) error = True break for r in results: From 2b31b6a306407f049b5e58daaf8a186ba53eea93 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sat, 12 Mar 2022 16:51:50 +0100 Subject: [PATCH 18/24] Fix for #2325 (author sort order differs from authors order with readonly database) --- cps/db.py | 25 +++++++---- cps/templates/author.html | 4 +- cps/templates/detail.html | 2 +- cps/templates/discover.html | 2 +- cps/templates/index.html | 4 +- cps/templates/listenmp3.html | 2 +- cps/templates/shelf.html | 2 +- cps/templates/shelfdown.html | 2 +- cps/web.py | 80 ++++++++++++++++++------------------ 9 files changed, 65 insertions(+), 58 deletions(-) diff --git a/cps/db.py b/cps/db.py index caf0ee6e..6ff0d03e 100644 --- a/cps/db.py +++ b/cps/db.py @@ -17,7 +17,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import sys +import copy import os import re import ast @@ -776,6 +776,8 @@ class CalibreDB(): # Orders all Authors in the list according to authors sort def order_authors(self, entries, list_return=False, combined=False): + # entries_copy = copy.deepcopy(entries) + # entries_copy =[] for entry in entries: if combined: sort_authors = entry.Books.author_sort.split('&') @@ -785,26 +787,31 @@ class CalibreDB(): sort_authors = entry.author_sort.split('&') ids = [a.id for a in entry.authors] authors_ordered = list() - error = False + # error = False for auth in sort_authors: results = self.session.query(Authors).filter(Authors.sort == auth.lstrip().strip()).all() # ToDo: How to handle not found author name if not len(results): log.error("Author {} not found to display name in right order".format(auth)) - error = True + # error = True break for r in results: if r.id in ids: authors_ordered.append(r) - if not error: + ids.remove(r.id) + for author_id in ids: + result = self.session.query(Authors).filter(Authors.id == author_id).first() + authors_ordered.append(result) + + if list_return: if combined: entry.Books.authors = authors_ordered else: - entry.authors = authors_ordered - if list_return: - return entries - else: - return authors_ordered + entry.ordered_authors = authors_ordered + else: + return authors_ordered + return entries + def get_typeahead(self, database, query, replace=('', ''), tag_filter=true()): query = query or '' diff --git a/cps/templates/author.html b/cps/templates/author.html index d832ad5c..fc811368 100644 --- a/cps/templates/author.html +++ b/cps/templates/author.html @@ -47,7 +47,7 @@

{{entry.title|shortentitle}}

- {% for author in entry.authors %} + {% for author in entry.ordered_authors %} {% if loop.index > g.config_authors_max and g.config_authors_max != 0 %} {% if not loop.first %} & @@ -110,7 +110,7 @@

{{entry.title|shortentitle}}

- {% for author in entry.authors %} + {% for author in entry.ordered_authors %} {% if loop.index > g.config_authors_max and g.config_authors_max != 0 %} {{author.name.replace('|',',')}} {% if loop.last %} diff --git a/cps/templates/detail.html b/cps/templates/detail.html index 06357ce8..c2153db8 100644 --- a/cps/templates/detail.html +++ b/cps/templates/detail.html @@ -99,7 +99,7 @@

{{entry.title}}

- {% for author in entry.authors %} + {% for author in entry.ordered_authors %} {{author.name.replace('|',',')}} {% if not loop.last %} & diff --git a/cps/templates/discover.html b/cps/templates/discover.html index 74448b98..bceb6a8a 100644 --- a/cps/templates/discover.html +++ b/cps/templates/discover.html @@ -20,7 +20,7 @@

{{entry.title|shortentitle}}

- {% for author in entry.authors %} + {% for author in entry.ordered_authors %} {% if loop.index > g.config_authors_max and g.config_authors_max != 0 %} {% if not loop.first %} & diff --git a/cps/templates/index.html b/cps/templates/index.html index 162adc7d..b69a9284 100644 --- a/cps/templates/index.html +++ b/cps/templates/index.html @@ -19,7 +19,7 @@

{{entry.title|shortentitle}}

- {% for author in entry.authors %} + {% for author in entry.ordered_authors %} {% if loop.index > g.config_authors_max and g.config_authors_max != 0 %} {% if not loop.first %} & @@ -101,7 +101,7 @@

{{entry.title|shortentitle}}

- {% for author in entry.authors %} + {% for author in entry.ordered_authors %} {% if loop.index > g.config_authors_max and g.config_authors_max != 0 %} {% if not loop.first %} & diff --git a/cps/templates/listenmp3.html b/cps/templates/listenmp3.html index 279ec28f..7da62a20 100644 --- a/cps/templates/listenmp3.html +++ b/cps/templates/listenmp3.html @@ -105,7 +105,7 @@

diff --git a/cps/templates/shelf.html b/cps/templates/shelf.html index adfead60..2e4cf906 100644 --- a/cps/templates/shelf.html +++ b/cps/templates/shelf.html @@ -44,7 +44,7 @@

{{entry.title|shortentitle}}

- {% for author in entry.authors %} + {% for author in entry.ordered_authors %} {% if loop.index > g.config_authors_max and g.config_authors_max != 0 %} {% if not loop.first %} & diff --git a/cps/templates/shelfdown.html b/cps/templates/shelfdown.html index 1d781310..78f00b5e 100644 --- a/cps/templates/shelfdown.html +++ b/cps/templates/shelfdown.html @@ -37,7 +37,7 @@

{{entry.title|shortentitle}}

- {% for author in entry.authors %} + {% for author in entry.ordered_authors %} {{author.name.replace('|',',')}} {% if not loop.last %} & diff --git a/cps/web.py b/cps/web.py index 6d33f809..94f511b1 100644 --- a/cps/web.py +++ b/cps/web.py @@ -300,43 +300,43 @@ def get_matching_tags(): return json_dumps -def get_sort_function(sort, data): +def get_sort_function(sort_param, data): order = [db.Books.timestamp.desc()] - if sort == 'stored': - sort = current_user.get_view_property(data, 'stored') + if sort_param == 'stored': + sort_param = current_user.get_view_property(data, 'stored') else: - current_user.set_view_property(data, 'stored', sort) - if sort == 'pubnew': + current_user.set_view_property(data, 'stored', sort_param) + if sort_param == 'pubnew': order = [db.Books.pubdate.desc()] - if sort == 'pubold': + if sort_param == 'pubold': order = [db.Books.pubdate] - if sort == 'abc': + if sort_param == 'abc': order = [db.Books.sort] - if sort == 'zyx': + if sort_param == 'zyx': order = [db.Books.sort.desc()] - if sort == 'new': + if sort_param == 'new': order = [db.Books.timestamp.desc()] - if sort == 'old': + if sort_param == 'old': order = [db.Books.timestamp] - if sort == 'authaz': + if sort_param == 'authaz': order = [db.Books.author_sort.asc(), db.Series.name, db.Books.series_index] - if sort == 'authza': + if sort_param == 'authza': order = [db.Books.author_sort.desc(), db.Series.name.desc(), db.Books.series_index.desc()] - if sort == 'seriesasc': + if sort_param == 'seriesasc': order = [db.Books.series_index.asc()] - if sort == 'seriesdesc': + if sort_param == 'seriesdesc': order = [db.Books.series_index.desc()] - if sort == 'hotdesc': + if sort_param == 'hotdesc': order = [func.count(ub.Downloads.book_id).desc()] - if sort == 'hotasc': + if sort_param == 'hotasc': order = [func.count(ub.Downloads.book_id).asc()] - if sort is None: - sort = "new" - return order, sort + if sort_param is None: + sort_param = "new" + return order, sort_param -def render_books_list(data, sort, book_id, page): - order = get_sort_function(sort, data) +def render_books_list(data, sort_param, book_id, page): + order = get_sort_function(sort_param, data) if data == "rated": return render_rated_books(page, book_id, order=order) elif data == "discover": @@ -604,7 +604,7 @@ def render_language_books(page, name, order): def render_read_books(page, are_read, as_xml=False, order=None): - sort = order[0] if order else [] + sort_param = order[0] if order else [] if not config.config_read_column: if are_read: db_filter = and_(ub.ReadBook.user_id == int(current_user.id), @@ -614,7 +614,7 @@ def render_read_books(page, are_read, as_xml=False, order=None): entries, random, pagination = calibre_db.fill_indexpage(page, 0, db.Books, db_filter, - sort, + sort_param, False, 0, db.books_series_link, db.Books.id == db.books_series_link.c.book, @@ -629,7 +629,7 @@ def render_read_books(page, are_read, as_xml=False, order=None): entries, random, pagination = calibre_db.fill_indexpage(page, 0, db.Books, db_filter, - sort, + sort_param, False, 0, db.books_series_link, db.Books.id == db.books_series_link.c.book, @@ -656,8 +656,8 @@ def render_read_books(page, are_read, as_xml=False, order=None): title=name, page=pagename, order=order[1]) -def render_archived_books(page, sort): - order = sort[0] or [] +def render_archived_books(page, sort_param): + order = sort_param[0] or [] archived_books = ( ub.session.query(ub.ArchivedBook) .filter(ub.ArchivedBook.user_id == int(current_user.id)) @@ -678,7 +678,7 @@ def render_archived_books(page, sort): name = _(u'Archived Books') + ' (' + str(len(archived_book_ids)) + ')' pagename = "archived" return render_title_template('index.html', random=random, entries=entries, pagination=pagination, - title=name, page=pagename, order=sort[1]) + title=name, page=pagename, order=sort_param[1]) def render_prepare_search_form(cc): @@ -767,32 +767,32 @@ def list_books(): off = int(request.args.get("offset") or 0) limit = int(request.args.get("limit") or config.config_books_per_page) search = request.args.get("search") - sort = request.args.get("sort", "id") + sort_param = request.args.get("sort", "id") order = request.args.get("order", "").lower() state = None join = tuple() - if sort == "state": + if sort_param == "state": state = json.loads(request.args.get("state", "[]")) - elif sort == "tags": + elif sort_param == "tags": order = [db.Tags.name.asc()] if order == "asc" else [db.Tags.name.desc()] join = db.books_tags_link, db.Books.id == db.books_tags_link.c.book, db.Tags - elif sort == "series": + elif sort_param == "series": order = [db.Series.name.asc()] if order == "asc" else [db.Series.name.desc()] join = db.books_series_link, db.Books.id == db.books_series_link.c.book, db.Series - elif sort == "publishers": + elif sort_param == "publishers": order = [db.Publishers.name.asc()] if order == "asc" else [db.Publishers.name.desc()] join = db.books_publishers_link, db.Books.id == db.books_publishers_link.c.book, db.Publishers - elif sort == "authors": + elif sort_param == "authors": order = [db.Authors.name.asc(), db.Series.name, db.Books.series_index] if order == "asc" \ else [db.Authors.name.desc(), db.Series.name.desc(), db.Books.series_index.desc()] join = db.books_authors_link, db.Books.id == db.books_authors_link.c.book, db.Authors, \ db.books_series_link, db.Books.id == db.books_series_link.c.book, db.Series - elif sort == "languages": + elif sort_param == "languages": order = [db.Languages.lang_code.asc()] if order == "asc" else [db.Languages.lang_code.desc()] join = db.books_languages_link, db.Books.id == db.books_languages_link.c.book, db.Languages - elif order and sort in ["sort", "title", "authors_sort", "series_index"]: - order = [text(sort + " " + order)] + elif order and sort_param in ["sort", "title", "authors_sort", "series_index"]: + order = [text(sort_param + " " + order)] elif not state: order = [db.Books.timestamp.desc()] @@ -817,7 +817,7 @@ def list_books(): except (KeyError, AttributeError): log.error("Custom Column No.%d is not existing in calibre database", read_column) # Skip linking read column and return None instead of read status - books =calibre_db.session.query(db.Books, None, ub.ArchivedBook.is_archived) + books = calibre_db.session.query(db.Books, None, ub.ArchivedBook.is_archived) books = (books.outerjoin(ub.ArchivedBook, and_(db.Books.id == ub.ArchivedBook.book_id, int(current_user.id) == ub.ArchivedBook.user_id)) .filter(calibre_db.common_filters(allow_show_archived=True)).all()) @@ -1263,7 +1263,7 @@ def extend_search_term(searchterm, def render_adv_search_results(term, offset=None, order=None, limit=None): - sort = order[0] if order else [db.Books.sort] + sort_param = order[0] if order else [db.Books.sort] pagination = None cc = get_cc_columns(filter_config_custom_read=True) @@ -1378,7 +1378,7 @@ def render_adv_search_results(term, offset=None, order=None, limit=None): log.debug_or_exception(ex) flash(_("Error on search for custom columns, please restart Calibre-Web"), category="error") - q = q.order_by(*sort).all() + q = q.order_by(*sort_param).all() flask_session['query'] = json.dumps(term) ub.store_combo_ids(q) result_count = len(q) @@ -1792,7 +1792,7 @@ def show_book(book_id): entry.tags = sort(entry.tags, key=lambda tag: tag.name) - entry.authors = calibre_db.order_authors([entry]) + entry.ordered_authors = calibre_db.order_authors([entry]) entry.kindle_list = check_send_to_kindle(entry) entry.reader_list = check_read_formats(entry) From 4379669cf89b440ecd83d562966d290d9a6ac9b3 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sat, 12 Mar 2022 17:14:54 +0100 Subject: [PATCH 19/24] Database error is more detailed renamed debug_or_exception to error_or_exception --- cps/admin.py | 34 ++++++++++++------------- cps/db.py | 8 +++--- cps/editbooks.py | 23 ++++++++++------- cps/gdrive.py | 2 +- cps/gdriveutils.py | 2 +- cps/helper.py | 4 +-- cps/logger.py | 2 +- cps/metadata_provider/amazon.py | 4 +-- cps/oauth_bb.py | 4 +-- cps/services/worker.py | 2 +- cps/shelf.py | 44 ++++++++++++++++----------------- cps/tasks/mail.py | 10 ++++---- cps/ub.py | 2 +- cps/updater.py | 2 +- cps/web.py | 4 +-- 15 files changed, 76 insertions(+), 71 deletions(-) diff --git a/cps/admin.py b/cps/admin.py index 18bd87ec..60a674e8 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -1216,10 +1216,10 @@ def _db_configuration_update_helper(): # gdrive_error drive setup gdrive_error = _configuration_gdrive_helper(to_save) - except (OperationalError, InvalidRequestError): + except (OperationalError, InvalidRequestError) as e: ub.session.rollback() - log.error("Settings DB is not Writeable") - _db_configuration_result(_("Settings DB is not Writeable"), gdrive_error) + log.error_or_exception("Settings Database error: {}".format(e)) + _db_configuration_result(_(u"Database error: %(error)s.", error=e.orig), gdrive_error) try: metadata_db = os.path.join(to_save['config_calibre_dir'], "metadata.db") if config.config_use_google_drive and is_gdrive_ready() and not os.path.exists(metadata_db): @@ -1332,10 +1332,10 @@ def _configuration_update_helper(): unrar_status = helper.check_unrar(config.config_rarfile_location) if unrar_status: return _configuration_result(unrar_status) - except (OperationalError, InvalidRequestError): + except (OperationalError, InvalidRequestError) as e: ub.session.rollback() - log.error("Settings DB is not Writeable") - _configuration_result(_("Settings DB is not Writeable")) + log.error_or_exception("Settings Database error: {}".format(e)) + _configuration_result(_(u"Database error: %(error)s.", error=e.orig)) config.save() if reboot_required: @@ -1430,10 +1430,10 @@ def _handle_new_user(to_save, content, languages, translations, kobo_support): ub.session.rollback() log.error("Found an existing account for {} or {}".format(content.name, content.email)) flash(_("Found an existing account for this e-mail address or name."), category="error") - except OperationalError: + except OperationalError as e: ub.session.rollback() - log.error("Settings DB is not Writeable") - flash(_("Settings DB is not Writeable"), category="error") + log.error_or_exception("Settings Database error: {}".format(e)) + flash(_(u"Database error: %(error)s.", error=e.orig), category="error") def _delete_user(content): @@ -1547,10 +1547,10 @@ def _handle_edit_user(to_save, content, languages, translations, kobo_support): ub.session.rollback() log.error("An unknown error occurred while changing user: {}".format(str(ex))) flash(_(u"An unknown error occurred. Please try again later."), category="error") - except OperationalError: + except OperationalError as e: ub.session.rollback() - log.error("Settings DB is not Writeable") - flash(_("Settings DB is not Writeable"), category="error") + log.error_or_exception("Settings Database error: {}".format(e)) + flash(_(u"Database error: %(error)s.", error=e.orig), category="error") return "" @@ -1616,10 +1616,10 @@ def update_mailsettings(): _config_int(to_save, "mail_size", lambda y: int(y)*1024*1024) try: config.save() - except (OperationalError, InvalidRequestError): + except (OperationalError, InvalidRequestError) as e: ub.session.rollback() - log.error("Settings DB is not Writeable") - flash(_("Settings DB is not Writeable"), category="error") + log.error_or_exception("Settings Database error: {}".format(e)) + flash(_(u"Database error: %(error)s.", error=e.orig), category="error") return edit_mailsettings() if to_save.get("test"): @@ -1851,7 +1851,7 @@ def import_ldap_users(): try: new_users = services.ldap.get_group_members(config.config_ldap_group_name) except (services.ldap.LDAPException, TypeError, AttributeError, KeyError) as e: - log.debug_or_exception(e) + log.error_or_exception(e) showtext['text'] = _(u'Error: %(ldaperror)s', ldaperror=e) return json.dumps(showtext) if not new_users: @@ -1879,7 +1879,7 @@ def import_ldap_users(): try: user_data = services.ldap.get_object_details(user=user_identifier, query_filter=query_filter) except AttributeError as ex: - log.debug_or_exception(ex) + log.error_or_exception(ex) continue if user_data: user_count, message = ldap_import_create_user(user, user_data) diff --git a/cps/db.py b/cps/db.py index 6ff0d03e..2292614c 100644 --- a/cps/db.py +++ b/cps/db.py @@ -597,7 +597,7 @@ class CalibreDB(): cc = conn.execute(text("SELECT id, datatype FROM custom_columns")) cls.setup_db_cc_classes(cc) except OperationalError as e: - log.debug_or_exception(e) + log.error_or_exception(e) cls.session_factory = scoped_session(sessionmaker(autocommit=False, autoflush=True, @@ -769,7 +769,7 @@ class CalibreDB(): len(query.all())) entries = query.order_by(*order).offset(off).limit(pagesize).all() except Exception as ex: - log.debug_or_exception(ex) + log.error_or_exception(ex) # display authors in right order entries = self.order_authors(entries, True, join_archive_read) return entries, randm, pagination @@ -792,7 +792,7 @@ class CalibreDB(): results = self.session.query(Authors).filter(Authors.sort == auth.lstrip().strip()).all() # ToDo: How to handle not found author name if not len(results): - log.error("Author {} not found to display name in right order".format(auth)) + log.error("Author {} not found to display name in right order".format(auth.strip())) # error = True break for r in results: @@ -974,5 +974,5 @@ def lcase(s): return unidecode.unidecode(s.lower()) except Exception as ex: log = logger.create() - log.debug_or_exception(ex) + log.error_or_exception(ex) return s.lower() diff --git a/cps/editbooks.py b/cps/editbooks.py index 7e251696..3a08f061 100755 --- a/cps/editbooks.py +++ b/cps/editbooks.py @@ -337,7 +337,7 @@ def delete_book_from_table(book_id, book_format, jsonResponse): kobo_sync_status.remove_synced_book(book.id, True) calibre_db.session.commit() except Exception as ex: - log.debug_or_exception(ex) + log.error_or_exception(ex) calibre_db.session.rollback() if jsonResponse: return json.dumps([{"location": url_for("editbook.edit_book", book_id=book_id), @@ -663,8 +663,8 @@ def upload_single_file(request, book, book_id): calibre_db.update_title_sort(config) except (OperationalError, IntegrityError) as e: calibre_db.session.rollback() - log.error('Database error: %s', e) - flash(_(u"Database error: %(error)s.", error=e), category="error") + log.error_or_exception("Database error: {}".format(e)) + flash(_(u"Database error: %(error)s.", error=e.orig), category="error") return redirect(url_for('web.show_book', book_id=book.id)) # Queue uploader info @@ -756,7 +756,7 @@ def edit_book(book_id): try: calibre_db.update_title_sort(config) except sqliteOperationalError as e: - log.debug_or_exception(e) + log.error_or_exception(e) calibre_db.session.rollback() # Show form @@ -864,8 +864,13 @@ def edit_book(book_id): calibre_db.session.rollback() flash(str(e), category="error") return redirect(url_for('web.show_book', book_id=book.id)) + except (OperationalError, IntegrityError) as e: + log.error_or_exception("Database error: {}".format(e)) + calibre_db.session.rollback() + flash(_(u"Database error: %(error)s.", error=e.orig), category="error") + return redirect(url_for('web.show_book', book_id=book.id)) except Exception as ex: - log.debug_or_exception(ex) + log.error_or_exception(ex) calibre_db.session.rollback() flash(_("Error editing book, please check logfile for details"), category="error") return redirect(url_for('web.show_book', book_id=book.id)) @@ -1103,8 +1108,8 @@ def upload(): return Response(json.dumps(resp), mimetype='application/json') except (OperationalError, IntegrityError) as e: calibre_db.session.rollback() - log.error("Database error: %s", e) - flash(_(u"Database error: %(error)s.", error=e), category="error") + log.error_or_exception("Database error: {}".format(e)) + flash(_(u"Database error: %(error)s.", error=e.orig), category="error") return Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json') @@ -1234,7 +1239,7 @@ def edit_list_book(param): calibre_db.session.commit() except (OperationalError, IntegrityError) as e: calibre_db.session.rollback() - log.error("Database error: {}".format(e)) + log.error_or_exception("Database error: {}".format(e)) ret = Response(json.dumps({'success': False, 'msg': 'Database error: {}'.format(e.orig)}), mimetype='application/json') @@ -1344,7 +1349,7 @@ def table_xchange_author_title(): calibre_db.session.commit() except (OperationalError, IntegrityError) as e: calibre_db.session.rollback() - log.error("Database error: %s", e) + log.error_or_exception("Database error: %s", e) return json.dumps({'success': False}) if config.config_use_google_drive: diff --git a/cps/gdrive.py b/cps/gdrive.py index e782cb9e..60e3d47b 100644 --- a/cps/gdrive.py +++ b/cps/gdrive.py @@ -152,7 +152,7 @@ try: move(os.path.join(tmp_dir, "tmp_metadata.db"), dbpath) calibre_db.reconnect_db(config, ub.app_DB_path) except Exception as ex: - log.debug_or_exception(ex) + log.error_or_exception(ex) return '' except AttributeError: pass diff --git a/cps/gdriveutils.py b/cps/gdriveutils.py index 57f272ff..0b704db4 100644 --- a/cps/gdriveutils.py +++ b/cps/gdriveutils.py @@ -215,7 +215,7 @@ def getDrive(drive=None, gauth=None): except RefreshError as e: log.error("Google Drive error: %s", e) except Exception as ex: - log.debug_or_exception(ex) + log.error_or_exception(ex) else: # Initialize the saved creds gauth.Authorize() diff --git a/cps/helper.py b/cps/helper.py index 03a2d03b..b0b28a99 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -713,7 +713,7 @@ def get_book_cover_internal(book, use_generic_cover_on_failure): log.error('%s/cover.jpg not found on Google Drive', book.path) return get_cover_on_failure(use_generic_cover_on_failure) except Exception as ex: - log.debug_or_exception(ex) + log.error_or_exception(ex) return get_cover_on_failure(use_generic_cover_on_failure) else: cover_file_path = os.path.join(config.config_calibre_dir, book.path) @@ -861,7 +861,7 @@ def check_unrar(unrarLocation): log.debug("unrar version %s", version) except (OSError, UnicodeDecodeError) as err: - log.debug_or_exception(err) + log.error_or_exception(err) return _('Error excecuting UnRar') diff --git a/cps/logger.py b/cps/logger.py index 053d0bd3..fcc25c27 100644 --- a/cps/logger.py +++ b/cps/logger.py @@ -42,7 +42,7 @@ logging.addLevelName(logging.CRITICAL, "CRIT") class _Logger(logging.Logger): - def debug_or_exception(self, message, stacklevel=2, *args, **kwargs): + def error_or_exception(self, message, stacklevel=2, *args, **kwargs): if sys.version_info > (3, 7): if is_debug_enabled(): self.exception(message, stacklevel=stacklevel, *args, **kwargs) diff --git a/cps/metadata_provider/amazon.py b/cps/metadata_provider/amazon.py index acea7e07..73c7e1fc 100644 --- a/cps/metadata_provider/amazon.py +++ b/cps/metadata_provider/amazon.py @@ -105,7 +105,7 @@ class Amazon(Metadata): match.cover = "" return match, index except Exception as e: - log.debug_or_exception(e) + log.error_or_exception(e) return val = list() @@ -126,5 +126,5 @@ class Amazon(Metadata): result = list(filter(lambda x: x, val)) return [x[0] for x in sorted(result, key=itemgetter(1))] #sort by amazons listing order for best relevance except requests.exceptions.HTTPError as e: - log.debug_or_exception(e) + log.error_or_exception(e) return [] diff --git a/cps/oauth_bb.py b/cps/oauth_bb.py index d9efd41e..d9a60c0e 100644 --- a/cps/oauth_bb.py +++ b/cps/oauth_bb.py @@ -149,7 +149,7 @@ def bind_oauth_or_register(provider_id, provider_user_id, redirect_url, provider log.info("Link to {} Succeeded".format(provider_name)) return redirect(url_for('web.profile')) except Exception as ex: - log.debug_or_exception(ex) + log.error_or_exception(ex) ub.session.rollback() else: flash(_(u"Login failed, No User Linked With OAuth Account"), category="error") @@ -197,7 +197,7 @@ def unlink_oauth(provider): flash(_(u"Unlink to %(oauth)s Succeeded", oauth=oauth_check[provider]), category="success") log.info("Unlink to {} Succeeded".format(oauth_check[provider])) except Exception as ex: - log.debug_or_exception(ex) + log.error_or_exception(ex) ub.session.rollback() flash(_(u"Unlink to %(oauth)s Failed", oauth=oauth_check[provider]), category="error") except NoResultFound: diff --git a/cps/services/worker.py b/cps/services/worker.py index 076c9104..aea6d640 100644 --- a/cps/services/worker.py +++ b/cps/services/worker.py @@ -178,7 +178,7 @@ class CalibreTask: self.run(*args) except Exception as ex: self._handleError(str(ex)) - log.debug_or_exception(ex) + log.error_or_exception(ex) self.end_time = datetime.now() diff --git a/cps/shelf.py b/cps/shelf.py index 27939cdc..0bf12164 100644 --- a/cps/shelf.py +++ b/cps/shelf.py @@ -94,10 +94,10 @@ def add_to_shelf(shelf_id, book_id): try: ub.session.merge(shelf) ub.session.commit() - except (OperationalError, InvalidRequestError): + except (OperationalError, InvalidRequestError) as e: ub.session.rollback() - log.error("Settings DB is not Writeable") - flash(_(u"Settings DB is not Writeable"), category="error") + log.error_or_exception("Settings Database error: {}".format(e)) + flash(_(u"Database error: %(error)s.", error=e.orig), category="error") if "HTTP_REFERER" in request.environ: return redirect(request.environ["HTTP_REFERER"]) else: @@ -154,10 +154,10 @@ def search_to_shelf(shelf_id): ub.session.merge(shelf) ub.session.commit() flash(_(u"Books have been added to shelf: %(sname)s", sname=shelf.name), category="success") - except (OperationalError, InvalidRequestError): + except (OperationalError, InvalidRequestError) as e: ub.session.rollback() - log.error("Settings DB is not Writeable") - flash(_("Settings DB is not Writeable"), category="error") + log.error_or_exception("Settings Database error: {}".format(e)) + flash(_(u"Database error: %(error)s.", error=e.orig), category="error") else: log.error("Could not add books to shelf: {}".format(shelf.name)) flash(_(u"Could not add books to shelf: %(sname)s", sname=shelf.name), category="error") @@ -197,10 +197,10 @@ def remove_from_shelf(shelf_id, book_id): ub.session.delete(book_shelf) shelf.last_modified = datetime.utcnow() ub.session.commit() - except (OperationalError, InvalidRequestError): + except (OperationalError, InvalidRequestError) as e: ub.session.rollback() - log.error("Settings DB is not Writeable") - flash(_("Settings DB is not Writeable"), category="error") + log.error_or_exception("Settings Database error: {}".format(e)) + flash(_(u"Database error: %(error)s.", error=e.orig), category="error") if "HTTP_REFERER" in request.environ: return redirect(request.environ["HTTP_REFERER"]) else: @@ -273,12 +273,12 @@ def create_edit_shelf(shelf, page_title, page, shelf_id=False): return redirect(url_for('shelf.show_shelf', shelf_id=shelf.id)) except (OperationalError, InvalidRequestError) as ex: ub.session.rollback() - log.debug_or_exception(ex) - log.error("Settings DB is not Writeable") - flash(_("Settings DB is not Writeable"), category="error") + log.error_or_exception(ex) + log.error_or_exception("Settings Database error: {}".format(ex)) + flash(_(u"Database error: %(error)s.", error=ex.orig), category="error") except Exception as ex: ub.session.rollback() - log.debug_or_exception(ex) + log.error_or_exception(ex) flash(_(u"There was an error"), category="error") return render_title_template('shelf_edit.html', shelf=shelf, @@ -337,10 +337,10 @@ def delete_shelf(shelf_id): flash(_("Error deleting Shelf"), category="error") else: flash(_("Shelf successfully deleted"), category="success") - except InvalidRequestError: + except InvalidRequestError as e: ub.session.rollback() - log.error("Settings DB is not Writeable") - flash(_("Settings DB is not Writeable"), category="error") + log.error_or_exception("Settings Database error: {}".format(e)) + flash(_(u"Database error: %(error)s.", error=e.orig), category="error") return redirect(url_for('web.index')) @@ -374,10 +374,10 @@ def order_shelf(shelf_id): # if order diffrent from before -> shelf.last_modified = datetime.utcnow() try: ub.session.commit() - except (OperationalError, InvalidRequestError): + except (OperationalError, InvalidRequestError) as e: ub.session.rollback() - log.error("Settings DB is not Writeable") - flash(_("Settings DB is not Writeable"), category="error") + log.error_or_exception("Settings Database error: {}".format(e)) + flash(_(u"Database error: %(error)s.", error=e.orig), category="error") result = list() if shelf: @@ -450,10 +450,10 @@ def render_show_shelf(shelf_type, shelf_id, page_no, sort_param): try: ub.session.query(ub.BookShelf).filter(ub.BookShelf.book_id == entry.book_id).delete() ub.session.commit() - except (OperationalError, InvalidRequestError): + except (OperationalError, InvalidRequestError) as e: ub.session.rollback() - log.error("Settings DB is not Writeable") - flash(_("Settings DB is not Writeable"), category="error") + log.error_or_exception("Settings Database error: {}".format(e)) + flash(_(u"Database error: %(error)s.", error=e.orig), category="error") return render_title_template(page, entries=result, diff --git a/cps/tasks/mail.py b/cps/tasks/mail.py index 80adbbcc..4954c2d6 100755 --- a/cps/tasks/mail.py +++ b/cps/tasks/mail.py @@ -167,10 +167,10 @@ class TaskEmail(CalibreTask): else: self.send_gmail_email(msg) except MemoryError as e: - log.debug_or_exception(e, stacklevel=3) + log.error_or_exception(e, stacklevel=3) self._handleError(u'MemoryError sending e-mail: {}'.format(str(e))) except (smtplib.SMTPException, smtplib.SMTPAuthenticationError) as e: - log.debug_or_exception(e, stacklevel=3) + log.error_or_exception(e, stacklevel=3) if hasattr(e, "smtp_error"): text = e.smtp_error.decode('utf-8').replace("\n", '. ') elif hasattr(e, "message"): @@ -181,10 +181,10 @@ class TaskEmail(CalibreTask): text = '' self._handleError(u'Smtplib Error sending e-mail: {}'.format(text)) except (socket.error) as e: - log.debug_or_exception(e, stacklevel=3) + log.error_or_exception(e, stacklevel=3) self._handleError(u'Socket Error sending e-mail: {}'.format(e.strerror)) except Exception as ex: - log.debug_or_exception(ex, stacklevel=3) + log.error_or_exception(ex, stacklevel=3) self._handleError(u'Error sending e-mail: {}'.format(ex)) def send_standard_email(self, msg): @@ -257,7 +257,7 @@ class TaskEmail(CalibreTask): data = file_.read() file_.close() except IOError as e: - log.debug_or_exception(e, stacklevel=3) + log.error_or_exception(e, stacklevel=3) log.error(u'The requested file could not be read. Maybe wrong permissions?') return None return data diff --git a/cps/ub.py b/cps/ub.py index 786dc909..a2be004c 100644 --- a/cps/ub.py +++ b/cps/ub.py @@ -853,5 +853,5 @@ def session_commit(success=None, _session=None): log.info(success) except (exc.OperationalError, exc.InvalidRequestError) as e: s.rollback() - log.debug_or_exception(e) + log.error_or_exception(e) return "" diff --git a/cps/updater.py b/cps/updater.py index 181c4374..f7341002 100644 --- a/cps/updater.py +++ b/cps/updater.py @@ -117,7 +117,7 @@ class Updater(threading.Thread): except (IOError, OSError) as ex: self.status = 12 log.error(u'Possible Reason for error: update file could not be saved in temp dir') - log.debug_or_exception(ex) + log.error_or_exception(ex) self.pause() return False diff --git a/cps/web.py b/cps/web.py index 94f511b1..e3ad4719 100644 --- a/cps/web.py +++ b/cps/web.py @@ -1375,7 +1375,7 @@ def render_adv_search_results(term, offset=None, order=None, limit=None): try: q = adv_search_custom_columns(cc, term, q) except AttributeError as ex: - log.debug_or_exception(ex) + log.error_or_exception(ex) flash(_("Error on search for custom columns, please restart Calibre-Web"), category="error") q = q.order_by(*sort_param).all() @@ -1437,7 +1437,7 @@ def serve_book(book_id, book_format, anyname): df = getFileFromEbooksFolder(book.path, data.name + "." + book_format) return do_gdrive_download(df, headers, (book_format.upper() == 'TXT')) except AttributeError as ex: - log.debug_or_exception(ex) + log.error_or_exception(ex) return "File Not Found" else: if book_format.upper() == 'TXT': From 8e2536c53b8978ec05c9cb84615a179c38aa5080 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sat, 12 Mar 2022 18:01:11 +0100 Subject: [PATCH 20/24] Improved cover extraction for epub files --- cps/comic.py | 78 ++++++++++++++++++---------------------------------- cps/cover.py | 48 ++++++++++++++++++++++++++++++++ cps/epub.py | 35 +++++++++++++---------- 3 files changed, 95 insertions(+), 66 deletions(-) create mode 100644 cps/cover.py diff --git a/cps/comic.py b/cps/comic.py index 9e7f4f8f..2549579e 100644 --- a/cps/comic.py +++ b/cps/comic.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) -# Copyright (C) 2018 OzzieIsaacs +# Copyright (C) 2018-2022 OzzieIsaacs # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -18,19 +18,16 @@ import os -from . import logger, isoLanguages +from . import logger, isoLanguages, cover from .constants import BookMeta - -log = logger.create() - - try: from wand.image import Image use_IM = True except (ImportError, RuntimeError) as e: use_IM = False +log = logger.create() try: from comicapi.comicarchive import ComicArchive, MetaDataStyle @@ -51,29 +48,8 @@ except (ImportError, LookupError) as e: use_rarfile = False use_comic_meta = False -NO_JPEG_EXTENSIONS = ['.png', '.webp', '.bmp'] -COVER_EXTENSIONS = ['.png', '.webp', '.bmp', '.jpg', '.jpeg'] - -def _cover_processing(tmp_file_name, img, extension): - tmp_cover_name = os.path.join(os.path.dirname(tmp_file_name), 'cover.jpg') - if extension in NO_JPEG_EXTENSIONS: - if use_IM: - with Image(blob=img) as imgc: - imgc.format = 'jpeg' - imgc.transform_colorspace('rgb') - imgc.save(filename=tmp_cover_name) - return tmp_cover_name - else: - return None - if img: - with open(tmp_cover_name, 'wb') as f: - f.write(img) - return tmp_cover_name - else: - return None - -def _extract_Cover_from_archive(original_file_extension, tmp_file_name, rarExecutable): +def _extract_cover_from_archive(original_file_extension, tmp_file_name, rar_executable): cover_data = extension = None if original_file_extension.upper() == '.CBZ': cf = zipfile.ZipFile(tmp_file_name) @@ -81,7 +57,7 @@ def _extract_Cover_from_archive(original_file_extension, tmp_file_name, rarExecu ext = os.path.splitext(name) if len(ext) > 1: extension = ext[1].lower() - if extension in COVER_EXTENSIONS: + if extension in cover.COVER_EXTENSIONS: cover_data = cf.read(name) break elif original_file_extension.upper() == '.CBT': @@ -90,44 +66,44 @@ def _extract_Cover_from_archive(original_file_extension, tmp_file_name, rarExecu ext = os.path.splitext(name) if len(ext) > 1: extension = ext[1].lower() - if extension in COVER_EXTENSIONS: + if extension in cover.COVER_EXTENSIONS: cover_data = cf.extractfile(name).read() break elif original_file_extension.upper() == '.CBR' and use_rarfile: try: - rarfile.UNRAR_TOOL = rarExecutable + rarfile.UNRAR_TOOL = rar_executable cf = rarfile.RarFile(tmp_file_name) for name in cf.namelist(): ext = os.path.splitext(name) if len(ext) > 1: extension = ext[1].lower() - if extension in COVER_EXTENSIONS: + if extension in cover.COVER_EXTENSIONS: cover_data = cf.read(name) break except Exception as ex: - log.debug('Rarfile failed with error: %s', ex) + log.debug('Rarfile failed with error: {}'.format(ex)) return cover_data, extension -def _extractCover(tmp_file_name, original_file_extension, rarExecutable): +def _extract_cover(tmp_file_name, original_file_extension, rar_executable): cover_data = extension = None if use_comic_meta: - archive = ComicArchive(tmp_file_name, rar_exe_path=rarExecutable) + archive = ComicArchive(tmp_file_name, rar_exe_path=rar_executable) for index, name in enumerate(archive.getPageNameList()): ext = os.path.splitext(name) if len(ext) > 1: extension = ext[1].lower() - if extension in COVER_EXTENSIONS: + if extension in cover.COVER_EXTENSIONS: cover_data = archive.getPage(index) break else: - cover_data, extension = _extract_Cover_from_archive(original_file_extension, tmp_file_name, rarExecutable) - return _cover_processing(tmp_file_name, cover_data, extension) + cover_data, extension = _extract_cover_from_archive(original_file_extension, tmp_file_name, rar_executable) + return cover.cover_processing(tmp_file_name, cover_data, extension) -def get_comic_info(tmp_file_path, original_file_name, original_file_extension, rarExecutable): +def get_comic_info(tmp_file_path, original_file_name, original_file_extension, rar_executable): if use_comic_meta: - archive = ComicArchive(tmp_file_path, rar_exe_path=rarExecutable) + archive = ComicArchive(tmp_file_path, rar_exe_path=rar_executable) if archive.seemsToBeAComicArchive(): if archive.hasMetadata(MetaDataStyle.CIX): style = MetaDataStyle.CIX @@ -137,23 +113,23 @@ def get_comic_info(tmp_file_path, original_file_name, original_file_extension, r style = None # if style is not None: - loadedMetadata = archive.readMetadata(style) + loaded_metadata = archive.readMetadata(style) - lang = loadedMetadata.language or "" - loadedMetadata.language = isoLanguages.get_lang3(lang) + lang = loaded_metadata.language or "" + loaded_metadata.language = isoLanguages.get_lang3(lang) return BookMeta( file_path=tmp_file_path, extension=original_file_extension, - title=loadedMetadata.title or original_file_name, + title=loaded_metadata.title or original_file_name, author=" & ".join([credit["person"] - for credit in loadedMetadata.credits if credit["role"] == "Writer"]) or u'Unknown', - cover=_extractCover(tmp_file_path, original_file_extension, rarExecutable), - description=loadedMetadata.comments or "", + for credit in loaded_metadata.credits if credit["role"] == "Writer"]) or 'Unknown', + cover=_extract_cover(tmp_file_path, original_file_extension, rar_executable), + description=loaded_metadata.comments or "", tags="", - series=loadedMetadata.series or "", - series_id=loadedMetadata.issue or "", - languages=loadedMetadata.language, + series=loaded_metadata.series or "", + series_id=loaded_metadata.issue or "", + languages=loaded_metadata.language, publisher="") return BookMeta( @@ -161,7 +137,7 @@ def get_comic_info(tmp_file_path, original_file_name, original_file_extension, r extension=original_file_extension, title=original_file_name, author=u'Unknown', - cover=_extractCover(tmp_file_path, original_file_extension, rarExecutable), + cover=_extract_cover(tmp_file_path, original_file_extension, rar_executable), description="", tags="", series="", diff --git a/cps/cover.py b/cps/cover.py new file mode 100644 index 00000000..5dd29534 --- /dev/null +++ b/cps/cover.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2022 OzzieIsaacs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os + +try: + from wand.image import Image + use_IM = True +except (ImportError, RuntimeError) as e: + use_IM = False + + +NO_JPEG_EXTENSIONS = ['.png', '.webp', '.bmp'] +COVER_EXTENSIONS = ['.png', '.webp', '.bmp', '.jpg', '.jpeg'] + + +def cover_processing(tmp_file_name, img, extension): + tmp_cover_name = os.path.join(os.path.dirname(tmp_file_name), 'cover.jpg') + if extension in NO_JPEG_EXTENSIONS: + if use_IM: + with Image(blob=img) as imgc: + imgc.format = 'jpeg' + imgc.transform_colorspace('rgb') + imgc.save(filename=tmp_cover_name) + return tmp_cover_name + else: + return None + if img: + with open(tmp_cover_name, 'wb') as f: + f.write(img) + return tmp_cover_name + else: + return None diff --git a/cps/epub.py b/cps/epub.py index b436a755..563590e8 100644 --- a/cps/epub.py +++ b/cps/epub.py @@ -20,23 +20,26 @@ import os import zipfile from lxml import etree -from . import isoLanguages +from . import isoLanguages, cover from .helper import split_authors from .constants import BookMeta -def extract_cover(zip_file, cover_file, cover_path, tmp_file_name): +def _extract_cover(zip_file, cover_file, cover_path, tmp_file_name): if cover_file is None: return None else: + cf = extension = None zip_cover_path = os.path.join(cover_path, cover_file).replace('\\', '/') - cf = zip_file.read(zip_cover_path) + prefix = os.path.splitext(tmp_file_name)[0] tmp_cover_name = prefix + '.' + os.path.basename(zip_cover_path) - image = open(tmp_cover_name, 'wb') - image.write(cf) - image.close() - return tmp_cover_name + ext = os.path.splitext(tmp_cover_name) + if len(ext) > 1: + extension = ext[1].lower() + if extension in cover.COVER_EXTENSIONS: + cf = zip_file.read(zip_cover_path) + return cover.cover_processing(tmp_file_name, cf, extension) def get_epub_info(tmp_file_path, original_file_name, original_file_extension): @@ -70,9 +73,9 @@ def get_epub_info(tmp_file_path, original_file_name, original_file_extension): else: epub_metadata[s] = tmp[0] else: - epub_metadata[s] = u'Unknown' + epub_metadata[s] = 'Unknown' - if epub_metadata['subject'] == u'Unknown': + if epub_metadata['subject'] == 'Unknown': epub_metadata['subject'] = '' if epub_metadata['description'] == u'Unknown': @@ -112,7 +115,7 @@ def parse_epub_cover(ns, tree, epub_zip, cover_path, tmp_file_path): cover_section = tree.xpath("/pkg:package/pkg:manifest/pkg:item[@id='cover-image']/@href", namespaces=ns) cover_file = None if len(cover_section) > 0: - cover_file = extract_cover(epub_zip, cover_section[0], cover_path, tmp_file_path) + cover_file = _extract_cover(epub_zip, cover_section[0], cover_path, tmp_file_path) else: meta_cover = tree.xpath("/pkg:package/pkg:metadata/pkg:meta[@name='cover']/@content", namespaces=ns) if len(meta_cover) > 0: @@ -123,10 +126,10 @@ def parse_epub_cover(ns, tree, epub_zip, cover_path, tmp_file_path): "/pkg:package/pkg:manifest/pkg:item[@properties='" + meta_cover[0] + "']/@href", namespaces=ns) else: cover_section = tree.xpath("/pkg:package/pkg:guide/pkg:reference/@href", namespaces=ns) - if len(cover_section) > 0: - filetype = cover_section[0].rsplit('.', 1)[-1] + for cs in cover_section: + filetype = cs.rsplit('.', 1)[-1] if filetype == "xhtml" or filetype == "html": # if cover is (x)html format - markup = epub_zip.read(os.path.join(cover_path, cover_section[0])) + markup = epub_zip.read(os.path.join(cover_path, cs)) markup_tree = etree.fromstring(markup) # no matter xhtml or html with no namespace img_src = markup_tree.xpath("//*[local-name() = 'img']/@src") @@ -137,9 +140,11 @@ def parse_epub_cover(ns, tree, epub_zip, cover_path, tmp_file_path): # img_src maybe start with "../"" so fullpath join then relpath to cwd filename = os.path.relpath(os.path.join(os.path.dirname(os.path.join(cover_path, cover_section[0])), img_src[0])) - cover_file = extract_cover(epub_zip, filename, "", tmp_file_path) + cover_file = _extract_cover(epub_zip, filename, "", tmp_file_path) else: - cover_file = extract_cover(epub_zip, cover_section[0], cover_path, tmp_file_path) + cover_file = _extract_cover(epub_zip, cs, cover_path, tmp_file_path) + if cover_file: + break return cover_file From 3b5e5f9b9099ae6e07834ddc0d7539a1a1c07d77 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sat, 12 Mar 2022 22:15:19 +0100 Subject: [PATCH 21/24] Undo check of read checkbox in case of error Display error message in details modal dialog Bugfix set archive bit in booktable Translate error message readstatus change --- cps/editbooks.py | 7 +++++-- cps/helper.py | 2 +- cps/static/js/details.js | 22 ++++++++++++++++------ cps/static/js/main.js | 1 + cps/static/js/table.js | 2 ++ 5 files changed, 25 insertions(+), 9 deletions(-) diff --git a/cps/editbooks.py b/cps/editbooks.py index 3a08f061..30535ef3 100755 --- a/cps/editbooks.py +++ b/cps/editbooks.py @@ -1212,8 +1212,11 @@ def edit_list_book(param): 'newValue': ' & '.join([author.replace('|',',') for author in input_authors])}), mimetype='application/json') elif param == 'is_archived': - change_archived_books(book.id, vals['value'] == "True") - ret = "" + is_archived = change_archived_books(book.id, vals['value'] == "True", + message="Book {} archivebit set to: {}".format(book.id, vals['value'])) + if is_archived: + kobo_sync_status.remove_synced_book(book.id) + return "" elif param == 'read_status': ret = helper.edit_book_read_status(book.id, vals['value'] == "True") if ret: diff --git a/cps/helper.py b/cps/helper.py index b0b28a99..74a8dbbb 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -335,7 +335,7 @@ def edit_book_read_status(book_id, read_status=None): except (OperationalError, InvalidRequestError) as e: calibre_db.session.rollback() log.error(u"Read status could not set: {}".format(e)) - return "Read status could not set: {}".format(e), 400 + return _("Read status could not set: {}".format(e.orig)) return "" # Deletes a book fro the local filestorage, returns True if deleting is successfull, otherwise false diff --git a/cps/static/js/details.js b/cps/static/js/details.js index 6f99595d..f0259f8c 100644 --- a/cps/static/js/details.js +++ b/cps/static/js/details.js @@ -28,14 +28,24 @@ $("#have_read_cb").on("change", function() { data: $(this).closest("form").serialize(), error: function(response) { var data = [{type:"danger", message:response.responseText}] - $("#flash_success").remove(); + // $("#flash_success").parent().remove(); $("#flash_danger").remove(); + $(".row-fluid.text-center").remove(); if (!jQuery.isEmptyObject(data)) { - data.forEach(function (item) { - $(".navbar").after('

' + - '
' + item.message + '
' + - '
'); - }); + $("#have_read_cb").prop("checked", !$("#have_read_cb").prop("checked")); + if($("#bookDetailsModal").is(":visible")) { + data.forEach(function (item) { + $(".modal-header").after('
' + item.message + '
'); + }); + } else + { + data.forEach(function (item) { + $(".navbar").after('
' + + '
' + item.message + '
' + + '
'); + }); + } } } }); diff --git a/cps/static/js/main.js b/cps/static/js/main.js index 75599d9b..b18cf229 100755 --- a/cps/static/js/main.js +++ b/cps/static/js/main.js @@ -515,6 +515,7 @@ $(function() { $("#bookDetailsModal") .on("show.bs.modal", function(e) { + $("#flash_danger").remove(); var $modalBody = $(this).find(".modal-body"); // Prevent static assets from loading multiple times diff --git a/cps/static/js/table.js b/cps/static/js/table.js index ae8c591b..8af7592f 100644 --- a/cps/static/js/table.js +++ b/cps/static/js/table.js @@ -812,11 +812,13 @@ function checkboxChange(checkbox, userId, field, field_index) { function BookCheckboxChange(checkbox, userId, field) { var value = checkbox.checked ? "True" : "False"; + var element = checkbox; $.ajax({ method: "post", url: getPath() + "/ajax/editbooks/" + field, data: {"pk": userId, "value": value}, error: function(data) { + element.checked = !element.checked; handleListServerResponse([{type:"danger", message:data.responseText}]) }, success: handleListServerResponse From 296f76b5fb96c6a0747af49fba81d01514ab2092 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sun, 13 Mar 2022 10:23:13 +0100 Subject: [PATCH 22/24] Fixes after testrun Code cosmetics --- cps/admin.py | 2 +- cps/templates/listenmp3.html | 2 +- cps/templates/read.html | 2 +- cps/web.py | 17 +- test/Calibre-Web TestSummary_Linux.html | 499 ++++++++++++++---------- 5 files changed, 295 insertions(+), 227 deletions(-) diff --git a/cps/admin.py b/cps/admin.py index 60a674e8..3d4ca609 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -499,7 +499,7 @@ def edit_list_user(param): else: return _("Parameter not found"), 400 except Exception as ex: - log.debug_or_exception(ex) + log.error_or_exception(ex) return str(ex), 400 ub.session_commit() return "" diff --git a/cps/templates/listenmp3.html b/cps/templates/listenmp3.html index 7da62a20..2067bf38 100644 --- a/cps/templates/listenmp3.html +++ b/cps/templates/listenmp3.html @@ -134,7 +134,7 @@ window.calibre = { filePath: "{{ url_for('static', filename='js/libs/') }}", cssPath: "{{ url_for('static', filename='css/') }}", bookUrl: "{{ url_for('static', filename=mp3file) }}/", - bookmarkUrl: "{{ url_for('web.bookmark', book_id=mp3file, book_format=audioformat.upper()) }}", + bookmarkUrl: "{{ url_for('web.set_bookmark', book_id=mp3file, book_format=audioformat.upper()) }}", bookmark: "{{ bookmark.bookmark_key if bookmark != None }}", useBookmarks: "{{ g.user.is_authenticated | tojson }}" }; diff --git a/cps/templates/read.html b/cps/templates/read.html index 1766eb1b..f69d662f 100644 --- a/cps/templates/read.html +++ b/cps/templates/read.html @@ -86,7 +86,7 @@ window.calibre = { filePath: "{{ url_for('static', filename='js/libs/') }}", cssPath: "{{ url_for('static', filename='css/') }}", - bookmarkUrl: "{{ url_for('web.bookmark', book_id=bookid, book_format='EPUB') }}", + bookmarkUrl: "{{ url_for('web.set_bookmark', book_id=bookid, book_format='EPUB') }}", bookUrl: "{{ url_for('web.serve_book', book_id=bookid, book_format='epub', anyname='file.epub') }}", bookmark: "{{ bookmark.bookmark_key if bookmark != None }}", useBookmarks: "{{ g.user.is_authenticated | tojson }}" diff --git a/cps/web.py b/cps/web.py index e3ad4719..8c057b9d 100644 --- a/cps/web.py +++ b/cps/web.py @@ -26,6 +26,7 @@ import json import mimetypes import chardet # dependency of requests import copy +from functools import wraps from babel.dates import format_date from babel import Locale as LC @@ -59,6 +60,7 @@ from .kobo_sync_status import remove_synced_book from .render_template import render_title_template from .kobo_sync_status import change_archived_books + feature_support = { 'ldap': bool(services.ldap), 'goodreads': bool(services.goodreads_support), @@ -72,11 +74,6 @@ except ImportError: feature_support['oauth'] = False oauth_check = {} -try: - from functools import wraps -except ImportError: - pass # We're not using Python 3 - try: from natsort import natsorted as sort except ImportError: @@ -134,7 +131,7 @@ def get_email_status_json(): @web.route("/ajax/bookmark//", methods=['POST']) @login_required -def bookmark(book_id, book_format): +def set_bookmark(book_id, book_format): bookmark_key = request.form["bookmark"] ub.session.query(ub.Bookmark).filter(and_(ub.Bookmark.user_id == int(current_user.id), ub.Bookmark.book_id == book_id, @@ -642,7 +639,8 @@ def render_read_books(page, are_read, as_xml=False, order=None): column=config.config_read_column), category="error") return redirect(url_for("web.index")) - # ToDo: Handle error Case for opds + return [] # ToDo: Handle error Case for opds + if as_xml: return entries, pagination else: @@ -809,6 +807,7 @@ def list_books(): and_(ub.ReadBook.user_id == int(current_user.id), ub.ReadBook.book_id == db.Books.id))) else: + read_column = "" try: read_column = db.cc_classes[config.config_read_column] books = (calibre_db.session.query(db.Books, read_column.value, ub.ArchivedBook.is_archived) @@ -1725,12 +1724,14 @@ def profile(): @viewer_required def read_book(book_id, book_format): book = calibre_db.get_filtered_book(book_id) + book.ordered_authors = calibre_db.order_authors([book], False) + if not book: flash(_(u"Oops! Selected book title is unavailable. File does not exist or is not accessible"), category="error") log.debug(u"Oops! Selected book title is unavailable. File does not exist or is not accessible") return redirect(url_for("web.index")) - # check if book has bookmark + # check if book has a bookmark bookmark = None if current_user.is_authenticated: bookmark = ub.session.query(ub.Bookmark).filter(and_(ub.Bookmark.user_id == int(current_user.id), diff --git a/test/Calibre-Web TestSummary_Linux.html b/test/Calibre-Web TestSummary_Linux.html index 5daf8267..580cf3c9 100644 --- a/test/Calibre-Web TestSummary_Linux.html +++ b/test/Calibre-Web TestSummary_Linux.html @@ -37,20 +37,20 @@
-

Start Time: 2022-03-02 20:56:18

+

Start Time: 2022-03-12 22:57:13

-

Stop Time: 2022-03-03 01:48:44

+

Stop Time: 2022-03-13 03:46:17

-

Duration: 4h 5 min

+

Duration: 4h 0 min

@@ -725,13 +725,13 @@ TestEditAdditionalBooks - 19 17 + 16 0 0 - 2 + 1 - Detail + Detail @@ -863,42 +863,7 @@ - - -
TestEditAdditionalBooks - test_writeonly_calibre_database
- - -
- SKIP -
- - - - - - - - - - -
TestEditAdditionalBooks - test_writeonly_path
- - PASS - - - - - +
TestEditAdditionalBooks - test_xss_author_edit
@@ -907,7 +872,7 @@ - +
TestEditAdditionalBooks - test_xss_comment_edit
@@ -916,7 +881,7 @@ - +
TestEditAdditionalBooks - test_xss_custom_comment_edit
@@ -1867,7 +1832,7 @@ AssertionError: 0.0 not greater than 0.02 
Traceback (most recent call last):
   File "/home/ozzie/Development/calibre-web-test/test/test_edit_ebooks_gdrive.py", line 947, in test_watch_metadata
     self.assertNotIn('series', book)
-AssertionError: 'series' unexpectedly found in {'id': 5, 'reader': [], 'title': 'testbook', 'author': ['John Döe'], 'rating': 0, 'languages': ['English'], 'identifier': [], 'cover': '/cover/5?edit=22ded0fa-26b4-429d-81fc-bc75707c4e4c', 'tag': [], 'publisher': ['Randomhäus'], 'pubdate': 'Jan 19, 2017', 'comment': 'Lorem ipsum dolor sit amet, consectetuer adipiscing elit.Aenean commodo ligula eget dolor.Aenean massa.Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus.Donec quam felis, ultricies nec, pellentesque eu, pretium quis, sem.Nulla consequat massa quis enim.Donec pede justo, fringilla vel, aliquet nec, vulputate', 'add_shelf': [], 'del_shelf': [], 'edit_enable': True, 'kindle': None, 'kindlebtn': None, 'download': ['EPUB (6.7 kB)'], 'read': False, 'archived': False, 'series_all': 'Book 1 of test', 'series_index': '1', 'series': 'test', 'cust_columns': []}
+AssertionError: 'series' unexpectedly found in {'id': 5, 'reader': [], 'title': 'testbook', 'author': ['John Döe'], 'rating': 0, 'languages': ['English'], 'identifier': [], 'cover': '/cover/5?edit=4a23d5c4-e97a-42e1-bd43-351fb1de43df', 'tag': [], 'publisher': ['Randomhäus'], 'pubdate': 'Jan 19, 2017', 'comment': 'Lorem ipsum dolor sit amet, consectetuer adipiscing elit.Aenean commodo ligula eget dolor.Aenean massa.Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus.Donec quam felis, ultricies nec, pellentesque eu, pretium quis, sem.Nulla consequat massa quis enim.Donec pede justo, fringilla vel, aliquet nec, vulputate', 'add_shelf': [], 'del_shelf': [], 'edit_enable': True, 'kindle': None, 'kindlebtn': None, 'download': ['EPUB (6.7 kB)'], 'read': False, 'archived': False, 'series_all': 'Book 1 of test', 'series_index': '1', 'series': 'test', 'cust_columns': []}
@@ -3172,11 +3137,11 @@ AssertionError: 'series' unexpectedly found in {'id': 5, 're - + TestReader 5 - 5 - 0 + 4 + 1 0 0 @@ -3213,11 +3178,35 @@ AssertionError: 'series' unexpectedly found in {'id': 5, 're - +
TestReader - test_sound_listener
- PASS + +
+ FAIL +
+ + + + @@ -3232,6 +3221,64 @@ AssertionError: 'series' unexpectedly found in {'id': 5, 're + + TestReadOnlyDatabase + 1 + 0 + 0 + 1 + 0 + + Detail + + + + + + + +
TestReadOnlyDatabase - test_readonly_path
+ + +
+ ERROR +
+ + + + + + + + + TestReconnect 1 @@ -3240,13 +3287,13 @@ AssertionError: 'series' unexpectedly found in {'id': 5, 're 0 0 - Detail + Detail - +
TestReconnect - test_reconnect_endpoint
@@ -3264,13 +3311,13 @@ AssertionError: 'series' unexpectedly found in {'id': 5, 're 0 0 - Detail + Detail - +
TestRegister - test_forgot_password
@@ -3279,7 +3326,7 @@ AssertionError: 'series' unexpectedly found in {'id': 5, 're - +
TestRegister - test_illegal_email
@@ -3288,7 +3335,7 @@ AssertionError: 'series' unexpectedly found in {'id': 5, 're - +
TestRegister - test_limit_domain
@@ -3297,7 +3344,7 @@ AssertionError: 'series' unexpectedly found in {'id': 5, 're - +
TestRegister - test_register_no_server
@@ -3306,7 +3353,7 @@ AssertionError: 'series' unexpectedly found in {'id': 5, 're - +
TestRegister - test_registering_only_email
@@ -3315,7 +3362,7 @@ AssertionError: 'series' unexpectedly found in {'id': 5, 're - +
TestRegister - test_registering_user
@@ -3324,7 +3371,7 @@ AssertionError: 'series' unexpectedly found in {'id': 5, 're - +
TestRegister - test_registering_user_fail
@@ -3333,7 +3380,7 @@ AssertionError: 'series' unexpectedly found in {'id': 5, 're - +
TestRegister - test_user_change_password
@@ -3351,13 +3398,13 @@ AssertionError: 'series' unexpectedly found in {'id': 5, 're 0 0 - Detail + Detail - +
TestReverseProxy - test_logout
@@ -3366,7 +3413,7 @@ AssertionError: 'series' unexpectedly found in {'id': 5, 're - +
TestReverseProxy - test_move_page
@@ -3375,7 +3422,7 @@ AssertionError: 'series' unexpectedly found in {'id': 5, 're - +
TestReverseProxy - test_reverse_about
@@ -3393,13 +3440,13 @@ AssertionError: 'series' unexpectedly found in {'id': 5, 're 0 1 - Detail + Detail - +
TestShelf - test_access_shelf
@@ -3408,7 +3455,7 @@ AssertionError: 'series' unexpectedly found in {'id': 5, 're - +
TestShelf - test_add_shelf_from_search
@@ -3417,7 +3464,7 @@ AssertionError: 'series' unexpectedly found in {'id': 5, 're - +
TestShelf - test_adv_search_shelf
@@ -3426,7 +3473,7 @@ AssertionError: 'series' unexpectedly found in {'id': 5, 're - +
TestShelf - test_arrange_shelf
@@ -3435,7 +3482,7 @@ AssertionError: 'series' unexpectedly found in {'id': 5, 're - +
TestShelf - test_create_public_shelf
@@ -3444,7 +3491,7 @@ AssertionError: 'series' unexpectedly found in {'id': 5, 're - +
TestShelf - test_create_public_shelf_no_permission
@@ -3453,7 +3500,7 @@ AssertionError: 'series' unexpectedly found in {'id': 5, 're - +
TestShelf - test_delete_book_of_shelf
@@ -3462,7 +3509,7 @@ AssertionError: 'series' unexpectedly found in {'id': 5, 're - +
TestShelf - test_private_shelf
@@ -3471,7 +3518,7 @@ AssertionError: 'series' unexpectedly found in {'id': 5, 're - +
TestShelf - test_public_private_shelf
@@ -3480,7 +3527,7 @@ AssertionError: 'series' unexpectedly found in {'id': 5, 're - +
TestShelf - test_public_shelf
@@ -3489,7 +3536,7 @@ AssertionError: 'series' unexpectedly found in {'id': 5, 're - +
TestShelf - test_rename_shelf
@@ -3498,7 +3545,7 @@ AssertionError: 'series' unexpectedly found in {'id': 5, 're - +
TestShelf - test_shelf_action_non_shelf_edit_role
@@ -3507,7 +3554,7 @@ AssertionError: 'series' unexpectedly found in {'id': 5, 're - +
TestShelf - test_shelf_anonymous
@@ -3516,19 +3563,19 @@ AssertionError: 'series' unexpectedly found in {'id': 5, 're - +
TestShelf - test_shelf_database_change
- SKIP + SKIP
-