diff --git a/README.md b/README.md index fb8bcd7f..94be5b33 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ Calibre-Web is a web app providing a clean interface for browsing, reading and d 3. Optional features can also be installed via pip, please refer to [this page](https://github.com/janeczku/calibre-web/wiki/Dependencies-in-Calibre-Web-Linux-Windows) for details 4. Calibre-Web can be started afterwards by typing `cps` -In the Wiki there are also examples for a [manual installation](https://github.com/janeczku/calibre-web/wiki/Manual-installation) and for installation on [Linux Mint](https://github.com/janeczku/calibre-web/wiki/How-To:Install-Calibre-Web-in-Linux-Mint-19-or-20) +In the Wiki there are also examples for: a [manual installation](https://github.com/janeczku/calibre-web/wiki/Manual-installation), [installation on Linux Mint](https://github.com/janeczku/calibre-web/wiki/How-To:Install-Calibre-Web-in-Linux-Mint-19-or-20), [installation on a Cloud Provider](https://github.com/janeczku/calibre-web/wiki/How-To:-Install-Calibre-Web-on-a-Cloud-Provider). ## Quick start diff --git a/cps.py b/cps.py index a99ee615..17cceb0a 100755 --- a/cps.py +++ b/cps.py @@ -16,11 +16,6 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . -try: - from gevent import monkey - monkey.patch_all() -except ImportError: - pass import sys import os diff --git a/cps/admin.py b/cps/admin.py index b486b689..1319c42a 100644 --- a/cps/admin.py +++ b/cps/admin.py @@ -1496,7 +1496,7 @@ def _handle_edit_user(to_save, content, languages, translations, kobo_support): content.role &= ~constants.ROLE_ANONYMOUS val = [int(k[5:]) for k in to_save if k.startswith('show_')] - sidebar = get_sidebar_config() + sidebar, __ = get_sidebar_config() for element in sidebar: value = element['visibility'] if value in val and not content.check_visibility(value): diff --git a/cps/db.py b/cps/db.py index 5049eccb..3f64fbb5 100644 --- a/cps/db.py +++ b/cps/db.py @@ -680,6 +680,25 @@ class CalibreDB: return and_(lang_filter, pos_content_tags_filter, ~neg_content_tags_filter, pos_content_cc_filter, ~neg_content_cc_filter, archived_filter) + def generate_linked_query(self, config_read_column, database): + if not config_read_column: + query = (self.session.query(database, ub.ArchivedBook.is_archived, ub.ReadBook.read_status) + .select_from(Books) + .outerjoin(ub.ReadBook, + and_(ub.ReadBook.user_id == int(current_user.id), ub.ReadBook.book_id == Books.id))) + else: + try: + read_column = cc_classes[config_read_column] + query = (self.session.query(database, ub.ArchivedBook.is_archived, read_column.value) + .select_from(Books) + .outerjoin(read_column, read_column.book == Books.id)) + except (KeyError, AttributeError, IndexError): + log.error("Custom Column No.{} is not existing in calibre database".format(config_read_column)) + # Skip linking read column and return None instead of read status + query = self.session.query(database, None, ub.ArchivedBook.is_archived) + return query.outerjoin(ub.ArchivedBook, and_(Books.id == ub.ArchivedBook.book_id, + int(current_user.id) == ub.ArchivedBook.user_id)) + @staticmethod def get_checkbox_sorted(inputlist, state, offset, limit, order, combo=False): outcome = list() @@ -709,31 +728,14 @@ class CalibreDB: join_archive_read, config_read_column, *join): pagesize = pagesize or self.config.config_books_per_page if current_user.show_detail_random(): - randm = self.session.query(Books) \ - .filter(self.common_filters(allow_show_archived)) \ - .order_by(func.random()) \ - .limit(self.config.config_random_books) \ - .all() + random_query = self.generate_linked_query(config_read_column, database) + randm = (random_query.filter(self.common_filters(allow_show_archived)) + .order_by(func.random()) + .limit(self.config.config_random_books).all()) else: randm = false() if join_archive_read: - if not config_read_column: - query = (self.session.query(database, ub.ReadBook.read_status, ub.ArchivedBook.is_archived) - .select_from(Books) - .outerjoin(ub.ReadBook, - and_(ub.ReadBook.user_id == int(current_user.id), ub.ReadBook.book_id == Books.id))) - else: - try: - read_column = cc_classes[config_read_column] - query = (self.session.query(database, read_column.value, ub.ArchivedBook.is_archived) - .select_from(Books) - .outerjoin(read_column, read_column.book == Books.id)) - except (KeyError, AttributeError, IndexError): - log.error("Custom Column No.{} is not existing in calibre database".format(read_column)) - # Skip linking read column and return None instead of read status - query = self.session.query(database, None, ub.ArchivedBook.is_archived) - query = query.outerjoin(ub.ArchivedBook, and_(Books.id == ub.ArchivedBook.book_id, - int(current_user.id) == ub.ArchivedBook.user_id)) + query = self.generate_linked_query(config_read_column, database) else: query = self.session.query(database) off = int(int(pagesize) * (page - 1)) @@ -817,36 +819,21 @@ class CalibreDB: def check_exists_book(self, authr, title): self.session.connection().connection.connection.create_function("lower", 1, lcase) q = list() - authorterms = re.split(r'\s*&\s*', authr) - for authorterm in authorterms: - q.append(Books.authors.any(func.lower(Authors.name).ilike("%" + authorterm + "%"))) + author_terms = re.split(r'\s*&\s*', authr) + for author_term in author_terms: + q.append(Books.authors.any(func.lower(Authors.name).ilike("%" + author_term + "%"))) return self.session.query(Books) \ .filter(and_(Books.authors.any(and_(*q)), func.lower(Books.title).ilike("%" + title + "%"))).first() - def search_query(self, term, config_read_column, *join): + def search_query(self, term, config, *join): term.strip().lower() self.session.connection().connection.connection.create_function("lower", 1, lcase) q = list() - authorterms = re.split("[, ]+", term) - for authorterm in authorterms: - q.append(Books.authors.any(func.lower(Authors.name).ilike("%" + authorterm + "%"))) - if not config_read_column: - query = (self.session.query(Books, ub.ArchivedBook.is_archived, ub.ReadBook).select_from(Books) - .outerjoin(ub.ReadBook, and_(Books.id == ub.ReadBook.book_id, - int(current_user.id) == ub.ReadBook.user_id))) - else: - try: - read_column = cc_classes[config_read_column] - query = (self.session.query(Books, ub.ArchivedBook.is_archived, read_column.value).select_from(Books) - .outerjoin(read_column, read_column.book == Books.id)) - except (KeyError, AttributeError, IndexError): - log.error("Custom Column No.{} is not existing in calibre database".format(config_read_column)) - # Skip linking read column - query = self.session.query(Books, ub.ArchivedBook.is_archived, None) - query = query.outerjoin(ub.ArchivedBook, and_(Books.id == ub.ArchivedBook.book_id, - int(current_user.id) == ub.ArchivedBook.user_id)) - + author_terms = re.split("[, ]+", term) + for author_term in author_terms: + q.append(Books.authors.any(func.lower(Authors.name).ilike("%" + author_term + "%"))) + query = self.generate_linked_query(config.config_read_column, Books) if len(join) == 6: query = query.outerjoin(join[0], join[1]).outerjoin(join[2]).outerjoin(join[3], join[4]).outerjoin(join[5]) if len(join) == 3: @@ -855,20 +842,42 @@ class CalibreDB: query = query.outerjoin(join[0], join[1]) elif len(join) == 1: query = query.outerjoin(join[0]) - return query.filter(self.common_filters(True)).filter( - or_(Books.tags.any(func.lower(Tags.name).ilike("%" + term + "%")), - Books.series.any(func.lower(Series.name).ilike("%" + term + "%")), - Books.authors.any(and_(*q)), - Books.publishers.any(func.lower(Publishers.name).ilike("%" + term + "%")), - func.lower(Books.title).ilike("%" + term + "%") - )) + + cc = self.get_cc_columns(config, filter_config_custom_read=True) + filter_expression = [Books.tags.any(func.lower(Tags.name).ilike("%" + term + "%")), + Books.series.any(func.lower(Series.name).ilike("%" + term + "%")), + Books.authors.any(and_(*q)), + Books.publishers.any(func.lower(Publishers.name).ilike("%" + term + "%")), + func.lower(Books.title).ilike("%" + term + "%")] + for c in cc: + if c.datatype not in ["datetime", "rating", "bool", "int", "float"]: + filter_expression.append( + getattr(Books, + 'custom_column_' + str(c.id)).any( + func.lower(cc_classes[c.id].value).ilike("%" + term + "%"))) + return query.filter(self.common_filters(True)).filter(or_(*filter_expression)) + + def get_cc_columns(self, config, filter_config_custom_read=False): + tmp_cc = self.session.query(CustomColumns).filter(CustomColumns.datatype.notin_(cc_exceptions)).all() + cc = [] + r = None + if config.config_columns_to_ignore: + r = re.compile(config.config_columns_to_ignore) + + for col in tmp_cc: + if filter_config_custom_read and config.config_read_column and config.config_read_column == col.id: + continue + if r and r.match(col.name): + continue + cc.append(col) + + return cc # read search results from calibre-database and return it (function is used for feed and simple search - def get_search_results(self, term, offset=None, order=None, limit=None, - config_read_column=False, *join): + def get_search_results(self, term, config, offset=None, order=None, limit=None, *join): order = order[0] if order else [Books.sort] pagination = None - result = self.search_query(term, config_read_column, *join).order_by(*order).all() + result = self.search_query(term, config, *join).order_by(*order).all() result_count = len(result) if offset != None and limit != None: offset = int(offset) diff --git a/cps/gdriveutils.py b/cps/gdriveutils.py index e75f3742..6073777e 100644 --- a/cps/gdriveutils.py +++ b/cps/gdriveutils.py @@ -32,7 +32,7 @@ try: from sqlalchemy.orm import declarative_base except ImportError: from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.exc import OperationalError, InvalidRequestError +from sqlalchemy.exc import OperationalError, InvalidRequestError, IntegrityError from sqlalchemy.sql.expression import text #try: @@ -81,7 +81,7 @@ if gdrive_support: if not logger.is_debug_enabled(): logger.get('googleapiclient.discovery').setLevel(logger.logging.ERROR) else: - log.debug("Cannot import pydrive, httplib2, using gdrive will not work: %s", importError) + log.debug("Cannot import pydrive, httplib2, using gdrive will not work: {}".format(importError)) class Singleton: @@ -213,7 +213,7 @@ def getDrive(drive=None, gauth=None): try: gauth.Refresh() except RefreshError as e: - log.error("Google Drive error: %s", e) + log.error("Google Drive error: {}".format(e)) except Exception as ex: log.error_or_exception(ex) else: @@ -225,7 +225,7 @@ def getDrive(drive=None, gauth=None): try: drive.auth.Refresh() except RefreshError as e: - log.error("Google Drive error: %s", e) + log.error("Google Drive error: {}".format(e)) return drive def listRootFolders(): @@ -234,7 +234,7 @@ def listRootFolders(): folder = "'root' in parents and mimeType = 'application/vnd.google-apps.folder' and trashed = false" fileList = drive.ListFile({'q': folder}).GetList() except (ServerNotFoundError, ssl.SSLError, RefreshError) as e: - log.info("GDrive Error %s" % e) + log.info("GDrive Error {}".format(e)) fileList = [] return fileList @@ -272,7 +272,7 @@ def getEbooksFolderId(drive=None): try: session.commit() except OperationalError as ex: - log.error_or_exception('Database error: %s', ex) + log.error_or_exception('Database error: {}'.format(ex)) session.rollback() return gDriveId.gdrive_id @@ -288,6 +288,7 @@ def getFile(pathId, fileName, drive): def getFolderId(path, drive): # drive = getDrive(drive) + currentFolderId = None try: currentFolderId = getEbooksFolderId(drive) sqlCheckPath = path if path[-1] == '/' else path + '/' @@ -320,8 +321,8 @@ def getFolderId(path, drive): session.commit() else: currentFolderId = storedPathName.gdrive_id - except OperationalError as ex: - log.error_or_exception('Database error: %s', ex) + except (OperationalError, IntegrityError) as ex: + log.error_or_exception('Database error: {}'.format(ex)) session.rollback() except ApiRequestError as ex: log.error('{} {}'.format(ex.error['message'], path)) @@ -545,7 +546,7 @@ def deleteDatabaseOnChange(): session.commit() except (OperationalError, InvalidRequestError) as ex: session.rollback() - log.error_or_exception('Database error: %s', ex) + log.error_or_exception('Database error: {}'.format(ex)) def updateGdriveCalibreFromLocal(): @@ -563,7 +564,7 @@ def updateDatabaseOnEdit(ID,newPath): try: session.commit() except OperationalError as ex: - log.error_or_exception('Database error: %s', ex) + log.error_or_exception('Database error: {}'.format(ex)) session.rollback() @@ -573,7 +574,7 @@ def deleteDatabaseEntry(ID): try: session.commit() except OperationalError as ex: - log.error_or_exception('Database error: %s', ex) + log.error_or_exception('Database error: {}'.format(ex)) session.rollback() @@ -594,7 +595,7 @@ def get_cover_via_gdrive(cover_path): try: session.commit() except OperationalError as ex: - log.error_or_exception('Database error: %s', ex) + log.error_or_exception('Database error: {}'.format(ex)) session.rollback() return df.metadata.get('webContentLink') else: @@ -616,7 +617,7 @@ def do_gdrive_download(df, headers, convert_encoding=False): def stream(convert_encoding): for byte in s: - headers = {"Range": 'bytes=%s-%s' % (byte[0], byte[1])} + headers = {"Range": 'bytes={}-{}'.format(byte[0], byte[1])} resp, content = df.auth.Get_Http_Object().request(download_url, headers=headers) if resp.status == 206: if convert_encoding: @@ -624,7 +625,7 @@ def do_gdrive_download(df, headers, convert_encoding=False): content = content.decode(result['encoding']).encode('utf-8') yield content else: - log.warning('An error occurred: %s', resp) + log.warning('An error occurred: {}'.format(resp)) return return Response(stream_with_context(stream(convert_encoding)), headers=headers) diff --git a/cps/gevent_wsgi.py b/cps/gevent_wsgi.py new file mode 100644 index 00000000..b044f31b --- /dev/null +++ b/cps/gevent_wsgi.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2022 OzzieIsaacs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +from gevent.pywsgi import WSGIHandler + +class MyWSGIHandler(WSGIHandler): + def get_environ(self): + env = super().get_environ() + path, __ = self.path.split('?', 1) if '?' in self.path else (self.path, '') + env['RAW_URI'] = path + return env + + diff --git a/cps/helper.py b/cps/helper.py index 8a261581..a98c986a 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -19,6 +19,7 @@ import os import io +import sys import mimetypes import re import shutil @@ -226,11 +227,23 @@ def send_mail(book_id, book_format, convert, kindle_mail, calibrepath, user_id): return _(u"The requested file could not be read. Maybe wrong permissions?") +def shorten_component(s, by_what): + l = len(s) + if l < by_what: + return s + l = (l - by_what)//2 + if l <= 0: + return s + return s[:l] + s[-l:] + + def get_valid_filename(value, replace_whitespace=True, chars=128): """ Returns the given string converted to a string that can be used for a clean filename. Limits num characters to 128 max. """ + + if value[-1:] == u'.': value = value[:-1]+u'_' value = value.replace("/", "_").replace(":", "_").strip('\0') @@ -241,7 +254,10 @@ def get_valid_filename(value, replace_whitespace=True, chars=128): value = re.sub(r'[*+:\\\"/<>?]+', u'_', value, flags=re.U) # pipe has to be replaced with comma value = re.sub(r'[|]+', u',', value, flags=re.U) - value = value[:chars].strip() + + filename_encoding_for_length = 'utf-16' if sys.platform == "win32" or sys.platform == "darwin" else 'utf-8' + value = value.encode(filename_encoding_for_length)[:chars].decode('utf-8', errors='ignore').strip() + if not value: raise ValueError("Filename cannot be empty") return value @@ -722,7 +738,7 @@ def get_book_cover_internal(book, use_generic_cover_on_failure, resolution=None) if path: return redirect(path) else: - log.error('%s/cover.jpg not found on Google Drive', book.path) + log.error('{}/cover.jpg not found on Google Drive'.format(book.path)) return get_cover_on_failure(use_generic_cover_on_failure) except Exception as ex: log.error_or_exception(ex) @@ -1029,24 +1045,6 @@ def check_valid_domain(domain_text): return not len(result) -def get_cc_columns(filter_config_custom_read=False): - tmpcc = calibre_db.session.query(db.CustomColumns)\ - .filter(db.CustomColumns.datatype.notin_(db.cc_exceptions)).all() - cc = [] - r = None - if config.config_columns_to_ignore: - r = re.compile(config.config_columns_to_ignore) - - for col in tmpcc: - if filter_config_custom_read and config.config_read_column and config.config_read_column == col.id: - continue - if r and r.match(col.name): - continue - cc.append(col) - - return cc - - def get_download_link(book_id, book_format, client): book_format = book_format.split(".")[0] book = calibre_db.get_filtered_book(book_id, allow_show_archived=True) diff --git a/cps/opds.py b/cps/opds.py index 180fcacb..cb8f397e 100644 --- a/cps/opds.py +++ b/cps/opds.py @@ -26,7 +26,8 @@ from functools import wraps from flask import Blueprint, request, render_template, Response, g, make_response, abort from flask_login import current_user -from sqlalchemy.sql.expression import func, text, or_, and_, any_, true +from sqlalchemy.sql.expression import func, text, or_, and_, true +from sqlalchemy.exc import InvalidRequestError, OperationalError from werkzeug.security import check_password_hash from . import constants, logger, config, db, calibre_db, ub, services, get_locale, isoLanguages from .helper import get_download_link, get_book_cover @@ -84,7 +85,7 @@ def feed_osd(): @requires_basic_auth_if_no_ano def feed_cc_search(query): # Handle strange query from Libera Reader with + instead of spaces - plus_query = unquote_plus(request.base_url.split('/opds/search/')[1]).strip() + plus_query = unquote_plus(request.environ['RAW_URI'].split('/opds/search/')[1]).strip() return feed_search(plus_query) @@ -108,7 +109,8 @@ def feed_letter_books(book_id): entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), 0, db.Books, letter, - [db.Books.sort]) + [db.Books.sort], + True, config.config_read_column) return render_xml_template('feed.xml', entries=entries, pagination=pagination) @@ -118,15 +120,16 @@ def feed_letter_books(book_id): def feed_new(): off = request.args.get("offset") or 0 entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), 0, - db.Books, True, [db.Books.timestamp.desc()]) + db.Books, True, [db.Books.timestamp.desc()], + True, config.config_read_column) return render_xml_template('feed.xml', entries=entries, pagination=pagination) @opds.route("/opds/discover") @requires_basic_auth_if_no_ano def feed_discover(): - entries = calibre_db.session.query(db.Books).filter(calibre_db.common_filters()).order_by(func.random())\ - .limit(config.config_books_per_page) + query = calibre_db.generate_linked_query(config.config_read_column, db.Books) + entries = query.filter(calibre_db.common_filters()).order_by(func.random()).limit(config.config_books_per_page) pagination = Pagination(1, config.config_books_per_page, int(config.config_books_per_page)) return render_xml_template('feed.xml', entries=entries, pagination=pagination) @@ -137,7 +140,8 @@ def feed_best_rated(): off = request.args.get("offset") or 0 entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), 0, db.Books, db.Books.ratings.any(db.Ratings.rating > 9), - [db.Books.timestamp.desc()]) + [db.Books.timestamp.desc()], + True, config.config_read_column) return render_xml_template('feed.xml', entries=entries, pagination=pagination) @@ -150,11 +154,11 @@ def feed_hot(): hot_books = all_books.offset(off).limit(config.config_books_per_page) entries = list() for book in hot_books: - download_book = calibre_db.get_book(book.Downloads.book_id) + query = calibre_db.generate_linked_query(config.config_read_column, db.Books) + download_book = query.filter(calibre_db.common_filters()).filter( + book.Downloads.book_id == db.Books.id).first() if download_book: - entries.append( - calibre_db.get_filtered_book(book.Downloads.book_id) - ) + entries.append(download_book) else: ub.delete_download(book.Downloads.book_id) num_books = entries.__len__() @@ -270,7 +274,8 @@ def feed_series(book_id): entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), 0, db.Books, db.Books.series.any(db.Series.id == book_id), - [db.Books.series_index]) + [db.Books.series_index], + True, config.config_read_column) return render_xml_template('feed.xml', entries=entries, pagination=pagination) @@ -324,7 +329,8 @@ def feed_format(book_id): entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), 0, db.Books, db.Books.data.any(db.Data.format == book_id.upper()), - [db.Books.timestamp.desc()]) + [db.Books.timestamp.desc()], + True, config.config_read_column) return render_xml_template('feed.xml', entries=entries, pagination=pagination) @@ -351,7 +357,8 @@ def feed_languages(book_id): entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), 0, db.Books, db.Books.languages.any(db.Languages.id == book_id), - [db.Books.timestamp.desc()]) + [db.Books.timestamp.desc()], + True, config.config_read_column) return render_xml_template('feed.xml', entries=entries, pagination=pagination) @@ -381,13 +388,25 @@ def feed_shelf(book_id): result = list() # user is allowed to access shelf if shelf: - books_in_shelf = ub.session.query(ub.BookShelf).filter(ub.BookShelf.shelf == book_id).order_by( - ub.BookShelf.order.asc()).all() - for book in books_in_shelf: - cur_book = calibre_db.get_book(book.book_id) - result.append(cur_book) - pagination = Pagination((int(off) / (int(config.config_books_per_page)) + 1), config.config_books_per_page, - len(result)) + result, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), + config.config_books_per_page, + db.Books, + ub.BookShelf.shelf == shelf.id, + [ub.BookShelf.order.asc()], + True, config.config_read_column, + ub.BookShelf, ub.BookShelf.book_id == db.Books.id) + # delete shelf entries where book is not existent anymore, can happen if book is deleted outside calibre-web + wrong_entries = calibre_db.session.query(ub.BookShelf) \ + .join(db.Books, ub.BookShelf.book_id == db.Books.id, isouter=True) \ + .filter(db.Books.id == None).all() + for entry in wrong_entries: + log.info('Not existing book {} in {} deleted'.format(entry.book_id, shelf)) + try: + ub.session.query(ub.BookShelf).filter(ub.BookShelf.book_id == entry.book_id).delete() + ub.session.commit() + except (OperationalError, InvalidRequestError) as e: + ub.session.rollback() + log.error_or_exception("Settings Database error: {}".format(e)) return render_xml_template('feed.xml', entries=result, pagination=pagination) @@ -448,11 +467,10 @@ def feed_unread_books(): def feed_search(term): if term: - entries, __, ___ = calibre_db.get_search_results(term, config_read_column=config.config_read_column) + entries, __, ___ = calibre_db.get_search_results(term, config=config) entries_count = len(entries) if len(entries) > 0 else 1 pagination = Pagination(1, entries_count, entries_count) - items = [entry[0] for entry in entries] - return render_xml_template('feed.xml', searchterm=term, entries=items, pagination=pagination) + return render_xml_template('feed.xml', searchterm=term, entries=entries, pagination=pagination) else: return render_xml_template('feed.xml', searchterm="") @@ -493,14 +511,16 @@ def render_xml_dataset(data_table, book_id): entries, __, pagination = calibre_db.fill_indexpage((int(off) / (int(config.config_books_per_page)) + 1), 0, db.Books, getattr(db.Books, data_table.__tablename__).any(data_table.id == book_id), - [db.Books.timestamp.desc()]) + [db.Books.timestamp.desc()], + True, config.config_read_column) return render_xml_template('feed.xml', entries=entries, pagination=pagination) def render_element_index(database_column, linked_table, folder): shift = 0 off = int(request.args.get("offset") or 0) - entries = calibre_db.session.query(func.upper(func.substr(database_column, 1, 1)).label('id')) + entries = calibre_db.session.query(func.upper(func.substr(database_column, 1, 1)).label('id'), None, None) + # query = calibre_db.generate_linked_query(config.config_read_column, db.Books) if linked_table is not None: entries = entries.join(linked_table).join(db.Books) entries = entries.filter(calibre_db.common_filters()).group_by(func.upper(func.substr(database_column, 1, 1))).all() diff --git a/cps/render_template.py b/cps/render_template.py index 09a32497..0c5423c9 100644 --- a/cps/render_template.py +++ b/cps/render_template.py @@ -16,7 +16,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from flask import render_template +from flask import render_template, request from flask_babel import gettext as _ from flask import g from werkzeug.local import LocalProxy @@ -30,6 +30,8 @@ log = logger.create() def get_sidebar_config(kwargs=None): kwargs = kwargs or [] + simple = bool([e for e in ['kindle', 'tolino', "kobo", "bookeen"] + if (e in request.headers.get('User-Agent', "").lower())]) if 'content' in kwargs: content = kwargs['content'] content = isinstance(content, (User, LocalProxy)) and not content.role_anonymous() @@ -93,14 +95,14 @@ def get_sidebar_config(kwargs=None): {"glyph": "glyphicon-trash", "text": _('Archived Books'), "link": 'web.books_list', "id": "archived", "visibility": constants.SIDEBAR_ARCHIVED, 'public': (not g.user.is_anonymous), "page": "archived", "show_text": _('Show archived books'), "config_show": content}) - sidebar.append( - {"glyph": "glyphicon-th-list", "text": _('Books List'), "link": 'web.books_table', "id": "list", - "visibility": constants.SIDEBAR_LIST, 'public': (not g.user.is_anonymous), "page": "list", - "show_text": _('Show Books List'), "config_show": content}) - - return sidebar + if not simple: + sidebar.append( + {"glyph": "glyphicon-th-list", "text": _('Books List'), "link": 'web.books_table', "id": "list", + "visibility": constants.SIDEBAR_LIST, 'public': (not g.user.is_anonymous), "page": "list", + "show_text": _('Show Books List'), "config_show": content}) + return sidebar, simple -def get_readbooks_ids(): +'''def get_readbooks_ids(): if not config.config_read_column: readBooks = ub.session.query(ub.ReadBook).filter(ub.ReadBook.user_id == int(current_user.id))\ .filter(ub.ReadBook.read_status == ub.ReadBook.STATUS_FINISHED).all() @@ -112,11 +114,11 @@ def get_readbooks_ids(): return frozenset([x.book for x in readBooks]) except (KeyError, AttributeError, IndexError): log.error("Custom Column No.{} is not existing in calibre database".format(config.config_read_column)) - return [] + return []''' # Returns the template for rendering and includes the instance name def render_title_template(*args, **kwargs): - sidebar = get_sidebar_config(kwargs) - return render_template(instance=config.config_calibre_web_title, sidebar=sidebar, - accept=constants.EXTENSIONS_UPLOAD, read_book_ids=get_readbooks_ids(), + sidebar, simple = get_sidebar_config(kwargs) + return render_template(instance=config.config_calibre_web_title, sidebar=sidebar, simple=simple, + accept=constants.EXTENSIONS_UPLOAD, # read_book_ids=get_readbooks_ids(), *args, **kwargs) diff --git a/cps/search_metadata.py b/cps/search_metadata.py index d72273f6..79e27554 100644 --- a/cps/search_metadata.py +++ b/cps/search_metadata.py @@ -23,7 +23,7 @@ import json import os import sys # from time import time -from dataclasses import asdict + from flask import Blueprint, Response, request, url_for from flask_login import current_user @@ -32,7 +32,7 @@ from sqlalchemy.exc import InvalidRequestError, OperationalError from sqlalchemy.orm.attributes import flag_modified from cps.services.Metadata import Metadata -from . import constants, get_locale, logger, ub +from . import constants, get_locale, logger, ub, web_server # current_milli_time = lambda: int(round(time() * 1000)) @@ -40,6 +40,14 @@ meta = Blueprint("metadata", __name__) log = logger.create() +try: + from dataclasses import asdict +except ImportError: + log.info('*** "dataclasses" is needed for calibre-web to run. Please install it using pip: "pip install dataclasses" ***') + print('*** "dataclasses" is needed for calibre-web to run. Please install it using pip: "pip install dataclasses" ***') + web_server.stop(True) + sys.exit(6) + new_list = list() meta_dir = os.path.join(constants.BASE_DIR, "cps", "metadata_provider") modules = os.listdir(os.path.join(constants.BASE_DIR, "cps", "metadata_provider")) diff --git a/cps/server.py b/cps/server.py index e261c50a..0ffdbd18 100644 --- a/cps/server.py +++ b/cps/server.py @@ -25,6 +25,7 @@ import subprocess # nosec try: from gevent.pywsgi import WSGIServer + from .gevent_wsgi import MyWSGIHandler from gevent.pool import Pool from gevent import __version__ as _version from greenlet import GreenletExit @@ -32,7 +33,7 @@ try: VERSION = 'Gevent ' + _version _GEVENT = True except ImportError: - from tornado.wsgi import WSGIContainer + from .tornado_wsgi import MyWSGIContainer from tornado.httpserver import HTTPServer from tornado.ioloop import IOLoop from tornado import version as _version @@ -202,7 +203,8 @@ class WebServer(object): if output is None: output = _readable_listen_address(self.listen_address, self.listen_port) log.info('Starting Gevent server on %s', output) - self.wsgiserver = WSGIServer(sock, self.app, log=self.access_logger, spawn=Pool(), **ssl_args) + self.wsgiserver = WSGIServer(sock, self.app, log=self.access_logger, handler_class=MyWSGIHandler, + spawn=Pool(), **ssl_args) if ssl_args: wrap_socket = self.wsgiserver.wrap_socket def my_wrap_socket(*args, **kwargs): @@ -225,8 +227,8 @@ class WebServer(object): asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) log.info('Starting Tornado server on %s', _readable_listen_address(self.listen_address, self.listen_port)) - # Max Buffersize set to 200MB ) - http_server = HTTPServer(WSGIContainer(self.app), + # Max Buffersize set to 200MB + http_server = HTTPServer(MyWSGIContainer(self.app), max_buffer_size=209700000, ssl_options=self.ssl_args) http_server.listen(self.listen_port, self.listen_address) diff --git a/cps/shelf.py b/cps/shelf.py index 0bf12164..35f2941d 100644 --- a/cps/shelf.py +++ b/cps/shelf.py @@ -439,7 +439,7 @@ def render_show_shelf(shelf_type, shelf_id, page_no, sort_param): db.Books, ub.BookShelf.shelf == shelf_id, [ub.BookShelf.order.asc()], - False, 0, + True, config.config_read_column, ub.BookShelf, ub.BookShelf.book_id == db.Books.id) # delete chelf entries where book is not existent anymore, can happen if book is deleted outside calibre-web wrong_entries = calibre_db.session.query(ub.BookShelf) \ diff --git a/cps/templates/admin.html b/cps/templates/admin.html index 8f749e0b..cdaec9bd 100644 --- a/cps/templates/admin.html +++ b/cps/templates/admin.html @@ -47,7 +47,9 @@ {% endfor %} {% endif %} - {{_('Edit Users')}} + {% if not simple %} + {{_('Edit Users')}} + {% endif %} {{_('Add New User')}} {% if (config.config_login_type == 1) %}
{{_('Import LDAP Users')}}
diff --git a/cps/templates/author.html b/cps/templates/author.html index 251a4348..b991e959 100644 --- a/cps/templates/author.html +++ b/cps/templates/author.html @@ -31,23 +31,22 @@
- {% if entries[0] %} {% for entry in entries %}
- - - {{ image.book_cover(entry, alt=author.name|safe) }} - {% if entry.id in read_book_ids %}{% endif %} + + + {{ image.book_cover(entry.Books, alt=author.name|safe) }} + {% if entry[2] == True %}{% endif %}
- -

{{entry.title|shortentitle}}

+ +

{{entry.Books.title|shortentitle}}

- {% for author in entry.ordered_authors %} + {% for author in entry.Books.authors %} {% if loop.index > g.config_authors_max and g.config_authors_max != 0 %} {% if not loop.first %} & @@ -63,23 +62,23 @@ {{author.name.replace('|',',')|shortentitle(30)}} {% endif %} {% endfor %} - {% for format in entry.data %} + {% for format in entry.Books.data %} {% if format.format|lower in g.constants.EXTENSIONS_AUDIO %} {% endif %} {% endfor %}

- {% if entry.series.__len__() > 0 %} + {% if entry.Books.series.__len__() > 0 %}

- - {{entry.series[0].name}} + + {{entry.Books.series[0].name}} - ({{entry.series_index|formatseriesindex}}) + ({{entry.Books.series_index|formatseriesindex}})

{% endif %} - {% if entry.ratings.__len__() > 0 %} + {% if entry.Books.ratings.__len__() > 0 %}
- {% for number in range((entry.ratings[0].rating/2)|int(2)) %} + {% for number in range((entry.Books.ratings[0].rating/2)|int(2)) %} {% if loop.last and loop.index < 5 %} {% for numer in range(5 - loop.index) %} @@ -92,7 +91,6 @@
{% endfor %} - {% endif %}
@@ -110,7 +108,7 @@

{{entry.title|shortentitle}}

- {% for author in entry.ordered_authors %} + {% for author in entry.authors %} {% if loop.index > g.config_authors_max and g.config_authors_max != 0 %} {{author.name.replace('|',',')}} {% if loop.last %} diff --git a/cps/templates/book_exists_flash.html b/cps/templates/book_exists_flash.html index b0855120..b55192ce 100644 --- a/cps/templates/book_exists_flash.html +++ b/cps/templates/book_exists_flash.html @@ -1,3 +1,3 @@ - + {{entry.title|shortentitle}} - \ No newline at end of file + diff --git a/cps/templates/config_view_edit.html b/cps/templates/config_view_edit.html index 1dc02a5e..e4fea44d 100644 --- a/cps/templates/config_view_edit.html +++ b/cps/templates/config_view_edit.html @@ -162,8 +162,10 @@

+ {% if not simple %} {{_('Add Allowed/Denied Tags')}} {{_('Add Allowed/Denied custom column values')}} + {% endif %} diff --git a/cps/templates/discover.html b/cps/templates/discover.html deleted file mode 100644 index 370b9820..00000000 --- a/cps/templates/discover.html +++ /dev/null @@ -1,66 +0,0 @@ -{% import 'image.html' as image %} -{% extends "layout.html" %} -{% block body %} -
-

{{title}}

-
- {% for entry in entries %} -
-
- {% if entry.has_cover is defined %} - - - {{ image.book_cover(entry) }} - {% if entry.id in read_book_ids %}{% endif %} - - - {% endif %} -
-
- -

{{entry.title|shortentitle}}

- -

- {% for author in entry.ordered_authors %} - {% if loop.index > g.config_authors_max and g.config_authors_max != 0 %} - {% if not loop.first %} - & - {% endif %} - {{author.name.replace('|',',')|shortentitle(30)}} - {% if loop.last %} - (...) - {% endif %} - {% else %} - {% if not loop.first %} - & - {% endif %} - {{author.name.replace('|',',')|shortentitle(30)}} - {% endif %} - {% endfor %} -

- {% if entry.series.__len__() > 0 %} -

- - {{entry.series[0].name}} - - ({{entry.series_index|formatseriesindex}}) -

- {% endif %} - {% if entry.ratings.__len__() > 0 %} -
- {% for number in range((entry.ratings[0].rating/2)|int(2)) %} - - {% if loop.last and loop.index < 5 %} - {% for numer in range(5 - loop.index) %} - - {% endfor %} - {% endif %} - {% endfor %} -
- {% endif %} -
-
- {% endfor %} -
-
-{% endblock %} diff --git a/cps/templates/email_edit.html b/cps/templates/email_edit.html index 9f23f78b..2a844209 100644 --- a/cps/templates/email_edit.html +++ b/cps/templates/email_edit.html @@ -69,7 +69,7 @@ {% endif %} {{_('Back')}} - {% if g.allow_registration %} + {% if g.allow_registration and not simple%}

{{_('Allowed Domains (Whitelist)')}}

diff --git a/cps/templates/feed.xml b/cps/templates/feed.xml index 9073142e..940fb0da 100644 --- a/cps/templates/feed.xml +++ b/cps/templates/feed.xml @@ -40,35 +40,35 @@ {% if entries and entries[0] %} {% for entry in entries %} - {{entry.title}} - urn:uuid:{{entry.uuid}} - {{entry.atom_timestamp}} - {% if entry.authors.__len__() > 0 %} + {{entry.Books.title}} + urn:uuid:{{entry.Books.uuid}} + {{entry.Books.atom_timestamp}} + {% if entry.Books.authors.__len__() > 0 %} - {{entry.authors[0].name}} + {{entry.Books.authors[0].name}} {% endif %} - {% if entry.publishers.__len__() > 0 %} + {% if entry.Books.publishers.__len__() > 0 %} - {{entry.publishers[0].name}} + {{entry.Books.publishers[0].name}} {% endif %} - {% for lang in entry.languages %} + {% for lang in entry.Books.languages %} {{lang.lang_code}} {% endfor %} - {% for tag in entry.tags %} + {% for tag in entry.Books.tags %} {% endfor %} - {% if entry.comments[0] %}{{entry.comments[0].text|striptags}}{% endif %} - {% if entry.has_cover %} - - + {% if entry.Books.comments[0] %}{{entry.Books.comments[0].text|striptags}}{% endif %} + {% if entry.Books.has_cover %} + + {% endif %} - {% for format in entry.data %} - + {% for format in entry.Books.data %} + {% endfor %} {% endfor %} diff --git a/cps/templates/index.html b/cps/templates/index.html index 18ec8adc..0bb3da72 100644 --- a/cps/templates/index.html +++ b/cps/templates/index.html @@ -1,31 +1,31 @@ {% import 'image.html' as image %} {% extends "layout.html" %} {% block body %} -{% if g.user.show_detail_random() %} +{% if g.user.show_detail_random() and page != "discover" %}

{{_('Discover (Random Books)')}}

{% for entry in random %}
- - - {{ image.book_cover(entry) }} - {% if entry.id in read_book_ids %}{% endif %} + + + {{ image.book_cover(entry.Books) }} + {% if entry[2] == True %}{% endif %}
- -

{{entry.title|shortentitle}}

+ +

{{entry.Books.title|shortentitle}}

- {% for author in entry.ordered_authors %} + {% for author in entry.Books.authors %} {% if loop.index > g.config_authors_max and g.config_authors_max != 0 %} {% if not loop.first %} & {% endif %} - {{author.name.replace('|',',')|shortentitle(30)}} + {{author.name.replace('|',',')|shortentitle(30)}} {% if loop.last %} (...) {% endif %} @@ -37,17 +37,17 @@ {% endif %} {% endfor %}

- {% if entry.series.__len__() > 0 %} + {% if entry.Books.series.__len__() > 0 %}

- - {{entry.series[0].name}} + + {{entry.Books.series[0].name}} - ({{entry.series_index|formatseriesindex}}) + ({{entry.Books.series_index|formatseriesindex}})

{% endif %} - {% if entry.ratings.__len__() > 0 %} + {% if entry.Books.ratings.__len__() > 0 %}
- {% for number in range((entry.ratings[0].rating/2)|int(2)) %} + {% for number in range((entry.Books.ratings[0].rating/2)|int(2)) %} {% if loop.last and loop.index < 5 %} {% for numer in range(5 - loop.index) %} @@ -65,6 +65,7 @@ {% endif %}

{{title}}

+ {% if page != 'discover' %} - + {% endif %}
{% if entries[0] %} {% for entry in entries %}
- - - {{ image.book_cover(entry) }} - {% if entry.id in read_book_ids %}{% endif %} + + + {{ image.book_cover(entry.Books) }} + {% if entry[2] == True %}{% endif %}
- -

{{entry.title|shortentitle}}

+ +

{{entry.Books.title|shortentitle}}

- {% for author in entry.ordered_authors %} + {% for author in entry.Books.authors %} {% if loop.index > g.config_authors_max and g.config_authors_max != 0 %} {% if not loop.first %} & @@ -118,23 +119,27 @@ {{author.name.replace('|',',')|shortentitle(30)}} {% endif %} {% endfor %} - {% for format in entry.data %} + {% for format in entry.Books.data %} {% if format.format|lower in g.constants.EXTENSIONS_AUDIO %} {% endif %} {%endfor%}

- {% if entry.series.__len__() > 0 %} + {% if entry.Books.series.__len__() > 0 %}

- - {{entry.series[0].name}} + {% if page != "series" %} + + {{entry.Books.series[0].name}} - ({{entry.series_index|formatseriesindex}}) + {% else %} + {{entry.Books.series[0].name}} + {% endif %} + ({{entry.Books.series_index|formatseriesindex}})

{% endif %} - {% if entry.ratings.__len__() > 0 %} + {% if entry.Books.ratings.__len__() > 0 %}
- {% for number in range((entry.ratings[0].rating/2)|int(2)) %} + {% for number in range((entry.Books.ratings[0].rating/2)|int(2)) %} {% if loop.last and loop.index < 5 %} {% for numer in range(5 - loop.index) %} diff --git a/cps/templates/layout.html b/cps/templates/layout.html index 778053ae..42012937 100644 --- a/cps/templates/layout.html +++ b/cps/templates/layout.html @@ -70,7 +70,7 @@ {% endif %} - {% if not g.user.is_anonymous %} + {% if not g.user.is_anonymous and not simple%}
    • {% endif %} {% if g.user.role_admin() %} diff --git a/cps/templates/search.html b/cps/templates/search.html index 6f1963f0..78e30494 100644 --- a/cps/templates/search.html +++ b/cps/templates/search.html @@ -44,16 +44,16 @@
    {% if entry.Books.has_cover is defined %} - + {{ image.book_cover(entry.Books) }} - {% if entry.Books.id in read_book_ids %}{% endif %} + {% if entry[2] == True %}{% endif %} {% endif %}
    - +

    {{entry.Books.title|shortentitle}}

    diff --git a/cps/templates/shelf.html b/cps/templates/shelf.html index ffb6953b..b4596f79 100644 --- a/cps/templates/shelf.html +++ b/cps/templates/shelf.html @@ -33,19 +33,19 @@ {% for entry in entries %}

    - - - {{ image.book_cover(entry) }} - {% if entry.id in read_book_ids %}{% endif %} + + + {{ image.book_cover(entry.Books) }} + {% if entry[2] == True %}{% endif %}
    - -

    {{entry.title|shortentitle}}

    +     +

    {{entry.Books.title|shortentitle}}

    - {% for author in entry.ordered_authors %} + {% for author in entry.Books.authors %} {% if loop.index > g.config_authors_max and g.config_authors_max != 0 %} {% if not loop.first %} & @@ -62,17 +62,17 @@ {% endif %} {% endfor %}

    - {% if entry.series.__len__() > 0 %} + {% if entry.Books.series.__len__() > 0 %}

    - - {{entry.series[0].name}} + + {{entry.Books.series[0].name}} - ({{entry.series_index|formatseriesindex}}) + ({{entry.Books.series_index|formatseriesindex}})

    {% endif %} - {% if entry.ratings.__len__() > 0 %} + {% if entry.Books.ratings.__len__() > 0 %}
    - {% for number in range((entry.ratings[0].rating/2)|int(2)) %} + {% for number in range((entry.Books.ratings[0].rating/2)|int(2)) %} {% if loop.last and loop.index < 5 %} {% for numer in range(5 - loop.index) %} diff --git a/cps/templates/shelfdown.html b/cps/templates/shelfdown.html index c800dca7..f1a0b137 100644 --- a/cps/templates/shelfdown.html +++ b/cps/templates/shelfdown.html @@ -35,31 +35,31 @@
    -

    {{entry.title|shortentitle}}

    +

    {{entry.Books.title|shortentitle}}

    - {% for author in entry.ordered_authors %} + {% for author in entry.Books.authors %} {{author.name.replace('|',',')}} {% if not loop.last %} & {% endif %} {% endfor %}

    - {% if entry.series.__len__() > 0 %} + {% if entry.Books.series.__len__() > 0 %}

    - - {{entry.series[0].name}} + + {{entry.Books.series[0].name}} - ({{entry.series_index}}) + ({{entry.Books.series_index}})

    {% endif %}
    {% if g.user.role_download() %} - {% if entry.data|length %} + {% if entry.Books.data|length %}
    - {% for format in entry.data %} - + {% for format in entry.Books.data %} + {{format.format}} ({{ format.uncompressed_size|filesizeformat }}) {% endfor %} diff --git a/cps/templates/user_edit.html b/cps/templates/user_edit.html index de7a4fb3..b489cbaf 100644 --- a/cps/templates/user_edit.html +++ b/cps/templates/user_edit.html @@ -83,7 +83,7 @@
    - {% if ( g.user and g.user.role_admin() and not new_user ) %} + {% if ( g.user and g.user.role_admin() and not new_user ) and not simple %} {{_('Add Allowed/Denied Tags')}} {{_('Add allowed/Denied Custom Column Values')}} {% endif %} @@ -131,7 +131,7 @@
    {% endif %} {% endif %} - {% if kobo_support and not content.role_anonymous() %} + {% if kobo_support and not content.role_anonymous() and not simple%}
    diff --git a/cps/tornado_wsgi.py b/cps/tornado_wsgi.py new file mode 100644 index 00000000..af93219c --- /dev/null +++ b/cps/tornado_wsgi.py @@ -0,0 +1,94 @@ +# -*- coding: utf-8 -*- + +# This file is part of the Calibre-Web (https://github.com/janeczku/calibre-web) +# Copyright (C) 2022 OzzieIsaacs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +from tornado.wsgi import WSGIContainer +import tornado + +from tornado import escape +from tornado import httputil + +from typing import List, Tuple, Optional, Callable, Any, Dict, Text +from types import TracebackType +import typing + +if typing.TYPE_CHECKING: + from typing import Type # noqa: F401 + from wsgiref.types import WSGIApplication as WSGIAppType # noqa: F4 + +class MyWSGIContainer(WSGIContainer): + + def __call__(self, request: httputil.HTTPServerRequest) -> None: + data = {} # type: Dict[str, Any] + response = [] # type: List[bytes] + + def start_response( + status: str, + headers: List[Tuple[str, str]], + exc_info: Optional[ + Tuple[ + "Optional[Type[BaseException]]", + Optional[BaseException], + Optional[TracebackType], + ] + ] = None, + ) -> Callable[[bytes], Any]: + data["status"] = status + data["headers"] = headers + return response.append + + app_response = self.wsgi_application( + MyWSGIContainer.environ(request), start_response + ) + try: + response.extend(app_response) + body = b"".join(response) + finally: + if hasattr(app_response, "close"): + app_response.close() # type: ignore + if not data: + raise Exception("WSGI app did not call start_response") + + status_code_str, reason = data["status"].split(" ", 1) + status_code = int(status_code_str) + headers = data["headers"] # type: List[Tuple[str, str]] + header_set = set(k.lower() for (k, v) in headers) + body = escape.utf8(body) + if status_code != 304: + if "content-length" not in header_set: + headers.append(("Content-Length", str(len(body)))) + if "content-type" not in header_set: + headers.append(("Content-Type", "text/html; charset=UTF-8")) + if "server" not in header_set: + headers.append(("Server", "TornadoServer/%s" % tornado.version)) + + start_line = httputil.ResponseStartLine("HTTP/1.1", status_code, reason) + header_obj = httputil.HTTPHeaders() + for key, value in headers: + header_obj.add(key, value) + assert request.connection is not None + request.connection.write_headers(start_line, header_obj, chunk=body) + request.connection.finish() + self._log(status_code, request) + + @staticmethod + def environ(request: httputil.HTTPServerRequest) -> Dict[Text, Any]: + environ = WSGIContainer.environ(request) + environ['RAW_URI'] = request.path + return environ + diff --git a/cps/web.py b/cps/web.py index babc3bcc..bc05ec66 100644 --- a/cps/web.py +++ b/cps/web.py @@ -49,7 +49,7 @@ from . import constants, logger, isoLanguages, services from . import babel, db, ub, config, get_locale, app from . import calibre_db, kobo_sync_status from .gdriveutils import getFileFromEbooksFolder, do_gdrive_download -from .helper import check_valid_domain, render_task_status, check_email, check_username, get_cc_columns, \ +from .helper import check_valid_domain, render_task_status, check_email, check_username, \ get_book_cover, get_series_cover_thumbnail, get_download_link, send_mail, generate_random_password, \ send_registration_mail, check_send_to_kindle, check_read_formats, tags_filters, reset_password, valid_email, \ edit_book_read_status @@ -85,7 +85,10 @@ except ImportError: def add_security_headers(resp): csp = "default-src 'self'" csp += ''.join([' ' + host for host in config.config_trustedhosts.strip().split(',')]) - csp += " 'unsafe-inline' 'unsafe-eval'; font-src 'self' data:; img-src 'self' data:" + csp += " 'unsafe-inline' 'unsafe-eval'; font-src 'self' data:; img-src 'self' " + if request.path.startswith("/author/") and config.config_use_goodreads: + csp += "images.gr-assets.com i.gr-assets.com s.gr-assets.com" + csp += " data:" resp.headers['Content-Security-Policy'] = csp if request.endpoint == "edit-book.show_edit_book" or config.config_use_google_drive: resp.headers['Content-Security-Policy'] += " *" @@ -350,7 +353,7 @@ def render_books_list(data, sort_param, book_id, page): if data == "rated": return render_rated_books(page, book_id, order=order) elif data == "discover": - return render_discover_books(page, book_id) + return render_discover_books(book_id) elif data == "unread": return render_read_books(page, False, order=order) elif data == "read": @@ -386,7 +389,7 @@ def render_books_list(data, sort_param, book_id, page): else: website = data or "newest" entries, random, pagination = calibre_db.fill_indexpage(page, 0, db.Books, True, order[0], - False, 0, + True, config.config_read_column, db.books_series_link, db.Books.id == db.books_series_link.c.book, db.Series) @@ -400,7 +403,7 @@ def render_rated_books(page, book_id, order): db.Books, db.Books.ratings.any(db.Ratings.rating > 9), order[0], - False, 0, + True, config.config_read_column, db.books_series_link, db.Books.id == db.books_series_link.c.book, db.Series) @@ -411,11 +414,13 @@ def render_rated_books(page, book_id, order): abort(404) -def render_discover_books(page, book_id): +def render_discover_books(book_id): if current_user.check_visibility(constants.SIDEBAR_RANDOM): - entries, __, pagination = calibre_db.fill_indexpage(page, 0, db.Books, True, [func.randomblob(2)]) + entries, __, ___ = calibre_db.fill_indexpage(1, 0, db.Books, True, [func.randomblob(2)], + join_archive_read=True, + config_read_column=config.config_read_column) pagination = Pagination(1, config.config_books_per_page, config.config_books_per_page) - return render_title_template('discover.html', entries=entries, pagination=pagination, id=book_id, + return render_title_template('index.html', random=false(), entries=entries, pagination=pagination, id=book_id, title=_(u"Discover (Random Books)"), page="discover") else: abort(404) @@ -429,18 +434,22 @@ def render_hot_books(page, order): # order[0][0].compare(func.count(ub.Downloads.book_id).asc())): order = [func.count(ub.Downloads.book_id).desc()], 'hotdesc' if current_user.show_detail_random(): - random = calibre_db.session.query(db.Books).filter(calibre_db.common_filters()) \ - .order_by(func.random()).limit(config.config_random_books) + random_query = calibre_db.generate_linked_query(config.config_read_column, db.Books) + random = (random_query.filter(calibre_db.common_filters()) + .order_by(func.random()) + .limit(config.config_random_books).all()) else: random = false() + off = int(int(config.config_books_per_page) * (page - 1)) all_books = ub.session.query(ub.Downloads, func.count(ub.Downloads.book_id)) \ .order_by(*order[0]).group_by(ub.Downloads.book_id) hot_books = all_books.offset(off).limit(config.config_books_per_page) entries = list() for book in hot_books: - download_book = calibre_db.session.query(db.Books).filter(calibre_db.common_filters()).filter( - db.Books.id == book.Downloads.book_id).first() + query = calibre_db.generate_linked_query(config.config_read_column, db.Books) + download_book = query.filter(calibre_db.common_filters()).filter( + book.Downloads.book_id == db.Books.id).first() if download_book: entries.append(download_book) else: @@ -459,26 +468,20 @@ def render_downloaded_books(page, order, user_id): else: user_id = current_user.id if current_user.check_visibility(constants.SIDEBAR_DOWNLOAD): - if current_user.show_detail_random(): - random = calibre_db.session.query(db.Books).filter(calibre_db.common_filters()) \ - .order_by(func.random()).limit(config.config_random_books) - else: - random = false() - - entries, __, pagination = calibre_db.fill_indexpage(page, + entries, random, pagination = calibre_db.fill_indexpage(page, 0, db.Books, ub.Downloads.user_id == user_id, order[0], - False, 0, + True, config.config_read_column, db.books_series_link, db.Books.id == db.books_series_link.c.book, db.Series, ub.Downloads, db.Books.id == ub.Downloads.book_id) for book in entries: - if not calibre_db.session.query(db.Books).\ - filter(calibre_db.common_filters()).filter(db.Books.id == book.id).first(): - ub.delete_download(book.id) + if not (calibre_db.session.query(db.Books).filter(calibre_db.common_filters()) + .filter(db.Books.id == book.Books.id).first()): + ub.delete_download(book.Books.id) user = ub.session.query(ub.User).filter(ub.User.id == user_id).first() return render_title_template('index.html', random=random, @@ -497,9 +500,9 @@ def render_author_books(page, author_id, order): db.Books, db.Books.authors.any(db.Authors.id == author_id), [order[0][0], db.Series.name, db.Books.series_index], - False, 0, + True, config.config_read_column, db.books_series_link, - db.Books.id == db.books_series_link.c.book, + db.books_series_link.c.book == db.Books.id, db.Series) if entries is None or not len(entries): flash(_(u"Oops! Selected book title is unavailable. File does not exist or is not accessible"), @@ -515,7 +518,8 @@ def render_author_books(page, author_id, order): other_books = [] if services.goodreads_support and config.config_use_goodreads: author_info = services.goodreads_support.get_author_info(author_name) - other_books = services.goodreads_support.get_other_books(author_info, entries) + book_entries = [entry.Books for entry in entries] + other_books = services.goodreads_support.get_other_books(author_info, book_entries) return render_title_template('author.html', entries=entries, pagination=pagination, id=author_id, title=_(u"Author: %(name)s", name=author_name), author=author_info, other_books=other_books, page="author", order=order[1]) @@ -528,7 +532,7 @@ def render_publisher_books(page, book_id, order): db.Books, db.Books.publishers.any(db.Publishers.id == book_id), [db.Series.name, order[0][0], db.Books.series_index], - False, 0, + True, config.config_read_column, db.books_series_link, db.Books.id == db.books_series_link.c.book, db.Series) @@ -546,7 +550,8 @@ def render_series_books(page, book_id, order): entries, random, pagination = calibre_db.fill_indexpage(page, 0, db.Books, db.Books.series.any(db.Series.id == book_id), - [order[0][0]]) + [order[0][0]], + True, config.config_read_column) return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id, title=_(u"Series: %(serie)s", serie=name.name), page="series", order=order[1]) else: @@ -558,7 +563,8 @@ def render_ratings_books(page, book_id, order): entries, random, pagination = calibre_db.fill_indexpage(page, 0, db.Books, db.Books.ratings.any(db.Ratings.id == book_id), - [order[0][0]]) + [order[0][0]], + True, config.config_read_column) if name and name.rating <= 10: return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id, title=_(u"Rating: %(rating)s stars", rating=int(name.rating / 2)), @@ -574,7 +580,8 @@ def render_formats_books(page, book_id, order): entries, random, pagination = calibre_db.fill_indexpage(page, 0, db.Books, db.Books.data.any(db.Data.format == book_id.upper()), - [order[0][0]]) + [order[0][0]], + True, config.config_read_column) return render_title_template('index.html', random=random, pagination=pagination, entries=entries, id=book_id, title=_(u"File format: %(format)s", format=name.format), page="formats", @@ -590,7 +597,7 @@ def render_category_books(page, book_id, order): db.Books, db.Books.tags.any(db.Tags.id == book_id), [order[0][0], db.Series.name, db.Books.series_index], - False, 0, + True, config.config_read_column, db.books_series_link, db.Books.id == db.books_series_link.c.book, db.Series) @@ -609,7 +616,8 @@ def render_language_books(page, name, order): entries, random, pagination = calibre_db.fill_indexpage(page, 0, db.Books, db.Books.languages.any(db.Languages.lang_code == name), - [order[0][0]]) + [order[0][0]], + True, config.config_read_column) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, id=name, title=_(u"Language: %(name)s", name=lang_name), page="language", order=order[1]) @@ -622,30 +630,12 @@ def render_read_books(page, are_read, as_xml=False, order=None): ub.ReadBook.read_status == ub.ReadBook.STATUS_FINISHED) else: db_filter = coalesce(ub.ReadBook.read_status, 0) != ub.ReadBook.STATUS_FINISHED - entries, random, pagination = calibre_db.fill_indexpage(page, 0, - db.Books, - db_filter, - sort_param, - False, 0, - db.books_series_link, - db.Books.id == db.books_series_link.c.book, - db.Series, - ub.ReadBook, db.Books.id == ub.ReadBook.book_id) else: try: if are_read: db_filter = db.cc_classes[config.config_read_column].value == True else: db_filter = coalesce(db.cc_classes[config.config_read_column].value, False) != True - entries, random, pagination = calibre_db.fill_indexpage(page, 0, - db.Books, - db_filter, - sort_param, - False, 0, - db.books_series_link, - db.Books.id == db.books_series_link.c.book, - db.Series, - db.cc_classes[config.config_read_column]) except (KeyError, AttributeError, IndexError): log.error("Custom Column No.{} is not existing in calibre database".format(config.config_read_column)) if not as_xml: @@ -655,6 +645,15 @@ def render_read_books(page, are_read, as_xml=False, order=None): return redirect(url_for("web.index")) return [] # ToDo: Handle error Case for opds + entries, random, pagination = calibre_db.fill_indexpage(page, 0, + db.Books, + db_filter, + sort_param, + True, config.config_read_column, + db.books_series_link, + db.Books.id == db.books_series_link.c.book, + db.Series) + if as_xml: return entries, pagination else: @@ -683,7 +682,7 @@ def render_archived_books(page, sort_param): archived_filter, order, True, - False, 0) + True, config.config_read_column) name = _(u'Archived Books') + ' (' + str(len(archived_book_ids)) + ')' page_name = "archived" @@ -723,12 +722,12 @@ def render_prepare_search_form(cc): def render_search_results(term, offset=None, order=None, limit=None): - join = db.books_series_link, db.Books.id == db.books_series_link.c.book, db.Series + join = db.books_series_link, db.books_series_link.c.book == db.Books.id, db.Series entries, result_count, pagination = calibre_db.get_search_results(term, + config, offset, order, limit, - config.config_read_column, *join) return render_title_template('search.html', searchterm=term, @@ -766,7 +765,7 @@ def books_list(data, sort_param, book_id, page): @login_required def books_table(): visibility = current_user.view_settings.get('table', {}) - cc = get_cc_columns(filter_config_custom_read=True) + cc = calibre_db.get_cc_columns(config, filter_config_custom_read=True) return render_title_template('book_table.html', title=_(u"Books List"), cc=cc, page="book_table", visiblility=visibility) @@ -810,37 +809,18 @@ def list_books(): calibre_db.common_filters(allow_show_archived=True)).count() if state is not None: if search_param: - books = calibre_db.search_query(search_param, config.config_read_column).all() + books = calibre_db.search_query(search_param, config).all() filtered_count = len(books) else: - if not config.config_read_column: - books = (calibre_db.session.query(db.Books, ub.ReadBook.read_status, ub.ArchivedBook.is_archived) - .select_from(db.Books) - .outerjoin(ub.ReadBook, - and_(ub.ReadBook.user_id == int(current_user.id), - ub.ReadBook.book_id == db.Books.id))) - else: - read_column = "" - try: - read_column = db.cc_classes[config.config_read_column] - books = (calibre_db.session.query(db.Books, read_column.value, ub.ArchivedBook.is_archived) - .select_from(db.Books) - .outerjoin(read_column, read_column.book == db.Books.id)) - except (KeyError, AttributeError, IndexError): - log.error( - "Custom Column No.{} is not existing in calibre database".format(config.config_read_column)) - # Skip linking read column and return None instead of read status - books = calibre_db.session.query(db.Books, None, ub.ArchivedBook.is_archived) - books = (books.outerjoin(ub.ArchivedBook, and_(db.Books.id == ub.ArchivedBook.book_id, - int(current_user.id) == ub.ArchivedBook.user_id)) - .filter(calibre_db.common_filters(allow_show_archived=True)).all()) + query = calibre_db.generate_linked_query(config.config_read_column, db.Books) + books = query.filter(calibre_db.common_filters(allow_show_archived=True)).all() entries = calibre_db.get_checkbox_sorted(books, state, off, limit, order, True) elif search_param: entries, filtered_count, __ = calibre_db.get_search_results(search_param, + config, off, [order, ''], limit, - config.config_read_column, *join) else: entries, __, __ = calibre_db.fill_indexpage_with_archived_books((int(off) / (int(limit)) + 1), @@ -856,8 +836,8 @@ def list_books(): result = list() for entry in entries: val = entry[0] - val.read_status = entry[1] == ub.ReadBook.STATUS_FINISHED - val.is_archived = entry[2] is True + val.is_archived = entry[1] is True + val.read_status = entry[2] == ub.ReadBook.STATUS_FINISHED for lang_index in range(0, len(val.languages)): val.languages[lang_index].language_name = isoLanguages.get_language_name(get_locale(), val.languages[ lang_index].lang_code) @@ -1252,26 +1232,10 @@ def render_adv_search_results(term, offset=None, order=None, limit=None): sort_param = order[0] if order else [db.Books.sort] pagination = None - cc = get_cc_columns(filter_config_custom_read=True) + cc = calibre_db.get_cc_columns(config, filter_config_custom_read=True) calibre_db.session.connection().connection.connection.create_function("lower", 1, db.lcase) - if not config.config_read_column: - query = (calibre_db.session.query(db.Books, ub.ArchivedBook.is_archived, ub.ReadBook).select_from(db.Books) - .outerjoin(ub.ReadBook, and_(db.Books.id == ub.ReadBook.book_id, - int(current_user.id) == ub.ReadBook.user_id))) - else: - try: - read_column = cc[config.config_read_column] - query = (calibre_db.session.query(db.Books, ub.ArchivedBook.is_archived, read_column.value) - .select_from(db.Books) - .outerjoin(read_column, read_column.book == db.Books.id)) - except (KeyError, AttributeError, IndexError): - log.error("Custom Column No.{} is not existing in calibre database".format(config.config_read_column)) - # Skip linking read column - query = calibre_db.session.query(db.Books, ub.ArchivedBook.is_archived, None) - query = query.outerjoin(ub.ArchivedBook, and_(db.Books.id == ub.ArchivedBook.book_id, - int(current_user.id) == ub.ArchivedBook.user_id)) - - q = query.outerjoin(db.books_series_link, db.Books.id == db.books_series_link.c.book) \ + query = calibre_db.generate_linked_query(config.config_read_column, db.Books) + q = query.outerjoin(db.books_series_link, db.books_series_link.c.book == db.Books.id) \ .outerjoin(db.Series) \ .filter(calibre_db.common_filters(True)) @@ -1357,7 +1321,7 @@ def render_adv_search_results(term, offset=None, order=None, limit=None): if description: q = q.filter(db.Books.comments.any(func.lower(db.Comments.text).ilike("%" + description + "%"))) - # search custom culumns + # search custom columns try: q = adv_search_custom_columns(cc, term, q) except AttributeError as ex: @@ -1390,7 +1354,7 @@ def render_adv_search_results(term, offset=None, order=None, limit=None): @login_required_if_no_ano def advanced_search_form(): # Build custom columns names - cc = get_cc_columns(filter_config_custom_read=True) + cc = calibre_db.get_cc_columns(config, filter_config_custom_read=True) return render_prepare_search_form(cc) @@ -1800,10 +1764,10 @@ def show_book(book_id): for lang_index in range(0, len(entry.languages)): entry.languages[lang_index].language_name = isoLanguages.get_language_name(get_locale(), entry.languages[ lang_index].lang_code) - cc = get_cc_columns(filter_config_custom_read=True) + cc = calibre_db.get_cc_columns(config, filter_config_custom_read=True) book_in_shelves = [] - shelfs = ub.session.query(ub.BookShelf).filter(ub.BookShelf.book_id == book_id).all() - for sh in shelfs: + shelves = ub.session.query(ub.BookShelf).filter(ub.BookShelf.book_id == book_id).all() + for sh in shelves: book_in_shelves.append(sh.shelf) entry.tags = sort(entry.tags, key=lambda tag: tag.name) diff --git a/optional-requirements.txt b/optional-requirements.txt index e54b0829..208d889d 100644 --- a/optional-requirements.txt +++ b/optional-requirements.txt @@ -1,5 +1,5 @@ # GDrive Integration -google-api-python-client>=1.7.11,<2.42.0 +google-api-python-client>=1.7.11,<2.43.0 gevent>20.6.0,<22.0.0 greenlet>=0.4.17,<1.2.0 httplib2>=0.9.2,<0.21.0 @@ -13,7 +13,7 @@ rsa>=3.4.2,<4.9.0 # Gmail google-auth-oauthlib>=0.4.3,<0.6.0 -google-api-python-client>=1.7.11,<2.42.0 +google-api-python-client>=1.7.11,<2.43.0 # goodreads goodreads>=0.3.2,<0.4.0 diff --git a/requirements.txt b/requirements.txt index 985bbaf1..7819afe3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ APScheduler>=3.6.3,<3.10.0 +werkzeug<2.1.0 Babel>=1.3,<3.0 Flask-Babel>=0.11.1,<2.1.0 Flask-Login>=0.3.2,<0.5.1 diff --git a/setup.cfg b/setup.cfg index 38fed3f6..4497839b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -60,7 +60,7 @@ install_requires = [options.extras_require] gdrive = - google-api-python-client>=1.7.11,<2.37.0 + google-api-python-client>=1.7.11,<2.43.0 gevent>20.6.0,<22.0.0 greenlet>=0.4.17,<1.2.0 httplib2>=0.9.2,<0.21.0 @@ -73,7 +73,7 @@ gdrive = rsa>=3.4.2,<4.9.0 gmail = google-auth-oauthlib>=0.4.3,<0.5.0 - google-api-python-client>=1.7.11,<2.37.0 + google-api-python-client>=1.7.11,<2.43.0 goodreads = goodreads>=0.3.2,<0.4.0 python-Levenshtein>=0.12.0,<0.13.0