Merge pull request #26 from iiab/deldesir-youtube-download

Youtube download support

cps/editbooks.py

@@ -311,6 +311,223 @@ def upload():
         return Response(json.dumps({"location": url_for("web.index")}), mimetype='application/json')
 
 
+@editbook.route("/youtube", methods=["POST"])
+@login_required_if_no_ano
+@upload_required
+def youtube():
+    if not config.config_uploading:
+        abort(404)
+    
+    def get_yb_executable():
+        yb_executable = os.getenv("YB_EXECUTABLE", "yb")
+        return yb_executable
+
+    def run_subprocess(command_args):
+        try:
+            completed_process = subprocess.run(
+                command_args,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                check=True,
+                text=True
+            )
+            return (True, completed_process.stdout, completed_process.stderr)
+
+        except subprocess.CalledProcessError as e:
+            error_message = f"Subprocess error (return code {e.returncode}): {e.stderr}, {e.stdout}"
+            log.error(error_message)
+            return False, error_message, e.stdout, e.stderr
+        except Exception as e:
+            error_message = f"An error occurred while running the subprocess: {e}"
+            log.error(error_message)
+            return False, error_message
+    
+    def process_youtube_download(youtube_url, video_quality):
+        yb_executable = get_yb_executable()
+
+        if youtube_url:
+            youtube_id = extract_youtube_url(youtube_url)
+
+            download_args = [
+                yb_executable,
+                youtube_id,
+                "/output",
+                video_quality,
+            ]
+            subprocess_result = run_subprocess(download_args)
+            
+            log.info("Subprocess result: {}".format(subprocess_result))
+
+            if subprocess_result[0]:
+                log.info("Renaming files in /output/{}/videos".format(youtube_id))
+                # make a list of requested files with names of directories found in /output/<youtube_id>/videos
+                requested_files = os.listdir("/output/{}/videos".format(youtube_id))
+                # remove "youtube-nsig" from the list of requested files
+                requested_files.remove("youtube-nsig")
+                log.info("Requested files: {}".format(requested_files))
+                renamed_files = rename_files(requested_files, youtube_id)
+                if renamed_files:    
+                    for requested_file in renamed_files:
+                        requested_file = open(requested_file, "rb")
+                        requested_file.filename = os.path.basename(requested_file.name)
+                        requested_file.save = lambda path: copyfile(requested_file.name, path)
+
+                        log.info("Processing file: {}".format(requested_file))
+                        try:
+                            modify_date = False
+                            calibre_db.update_title_sort(config)
+                            calibre_db.session.connection().connection.connection.create_function(
+                                "uuid4", 0, lambda: str(uuid4())
+                            )
+
+                            meta, error = file_handling_on_upload(requested_file)
+                            if error:
+                                return error
+
+                            (
+                                db_book,
+                                input_authors,
+                                title_dir,
+                                renamed_authors,
+                            ) = create_book_on_upload(modify_date, meta)
+
+                            # Comments need book id therefore only possible after flush
+                            modify_date |= edit_book_comments(
+                                Markup(meta.description).unescape(), db_book
+                            )
+
+                            book_id = db_book.id
+                            title = db_book.title
+                           
+                            error = helper.update_dir_structure(
+                                book_id,
+                                config.config_calibre_dir,
+                                input_authors[0],
+                                meta.file_path,
+                                title_dir + meta.extension.lower(),
+                                renamed_author=renamed_authors,
+                            )
+
+                            move_coverfile(meta, db_book)
+
+                            if modify_date:
+                                calibre_db.set_metadata_dirty(book_id)
+                            # save data to database, reread data
+                            calibre_db.session.commit()
+
+                            if error:
+                                flash(error, category="error")
+                            link = '<a href="{}">{}</a>'.format(
+                                url_for("web.show_book", book_id=book_id), escape(title)
+                            )
+                            upload_text = N_("File %(file)s uploaded", file=link)
+                            WorkerThread.add(
+                                current_user.name, TaskUpload(upload_text, escape(title))
+                            )
+                            helper.add_book_to_thumbnail_cache(book_id)
+
+                            if len(renamed_files) < 2:
+                                if current_user.role_edit() or current_user.role_admin():
+                                    resp = {
+                                        "location": url_for(
+                                            "edit-book.show_edit_book", book_id=book_id
+                                        )
+                                    }
+                                    return Response(json.dumps(resp), mimetype="application/json")
+                                else:
+                                    resp = {"location": url_for("web.show_book", book_id=book_id)}
+                                    return Response(json.dumps(resp), mimetype="application/json")
+
+                        except (OperationalError, IntegrityError, StaleDataError) as e:
+                            calibre_db.session.rollback()
+                            log.error_or_exception("Database error: {}".format(e))
+                            flash(
+                                _(
+                                    "Oops! Database Error: %(error)s.",
+                                    error=e.orig if hasattr(e, "orig") else e,
+                                ),
+                                category="error",
+                            )
+        else:
+            flash("Error: 'poetry' executable not found in PATH", category="error")
+            return False
+
+    if request.method == "POST" and "youtubeURL" in request.form:
+        youtube_url = request.form["youtubeURL"]
+        video_quality = request.form.get("videoQuality", "720")
+
+        if process_youtube_download(youtube_url, video_quality):
+            response = {
+                "success": "Downloaded YouTube media successfully",
+            }
+            return jsonify(response)
+        else:
+            response = {
+                "error": "Failed to download YouTube media",
+            }
+            return jsonify(response), 500
+
+def extract_youtube_url(url):
+        try:
+            if "youtube.com" in url:
+                if "watch?v=" in url:
+                    return url.split("watch?v=")[1]
+                elif "playlist?list=" in url:
+                    return url.split("playlist?list=")[1]
+                elif "channel/" in url:
+                    return url.split("channel/")[1]
+                elif "user/" in url:
+                    return url.split("user/")[1]
+                elif "@" in url:
+                    return extract_channel_id_from_handle(url)
+            elif "youtu.be" in url:
+                return url.split("youtu.be/")[1]
+
+            flash("Error: Invalid YouTube URL", category="error")
+            return None
+        except Exception as e:
+            flash("An error occurred while processing the YouTube URL: {}".format(e), category="error")
+            return None
+
+def extract_channel_id_from_handle(url):
+    handle = url.split("@")[1]
+    operational_api_url = "https://yt.lemnoslife.com/channels?handle=" + handle
+    response = requests.get(operational_api_url)
+
+    if response.status_code == 200:
+        return response.json()["items"][0]["id"]
+    else:
+        flash("Error: Failed to retrieve YouTube channel ID from API", category="error")
+        return None
+
+def rename_files(requested_files, youtube_id):
+    # cache_directory_path = "/output/{}/cache".format(youtube_id)
+    video_dir_path = "/output/{}/videos".format(youtube_id)
+    renamed_files = []
+    if not os.path.exists("/tmp/calibre_web"):
+        os.makedirs("/tmp/calibre_web")
+
+    for video_id in requested_files:
+        # video_json_path = os.path.join(cache_directory_path, "videos.json")
+        video_webm_path = os.path.join(video_dir_path, video_id, "video.webm")
+        thumbnail_path = os.path.join(video_dir_path, video_id, "video.webp")
+
+        try:
+            thumbnail_path_new = os.path.join("/tmp", "calibre_web", "{}.webp".format(video_id + youtube_id))
+            move(thumbnail_path, thumbnail_path_new)
+            video_webm_path_new = os.path.join("/tmp", "calibre_web", "{}.webm".format(video_id + youtube_id))
+            move(video_webm_path, video_webm_path_new)
+            renamed_files.append(video_webm_path_new)
+
+            if not os.listdir(video_dir_path):
+                os.rmdir(video_dir_path)
+
+        except Exception as e:
+            flash("An error occurred while renaming the YouTube video file: {}".format(e), category="error")
+            return None
+        
+    return renamed_files
+
 @editbook.route("/admin/book/convert/<int:book_id>", methods=['POST'])
 @login_required_if_no_ano
 @edit_required

cps/static/js/main.js

@@ -149,6 +149,133 @@ $(document).ready(function() {
             inp.val('').blur().focus().val(val)
         }
     }
+
+    // Function to toggle advanced options visibility
+    function toggleAdvancedOptions() {
+        var advancedOptions = $("#advancedOptions");
+        if (advancedOptions.is(":visible")) {
+            advancedOptions.hide();
+            $("#advancedOptionsToggle").text("Show advanced options")
+        } else {
+            advancedOptions.show();
+            $("#advancedOptionsToggle").text("Hide advanced options")
+        }
+    }
+
+    // Handle click event for the advanced options toggle
+    $("#advancedOptionsToggle").click(function(event) {
+        event.preventDefault();
+        toggleAdvancedOptions();
+    });
+
+    // Function to initiate the YouTube download AJAX request
+    function initiateYoutubeDownload() {
+        var url = $("#youtubeURL").val();
+        var videoQuality = $("input[name='videoQuality']:checked").val();
+        var maxVideos = $("#maxVideos").val();
+        var maxVideosSize = $("#maxVideosSize").val();
+        var addToBookshelf = $("#addToBookshelf").is(":checked");
+
+        // Set empty number values to zero
+        maxVideos = maxVideos === "" ? 0 : parseInt(maxVideos);
+        maxVideosSize = maxVideosSize === "" ? 0 : parseInt(maxVideosSize);
+
+        // Check if the input URL is a valid YouTube URL
+        if (!isValidYoutubeURL(url)) {
+            alert("Invalid YouTube URL");
+            return;
+        }
+
+        $.ajax({
+            url: "/books/youtube",
+            method: "POST",
+            data: {
+                csrf_token: $("#youtubeDownloadForm input[name=csrf_token]").val(),
+                youtubeURL: url,
+                videoQuality: videoQuality,
+                maxVideos: maxVideos,
+                maxVideosSize: maxVideosSize,
+                addToBookshelf: addToBookshelf
+            },
+            success: function(response) {
+                // Handle success response here
+                if (response && response.location) {
+                    // Redirect to the specified location
+                    window.location.href = response.location;
+                } else {
+                    // Handle any specific success behavior
+                    console.log("YouTube download request successful.");
+                }
+            },
+            error: function(xhr, status, error) {
+                // Handle error here
+                console.log("YouTube download request failed:", error);
+                $("#youtubeDownloadForm .error-message").text("YouTube download request failed.");
+            }
+        });
+    }
+
+    // Handle Enter key press event in the input field
+    $(document).on('keydown', function(event) {
+        // Check if the pressed key is Enter (key code 13)
+        if (event.which === 13 && $("#youtubeDownloadModal").is(":visible")) {
+            initiateYoutubeDownload();
+        }
+    });
+
+    // Handle the "Start" button click event
+    $("#btn-download-youtube-submit").click(function() {
+        initiateYoutubeDownload();
+    });
+
+    // Handle change event for the video quality radio buttons
+    $("input[name='videoQuality']").change(function() {
+        // Handle change event
+    });
+
+    // Handle input event for the max videos input
+    $("#maxVideos").on('input', function() {
+        var inputValue = $(this).val();
+        if (!/^\d*$/.test(inputValue)) {
+            alert("Please enter a valid number.");
+            $(this).val("");
+        }
+
+        // If maxVideos is changed, disable and clear maxVideosSize
+        if (inputValue) {
+            $("#maxVideosSize").prop("disabled", true).val("");
+        } else {
+            $("#maxVideosSize").prop("disabled", false);
+        }
+    });
+
+    // Handle input event for the max size input
+    $("#maxVideosSize").on('input', function() {
+        var inputValue = $(this).val();
+        if (!/^\d*$/.test(inputValue)) {
+            alert("Please enter a valid number.");
+            $(this).val("");
+        }
+
+        // If maxVideosSize is changed, disable and clear maxVideos
+        if (inputValue) {
+            $("#maxVideos").prop("disabled", true).val("");
+        } else {
+            $("#maxVideos").prop("disabled", false);
+        }
+    });
+
+    // Handle change event for the add to bookshelf checkbox
+    $("#addToBookshelf").change(function() {
+        // Handle change event
+    });
+
+    // Function to validate YouTube URL (updated to handle https://youtube.com/@handle)
+    function isValidYoutubeURL(url) {
+        var youtubeURLPattern = /^(https?:\/\/)?(www\.)?(youtube\.com\/watch\?v=|youtu\.be\/|youtube\.com\/playlist\?list=|youtube\.com\/channel\/|youtube\.com\/@)([a-zA-Z0-9_-]{11}|[a-zA-Z0-9_-]{34})/;
+        return youtubeURLPattern.test(url);
+    }
+
 });
 
 $(".session").click(function() {

cps/templates/layout.html

@@ -1,4 +1,4 @@
-{% from 'modal_dialogs.html' import restrict_modal, delete_book, filechooser_modal, delete_confirm_modal, change_confirm_modal %}
+{% from 'modal_dialogs.html' import restrict_modal, delete_book, filechooser_modal, delete_confirm_modal, change_confirm_modal, youtube_download_modal %}
 {% import 'image.html' as image %}
 <!DOCTYPE html>
 <html lang="{{ current_user.locale }}">
@@ -83,6 +83,13 @@
                       </div>
                     </form>
                   </li>
+                  <li>
+                  <button id="btn-download-youtube" class="btn btn-default navbar-btn" data-toggle="modal"
+                    data-target="#youtubeDownloadModal">
+                    <span class="glyphicon glyphicon-download-alt"></span>
+                    {{_(' Add YouTube Videos')}}
+                  </button>
+                </li>
               {% endif %}
               {% if not current_user.is_anonymous and not simple%}
                 <li class="top_tasks"><a id="top_tasks" href="{{url_for('tasks.get_tasks_status')}}"><span class="glyphicon glyphicon-tasks"></span> <span class="hidden-sm">{{_('Tasks')}}</span></a></li>
@@ -205,7 +212,9 @@
         </div>
       </div>
     </div>
-    {% block modal %}{% endblock %}
+    {% block modal %}
+    {{ youtube_download_modal() }}
+    {% endblock %}
     <!-- jQuery (necessary for Bootstrap's JavaScript plugins) -->
     <script src="{{ url_for('static', filename='js/libs/jquery.min.js') }}"></script>
     <!-- Include all compiled plugins (below), or include individual files as needed -->

cps/templates/modal_dialogs.html

@@ -139,3 +139,84 @@
 </div>
 
 {% endmacro %}
+
+{% macro youtube_download_modal() %}
+<div class="modal fade" id="youtubeDownloadModal" tabindex="-1" role="dialog"
+  aria-labelledby="youtubeDownloadModalLabel">
+  <div class="modal-dialog" role="document">
+    <div class="modal-content">
+      <div class="modal-header">
+        <button type="button" class="close" data-dismiss="modal" aria-label="Close"><span
+            aria-hidden="true">&times;</span></button>
+        <h4 class="modal-title" id="youtubeDownloadModalLabel">{{_('Download a YouTube Channel or Playlist to your
+          Internet-in-a-Box')}}</h4>
+      </div>
+      <div class="modal-body">
+        <form id="youtubeDownloadForm" class="form-horizontal" action="{{ url_for('edit-book.youtube') }}"
+          method="post">
+          <input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
+          <div class="form-group">
+            <label for="youtubeURL" class="col-sm-2 control-label">{{_('YouTube URL')}}</label>
+            <div class="col-sm-10">
+              <input type="text" class="form-control" id="youtubeURL" name="youtubeURL"
+                placeholder="{{_('Enter YouTube URL')}}">
+            </div>
+          </div>
+          <!-- Advanced Options Toggle Button -->
+          <div class="form-group">
+            <div class="col-sm-offset-2 col-sm-10">
+              <a href="#" id="advancedOptionsToggle">Show Advanced Options</a>
+            </div>
+          </div>
+          <!-- Advanced Options (Initially Hidden) -->
+          <div id="advancedOptions" style="display: none;">
+            <!-- Video Quality -->
+            <div class="form-group">
+              <label class="col-sm-2 control-label">{{_('Video Quality')}}</label>
+              <div class="col-sm-10">
+                <label class="radio-inline">
+                  <input type="radio" name="videoQuality" value="480">480p
+                </label>
+                <label class="radio-inline">
+                  <input type="radio" name="videoQuality" value="720" checked>720p
+                </label>
+                <label class="radio-inline">
+                  <input type="radio" name="videoQuality" value="1080">1080p
+                </label>
+                <!-- Add more quality options as needed -->
+              </div>
+            </div>
+            <!-- Maximum Number of Videos to Download -->
+            <div class="form-group">
+              <label for="maxVideos" class="col-sm-2 control-label">{{_('Max Videos')}}</label>
+              <div class="col-sm-10">
+                <input type="number" class="form-control" id="maxVideos" name="maxVideos" min="1" max="100">
+              </div>
+            </div>
+            <!-- Maximum Size of All Videos -->
+            <div class="form-group">
+              <label for="maxVideosSize" class="col-sm-2 control-label">{{_('Max Size (GB)')}}</label>
+              <div class="col-sm-10">
+                <input type="number" class="form-control" id="maxVideosSize" name="maxVideosSize" min="1" max="5000">
+              </div>
+            </div>
+            <!-- Add All Videos to Bookshelf -->
+            <div class="form-group">
+              <div class="col-sm-offset-2 col-sm-10">
+                <label class="checkbox-inline">
+                  <input type="checkbox" id="addToBookshelf" name="addToBookshelf"> Add All Videos to Bookshelf
+                </label>
+              </div>
+            </div>
+        </form>
+      </div>
+      <div class="modal-footer">
+        <button type="button" class="btn btn-default" data-dismiss="modal">{{_('Cancel')}}</button>
+        <button type="button" id="btn-download-youtube-submit" class="btn btn-primary"
+          onclick="$('#youtubeDownloadModal').modal('hide')">{{_('Start')}}</button>
+      </div>
+    </div>
+  </div>
+</div>
+{% endmacro %}
+

cps/uploader.py

@@ -16,6 +16,7 @@
 #  You should have received a copy of the GNU General Public License
 #  along with this program. If not, see <http://www.gnu.org/licenses/>.
 
+import json
 import os
 import hashlib
 import shutil
@@ -215,7 +216,7 @@ def pdf_meta(tmp_file_path, original_file_name, original_file_extension):
         cover=pdf_preview(tmp_file_path, original_file_name),
         description=subject,
         tags=tags,
-        series="",
+        series="",Deldesir y
         series_id="",
         languages=','.join(languages),
         publisher=publisher,
@@ -303,6 +304,94 @@ def image_metadata(tmp_file_path, original_file_name, original_file_extension):
     return meta
 
 
+def video_metadata(tmp_file_path, original_file_name, original_file_extension):
+    video_id = os.path.splitext(original_file_name)[0][:11]
+    youtube_id = os.path.splitext(original_file_name)[0][11:]
+    json_file_path = os.path.join("/output", youtube_id, "cache", "videos.json")
+    coverfile_path = os.path.splitext(original_file_name)[0] + '.webp'
+    if os.path.isfile(coverfile_path):
+        coverfile_path = os.path.splitext(tmp_file_path)[0] + '.cover.webp'
+        os.rename(os.path.splitext(original_file_name)[0] + '.webp', coverfile_path)
+        os.remove(os.path.splitext(original_file_name)[0] + '.webm')
+        return coverfile_path  
+    if os.path.isfile(json_file_path):
+        with open(json_file_path) as json_file:
+            data = json.load(json_file)
+            title = data[video_id]['snippet']['title']
+            author = data[video_id]['snippet']['videoOwnerChannelTitle']
+            description = data[video_id]['snippet']['description']
+            publisher = 'YouTube'
+            pubdate = data[video_id]['contentDetails']['videoPublishedAt'][0:10]
+
+            meta = BookMeta(
+                file_path=tmp_file_path,
+                extension=original_file_extension,
+                title=title,
+                author=author,
+                cover=coverfile_path,
+                description=description,
+                tags='',
+                series="",
+                series_id="",
+                languages="",
+                publisher=publisher,
+                pubdate=pubdate,
+                identifiers=[])
+            return meta
+    else:
+        ffmpeg_executable = os.getenv('FFMPEG_PATH', 'ffmpeg')
+        ffmpeg_output_file = os.path.splitext(tmp_file_path)[0] + '.cover.jpg'
+        ffmpeg_args = [
+            ffmpeg_executable,
+            '-i', tmp_file_path,
+            '-vframes', '1',
+            '-y', ffmpeg_output_file
+        ]
+
+        try:
+            ffmpeg_result = run(ffmpeg_args, capture_output=True, check=True)
+            log.debug(f"ffmpeg output: {ffmpeg_result.stdout}")
+
+        except Exception as e:
+            log.warning(f"ffmpeg failed: {e}")
+            return None
+        
+        meta = BookMeta(
+            file_path=tmp_file_path,
+            extension=original_file_extension,
+            title=original_file_name,
+            author='Unknown',
+            cover=os.path.splitext(tmp_file_path)[0] + '.cover.jpg',
+            description='',
+            tags='',
+            series="",
+            series_id="",
+            languages="",
+            publisher="",
+            pubdate="",
+            identifiers=[])
+        return meta
+
+
+def image_metadata(tmp_file_path, original_file_name, original_file_extension):
+    shutil.copyfile(tmp_file_path, os.path.splitext(tmp_file_path)[0] + '.cover.jpg')
+    meta = BookMeta(
+        file_path=tmp_file_path,
+        extension=original_file_extension,
+        title=original_file_name,
+        author='Unknown',
+        cover=os.path.splitext(tmp_file_path)[0] + '.cover.jpg',
+        description='',
+        tags='',
+        series="",
+        series_id="",
+        languages="",
+        publisher="",
+        pubdate="",
+        identifiers=[])
+    return meta
+
+
 def get_magick_version():
     ret = dict()
     if not use_generic_pdf_cover: