[cleanup] Misc

devscripts/run_tests.sh

@@ -11,5 +11,4 @@ else
     exit 1
 fi
 
-echo python3 -m pytest -k $test_set
 python3 -m pytest -k "$test_set"

test/test_overwrites.py

@@ -8,13 +8,14 @@ import sys
 import unittest
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-from test.helper import try_rm
+from test.helper import is_download_test, try_rm
 
 
 root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 download_file = join(root_dir, 'test.webm')
 
 
+@is_download_test
 class TestOverwrites(unittest.TestCase):
     def setUp(self):
         # create an empty file

yt_dlp/YoutubeDL.py

@@ -515,8 +515,15 @@ class YoutubeDL(object):
                 self.report_warning('--merge-output-format will be ignored since --remux-video or --recode-video is given')
             self.params['merge_output_format'] = self.params['final_ext']
 
-        if 'overwrites' in self.params and self.params['overwrites'] is None:
-            del self.params['overwrites']
+        if self.params.get('overwrites') is None:
+            self.params.pop('overwrites', None)
+        elif self.params.get('nooverwrites') is not None:
+            # nooverwrites was unnecessarily changed to overwrites
+            # in 0c3d0f51778b153f65c21906031c2e091fcfb641
+            # This ensures compatibility with both keys
+            self.params['overwrites'] = not self.params['nooverwrites']
+        else:
+            self.params['nooverwrites'] = not self.params['overwrites']
 
         if params.get('bidi_workaround', False):
             try:
@@ -889,7 +896,6 @@ class YoutubeDL(object):
     def prepare_outtmpl(self, outtmpl, info_dict, sanitize=None):
         """ Make the template and info_dict suitable for substitution : ydl.outtmpl_escape(outtmpl) % info_dict """
         info_dict.setdefault('epoch', int(time.time()))  # keep epoch consistent once set
-        na = self.params.get('outtmpl_na_placeholder', 'NA')
 
         info_dict = dict(info_dict)  # Do not sanitize so as not to consume LazyList
         for key in ('__original_infodict', '__postprocessors'):
@@ -970,6 +976,8 @@ class YoutubeDL(object):
 
             return value
 
+        na = self.params.get('outtmpl_na_placeholder', 'NA')
+
         def _dumpjson_default(obj):
             if isinstance(obj, (set, LazyList)):
                 return list(obj)
@@ -978,10 +986,7 @@ class YoutubeDL(object):
         def create_key(outer_mobj):
             if not outer_mobj.group('has_key'):
                 return f'%{outer_mobj.group(0)}'
-
-            prefix = outer_mobj.group('prefix')
             key = outer_mobj.group('key')
-            original_fmt = fmt = outer_mobj.group('format')
             mobj = re.match(INTERNAL_FORMAT_RE, key)
             if mobj is None:
                 value, default, mobj = None, na, {'fields': ''}
@@ -990,6 +995,7 @@ class YoutubeDL(object):
                 default = mobj['default'] if mobj['default'] is not None else na
                 value = get_value(mobj)
 
+            fmt = outer_mobj.group('format')
             if fmt == 's' and value is not None and key in field_size_compat_map.keys():
                 fmt = '0{:d}d'.format(field_size_compat_map[key])
 
@@ -1021,9 +1027,9 @@ class YoutubeDL(object):
                 if fmt[-1] in 'csr':
                     value = sanitize(mobj['fields'].split('.')[-1], value)
 
-            key = '%s\0%s' % (key.replace('%', '%\0'), original_fmt)
+            key = '%s\0%s' % (key.replace('%', '%\0'), outer_mobj.group('format'))
             TMPL_DICT[key] = value
-            return f'{prefix}%({key}){fmt}'
+            return '{prefix}%({key}){fmt}'.format(key=key, fmt=fmt, prefix=outer_mobj.group('prefix'))
 
         return EXTERNAL_FORMAT_RE.sub(create_key, outtmpl), TMPL_DICT
 
@@ -1069,7 +1075,6 @@ class YoutubeDL(object):
                 self.report_warning('--paths is ignored when an outputting to stdout', only_once=True)
             elif os.path.isabs(filename):
                 self.report_warning('--paths is ignored since an absolute path is given in output template', only_once=True)
-            self.__prepare_filename_warned = True
         if filename == '-' or not filename:
             return filename
 
@@ -1348,15 +1353,12 @@ class YoutubeDL(object):
                 'It needs to be updated.' % ie_result.get('extractor'))
 
             def _fixup(r):
-                self.add_extra_info(
-                    r,
-                    {
-                        'extractor': ie_result['extractor'],
-                        'webpage_url': ie_result['webpage_url'],
-                        'webpage_url_basename': url_basename(ie_result['webpage_url']),
-                        'extractor_key': ie_result['extractor_key'],
-                    }
-                )
+                self.add_extra_info(r, {
+                    'extractor': ie_result['extractor'],
+                    'webpage_url': ie_result['webpage_url'],
+                    'webpage_url_basename': url_basename(ie_result['webpage_url']),
+                    'extractor_key': ie_result['extractor_key'],
+                })
                 return r
             ie_result['entries'] = [
                 self.process_ie_result(_fixup(r), download, extra_info)
@@ -2193,7 +2195,7 @@ class YoutubeDL(object):
                 format['format'] = '{id} - {res}{note}'.format(
                     id=format['format_id'],
                     res=self.format_resolution(format),
-                    note=' ({0})'.format(format['format_note']) if format.get('format_note') is not None else '',
+                    note=format_field(format, 'format_note', ' (%s)'),
                 )
             # Automatically determine file extension if missing
             if format.get('ext') is None:
@@ -2395,7 +2397,7 @@ class YoutubeDL(object):
         print_optional('thumbnail')
         print_optional('description')
         print_optional('filename')
-        if self.params.get('forceduration', False) and info_dict.get('duration') is not None:
+        if self.params.get('forceduration') and info_dict.get('duration') is not None:
             self.to_stdout(formatSeconds(info_dict['duration']))
         print_mandatory('format')
 
@@ -2435,8 +2437,6 @@ class YoutubeDL(object):
 
         assert info_dict.get('_type', 'video') == 'video'
 
-        info_dict.setdefault('__postprocessors', [])
-
         max_downloads = self.params.get('max_downloads')
         if max_downloads is not None:
             if self._num_downloads >= int(max_downloads):
@@ -2637,6 +2637,7 @@ class YoutubeDL(object):
             info_dict = self.run_pp(MoveFilesAfterDownloadPP(self, False), info_dict)
         else:
             # Download
+            info_dict.setdefault('__postprocessors', [])
             try:
 
                 def existing_file(*filepaths):

yt_dlp/downloader/common.py

@@ -320,12 +320,9 @@ class FileDownloader(object):
             '[download] Got server HTTP error: %s. Retrying (attempt %d of %s) ...'
             % (error_to_compat_str(err), count, self.format_retries(retries)))
 
-    def report_file_already_downloaded(self, file_name):
+    def report_file_already_downloaded(self, *args, **kwargs):
         """Report file has already been fully downloaded."""
-        try:
-            self.to_screen('[download] %s has already been downloaded' % file_name)
-        except UnicodeEncodeError:
-            self.to_screen('[download] The file has already been downloaded')
+        return self.ydl.report_file_already_downloaded(*args, **kwargs)
 
     def report_unable_to_resume(self):
         """Report it was impossible to resume download."""

yt_dlp/extractor/common.py

@@ -35,7 +35,6 @@ from ..downloader.f4m import (
     remove_encrypted_media,
 )
 from ..utils import (
-    NO_DEFAULT,
     age_restricted,
     base_url,
     bug_reports_message,
@@ -45,10 +44,11 @@ from ..utils import (
     determine_protocol,
     dict_get,
     error_to_compat_str,
-    ExtractorError,
     extract_attributes,
+    ExtractorError,
     fix_xml_ampersands,
     float_or_none,
+    format_field,
     GeoRestrictedError,
     GeoUtils,
     int_or_none,
@@ -56,6 +56,7 @@ from ..utils import (
     JSON_LD_RE,
     mimetype2ext,
     network_exceptions,
+    NO_DEFAULT,
     orderedSet,
     parse_bitrate,
     parse_codecs,
@@ -64,8 +65,8 @@ from ..utils import (
     parse_m3u8_attributes,
     parse_resolution,
     RegexNotFoundError,
-    sanitized_Request,
     sanitize_filename,
+    sanitized_Request,
     str_or_none,
     str_to_int,
     strip_or_none,
@@ -75,9 +76,9 @@ from ..utils import (
     unified_timestamp,
     update_Request,
     update_url_query,
-    urljoin,
     url_basename,
     url_or_none,
+    urljoin,
     variadic,
     xpath_element,
     xpath_text,
@@ -1002,7 +1003,7 @@ class InfoExtractor(object):
         return res if res is False else res[0]
 
     def report_warning(self, msg, video_id=None, *args, only_once=False, **kwargs):
-        idstr = '' if video_id is None else '%s: ' % video_id
+        idstr = format_field(video_id, template='%s: ')
         msg = f'[{self.IE_NAME}] {idstr}{msg}'
         if only_once:
             if f'WARNING: {msg}' in self._printed_messages:

yt_dlp/extractor/niconico.py

@@ -13,16 +13,16 @@ from ..compat import (
     compat_urllib_parse_urlparse,
 )
 from ..utils import (
-    dict_get,
     ExtractorError,
-    int_or_none,
+    dict_get,
     float_or_none,
+    int_or_none,
     OnDemandPagedList,
     parse_duration,
     parse_iso8601,
     PostProcessingError,
-    str_or_none,
     remove_start,
+    str_or_none,
     try_get,
     unified_timestamp,
     urlencode_postdata,

yt_dlp/utils.py

@@ -6161,8 +6161,11 @@ def to_high_limit_path(path):
     return path
 
 
-def format_field(obj, field, template='%s', ignore=(None, ''), default='', func=None):
-    val = obj.get(field, default)
+def format_field(obj, field=None, template='%s', ignore=(None, ''), default='', func=None):
+    if field is None:
+        val = obj if obj is not None else default
+    else:
+        val = obj.get(field, default)
     if func and val not in ignore:
         val = func(val)
     return template % val if val not in ignore else default