From 7c5e1701f6e948c83a928b6657542036c1d7516e Mon Sep 17 00:00:00 2001 From: bashonly <88596187+bashonly@users.noreply.github.com> Date: Fri, 9 Dec 2022 23:43:10 +0000 Subject: [PATCH] [extractor/foxsports] Fix extractor (#5719) Closes #5714 Authored by: bashonly --- yt_dlp/extractor/foxsports.py | 56 ++++++++++++++++-------- yt_dlp/extractor/uplynk.py | 80 ++++++++++++++++++++++------------- 2 files changed, 89 insertions(+), 47 deletions(-) diff --git a/yt_dlp/extractor/foxsports.py b/yt_dlp/extractor/foxsports.py index f9d7fe52a..f906a1718 100644 --- a/yt_dlp/extractor/foxsports.py +++ b/yt_dlp/extractor/foxsports.py @@ -1,31 +1,51 @@ from .common import InfoExtractor +from .uplynk import UplynkPreplayIE +from ..utils import HEADRequest, float_or_none, make_archive_id, smuggle_url class FoxSportsIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?foxsports\.com/(?:[^/]+/)*video/(?P\d+)' - - _TEST = { - 'url': 'http://www.foxsports.com/tennessee/video/432609859715', - 'md5': 'b49050e955bebe32c301972e4012ac17', + _VALID_URL = r'https?://(?:www\.)?foxsports\.com/watch/(?P[\w-]+)' + _TESTS = [{ + 'url': 'https://www.foxsports.com/watch/play-612168c6700004b', 'info_dict': { - 'id': '432609859715', + 'id': 'b72f5bd8658140baa5791bb676433733', 'ext': 'mp4', - 'title': 'Courtney Lee on going up 2-0 in series vs. Blazers', - 'description': 'Courtney Lee talks about Memphis being focused.', - # TODO: fix timestamp - 'upload_date': '19700101', # '20150423', - # 'timestamp': 1429761109, - 'uploader': 'NEWA-FNG-FOXSPORTS', + 'display_id': 'play-612168c6700004b', + 'title': 'md5:e0c4ecac3a1f25295b4fae22fb5c126a', + 'description': 'md5:371bc43609708ae2b9e1a939229762af', + 'uploader_id': '06b4a36349624051a9ba52ac3a91d268', + 'upload_date': '20221205', + 'timestamp': 1670262586, + 'duration': 31.7317, + 'thumbnail': r're:^https?://.*\.jpg$', + 'extra_param_to_segment_url': str, }, 'params': { - # m3u8 download - 'skip_download': True, + 'skip_download': 'm3u8', }, - 'add_ie': ['ThePlatform'], - } + }] def _real_extract(self, url): video_id = self._match_id(url) + webpage = self._download_webpage(url, video_id) + json_ld = self._search_json_ld(webpage, video_id, expected_type='VideoObject', default={}) + data = self._download_json( + f'https://api3.fox.com/v2.0/vodplayer/sportsclip/{video_id}', + video_id, note='Downloading API JSON', headers={ + 'x-api-key': 'cf289e299efdfa39fb6316f259d1de93', + }) + preplay_url = self._request_webpage( + HEADRequest(data['url']), video_id, 'Fetching preplay URL').geturl() - return self.url_result( - 'https://feed.theplatform.com/f/BKQ29B/foxsports-all?byId=' + video_id, 'ThePlatformFeed') + return { + '_type': 'url_transparent', + 'ie_key': UplynkPreplayIE.ie_key(), + 'url': smuggle_url(preplay_url, {'Origin': 'https://www.foxsports.com'}), + 'display_id': video_id, + 'title': data.get('name') or json_ld.get('title'), + 'description': data.get('description') or json_ld.get('description'), + 'duration': float_or_none(data.get('durationInSeconds')), + 'timestamp': json_ld.get('timestamp'), + 'thumbnails': json_ld.get('thumbnails'), + '_old_archive_ids': [make_archive_id(self, video_id)], + } diff --git a/yt_dlp/extractor/uplynk.py b/yt_dlp/extractor/uplynk.py index 87c427f63..e7d816ef4 100644 --- a/yt_dlp/extractor/uplynk.py +++ b/yt_dlp/extractor/uplynk.py @@ -2,40 +2,42 @@ import re from .common import InfoExtractor from ..utils import ( - float_or_none, ExtractorError, + float_or_none, + smuggle_url, + traverse_obj, + unsmuggle_url, + update_url_query, ) -class UplynkIE(InfoExtractor): - IE_NAME = 'uplynk' - _VALID_URL = r'https?://.*?\.uplynk\.com/(?Pext/[0-9a-f]{32}/(?P[^/?&]+)|(?P[0-9a-f]{32}))\.(?:m3u8|json)(?:.*?\bpbs=(?P[^&]+))?' - _TEST = { - 'url': 'http://content.uplynk.com/e89eaf2ce9054aa89d92ddb2d817a52e.m3u8', - 'info_dict': { - 'id': 'e89eaf2ce9054aa89d92ddb2d817a52e', - 'ext': 'mp4', - 'title': '030816-kgo-530pm-solar-eclipse-vid_web.mp4', - 'uploader_id': '4413701bf5a1488db55b767f8ae9d4fa', - }, - 'params': { - # m3u8 download - 'skip_download': True, - }, - } +class UplynkBaseIE(InfoExtractor): + _UPLYNK_URL_RE = r'''(?x) + https?://[\w-]+\.uplynk\.com/(?P + ext/[0-9a-f]{32}/(?P[^/?&]+)| + (?P[0-9a-f]{32}) + )\.(?:m3u8|json) + (?:.*?\bpbs=(?P[^&]+))?''' - def _extract_uplynk_info(self, uplynk_content_url): - path, external_id, video_id, session_id = re.match(UplynkIE._VALID_URL, uplynk_content_url).groups() + def _extract_uplynk_info(self, url): + uplynk_content_url, smuggled_data = unsmuggle_url(url, {}) + mobj = re.match(self._UPLYNK_URL_RE, uplynk_content_url) + if not mobj: + raise ExtractorError('Necessary parameters not found in Uplynk URL') + path, external_id, video_id, session_id = mobj.group('path', 'external_id', 'id', 'session_id') display_id = video_id or external_id + headers = traverse_obj( + smuggled_data, {'Referer': 'Referer', 'Origin': 'Origin'}, casesense=False) formats, subtitles = self._extract_m3u8_formats_and_subtitles( - 'http://content.uplynk.com/%s.m3u8' % path, - display_id, 'mp4', 'm3u8_native') + f'http://content.uplynk.com/{path}.m3u8', display_id, 'mp4', headers=headers) if session_id: for f in formats: - f['extra_param_to_segment_url'] = 'pbs=' + session_id - asset = self._download_json('http://content.uplynk.com/player/assetinfo/%s.json' % path, display_id) + f['extra_param_to_segment_url'] = f'pbs={session_id}' + asset = self._download_json( + f'http://content.uplynk.com/player/assetinfo/{path}.json', display_id) if asset.get('error') == 1: - raise ExtractorError('% said: %s' % (self.IE_NAME, asset['msg']), expected=True) + msg = asset.get('msg') or 'unknown error' + raise ExtractorError(f'{self.IE_NAME} said: {msg}', expected=True) return { 'id': asset['asset'], @@ -47,20 +49,40 @@ class UplynkIE(InfoExtractor): 'subtitles': subtitles, } + +class UplynkIE(UplynkBaseIE): + IE_NAME = 'uplynk' + _VALID_URL = UplynkBaseIE._UPLYNK_URL_RE + _TEST = { + 'url': 'http://content.uplynk.com/e89eaf2ce9054aa89d92ddb2d817a52e.m3u8', + 'info_dict': { + 'id': 'e89eaf2ce9054aa89d92ddb2d817a52e', + 'ext': 'mp4', + 'title': '030816-kgo-530pm-solar-eclipse-vid_web.mp4', + 'uploader_id': '4413701bf5a1488db55b767f8ae9d4fa', + 'duration': 530.2739166666679, + 'thumbnail': r're:^https?://.*\.jpg$', + }, + 'params': { + 'skip_download': 'm3u8', + }, + } + def _real_extract(self, url): return self._extract_uplynk_info(url) -class UplynkPreplayIE(UplynkIE): # XXX: Do not subclass from concrete IE +class UplynkPreplayIE(UplynkBaseIE): IE_NAME = 'uplynk:preplay' - _VALID_URL = r'https?://.*?\.uplynk\.com/preplay2?/(?Pext/[0-9a-f]{32}/(?P[^/?&]+)|(?P[0-9a-f]{32}))\.json' + _VALID_URL = r'https?://[\w-]+\.uplynk\.com/preplay2?/(?Pext/[0-9a-f]{32}/(?P[^/?&]+)|(?P[0-9a-f]{32}))\.json' def _real_extract(self, url): + url, smuggled_data = unsmuggle_url(url, {}) path, external_id, video_id = self._match_valid_url(url).groups() display_id = video_id or external_id preplay = self._download_json(url, display_id) - content_url = 'http://content.uplynk.com/%s.m3u8' % path + content_url = f'http://content.uplynk.com/{path}.m3u8' session_id = preplay.get('sid') if session_id: - content_url += '?pbs=' + session_id - return self._extract_uplynk_info(content_url) + content_url = update_url_query(content_url, {'pbs': session_id}) + return self._extract_uplynk_info(smuggle_url(content_url, smuggled_data))