From 1f6fb1d348a9754a5f9d8d9285507fd63375d597 Mon Sep 17 00:00:00 2001 From: Mikhail Novosyolov Date: Sat, 16 Oct 2021 21:07:26 +0300 Subject: [PATCH] Fix regression, fix interdeps --- ...de722a6cf86dbcc6d17a63145ec59a573bf6.patch | 155 ++++++++++++++++++ ...10ee8adcf61e1136a252462670ec230e9439.patch | 72 ++++++++ ...3274e50d9cca79189689ba53db7295ea267c.patch | 49 ++++++ yt-dlp.spec | 11 +- 4 files changed, 285 insertions(+), 2 deletions(-) create mode 100644 03b4de722a6cf86dbcc6d17a63145ec59a573bf6.patch create mode 100644 48ee10ee8adcf61e1136a252462670ec230e9439.patch create mode 100644 580d3274e50d9cca79189689ba53db7295ea267c.patch diff --git a/03b4de722a6cf86dbcc6d17a63145ec59a573bf6.patch b/03b4de722a6cf86dbcc6d17a63145ec59a573bf6.patch new file mode 100644 index 0000000..2603f83 --- /dev/null +++ b/03b4de722a6cf86dbcc6d17a63145ec59a573bf6.patch @@ -0,0 +1,155 @@ +From 03b4de722a6cf86dbcc6d17a63145ec59a573bf6 Mon Sep 17 00:00:00 2001 +From: pukkandan +Date: Sat, 16 Oct 2021 18:31:00 +0530 +Subject: [PATCH] [downloader] Fix slow progress hooks Closes #1301 + +--- + yt_dlp/YoutubeDL.py | 16 +++++++++++----- + yt_dlp/downloader/common.py | 5 +---- + yt_dlp/downloader/dash.py | 5 ++--- + yt_dlp/downloader/hls.py | 5 ++--- + yt_dlp/postprocessor/common.py | 13 +++++++------ + 5 files changed, 23 insertions(+), 21 deletions(-) + +diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py +index aff7d6ddb7..fd8ad0f983 100644 +--- a/yt_dlp/YoutubeDL.py ++++ b/yt_dlp/YoutubeDL.py +@@ -950,13 +950,18 @@ def validate_outtmpl(cls, outtmpl): + except ValueError as err: + return err + ++ @staticmethod ++ def _copy_infodict(info_dict): ++ info_dict = dict(info_dict) ++ for key in ('__original_infodict', '__postprocessors'): ++ info_dict.pop(key, None) ++ return info_dict ++ + def prepare_outtmpl(self, outtmpl, info_dict, sanitize=None): + """ Make the outtmpl and info_dict suitable for substitution: ydl.escape_outtmpl(outtmpl) % info_dict """ + info_dict.setdefault('epoch', int(time.time())) # keep epoch consistent once set + +- info_dict = dict(info_dict) # Do not sanitize so as not to consume LazyList +- for key in ('__original_infodict', '__postprocessors'): +- info_dict.pop(key, None) ++ info_dict = self._copy_infodict(info_dict) + info_dict['duration_string'] = ( # %(duration>%H-%M-%S)s is wrong if duration > 24hrs + formatSeconds(info_dict['duration'], '-' if sanitize else ':') + if info_dict.get('duration', None) is not None +@@ -2265,7 +2270,7 @@ def is_wellformed(f): + formats_dict[format_id].append(format) + + # Make sure all formats have unique format_id +- common_exts = set(ext for exts in self._format_selection_exts.values() for ext in exts) ++ common_exts = set(itertools.chain(*self._format_selection_exts.values())) + for format_id, ambiguous_formats in formats_dict.items(): + ambigious_id = len(ambiguous_formats) > 1 + for i, format in enumerate(ambiguous_formats): +@@ -2523,7 +2528,8 @@ def dl(self, name, info, subtitle=False, test=False): + fd.add_progress_hook(ph) + urls = '", "'.join([f['url'] for f in info.get('requested_formats', [])] or [info['url']]) + self.write_debug('Invoking downloader on "%s"' % urls) +- new_info = dict(info) ++ ++ new_info = copy.deepcopy(self._copy_infodict(info)) + if new_info.get('http_headers') is None: + new_info['http_headers'] = self._calc_headers(new_info) + return fd.download(name, new_info, subtitle) +diff --git a/yt_dlp/downloader/common.py b/yt_dlp/downloader/common.py +index 89cdffd246..96b78a968c 100644 +--- a/yt_dlp/downloader/common.py ++++ b/yt_dlp/downloader/common.py +@@ -405,13 +405,10 @@ def real_download(self, filename, info_dict): + def _hook_progress(self, status, info_dict): + if not self._progress_hooks: + return +- info_dict = dict(info_dict) +- for key in ('__original_infodict', '__postprocessors'): +- info_dict.pop(key, None) ++ status['info_dict'] = info_dict + # youtube-dl passes the same status object to all the hooks. + # Some third party scripts seems to be relying on this. + # So keep this behavior if possible +- status['info_dict'] = copy.deepcopy(info_dict) + for ph in self._progress_hooks: + ph(status) + +diff --git a/yt_dlp/downloader/dash.py b/yt_dlp/downloader/dash.py +index 734eab3ef2..6444ad6928 100644 +--- a/yt_dlp/downloader/dash.py ++++ b/yt_dlp/downloader/dash.py +@@ -55,9 +55,8 @@ def real_download(self, filename, info_dict): + if real_downloader: + self.to_screen( + '[%s] Fragment downloads will be delegated to %s' % (self.FD_NAME, real_downloader.get_basename())) +- info_copy = info_dict.copy() +- info_copy['fragments'] = fragments_to_download ++ info_dict['fragments'] = fragments_to_download + fd = real_downloader(self.ydl, self.params) +- return fd.real_download(filename, info_copy) ++ return fd.real_download(filename, info_dict) + + return self.download_and_append_fragments(ctx, fragments_to_download, info_dict) +diff --git a/yt_dlp/downloader/hls.py b/yt_dlp/downloader/hls.py +index 3c5a2617d0..61312c5ba5 100644 +--- a/yt_dlp/downloader/hls.py ++++ b/yt_dlp/downloader/hls.py +@@ -245,13 +245,12 @@ def is_ad_fragment_end(s): + fragments = [fragments[0] if fragments else None] + + if real_downloader: +- info_copy = info_dict.copy() +- info_copy['fragments'] = fragments ++ info_dict['fragments'] = fragments + fd = real_downloader(self.ydl, self.params) + # TODO: Make progress updates work without hooking twice + # for ph in self._progress_hooks: + # fd.add_progress_hook(ph) +- return fd.real_download(filename, info_copy) ++ return fd.real_download(filename, info_dict) + + if is_webvtt: + def pack_fragment(frag_content, frag_index): +diff --git a/yt_dlp/postprocessor/common.py b/yt_dlp/postprocessor/common.py +index d2daeb0fba..b367167432 100644 +--- a/yt_dlp/postprocessor/common.py ++++ b/yt_dlp/postprocessor/common.py +@@ -17,11 +17,12 @@ class PostProcessorMetaClass(type): + def run_wrapper(func): + @functools.wraps(func) + def run(self, info, *args, **kwargs): +- self._hook_progress({'status': 'started'}, info) ++ info_copy = copy.deepcopy(self._copy_infodict(info)) ++ self._hook_progress({'status': 'started'}, info_copy) + ret = func(self, info, *args, **kwargs) + if ret is not None: + _, info = ret +- self._hook_progress({'status': 'finished'}, info) ++ self._hook_progress({'status': 'finished'}, info_copy) + return ret + return run + +@@ -93,6 +94,9 @@ def set_downloader(self, downloader): + for ph in getattr(downloader, '_postprocessor_hooks', []): + self.add_progress_hook(ph) + ++ def _copy_infodict(self, info_dict): ++ return getattr(self._downloader, '_copy_infodict', dict)(info_dict) ++ + @staticmethod + def _restrict_to(*, video=True, audio=True, images=True): + allowed = {'video': video, 'audio': audio, 'images': images} +@@ -142,11 +146,8 @@ def _configuration_args(self, exe, *args, **kwargs): + def _hook_progress(self, status, info_dict): + if not self._progress_hooks: + return +- info_dict = dict(info_dict) +- for key in ('__original_infodict', '__postprocessors'): +- info_dict.pop(key, None) + status.update({ +- 'info_dict': copy.deepcopy(info_dict), ++ 'info_dict': info_dict, + 'postprocessor': self.pp_key(), + }) + for ph in self._progress_hooks: diff --git a/48ee10ee8adcf61e1136a252462670ec230e9439.patch b/48ee10ee8adcf61e1136a252462670ec230e9439.patch new file mode 100644 index 0000000..6a6c131 --- /dev/null +++ b/48ee10ee8adcf61e1136a252462670ec230e9439.patch @@ -0,0 +1,72 @@ +From 48ee10ee8adcf61e1136a252462670ec230e9439 Mon Sep 17 00:00:00 2001 +From: pukkandan +Date: Fri, 15 Oct 2021 18:50:28 +0530 +Subject: [PATCH] Fix conflict b/w id and ext in format selection Closes #1282 + +--- + yt_dlp/YoutubeDL.py | 27 +++++++++++++++++++-------- + 1 file changed, 19 insertions(+), 8 deletions(-) + +diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py +index 542a97794..aff7d6ddb 100644 +--- a/yt_dlp/YoutubeDL.py ++++ b/yt_dlp/YoutubeDL.py +@@ -483,6 +483,12 @@ class YoutubeDL(object): + 'track_number', 'disc_number', 'release_year', + )) + ++ _format_selection_exts = { ++ 'audio': {'m4a', 'mp3', 'ogg', 'aac'}, ++ 'video': {'mp4', 'flv', 'webm', '3gp'}, ++ 'storyboards': {'mhtml'}, ++ } ++ + params = None + _ies = {} + _pps = {'pre_process': [], 'before_dl': [], 'after_move': [], 'post_process': []} +@@ -1980,11 +1986,11 @@ def selector_function(ctx): + filter_f = lambda f: _filter_f(f) and ( + f.get('vcodec') != 'none' or f.get('acodec') != 'none') + else: +- if format_spec in ('m4a', 'mp3', 'ogg', 'aac'): # audio extension ++ if format_spec in self._format_selection_exts['audio']: + filter_f = lambda f: f.get('ext') == format_spec and f.get('acodec') != 'none' +- elif format_spec in ('mp4', 'flv', 'webm', '3gp'): # video extension ++ elif format_spec in self._format_selection_exts['video']: + filter_f = lambda f: f.get('ext') == format_spec and f.get('acodec') != 'none' and f.get('vcodec') != 'none' +- elif format_spec in ('mhtml', ): # storyboards extension ++ elif format_spec in self._format_selection_exts['storyboards']: + filter_f = lambda f: f.get('ext') == format_spec and f.get('acodec') == 'none' and f.get('vcodec') == 'none' + else: + filter_f = lambda f: f.get('format_id') == format_spec # id +@@ -2259,10 +2265,18 @@ def is_wellformed(f): + formats_dict[format_id].append(format) + + # Make sure all formats have unique format_id ++ common_exts = set(ext for exts in self._format_selection_exts.values() for ext in exts) + for format_id, ambiguous_formats in formats_dict.items(): +- if len(ambiguous_formats) > 1: +- for i, format in enumerate(ambiguous_formats): ++ ambigious_id = len(ambiguous_formats) > 1 ++ for i, format in enumerate(ambiguous_formats): ++ if ambigious_id: + format['format_id'] = '%s-%d' % (format_id, i) ++ if format.get('ext') is None: ++ format['ext'] = determine_ext(format['url']).lower() ++ # Ensure there is no conflict between id and ext in format selection ++ # See https://github.com/yt-dlp/yt-dlp/issues/1282 ++ if format['format_id'] != format['ext'] and format['format_id'] in common_exts: ++ format['format_id'] = 'f%s' % format['format_id'] + + for i, format in enumerate(formats): + if format.get('format') is None: +@@ -2271,9 +2285,6 @@ def is_wellformed(f): + res=self.format_resolution(format), + note=format_field(format, 'format_note', ' (%s)'), + ) +- # Automatically determine file extension if missing +- if format.get('ext') is None: +- format['ext'] = determine_ext(format['url']).lower() + # Automatically determine protocol if missing (useful for format + # selection purposes) + if format.get('protocol') is None: diff --git a/580d3274e50d9cca79189689ba53db7295ea267c.patch b/580d3274e50d9cca79189689ba53db7295ea267c.patch new file mode 100644 index 0000000..65ebf37 --- /dev/null +++ b/580d3274e50d9cca79189689ba53db7295ea267c.patch @@ -0,0 +1,49 @@ +From 580d3274e50d9cca79189689ba53db7295ea267c Mon Sep 17 00:00:00 2001 +From: pukkandan +Date: Sat, 16 Oct 2021 20:13:23 +0530 +Subject: [PATCH] [youtube] Expose different formats with same itag + +--- + yt_dlp/downloader/common.py | 1 - + yt_dlp/extractor/youtube.py | 9 +++++++-- + 2 files changed, 7 insertions(+), 3 deletions(-) + +diff --git a/yt_dlp/downloader/common.py b/yt_dlp/downloader/common.py +index 96b78a968..9081794db 100644 +--- a/yt_dlp/downloader/common.py ++++ b/yt_dlp/downloader/common.py +@@ -1,6 +1,5 @@ + from __future__ import division, unicode_literals + +-import copy + import os + import re + import time +diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py +index 1ef80445e..dc9aa8ab7 100644 +--- a/yt_dlp/extractor/youtube.py ++++ b/yt_dlp/extractor/youtube.py +@@ -2692,7 +2692,9 @@ def guess_quality(f): + itag = self._search_regex( + r'/itag/(\d+)', f['url'], 'itag', default=None) + if itag in itags: +- continue ++ itag += '-hls' ++ if itag in itags: ++ continue + if itag: + f['format_id'] = itag + itags.append(itag) +@@ -2704,8 +2706,11 @@ def guess_quality(f): + for f in self._extract_mpd_formats(dash_manifest_url, video_id, fatal=False): + itag = f['format_id'] + if itag in itags: +- continue ++ itag += '-dash' ++ if itag in itags: ++ continue + if itag: ++ f['format_id'] = itag + itags.append(itag) + f['quality'] = guess_quality(f) + filesize = int_or_none(self._search_regex( diff --git a/yt-dlp.spec b/yt-dlp.spec index 82896d8..690c196 100644 --- a/yt-dlp.spec +++ b/yt-dlp.spec @@ -4,19 +4,26 @@ Summary: Small command-line program to download videos and audio from different websites Name: yt-dlp Version: 2021.10.10 -Release: 1 +Release: 2 License: Public Domain and GPLv2+ Group: Video Url: https://github.com/yt-dlp/yt-dlp # source from Github requires at least pandoc which is not packaged Source0: %{pypi_source} +# Pick upstream commits to fix https://github.com/yt-dlp/yt-dlp/issues/1301 +# https://github.com/yt-dlp/yt-dlp/commit/48ee10ee8adcf61e1136a252462670ec230e9439 +Patch1: 48ee10ee8adcf61e1136a252462670ec230e9439.patch +# https://github.com/yt-dlp/yt-dlp/commit/03b4de722a6cf86dbcc6d17a63145ec59a573bf6 +Patch2: 03b4de722a6cf86dbcc6d17a63145ec59a573bf6.patch +# https://github.com/yt-dlp/yt-dlp/commit/580d3274e50d9cca79189689ba53db7295ea267c +Patch3: 580d3274e50d9cca79189689ba53db7295ea267c.patch BuildRequires: ffmpeg BuildRequires: gnupg2 BuildRequires: pkgconfig(python3) BuildRequires: python3egg(setuptools) BuildRequires: zip BuildArch: noarch -Requires: python3 +Requires: python3-%{name} = %{EVRD} Recommends: ffmpeg # youtube-dl is not developed anymore, it is a fork of it Obsoletes: youtube-dl < 2021.10.10