Djam/yt-dlp
1
0
Fork 0
mirror of https://abf.rosa.ru/djam/yt-dlp.git synced 2025-02-23 07:12:55 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fix regression, fix interdeps

Browse source
This commit is contained in:
Mikhail Novosyolov 2021-10-16 21:07:26 +03:00
parent cb8ffdf8e3
commit 1f6fb1d348
4 changed files with 285 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

155
03b4de722a6cf86dbcc6d17a63145ec59a573bf6.patch Normal file
View file

@ -0,0 +1,155 @@
From 03b4de722a6cf86dbcc6d17a63145ec59a573bf6 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sat, 16 Oct 2021 18:31:00 +0530
Subject: [PATCH] [downloader] Fix slow progress hooks Closes #1301
---
yt_dlp/YoutubeDL.py | 16 +++++++++++-----
yt_dlp/downloader/common.py | 5 +----
yt_dlp/downloader/dash.py | 5 ++---
yt_dlp/downloader/hls.py | 5 ++---
yt_dlp/postprocessor/common.py | 13 +++++++------
5 files changed, 23 insertions(+), 21 deletions(-)
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index aff7d6ddb7..fd8ad0f983 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -950,13 +950,18 @@ def validate_outtmpl(cls, outtmpl):
except ValueError as err:
return err
+ @staticmethod
+ def _copy_infodict(info_dict):
+ info_dict = dict(info_dict)
+ for key in ('__original_infodict', '__postprocessors'):
+ info_dict.pop(key, None)
+ return info_dict
+
def prepare_outtmpl(self, outtmpl, info_dict, sanitize=None):
""" Make the outtmpl and info_dict suitable for substitution: ydl.escape_outtmpl(outtmpl) % info_dict """
info_dict.setdefault('epoch', int(time.time())) # keep epoch consistent once set
- info_dict = dict(info_dict) # Do not sanitize so as not to consume LazyList
- for key in ('__original_infodict', '__postprocessors'):
- info_dict.pop(key, None)
+ info_dict = self._copy_infodict(info_dict)
info_dict['duration_string'] = ( # %(duration>%H-%M-%S)s is wrong if duration > 24hrs
formatSeconds(info_dict['duration'], '-' if sanitize else ':')
if info_dict.get('duration', None) is not None
@@ -2265,7 +2270,7 @@ def is_wellformed(f):
formats_dict[format_id].append(format)
# Make sure all formats have unique format_id
- common_exts = set(ext for exts in self._format_selection_exts.values() for ext in exts)
+ common_exts = set(itertools.chain(*self._format_selection_exts.values()))
for format_id, ambiguous_formats in formats_dict.items():
ambigious_id = len(ambiguous_formats) > 1
for i, format in enumerate(ambiguous_formats):
@@ -2523,7 +2528,8 @@ def dl(self, name, info, subtitle=False, test=False):
fd.add_progress_hook(ph)
urls = '", "'.join([f['url'] for f in info.get('requested_formats', [])] or [info['url']])
self.write_debug('Invoking downloader on "%s"' % urls)
- new_info = dict(info)
+
+ new_info = copy.deepcopy(self._copy_infodict(info))
if new_info.get('http_headers') is None:
new_info['http_headers'] = self._calc_headers(new_info)
return fd.download(name, new_info, subtitle)
diff --git a/yt_dlp/downloader/common.py b/yt_dlp/downloader/common.py
index 89cdffd246..96b78a968c 100644
--- a/yt_dlp/downloader/common.py
+++ b/yt_dlp/downloader/common.py
@@ -405,13 +405,10 @@ def real_download(self, filename, info_dict):
def _hook_progress(self, status, info_dict):
if not self._progress_hooks:
return
- info_dict = dict(info_dict)
- for key in ('__original_infodict', '__postprocessors'):
- info_dict.pop(key, None)
+ status['info_dict'] = info_dict
# youtube-dl passes the same status object to all the hooks.
# Some third party scripts seems to be relying on this.
# So keep this behavior if possible
- status['info_dict'] = copy.deepcopy(info_dict)
for ph in self._progress_hooks:
ph(status)
diff --git a/yt_dlp/downloader/dash.py b/yt_dlp/downloader/dash.py
index 734eab3ef2..6444ad6928 100644
--- a/yt_dlp/downloader/dash.py
+++ b/yt_dlp/downloader/dash.py
@@ -55,9 +55,8 @@ def real_download(self, filename, info_dict):
if real_downloader:
self.to_screen(
'[%s] Fragment downloads will be delegated to %s' % (self.FD_NAME, real_downloader.get_basename()))
- info_copy = info_dict.copy()
- info_copy['fragments'] = fragments_to_download
+ info_dict['fragments'] = fragments_to_download
fd = real_downloader(self.ydl, self.params)
- return fd.real_download(filename, info_copy)
+ return fd.real_download(filename, info_dict)
return self.download_and_append_fragments(ctx, fragments_to_download, info_dict)
diff --git a/yt_dlp/downloader/hls.py b/yt_dlp/downloader/hls.py
index 3c5a2617d0..61312c5ba5 100644
--- a/yt_dlp/downloader/hls.py
+++ b/yt_dlp/downloader/hls.py
@@ -245,13 +245,12 @@ def is_ad_fragment_end(s):
fragments = [fragments[0] if fragments else None]
if real_downloader:
- info_copy = info_dict.copy()
- info_copy['fragments'] = fragments
+ info_dict['fragments'] = fragments
fd = real_downloader(self.ydl, self.params)
# TODO: Make progress updates work without hooking twice
# for ph in self._progress_hooks:
# fd.add_progress_hook(ph)
- return fd.real_download(filename, info_copy)
+ return fd.real_download(filename, info_dict)
if is_webvtt:
def pack_fragment(frag_content, frag_index):
diff --git a/yt_dlp/postprocessor/common.py b/yt_dlp/postprocessor/common.py
index d2daeb0fba..b367167432 100644
--- a/yt_dlp/postprocessor/common.py
+++ b/yt_dlp/postprocessor/common.py
@@ -17,11 +17,12 @@ class PostProcessorMetaClass(type):
def run_wrapper(func):
@functools.wraps(func)
def run(self, info, *args, **kwargs):
- self._hook_progress({'status': 'started'}, info)
+ info_copy = copy.deepcopy(self._copy_infodict(info))
+ self._hook_progress({'status': 'started'}, info_copy)
ret = func(self, info, *args, **kwargs)
if ret is not None:
_, info = ret
- self._hook_progress({'status': 'finished'}, info)
+ self._hook_progress({'status': 'finished'}, info_copy)
return ret
return run
@@ -93,6 +94,9 @@ def set_downloader(self, downloader):
for ph in getattr(downloader, '_postprocessor_hooks', []):
self.add_progress_hook(ph)
+ def _copy_infodict(self, info_dict):
+ return getattr(self._downloader, '_copy_infodict', dict)(info_dict)
+
@staticmethod
def _restrict_to(*, video=True, audio=True, images=True):
allowed = {'video': video, 'audio': audio, 'images': images}
@@ -142,11 +146,8 @@ def _configuration_args(self, exe, *args, **kwargs):
def _hook_progress(self, status, info_dict):
if not self._progress_hooks:
return
- info_dict = dict(info_dict)
- for key in ('__original_infodict', '__postprocessors'):
- info_dict.pop(key, None)
status.update({
- 'info_dict': copy.deepcopy(info_dict),
+ 'info_dict': info_dict,
'postprocessor': self.pp_key(),
})
for ph in self._progress_hooks:

72
48ee10ee8adcf61e1136a252462670ec230e9439.patch Normal file
View file

@ -0,0 +1,72 @@
From 48ee10ee8adcf61e1136a252462670ec230e9439 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Fri, 15 Oct 2021 18:50:28 +0530
Subject: [PATCH] Fix conflict b/w id and ext in format selection Closes #1282
---
yt_dlp/YoutubeDL.py | 27 +++++++++++++++++++--------
1 file changed, 19 insertions(+), 8 deletions(-)
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 542a97794..aff7d6ddb 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -483,6 +483,12 @@ class YoutubeDL(object):
'track_number', 'disc_number', 'release_year',
))
+ _format_selection_exts = {
+ 'audio': {'m4a', 'mp3', 'ogg', 'aac'},
+ 'video': {'mp4', 'flv', 'webm', '3gp'},
+ 'storyboards': {'mhtml'},
+ }
+
params = None
_ies = {}
_pps = {'pre_process': [], 'before_dl': [], 'after_move': [], 'post_process': []}
@@ -1980,11 +1986,11 @@ def selector_function(ctx):
filter_f = lambda f: _filter_f(f) and (
f.get('vcodec') != 'none' or f.get('acodec') != 'none')
else:
- if format_spec in ('m4a', 'mp3', 'ogg', 'aac'): # audio extension
+ if format_spec in self._format_selection_exts['audio']:
filter_f = lambda f: f.get('ext') == format_spec and f.get('acodec') != 'none'
- elif format_spec in ('mp4', 'flv', 'webm', '3gp'): # video extension
+ elif format_spec in self._format_selection_exts['video']:
filter_f = lambda f: f.get('ext') == format_spec and f.get('acodec') != 'none' and f.get('vcodec') != 'none'
- elif format_spec in ('mhtml', ): # storyboards extension
+ elif format_spec in self._format_selection_exts['storyboards']:
filter_f = lambda f: f.get('ext') == format_spec and f.get('acodec') == 'none' and f.get('vcodec') == 'none'
else:
filter_f = lambda f: f.get('format_id') == format_spec # id
@@ -2259,10 +2265,18 @@ def is_wellformed(f):
formats_dict[format_id].append(format)
# Make sure all formats have unique format_id
+ common_exts = set(ext for exts in self._format_selection_exts.values() for ext in exts)
for format_id, ambiguous_formats in formats_dict.items():
- if len(ambiguous_formats) > 1:
- for i, format in enumerate(ambiguous_formats):
+ ambigious_id = len(ambiguous_formats) > 1
+ for i, format in enumerate(ambiguous_formats):
+ if ambigious_id:
format['format_id'] = '%s-%d' % (format_id, i)
+ if format.get('ext') is None:
+ format['ext'] = determine_ext(format['url']).lower()
+ # Ensure there is no conflict between id and ext in format selection
+ # See https://github.com/yt-dlp/yt-dlp/issues/1282
+ if format['format_id'] != format['ext'] and format['format_id'] in common_exts:
+ format['format_id'] = 'f%s' % format['format_id']
for i, format in enumerate(formats):
if format.get('format') is None:
@@ -2271,9 +2285,6 @@ def is_wellformed(f):
res=self.format_resolution(format),
note=format_field(format, 'format_note', ' (%s)'),
)
- # Automatically determine file extension if missing
- if format.get('ext') is None:
- format['ext'] = determine_ext(format['url']).lower()
# Automatically determine protocol if missing (useful for format
# selection purposes)
if format.get('protocol') is None:

49
580d3274e50d9cca79189689ba53db7295ea267c.patch Normal file
View file

@ -0,0 +1,49 @@
From 580d3274e50d9cca79189689ba53db7295ea267c Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sat, 16 Oct 2021 20:13:23 +0530
Subject: [PATCH] [youtube] Expose different formats with same itag
---
yt_dlp/downloader/common.py | 1 -
yt_dlp/extractor/youtube.py | 9 +++++++--
2 files changed, 7 insertions(+), 3 deletions(-)
diff --git a/yt_dlp/downloader/common.py b/yt_dlp/downloader/common.py
index 96b78a968..9081794db 100644
--- a/yt_dlp/downloader/common.py
+++ b/yt_dlp/downloader/common.py
@@ -1,6 +1,5 @@
from __future__ import division, unicode_literals
-import copy
import os
import re
import time
diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py
index 1ef80445e..dc9aa8ab7 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube.py
@@ -2692,7 +2692,9 @@ def guess_quality(f):
itag = self._search_regex(
r'/itag/(\d+)', f['url'], 'itag', default=None)
if itag in itags:
- continue
+ itag += '-hls'
+ if itag in itags:
+ continue
if itag:
f['format_id'] = itag
itags.append(itag)
@@ -2704,8 +2706,11 @@ def guess_quality(f):
for f in self._extract_mpd_formats(dash_manifest_url, video_id, fatal=False):
itag = f['format_id']
if itag in itags:
- continue
+ itag += '-dash'
+ if itag in itags:
+ continue
if itag:
+ f['format_id'] = itag
itags.append(itag)
f['quality'] = guess_quality(f)
filesize = int_or_none(self._search_regex(

11
yt-dlp.spec
View file

@ -4,19 +4,26 @@
Summary: Small command-line program to download videos and audio from different websites
Name: yt-dlp
Version: 2021.10.10
Release: 1
Release: 2
License: Public Domain and GPLv2+
Group: Video
Url: https://github.com/yt-dlp/yt-dlp
# source from Github requires at least pandoc which is not packaged
Source0: %{pypi_source}
# Pick upstream commits to fix https://github.com/yt-dlp/yt-dlp/issues/1301
# https://github.com/yt-dlp/yt-dlp/commit/48ee10ee8adcf61e1136a252462670ec230e9439
Patch1: 48ee10ee8adcf61e1136a252462670ec230e9439.patch
# https://github.com/yt-dlp/yt-dlp/commit/03b4de722a6cf86dbcc6d17a63145ec59a573bf6
Patch2: 03b4de722a6cf86dbcc6d17a63145ec59a573bf6.patch
# https://github.com/yt-dlp/yt-dlp/commit/580d3274e50d9cca79189689ba53db7295ea267c
Patch3: 580d3274e50d9cca79189689ba53db7295ea267c.patch
BuildRequires: ffmpeg
BuildRequires: gnupg2
BuildRequires: pkgconfig(python3)
BuildRequires: python3egg(setuptools)
BuildRequires: zip
BuildArch: noarch
Requires: python3
Requires: python3-%{name} = %{EVRD}
Recommends: ffmpeg
# youtube-dl is not developed anymore, it is a fork of it
Obsoletes: youtube-dl < 2021.10.10