Merge pull request #892 from ivanbarsukov/template_substitution

Refactor output template field substitution logic

README.md

@@ -56,6 +56,7 @@ Certain values can be set via environment variables, using the `-e` parameter on
 * __OUTPUT_TEMPLATE__: The template for the filenames of the downloaded videos, formatted according to [this spec](https://github.com/yt-dlp/yt-dlp/blob/master/README.md#output-template). Defaults to `%(title)s.%(ext)s`.
 * __OUTPUT_TEMPLATE_CHAPTER__: The template for the filenames of the downloaded videos when split into chapters via postprocessors. Defaults to `%(title)s - %(section_number)s %(section_title)s.%(ext)s`.
 * __OUTPUT_TEMPLATE_PLAYLIST__: The template for the filenames of the downloaded videos when downloaded as a playlist. Defaults to `%(playlist_title)s/%(title)s.%(ext)s`. When empty, then `OUTPUT_TEMPLATE` is used.
+* __OUTPUT_TEMPLATE_CHANNEL__: The template for the filenames of the downloaded videos when downloaded as a channel. Defaults to `%(channel)s/%(title)s.%(ext)s`. When empty, then `OUTPUT_TEMPLATE` is used.
 * __YTDL_OPTIONS__: Additional options to pass to yt-dlp in JSON format. [See available options here](https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/YoutubeDL.py#L222). They roughly correspond to command-line options, though some do not have exact equivalents here. For example, `--recode-video` has to be specified via `postprocessors`. Also note that dashes are replaced with underscores. You may find [this script](https://github.com/yt-dlp/yt-dlp/blob/master/devscripts/cli_to_api.py) helpful for converting from command-line options to `YTDL_OPTIONS`.
 * __YTDL_OPTIONS_FILE__: A path to a JSON file that will be loaded and used for populating `YTDL_OPTIONS` above. Please note that if both `YTDL_OPTIONS_FILE` and `YTDL_OPTIONS` are specified, the options in `YTDL_OPTIONS` take precedence. The file will be monitored for changes and reloaded automatically when changes are detected.
 

app/main.py

@@ -58,6 +58,7 @@ class Config:
         'OUTPUT_TEMPLATE': '%(title)s.%(ext)s',
         'OUTPUT_TEMPLATE_CHAPTER': '%(title)s - %(section_number)02d - %(section_title)s.%(ext)s',
         'OUTPUT_TEMPLATE_PLAYLIST': '%(playlist_title)s/%(title)s.%(ext)s',
+        'OUTPUT_TEMPLATE_CHANNEL': '%(channel)s/%(title)s.%(ext)s',
         'DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT' : '0',
         'YTDL_OPTIONS': '{}',
         'YTDL_OPTIONS_FILE': '',

app/ytdl.py

@@ -11,13 +11,53 @@ import re
 import types
 import dbm
 import subprocess
+from typing import Any
+from functools import lru_cache
 
 import yt_dlp.networking.impersonate
+from yt_dlp.utils import STR_FORMAT_RE_TMPL, STR_FORMAT_TYPES
 from dl_formats import get_format, get_opts, AUDIO_FORMATS
 from datetime import datetime
 
 log = logging.getLogger('ytdl')
 
+
+@lru_cache(maxsize=None)
+def _compile_outtmpl_pattern(field: str) -> re.Pattern:
+    """Compile a regex pattern to match a specific field in an output template, including optional format specifiers."""
+    conversion_types = f"[{re.escape(STR_FORMAT_TYPES)}]"
+    return re.compile(STR_FORMAT_RE_TMPL.format(re.escape(field), conversion_types))
+
+
+def _outtmpl_substitute_field(template: str, field: str, value: Any) -> str:
+    """Substitute a single field in an output template, applying any format specifiers to the value."""
+    pattern = _compile_outtmpl_pattern(field)
+
+    def replacement(match: re.Match) -> str:
+        if match.group("has_key") is None:
+            return match.group(0)
+
+        prefix = match.group("prefix") or ""
+        format_spec = match.group("format")
+
+        if not format_spec:
+            return f"{prefix}{value}"
+
+        conversion_type = format_spec[-1]
+        try:
+            if conversion_type in "diouxX":
+                coerced_value = int(value)
+            elif conversion_type in "eEfFgG":
+                coerced_value = float(value)
+            else:
+                coerced_value = value
+
+            return f"{prefix}{('%' + format_spec) % coerced_value}"
+        except (ValueError, TypeError):
+            return f"{prefix}{value}"
+
+    return pattern.sub(replacement, template)
+
 def _convert_generators_to_lists(obj):
     """Recursively convert generators to lists in a dictionary to make it pickleable."""
     if isinstance(obj, types.GeneratorType):
@@ -426,7 +466,7 @@ class DownloadQueue:
         base_directory = self.config.DOWNLOAD_DIR if (quality != 'audio' and format not in AUDIO_FORMATS) else self.config.AUDIO_DOWNLOAD_DIR
         if folder:
             if not self.config.CUSTOM_DIRS:
-                return None, {'status': 'error', 'msg': f'A folder for the download was specified but CUSTOM_DIRS is not true in the configuration.'}
+                return None, {'status': 'error', 'msg': 'A folder for the download was specified but CUSTOM_DIRS is not true in the configuration.'}
             dldirectory = os.path.realpath(os.path.join(base_directory, folder))
             real_base_directory = os.path.realpath(base_directory)
             if not dldirectory.startswith(real_base_directory):
@@ -446,12 +486,18 @@ class DownloadQueue:
         output = self.config.OUTPUT_TEMPLATE if len(dl.custom_name_prefix) == 0 else f'{dl.custom_name_prefix}.{self.config.OUTPUT_TEMPLATE}'
         output_chapter = self.config.OUTPUT_TEMPLATE_CHAPTER
         entry = getattr(dl, 'entry', None)
-        if entry is not None and 'playlist' in entry and entry['playlist'] is not None:
+        if entry is not None and entry.get('playlist_index') is not None:
             if len(self.config.OUTPUT_TEMPLATE_PLAYLIST):
                 output = self.config.OUTPUT_TEMPLATE_PLAYLIST
             for property, value in entry.items():
                 if property.startswith("playlist"):
-                    output = output.replace(f"%({property})s", str(value))
+                    output = _outtmpl_substitute_field(output, property, value)
+        if entry is not None and entry.get('channel_index') is not None:
+            if len(self.config.OUTPUT_TEMPLATE_CHANNEL):
+                output = self.config.OUTPUT_TEMPLATE_CHANNEL
+            for property, value in entry.items():
+                if property.startswith("channel"):
+                    output = _outtmpl_substitute_field(output, property, value)
         ytdl_options = dict(self.config.YTDL_OPTIONS)
         playlist_item_limit = getattr(dl, 'playlist_item_limit', 0)
         if playlist_item_limit > 0:
@@ -480,27 +526,27 @@ class DownloadQueue:
         etype = entry.get('_type') or 'video'
 
         if etype.startswith('url'):
-            log.debug('Processing as an url')
+            log.debug('Processing as a url')
             return await self.add(entry['url'], quality, format, folder, custom_name_prefix, playlist_item_limit, auto_start, split_by_chapters, chapter_template, already)
-        elif etype == 'playlist':
+        elif etype == 'playlist' or etype == 'channel':
-            log.debug('Processing as a playlist')
+            log.debug(f'Processing as a {etype}')
             entries = entry['entries']
             # Convert generator to list if needed (for len() and slicing operations)
             if isinstance(entries, types.GeneratorType):
                 entries = list(entries)
-            log.info(f'playlist detected with {len(entries)} entries')
+            log.info(f'{etype} detected with {len(entries)} entries')
-            playlist_index_digits = len(str(len(entries)))
+            index_digits = len(str(len(entries)))
             results = []
             if playlist_item_limit > 0:
-                log.info(f'Playlist item limit is set. Processing only first {playlist_item_limit} entries')
+                log.info(f'Item limit is set. Processing only first {playlist_item_limit} entries')
                 entries = entries[:playlist_item_limit]
             for index, etr in enumerate(entries, start=1):
                 etr["_type"] = "video"
-                etr["playlist"] = entry["id"]
+                etr[etype] = entry.get("id") or entry.get("channel_id") or entry.get("channel")
-                etr["playlist_index"] = '{{0:0{0:d}d}}'.format(playlist_index_digits).format(index)
+                etr[f"{etype}_index"] = '{{0:0{0:d}d}}'.format(index_digits).format(index)
                 for property in ("id", "title", "uploader", "uploader_id"):
                     if property in entry:
-                        etr[f"playlist_{property}"] = entry[property]
+                        etr[f"{etype}_{property}"] = entry[property]
                 results.append(await self.__add_entry(etr, quality, format, folder, custom_name_prefix, playlist_item_limit, auto_start, split_by_chapters, chapter_template, already))
             if any(res['status'] == 'error' for res in results):
                 return {'status': 'error', 'msg': ', '.join(res['msg'] for res in results if res['status'] == 'error' and 'msg' in res)}

ui/src/app/app.html

@@ -94,7 +94,7 @@
               autocomplete="off"
               spellcheck="false"
               class="form-control form-control-lg"
-              placeholder="Enter video or playlist URL"
+              placeholder="Enter video, channel, or playlist URL"
               name="addUrl"
               [(ngModel)]="addUrl"
               [disabled]="addInProgress || downloads.loading">
@@ -215,7 +215,7 @@
                       (keydown)="isNumber($event)"
                       [(ngModel)]="playlistItemLimit"
                       [disabled]="addInProgress || downloads.loading"
-                      ngbTooltip="Maximum number of items to download from a playlist (0 = no limit)">
+                      ngbTooltip="Maximum number of items to download from a playlist or channel (0 = no limit)">
                   </div>
                 </div>
                 <div class="col-12">