Refactor download status handling to ensure correct file path processing and task synchronization (closes #872)

Refactor output template field substitution logic

README.md

@@ -56,6 +56,7 @@ Certain values can be set via environment variables, using the `-e` parameter on
 * __OUTPUT_TEMPLATE__: The template for the filenames of the downloaded videos, formatted according to [this spec](https://github.com/yt-dlp/yt-dlp/blob/master/README.md#output-template). Defaults to `%(title)s.%(ext)s`.
 * __OUTPUT_TEMPLATE_CHAPTER__: The template for the filenames of the downloaded videos when split into chapters via postprocessors. Defaults to `%(title)s - %(section_number)s %(section_title)s.%(ext)s`.
 * __OUTPUT_TEMPLATE_PLAYLIST__: The template for the filenames of the downloaded videos when downloaded as a playlist. Defaults to `%(playlist_title)s/%(title)s.%(ext)s`. When empty, then `OUTPUT_TEMPLATE` is used.
+* __OUTPUT_TEMPLATE_CHANNEL__: The template for the filenames of the downloaded videos when downloaded as a channel. Defaults to `%(channel)s/%(title)s.%(ext)s`. When empty, then `OUTPUT_TEMPLATE` is used.
 * __YTDL_OPTIONS__: Additional options to pass to yt-dlp in JSON format. [See available options here](https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/YoutubeDL.py#L222). They roughly correspond to command-line options, though some do not have exact equivalents here. For example, `--recode-video` has to be specified via `postprocessors`. Also note that dashes are replaced with underscores. You may find [this script](https://github.com/yt-dlp/yt-dlp/blob/master/devscripts/cli_to_api.py) helpful for converting from command-line options to `YTDL_OPTIONS`.
 * __YTDL_OPTIONS_FILE__: A path to a JSON file that will be loaded and used for populating `YTDL_OPTIONS` above. Please note that if both `YTDL_OPTIONS_FILE` and `YTDL_OPTIONS` are specified, the options in `YTDL_OPTIONS` take precedence. The file will be monitored for changes and reloaded automatically when changes are detected.
 

app/main.py

@@ -58,6 +58,7 @@ class Config:
         'OUTPUT_TEMPLATE': '%(title)s.%(ext)s',
         'OUTPUT_TEMPLATE_CHAPTER': '%(title)s - %(section_number)02d - %(section_title)s.%(ext)s',
         'OUTPUT_TEMPLATE_PLAYLIST': '%(playlist_title)s/%(title)s.%(ext)s',
+        'OUTPUT_TEMPLATE_CHANNEL': '%(channel)s/%(title)s.%(ext)s',
         'DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT' : '0',
         'YTDL_OPTIONS': '{}',
         'YTDL_OPTIONS_FILE': '',
@@ -401,7 +402,7 @@ async def on_prepare(request, response):
         response.headers['Access-Control-Allow-Headers'] = 'Content-Type'
 
 app.on_response_prepare.append(on_prepare)
- 
+
 def supports_reuse_port():
     try:
         sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

app/ytdl.py

@@ -11,13 +11,53 @@ import re
 import types
 import dbm
 import subprocess
+from typing import Any
+from functools import lru_cache
 
 import yt_dlp.networking.impersonate
+from yt_dlp.utils import STR_FORMAT_RE_TMPL, STR_FORMAT_TYPES
 from dl_formats import get_format, get_opts, AUDIO_FORMATS
 from datetime import datetime
 
 log = logging.getLogger('ytdl')
 
+
+@lru_cache(maxsize=None)
+def _compile_outtmpl_pattern(field: str) -> re.Pattern:
+    """Compile a regex pattern to match a specific field in an output template, including optional format specifiers."""
+    conversion_types = f"[{re.escape(STR_FORMAT_TYPES)}]"
+    return re.compile(STR_FORMAT_RE_TMPL.format(re.escape(field), conversion_types))
+
+
+def _outtmpl_substitute_field(template: str, field: str, value: Any) -> str:
+    """Substitute a single field in an output template, applying any format specifiers to the value."""
+    pattern = _compile_outtmpl_pattern(field)
+
+    def replacement(match: re.Match) -> str:
+        if match.group("has_key") is None:
+            return match.group(0)
+
+        prefix = match.group("prefix") or ""
+        format_spec = match.group("format")
+
+        if not format_spec:
+            return f"{prefix}{value}"
+
+        conversion_type = format_spec[-1]
+        try:
+            if conversion_type in "diouxX":
+                coerced_value = int(value)
+            elif conversion_type in "eEfFgG":
+                coerced_value = float(value)
+            else:
+                coerced_value = value
+
+            return f"{prefix}{('%' + format_spec) % coerced_value}"
+        except (ValueError, TypeError):
+            return f"{prefix}{value}"
+
+    return pattern.sub(replacement, template)
+
 def _convert_generators_to_lists(obj):
     """Recursively convert generators to lists in a dictionary to make it pickleable."""
     if isinstance(obj, types.GeneratorType):
@@ -104,12 +144,9 @@ class Download:
 
             def put_status_postprocessor(d):
                 if d['postprocessor'] == 'MoveFiles' and d['status'] == 'finished':
-                    if '__finaldir' in d['info_dict']:
+                    filename = d['info_dict']['filepath']
-                        filename = os.path.join(d['info_dict']['__finaldir'], os.path.basename(d['info_dict']['filepath']))
-                    else:
-                        filename = d['info_dict']['filepath']
                     self.status_queue.put({'status': 'finished', 'filename': filename})
-                
+
                 # Capture all chapter files when SplitChapters finishes
                 elif d.get('postprocessor') == 'SplitChapters' and d.get('status') == 'finished':
                     chapters = d.get('info_dict', {}).get('chapters', [])
@@ -134,7 +171,7 @@ class Download:
                 'postprocessor_hooks': [put_status_postprocessor],
                 **self.ytdl_opts,
             }
-            
+
             # Add chapter splitting options if enabled
             if self.info.split_by_chapters:
                 ytdl_params['outtmpl']['chapter'] = self.info.chapter_template
@@ -144,7 +181,7 @@ class Download:
                     'key': 'FFmpegSplitChapters',
                     'force_keyframes': False
                 })
-            
+
             ret = yt_dlp.YoutubeDL(params=ytdl_params).download([self.info.url])
             self.status_queue.put({'status': 'finished' if ret == 0 else 'error'})
             log.info(f"Finished download for: {self.info.title}")
@@ -163,8 +200,14 @@ class Download:
         self.notifier = notifier
         self.info.status = 'preparing'
         await self.notifier.updated(self.info)
-        asyncio.create_task(self.update_status())
+        self.status_task = asyncio.create_task(self.update_status())
-        return await self.loop.run_in_executor(None, self.proc.join)
+        await self.loop.run_in_executor(None, self.proc.join)
+        # Signal update_status to stop and wait for it to finish
+        # so that all status updates (including MoveFiles with correct
+        # file size) are processed before _post_download_cleanup runs.
+        if self.status_queue is not None:
+            self.status_queue.put(None)
+        await self.status_task
 
     def cancel(self):
         log.info(f"Cancelling download: {self.info.title}")
@@ -181,8 +224,6 @@ class Download:
         log.info(f"Closing download process for: {self.info.title}")
         if self.started():
             self.proc.close()
-            if self.status_queue is not None:
-                self.status_queue.put(None)
 
     def running(self):
         try:
@@ -211,7 +252,7 @@ class Download:
                     self.info.filename = re.sub(r'\.webm$', '.jpg', self.info.filename)
 
             # Handle chapter files
-            log.debug(f"Update status for {self.info.title}: {status}") 
+            log.debug(f"Update status for {self.info.title}: {status}")
             if 'chapter_file' in status:
                 chapter_file = status.get('chapter_file')
                 if not hasattr(self.info, 'chapter_files'):
@@ -224,7 +265,7 @@ class Download:
                     self.info.chapter_files.append({'filename': rel_path, 'size': file_size})
                 # Skip the rest of status processing for chapter files
                 continue
-            
+
             self.info.status = status['status']
             self.info.msg = status.get('msg')
             if 'downloaded_bytes' in status:
@@ -355,7 +396,7 @@ class PersistentQueue:
                     log.debug(f"{log_prefix}{result.stdout or " was successful, no output"}")
             except FileNotFoundError:
                 log.debug(f"{log_prefix} failed: 'sqlite3' was not found")
-                
+
 class DownloadQueue:
     def __init__(self, config, notifier):
         self.config = config
@@ -426,7 +467,7 @@ class DownloadQueue:
         base_directory = self.config.DOWNLOAD_DIR if (quality != 'audio' and format not in AUDIO_FORMATS) else self.config.AUDIO_DOWNLOAD_DIR
         if folder:
             if not self.config.CUSTOM_DIRS:
-                return None, {'status': 'error', 'msg': f'A folder for the download was specified but CUSTOM_DIRS is not true in the configuration.'}
+                return None, {'status': 'error', 'msg': 'A folder for the download was specified but CUSTOM_DIRS is not true in the configuration.'}
             dldirectory = os.path.realpath(os.path.join(base_directory, folder))
             real_base_directory = os.path.realpath(base_directory)
             if not dldirectory.startswith(real_base_directory):
@@ -446,12 +487,18 @@ class DownloadQueue:
         output = self.config.OUTPUT_TEMPLATE if len(dl.custom_name_prefix) == 0 else f'{dl.custom_name_prefix}.{self.config.OUTPUT_TEMPLATE}'
         output_chapter = self.config.OUTPUT_TEMPLATE_CHAPTER
         entry = getattr(dl, 'entry', None)
-        if entry is not None and 'playlist' in entry and entry['playlist'] is not None:
+        if entry is not None and entry.get('playlist_index') is not None:
             if len(self.config.OUTPUT_TEMPLATE_PLAYLIST):
                 output = self.config.OUTPUT_TEMPLATE_PLAYLIST
             for property, value in entry.items():
                 if property.startswith("playlist"):
-                    output = output.replace(f"%({property})s", str(value))
+                    output = _outtmpl_substitute_field(output, property, value)
+        if entry is not None and entry.get('channel_index') is not None:
+            if len(self.config.OUTPUT_TEMPLATE_CHANNEL):
+                output = self.config.OUTPUT_TEMPLATE_CHANNEL
+            for property, value in entry.items():
+                if property.startswith("channel"):
+                    output = _outtmpl_substitute_field(output, property, value)
         ytdl_options = dict(self.config.YTDL_OPTIONS)
         playlist_item_limit = getattr(dl, 'playlist_item_limit', 0)
         if playlist_item_limit > 0:
@@ -480,27 +527,27 @@ class DownloadQueue:
         etype = entry.get('_type') or 'video'
 
         if etype.startswith('url'):
-            log.debug('Processing as an url')
+            log.debug('Processing as a url')
             return await self.add(entry['url'], quality, format, folder, custom_name_prefix, playlist_item_limit, auto_start, split_by_chapters, chapter_template, already)
-        elif etype == 'playlist':
+        elif etype == 'playlist' or etype == 'channel':
-            log.debug('Processing as a playlist')
+            log.debug(f'Processing as a {etype}')
             entries = entry['entries']
             # Convert generator to list if needed (for len() and slicing operations)
             if isinstance(entries, types.GeneratorType):
                 entries = list(entries)
-            log.info(f'playlist detected with {len(entries)} entries')
+            log.info(f'{etype} detected with {len(entries)} entries')
-            playlist_index_digits = len(str(len(entries)))
+            index_digits = len(str(len(entries)))
             results = []
             if playlist_item_limit > 0:
-                log.info(f'Playlist item limit is set. Processing only first {playlist_item_limit} entries')
+                log.info(f'Item limit is set. Processing only first {playlist_item_limit} entries')
                 entries = entries[:playlist_item_limit]
             for index, etr in enumerate(entries, start=1):
                 etr["_type"] = "video"
-                etr["playlist"] = entry["id"]
+                etr[etype] = entry.get("id") or entry.get("channel_id") or entry.get("channel")
-                etr["playlist_index"] = '{{0:0{0:d}d}}'.format(playlist_index_digits).format(index)
+                etr[f"{etype}_index"] = '{{0:0{0:d}d}}'.format(index_digits).format(index)
                 for property in ("id", "title", "uploader", "uploader_id"):
                     if property in entry:
-                        etr[f"playlist_{property}"] = entry[property]
+                        etr[f"{etype}_{property}"] = entry[property]
                 results.append(await self.__add_entry(etr, quality, format, folder, custom_name_prefix, playlist_item_limit, auto_start, split_by_chapters, chapter_template, already))
             if any(res['status'] == 'error' for res in results):
                 return {'status': 'error', 'msg': ', '.join(res['msg'] for res in results if res['status'] == 'error' and 'msg' in res)}

ui/src/app/app.html

@@ -63,7 +63,7 @@
         <ul class="dropdown-menu dropdown-menu-end position-absolute" aria-labelledby="theme-select">
           @for (theme of themes; track theme) {
             <li>
-              <button type="button" class="dropdown-item d-flex align-items-center" 
+              <button type="button" class="dropdown-item d-flex align-items-center"
               [class.active]="activeTheme === theme"
               (click)="themeChanged(theme)">
                 <span class="me-2 opacity-50">
@@ -94,7 +94,7 @@
               autocomplete="off"
               spellcheck="false"
               class="form-control form-control-lg"
-              placeholder="Enter video or playlist URL"
+              placeholder="Enter video, channel, or playlist URL"
               name="addUrl"
               [(ngModel)]="addUrl"
               [disabled]="addInProgress || downloads.loading">
@@ -190,7 +190,7 @@
                       ngbTooltip="Choose where to save downloads. Type to create a new folder." />
                   }
                   </div>
-           
+
                 </div>
                 <div class="col-md-6">
                   <div class="input-group">
@@ -215,7 +215,7 @@
                       (keydown)="isNumber($event)"
                       [(ngModel)]="playlistItemLimit"
                       [disabled]="addInProgress || downloads.loading"
-                      ngbTooltip="Maximum number of items to download from a playlist (0 = no limit)">
+                      ngbTooltip="Maximum number of items to download from a playlist or channel (0 = no limit)">
                   </div>
                 </div>
                 <div class="col-12">
@@ -283,7 +283,7 @@
   </form>
 
   <!-- Batch Import Modal -->
-  <div class="modal fade" tabindex="-1" role="dialog" 
+  <div class="modal fade" tabindex="-1" role="dialog"
   [class.show]="batchImportModalOpen"
   [style.display]="batchImportModalOpen ? 'block' : 'none'">
     <div class="modal-dialog" role="document">
@@ -349,7 +349,7 @@
             <td title="{{ download.value.filename }}">
               <div class="d-flex flex-column flex-sm-row align-items-center row-gap-2 column-gap-3">
                 <div>{{ download.value.title }} </div>
-                <ngb-progressbar height="1.5rem" [showValue]="download.value.status !== 'preparing'" [striped]="download.value.status === 'preparing'" [animated]="download.value.status === 'preparing'" type="success" 
+                <ngb-progressbar height="1.5rem" [showValue]="download.value.status !== 'preparing'" [striped]="download.value.status === 'preparing'" [animated]="download.value.status === 'preparing'" type="success"
                 [value]="download.value.status === 'preparing' ? 100 : download.value.percent" class="download-progressbar" />
               </div>
             </td>