Refactor download status handling to ensure correct file path processing and task synchronization (closes #872)

Refactor output template field substitution logic

Dockerfile

@@ -69,8 +69,8 @@ RUN BGUTIL_TAG="$(curl -Ls -o /dev/null -w '%{url_effective}' https://github.com
 COPY app ./app
 COPY --from=builder /metube/dist/metube ./ui/dist/metube
 
-ENV UID=1000
+ENV PUID=1000
-ENV GID=1000
+ENV PGID=1000
 ENV UMASK=022
 
 ENV DOWNLOAD_DIR /downloads

README.md

@@ -56,6 +56,7 @@ Certain values can be set via environment variables, using the `-e` parameter on
 * __OUTPUT_TEMPLATE__: The template for the filenames of the downloaded videos, formatted according to [this spec](https://github.com/yt-dlp/yt-dlp/blob/master/README.md#output-template). Defaults to `%(title)s.%(ext)s`.
 * __OUTPUT_TEMPLATE_CHAPTER__: The template for the filenames of the downloaded videos when split into chapters via postprocessors. Defaults to `%(title)s - %(section_number)s %(section_title)s.%(ext)s`.
 * __OUTPUT_TEMPLATE_PLAYLIST__: The template for the filenames of the downloaded videos when downloaded as a playlist. Defaults to `%(playlist_title)s/%(title)s.%(ext)s`. When empty, then `OUTPUT_TEMPLATE` is used.
+* __OUTPUT_TEMPLATE_CHANNEL__: The template for the filenames of the downloaded videos when downloaded as a channel. Defaults to `%(channel)s/%(title)s.%(ext)s`. When empty, then `OUTPUT_TEMPLATE` is used.
 * __YTDL_OPTIONS__: Additional options to pass to yt-dlp in JSON format. [See available options here](https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/YoutubeDL.py#L222). They roughly correspond to command-line options, though some do not have exact equivalents here. For example, `--recode-video` has to be specified via `postprocessors`. Also note that dashes are replaced with underscores. You may find [this script](https://github.com/yt-dlp/yt-dlp/blob/master/devscripts/cli_to_api.py) helpful for converting from command-line options to `YTDL_OPTIONS`.
 * __YTDL_OPTIONS_FILE__: A path to a JSON file that will be loaded and used for populating `YTDL_OPTIONS` above. Please note that if both `YTDL_OPTIONS_FILE` and `YTDL_OPTIONS` are specified, the options in `YTDL_OPTIONS` take precedence. The file will be monitored for changes and reloaded automatically when changes are detected.
 
@@ -73,8 +74,8 @@ Certain values can be set via environment variables, using the `-e` parameter on
 
 ### 🏠 Basic Setup
 
-* __UID__: User under which MeTube will run. Defaults to `1000`.
+* __PUID__: User under which MeTube will run. Defaults to `1000` (legacy `UID` also supported).
-* __GID__: Group under which MeTube will run. Defaults to `1000`.
+* __PGID__: Group under which MeTube will run. Defaults to `1000` (legacy `GID` also supported).
 * __UMASK__: Umask value used by MeTube. Defaults to `022`.
 * __DEFAULT_THEME__: Default theme to use for the UI, can be set to `light`, `dark`, or `auto`. Defaults to `auto`.
 * __LOGLEVEL__: Log level, can be set to `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL`, or `NONE`. Defaults to `INFO`. 

app/main.py

@@ -58,6 +58,7 @@ class Config:
         'OUTPUT_TEMPLATE': '%(title)s.%(ext)s',
         'OUTPUT_TEMPLATE_CHAPTER': '%(title)s - %(section_number)02d - %(section_title)s.%(ext)s',
         'OUTPUT_TEMPLATE_PLAYLIST': '%(playlist_title)s/%(title)s.%(ext)s',
+        'OUTPUT_TEMPLATE_CHANNEL': '%(channel)s/%(title)s.%(ext)s',
         'DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT' : '0',
         'YTDL_OPTIONS': '{}',
         'YTDL_OPTIONS_FILE': '',

app/ytdl.py

@@ -11,13 +11,53 @@ import re
 import types
 import dbm
 import subprocess
+from typing import Any
+from functools import lru_cache
 
 import yt_dlp.networking.impersonate
+from yt_dlp.utils import STR_FORMAT_RE_TMPL, STR_FORMAT_TYPES
 from dl_formats import get_format, get_opts, AUDIO_FORMATS
 from datetime import datetime
 
 log = logging.getLogger('ytdl')
 
+
+@lru_cache(maxsize=None)
+def _compile_outtmpl_pattern(field: str) -> re.Pattern:
+    """Compile a regex pattern to match a specific field in an output template, including optional format specifiers."""
+    conversion_types = f"[{re.escape(STR_FORMAT_TYPES)}]"
+    return re.compile(STR_FORMAT_RE_TMPL.format(re.escape(field), conversion_types))
+
+
+def _outtmpl_substitute_field(template: str, field: str, value: Any) -> str:
+    """Substitute a single field in an output template, applying any format specifiers to the value."""
+    pattern = _compile_outtmpl_pattern(field)
+
+    def replacement(match: re.Match) -> str:
+        if match.group("has_key") is None:
+            return match.group(0)
+
+        prefix = match.group("prefix") or ""
+        format_spec = match.group("format")
+
+        if not format_spec:
+            return f"{prefix}{value}"
+
+        conversion_type = format_spec[-1]
+        try:
+            if conversion_type in "diouxX":
+                coerced_value = int(value)
+            elif conversion_type in "eEfFgG":
+                coerced_value = float(value)
+            else:
+                coerced_value = value
+
+            return f"{prefix}{('%' + format_spec) % coerced_value}"
+        except (ValueError, TypeError):
+            return f"{prefix}{value}"
+
+    return pattern.sub(replacement, template)
+
 def _convert_generators_to_lists(obj):
     """Recursively convert generators to lists in a dictionary to make it pickleable."""
     if isinstance(obj, types.GeneratorType):
@@ -104,10 +144,7 @@ class Download:
 
             def put_status_postprocessor(d):
                 if d['postprocessor'] == 'MoveFiles' and d['status'] == 'finished':
-                    if '__finaldir' in d['info_dict']:
+                    filename = d['info_dict']['filepath']
-                        filename = os.path.join(d['info_dict']['__finaldir'], os.path.basename(d['info_dict']['filepath']))
-                    else:
-                        filename = d['info_dict']['filepath']
                     self.status_queue.put({'status': 'finished', 'filename': filename})
 
                 # Capture all chapter files when SplitChapters finishes
@@ -163,8 +200,14 @@ class Download:
         self.notifier = notifier
         self.info.status = 'preparing'
         await self.notifier.updated(self.info)
-        asyncio.create_task(self.update_status())
+        self.status_task = asyncio.create_task(self.update_status())
-        return await self.loop.run_in_executor(None, self.proc.join)
+        await self.loop.run_in_executor(None, self.proc.join)
+        # Signal update_status to stop and wait for it to finish
+        # so that all status updates (including MoveFiles with correct
+        # file size) are processed before _post_download_cleanup runs.
+        if self.status_queue is not None:
+            self.status_queue.put(None)
+        await self.status_task
 
     def cancel(self):
         log.info(f"Cancelling download: {self.info.title}")
@@ -181,8 +224,6 @@ class Download:
         log.info(f"Closing download process for: {self.info.title}")
         if self.started():
             self.proc.close()
-            if self.status_queue is not None:
-                self.status_queue.put(None)
 
     def running(self):
         try:
@@ -426,7 +467,7 @@ class DownloadQueue:
         base_directory = self.config.DOWNLOAD_DIR if (quality != 'audio' and format not in AUDIO_FORMATS) else self.config.AUDIO_DOWNLOAD_DIR
         if folder:
             if not self.config.CUSTOM_DIRS:
-                return None, {'status': 'error', 'msg': f'A folder for the download was specified but CUSTOM_DIRS is not true in the configuration.'}
+                return None, {'status': 'error', 'msg': 'A folder for the download was specified but CUSTOM_DIRS is not true in the configuration.'}
             dldirectory = os.path.realpath(os.path.join(base_directory, folder))
             real_base_directory = os.path.realpath(base_directory)
             if not dldirectory.startswith(real_base_directory):
@@ -446,12 +487,18 @@ class DownloadQueue:
         output = self.config.OUTPUT_TEMPLATE if len(dl.custom_name_prefix) == 0 else f'{dl.custom_name_prefix}.{self.config.OUTPUT_TEMPLATE}'
         output_chapter = self.config.OUTPUT_TEMPLATE_CHAPTER
         entry = getattr(dl, 'entry', None)
-        if entry is not None and 'playlist' in entry and entry['playlist'] is not None:
+        if entry is not None and entry.get('playlist_index') is not None:
             if len(self.config.OUTPUT_TEMPLATE_PLAYLIST):
                 output = self.config.OUTPUT_TEMPLATE_PLAYLIST
             for property, value in entry.items():
                 if property.startswith("playlist"):
-                    output = output.replace(f"%({property})s", str(value))
+                    output = _outtmpl_substitute_field(output, property, value)
+        if entry is not None and entry.get('channel_index') is not None:
+            if len(self.config.OUTPUT_TEMPLATE_CHANNEL):
+                output = self.config.OUTPUT_TEMPLATE_CHANNEL
+            for property, value in entry.items():
+                if property.startswith("channel"):
+                    output = _outtmpl_substitute_field(output, property, value)
         ytdl_options = dict(self.config.YTDL_OPTIONS)
         playlist_item_limit = getattr(dl, 'playlist_item_limit', 0)
         if playlist_item_limit > 0:
@@ -480,27 +527,27 @@ class DownloadQueue:
         etype = entry.get('_type') or 'video'
 
         if etype.startswith('url'):
-            log.debug('Processing as an url')
+            log.debug('Processing as a url')
             return await self.add(entry['url'], quality, format, folder, custom_name_prefix, playlist_item_limit, auto_start, split_by_chapters, chapter_template, already)
-        elif etype == 'playlist':
+        elif etype == 'playlist' or etype == 'channel':
-            log.debug('Processing as a playlist')
+            log.debug(f'Processing as a {etype}')
             entries = entry['entries']
             # Convert generator to list if needed (for len() and slicing operations)
             if isinstance(entries, types.GeneratorType):
                 entries = list(entries)
-            log.info(f'playlist detected with {len(entries)} entries')
+            log.info(f'{etype} detected with {len(entries)} entries')
-            playlist_index_digits = len(str(len(entries)))
+            index_digits = len(str(len(entries)))
             results = []
             if playlist_item_limit > 0:
-                log.info(f'Playlist item limit is set. Processing only first {playlist_item_limit} entries')
+                log.info(f'Item limit is set. Processing only first {playlist_item_limit} entries')
                 entries = entries[:playlist_item_limit]
             for index, etr in enumerate(entries, start=1):
                 etr["_type"] = "video"
-                etr["playlist"] = entry["id"]
+                etr[etype] = entry.get("id") or entry.get("channel_id") or entry.get("channel")
-                etr["playlist_index"] = '{{0:0{0:d}d}}'.format(playlist_index_digits).format(index)
+                etr[f"{etype}_index"] = '{{0:0{0:d}d}}'.format(index_digits).format(index)
                 for property in ("id", "title", "uploader", "uploader_id"):
                     if property in entry:
-                        etr[f"playlist_{property}"] = entry[property]
+                        etr[f"{etype}_{property}"] = entry[property]
                 results.append(await self.__add_entry(etr, quality, format, folder, custom_name_prefix, playlist_item_limit, auto_start, split_by_chapters, chapter_template, already))
             if any(res['status'] == 'error' for res in results):
                 return {'status': 'error', 'msg': ', '.join(res['msg'] for res in results if res['status'] == 'error' and 'msg' in res)}

docker-entrypoint.sh

@@ -1,22 +1,25 @@
 #!/bin/sh
 
+PUID="${UID:-$PUID}"
+PGID="${GID:-$PGID}"
+
 echo "Setting umask to ${UMASK}"
 umask ${UMASK}
 echo "Creating download directory (${DOWNLOAD_DIR}), state directory (${STATE_DIR}), and temp dir (${TEMP_DIR})"
 mkdir -p "${DOWNLOAD_DIR}" "${STATE_DIR}" "${TEMP_DIR}"
 
 if [ `id -u` -eq 0 ] && [ `id -g` -eq 0 ]; then
-    if [ "${UID}" -eq 0 ]; then
+    if [ "${PUID}" -eq 0 ]; then
-        echo "Warning: it is not recommended to run as root user, please check your setting of the UID environment variable"
+        echo "Warning: it is not recommended to run as root user, please check your setting of the PUID/PGID (or legacy UID/GID) environment variables"
     fi
     if [ "${CHOWN_DIRS:-true}" != "false" ]; then
-        echo "Changing ownership of download and state directories to ${UID}:${GID}"
+        echo "Changing ownership of download and state directories to ${PUID}:${PGID}"
-        chown -R "${UID}":"${GID}" /app "${DOWNLOAD_DIR}" "${STATE_DIR}" "${TEMP_DIR}"
+        chown -R "${PUID}":"${PGID}" /app "${DOWNLOAD_DIR}" "${STATE_DIR}" "${TEMP_DIR}"
     fi
     echo "Starting BgUtils POT Provider"
-    gosu "${UID}":"${GID}" bgutil-pot server >/tmp/bgutil-pot.log 2>&1 &
+    gosu "${PUID}":"${PGID}" bgutil-pot server >/tmp/bgutil-pot.log 2>&1 &
-    echo "Running MeTube as user ${UID}:${GID}"
+    echo "Running MeTube as user ${PUID}:${PGID}"
-    exec gosu "${UID}":"${GID}" python3 app/main.py
+    exec gosu "${PUID}":"${PGID}" python3 app/main.py
 else
     echo "User set by docker; running MeTube as `id -u`:`id -g`"
     echo "Starting BgUtils POT Provider"

ui/src/app/app.html

@@ -94,7 +94,7 @@
               autocomplete="off"
               spellcheck="false"
               class="form-control form-control-lg"
-              placeholder="Enter video or playlist URL"
+              placeholder="Enter video, channel, or playlist URL"
               name="addUrl"
               [(ngModel)]="addUrl"
               [disabled]="addInProgress || downloads.loading">
@@ -215,7 +215,7 @@
                       (keydown)="isNumber($event)"
                       [(ngModel)]="playlistItemLimit"
                       [disabled]="addInProgress || downloads.loading"
-                      ngbTooltip="Maximum number of items to download from a playlist (0 = no limit)">
+                      ngbTooltip="Maximum number of items to download from a playlist or channel (0 = no limit)">
                   </div>
                 </div>
                 <div class="col-12">