Merge pull request #876 from alexta69/copilot/fix-clear-completed-error

Fix undefined access error when bulk deleting downloads

.gitignore

@@ -7,6 +7,7 @@
 
 # dependencies
 /ui/node_modules
+/ui/package-lock.json
 
 # profiling files
 chrome-profiler-events*.json

README.md

@@ -33,14 +33,8 @@ Certain values can be set via environment variables, using the `-e` parameter on
 
 ### ⬇️ Download Behavior
 
-* __DOWNLOAD_MODE__: This flag controls how downloads are scheduled and executed. Options are `sequential`, `concurrent`, and `limited`.  Defaults to `limited`:
-    *   `sequential`: Downloads are processed one at a time. A new download won't start until the previous one has finished. This mode is useful for conserving system resources or ensuring downloads occur in strict order.
-    *   `concurrent`: Downloads are started immediately as they are added, with no built-in limit on how many run simultaneously. This mode may overwhelm your system if too many downloads start at once.
-    *   `limited`: Downloads are started concurrently but are capped by a concurrency limit. In this mode, a semaphore is used so that at most a fixed number of downloads run at any given time.
-* __MAX_CONCURRENT_DOWNLOADS__: This flag is used only when `DOWNLOAD_MODE` is set to `limited`.  
-    It specifies the maximum number of simultaneous downloads allowed. For example, if set to `5`, then at most five downloads will run concurrently, and any additional downloads will wait until one of the active downloads completes. Defaults to `3`. 
+* __MAX_CONCURRENT_DOWNLOADS__: Maximum number of simultaneous downloads allowed. For example, if set to `5`, then at most five downloads will run concurrently, and any additional downloads will wait until one of the active downloads completes. Defaults to `3`. 
 * __DELETE_FILE_ON_TRASHCAN__: if `true`, downloaded files are deleted on the server, when they are trashed from the "Completed" section of the UI. Defaults to `false`.
-* __DEFAULT_OPTION_PLAYLIST_STRICT_MODE__: if `true`, the "Strict Playlist mode" switch will be enabled by default. In this mode the playlists will be downloaded only if the URL strictly points to a playlist. URLs to videos inside a playlist will be treated same as direct video URL. Defaults to `false` .
 * __DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT__: Maximum number of playlist items that can be downloaded. Defaults to `0` (no limit).
 
 ### 📁 Storage & Directories
@@ -66,6 +60,8 @@ Certain values can be set via environment variables, using the `-e` parameter on
 
 ### 🌐 Web Server & URLs
 
+* __HOST__: The host address the web server will bind to. Defaults to `0.0.0.0` (all interfaces).
+* __PORT__: The port number the web server will listen on. Defaults to `8081`.
 * __URL_PREFIX__: Base path for the web server (for use when hosting behind a reverse proxy). Defaults to `/`.
 * __PUBLIC_HOST_URL__: Base URL for the download links shown in the UI for completed files. By default, MeTube serves them under its own URL. If your download directory is accessible on another URL and you want the download links to be based there, use this variable to set it.
 * __PUBLIC_HOST_AUDIO_URL__: Same as PUBLIC_HOST_URL but for audio downloads.

app/main.py

@@ -58,7 +58,6 @@ class Config:
         'OUTPUT_TEMPLATE': '%(title)s.%(ext)s',
         'OUTPUT_TEMPLATE_CHAPTER': '%(title)s - %(section_number)02d - %(section_title)s.%(ext)s',
         'OUTPUT_TEMPLATE_PLAYLIST': '%(playlist_title)s/%(title)s.%(ext)s',
-        'DEFAULT_OPTION_PLAYLIST_STRICT_MODE' : 'false',
         'DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT' : '0',
         'YTDL_OPTIONS': '{}',
         'YTDL_OPTIONS_FILE': '',
@@ -70,13 +69,12 @@ class Config:
         'KEYFILE': '',
         'BASE_DIR': '',
         'DEFAULT_THEME': 'auto',
-        'DOWNLOAD_MODE': 'limited',
         'MAX_CONCURRENT_DOWNLOADS': 3,
         'LOGLEVEL': 'INFO',
         'ENABLE_ACCESSLOG': 'false',
     }
 
-    _BOOLEAN = ('DOWNLOAD_DIRS_INDEXABLE', 'CUSTOM_DIRS', 'CREATE_CUSTOM_DIRS', 'DELETE_FILE_ON_TRASHCAN', 'DEFAULT_OPTION_PLAYLIST_STRICT_MODE', 'HTTPS', 'ENABLE_ACCESSLOG')
+    _BOOLEAN = ('DOWNLOAD_DIRS_INDEXABLE', 'CUSTOM_DIRS', 'CREATE_CUSTOM_DIRS', 'DELETE_FILE_ON_TRASHCAN', 'HTTPS', 'ENABLE_ACCESSLOG')
 
     def __init__(self):
         for k, v in self._DEFAULTS.items():
@@ -244,7 +242,6 @@ async def add(request):
     format = post.get('format')
     folder = post.get('folder')
     custom_name_prefix = post.get('custom_name_prefix')
-    playlist_strict_mode = post.get('playlist_strict_mode')
     playlist_item_limit = post.get('playlist_item_limit')
     auto_start = post.get('auto_start')
     split_by_chapters = post.get('split_by_chapters')
@@ -254,8 +251,6 @@ async def add(request):
         custom_name_prefix = ''
     if auto_start is None:
         auto_start = True
-    if playlist_strict_mode is None:
-        playlist_strict_mode = config.DEFAULT_OPTION_PLAYLIST_STRICT_MODE
     if playlist_item_limit is None:
         playlist_item_limit = config.DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT
     if split_by_chapters is None:
@@ -265,7 +260,7 @@ async def add(request):
 
     playlist_item_limit = int(playlist_item_limit)
 
-    status = await dqueue.add(url, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, split_by_chapters, chapter_template)
+    status = await dqueue.add(url, quality, format, folder, custom_name_prefix, playlist_item_limit, auto_start, split_by_chapters, chapter_template)
     return web.Response(text=serializer.encode(status))
 
 @routes.post(config.URL_PREFIX + 'delete')

app/ytdl.py

@@ -364,13 +364,7 @@ class DownloadQueue:
         self.done = PersistentQueue("completed", self.config.STATE_DIR + '/completed')
         self.pending = PersistentQueue("pending", self.config.STATE_DIR + '/pending')
         self.active_downloads = set()
-        self.semaphore = None
-        # For sequential mode, use an asyncio lock to ensure one-at-a-time execution.
-        if self.config.DOWNLOAD_MODE == 'sequential':
-            self.seq_lock = asyncio.Lock()
-        elif self.config.DOWNLOAD_MODE == 'limited':
-            self.semaphore = asyncio.Semaphore(int(self.config.MAX_CONCURRENT_DOWNLOADS))
-        
+        self.semaphore = asyncio.Semaphore(int(self.config.MAX_CONCURRENT_DOWNLOADS))
         self.done.load()
 
     async def __import_queue(self):
@@ -390,31 +384,12 @@ class DownloadQueue:
         if download.canceled:
             log.info(f"Download {download.info.title} was canceled, skipping start.")
             return
-        if self.config.DOWNLOAD_MODE == 'sequential':
-            async with self.seq_lock:
-                log.info("Starting sequential download.")
-                await download.start(self.notifier)
-                self._post_download_cleanup(download)
-        elif self.config.DOWNLOAD_MODE == 'limited' and self.semaphore is not None:
-            await self.__limited_concurrent_download(download)
-        else:
-            await self.__concurrent_download(download)
-
-    async def __concurrent_download(self, download):
-        log.info("Starting concurrent download without limits.")
-        asyncio.create_task(self._run_download(download))
-
-    async def __limited_concurrent_download(self, download):
-        log.info("Starting limited concurrent download.")
         async with self.semaphore:
-            await self._run_download(download)
-
-    async def _run_download(self, download):
-        if download.canceled:
-            log.info(f"Download {download.info.title} is canceled; skipping start.")
-            return
-        await download.start(self.notifier)
-        self._post_download_cleanup(download)
+            if download.canceled:
+                log.info(f"Download {download.info.title} was canceled, skipping start.")
+                return
+            await download.start(self.notifier)
+            self._post_download_cleanup(download)
 
     def _post_download_cleanup(self, download):
         if download.info.status != 'finished':
@@ -433,7 +408,7 @@ class DownloadQueue:
                 self.done.put(download)
                 asyncio.create_task(self.notifier.completed(download.info))
 
-    def __extract_info(self, url, playlist_strict_mode):
+    def __extract_info(self, url):
         debug_logging = logging.getLogger().isEnabledFor(logging.DEBUG)
         return yt_dlp.YoutubeDL(params={
             'quiet': not debug_logging,
@@ -441,7 +416,7 @@ class DownloadQueue:
             'no_color': True,
             'extract_flat': True,
             'ignore_no_formats_error': True,
-            'noplaylist': playlist_strict_mode,
+            'noplaylist': True,
             'paths': {"home": self.config.DOWNLOAD_DIR, "temp": self.config.TEMP_DIR},
             **self.config.YTDL_OPTIONS,
             **({'impersonate': yt_dlp.networking.impersonate.ImpersonateTarget.from_str(self.config.YTDL_OPTIONS['impersonate'])} if 'impersonate' in self.config.YTDL_OPTIONS else {}),
@@ -490,7 +465,7 @@ class DownloadQueue:
             self.pending.put(download)
         await self.notifier.added(dl)
 
-    async def __add_entry(self, entry, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, split_by_chapters, chapter_template, already):
+    async def __add_entry(self, entry, quality, format, folder, custom_name_prefix, playlist_item_limit, auto_start, split_by_chapters, chapter_template, already):
         if not entry:
             return {'status': 'error', 'msg': "Invalid/empty data was given."}
 
@@ -506,7 +481,7 @@ class DownloadQueue:
 
         if etype.startswith('url'):
             log.debug('Processing as an url')
-            return await self.add(entry['url'], quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, split_by_chapters, chapter_template, already)
+            return await self.add(entry['url'], quality, format, folder, custom_name_prefix, playlist_item_limit, auto_start, split_by_chapters, chapter_template, already)
         elif etype == 'playlist':
             log.debug('Processing as a playlist')
             entries = entry['entries']
@@ -526,7 +501,7 @@ class DownloadQueue:
                 for property in ("id", "title", "uploader", "uploader_id"):
                     if property in entry:
                         etr[f"playlist_{property}"] = entry[property]
-                results.append(await self.__add_entry(etr, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, split_by_chapters, chapter_template, already))
+                results.append(await self.__add_entry(etr, quality, format, folder, custom_name_prefix, playlist_item_limit, auto_start, split_by_chapters, chapter_template, already))
             if any(res['status'] == 'error' for res in results):
                 return {'status': 'error', 'msg': ', '.join(res['msg'] for res in results if res['status'] == 'error' and 'msg' in res)}
             return {'status': 'ok'}
@@ -539,8 +514,8 @@ class DownloadQueue:
             return {'status': 'ok'}
         return {'status': 'error', 'msg': f'Unsupported resource "{etype}"'}
 
-    async def add(self, url, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start=True, split_by_chapters=False, chapter_template=None, already=None):
-        log.info(f'adding {url}: {quality=} {format=} {already=} {folder=} {custom_name_prefix=} {playlist_strict_mode=} {playlist_item_limit=} {auto_start=} {split_by_chapters=} {chapter_template=}')
+    async def add(self, url, quality, format, folder, custom_name_prefix, playlist_item_limit, auto_start=True, split_by_chapters=False, chapter_template=None, already=None):
+        log.info(f'adding {url}: {quality=} {format=} {already=} {folder=} {custom_name_prefix=} {playlist_item_limit=} {auto_start=} {split_by_chapters=} {chapter_template=}')
         already = set() if already is None else already
         if url in already:
             log.info('recursion detected, skipping')
@@ -548,10 +523,10 @@ class DownloadQueue:
         else:
             already.add(url)
         try:
-            entry = await asyncio.get_running_loop().run_in_executor(None, self.__extract_info, url, playlist_strict_mode)
+            entry = await asyncio.get_running_loop().run_in_executor(None, self.__extract_info, url)
         except yt_dlp.utils.YoutubeDLError as exc:
             return {'status': 'error', 'msg': str(exc)}
-        return await self.__add_entry(entry, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, split_by_chapters, chapter_template, already)
+        return await self.__add_entry(entry, quality, format, folder, custom_name_prefix, playlist_item_limit, auto_start, split_by_chapters, chapter_template, already)
 
     async def start_pending(self, ids):
         for id in ids:

ui/package.json

@@ -38,6 +38,7 @@
     "@fortawesome/free-solid-svg-icons": "^7.1.0",
     "@ng-bootstrap/ng-bootstrap": "^20.0.0",
     "@ng-select/ng-select": "^21.1.0",
+    "@popperjs/core": "^2.11.8",
     "bootstrap": "^5.3.6",
     "ngx-cookie-service": "^21.1.0",
     "ngx-socket-io": "~4.9.3",

ui/src/app/app.html

@@ -218,19 +218,6 @@
                       ngbTooltip="Maximum number of items to download from a playlist (0 = no limit)">
                   </div>
                 </div>
-                <div class="col-12">
-                  <div class="form-check form-switch">
-                    <input class="form-check-input"
-                      type="checkbox"
-                      role="switch"
-                      id="checkbox-strict-mode"
-                      name="playlistStrictMode"
-                      [(ngModel)]="playlistStrictMode"
-                      [disabled]="addInProgress || downloads.loading"
-                      ngbTooltip="Only download playlists when URL explicitly points to a playlist">
-                    <label class="form-check-label" for="checkbox-strict-mode">Strict Playlist Mode</label>
-                  </div>
-                </div>
                 <div class="col-12">
                   <div class="row g-2 align-items-center">
                     <div class="col-auto">

ui/src/app/app.ts

@@ -46,7 +46,6 @@ export class App implements AfterViewInit, OnInit {
   folder!: string;
   customNamePrefix!: string;
   autoStart: boolean;
-  playlistStrictMode!: boolean;
   playlistItemLimit!: number;
   splitByChapters: boolean;
   chapterTemplate: string;
@@ -221,7 +220,6 @@ export class App implements AfterViewInit, OnInit {
     this.downloads.configurationChanged.subscribe({
        // eslint-disable-next-line @typescript-eslint/no-explicit-any
       next: (config: any) => {
-        this.playlistStrictMode = config['DEFAULT_OPTION_PLAYLIST_STRICT_MODE'];
         const playlistItemLimit = config['DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT'];
         if (playlistItemLimit !== '0') {
           this.playlistItemLimit = playlistItemLimit;
@@ -301,13 +299,12 @@ export class App implements AfterViewInit, OnInit {
   }
 }
 
-  addDownload(url?: string, quality?: string, format?: string, folder?: string, customNamePrefix?: string, playlistStrictMode?: boolean, playlistItemLimit?: number, autoStart?: boolean, splitByChapters?: boolean, chapterTemplate?: string) {
+  addDownload(url?: string, quality?: string, format?: string, folder?: string, customNamePrefix?: string, playlistItemLimit?: number, autoStart?: boolean, splitByChapters?: boolean, chapterTemplate?: string) {
     url = url ?? this.addUrl
     quality = quality ?? this.quality
     format = format ?? this.format
     folder = folder ?? this.folder
     customNamePrefix = customNamePrefix ?? this.customNamePrefix
-    playlistStrictMode = playlistStrictMode ?? this.playlistStrictMode
     playlistItemLimit = playlistItemLimit ?? this.playlistItemLimit
     autoStart = autoStart ?? this.autoStart
     splitByChapters = splitByChapters ?? this.splitByChapters
@@ -319,9 +316,9 @@ export class App implements AfterViewInit, OnInit {
       return;
     }
 
-    console.debug('Downloading: url=' + url + ' quality=' + quality + ' format=' + format + ' folder=' + folder + ' customNamePrefix=' + customNamePrefix + ' playlistStrictMode=' + playlistStrictMode + ' playlistItemLimit=' + playlistItemLimit + ' autoStart=' + autoStart + ' splitByChapters=' + splitByChapters + ' chapterTemplate=' + chapterTemplate);
+    console.debug('Downloading: url=' + url + ' quality=' + quality + ' format=' + format + ' folder=' + folder + ' customNamePrefix=' + customNamePrefix + ' playlistItemLimit=' + playlistItemLimit + ' autoStart=' + autoStart + ' splitByChapters=' + splitByChapters + ' chapterTemplate=' + chapterTemplate);
     this.addInProgress = true;
-    this.downloads.add(url, quality, format, folder, customNamePrefix, playlistStrictMode, playlistItemLimit, autoStart, splitByChapters, chapterTemplate).subscribe((status: Status) => {
+    this.downloads.add(url, quality, format, folder, customNamePrefix, playlistItemLimit, autoStart, splitByChapters, chapterTemplate).subscribe((status: Status) => {
       if (status.status === 'error') {
         alert(`Error adding URL: ${status.msg}`);
       } else {
@@ -336,7 +333,7 @@ export class App implements AfterViewInit, OnInit {
   }
 
   retryDownload(key: string, download: Download) {
-    this.addDownload(download.url, download.quality, download.format, download.folder, download.custom_name_prefix, download.playlist_strict_mode, download.playlist_item_limit, true, download.split_by_chapters, download.chapter_template);
+    this.addDownload(download.url, download.quality, download.format, download.folder, download.custom_name_prefix, download.playlist_item_limit, true, download.split_by_chapters, download.chapter_template);
     this.downloads.delById('done', [key]).subscribe();
   }
 
@@ -482,7 +479,7 @@ export class App implements AfterViewInit, OnInit {
       this.batchImportStatus = `Importing URL ${index + 1} of ${urls.length}: ${url}`;
       // Now pass the selected quality, format, folder, etc. to the add() method
       this.downloads.add(url, this.quality, this.format, this.folder, this.customNamePrefix,
-        this.playlistStrictMode, this.playlistItemLimit, this.autoStart, this.splitByChapters, this.chapterTemplate)
+        this.playlistItemLimit, this.autoStart, this.splitByChapters, this.chapterTemplate)
         .subscribe({
           next: (status: Status) => {
             if (status.status === 'error') {

ui/src/app/interfaces/download.ts

@@ -7,7 +7,6 @@ export interface Download {
   format: string;
   folder: string;
   custom_name_prefix: string;
-  playlist_strict_mode: boolean;
   playlist_item_limit: number;
   split_by_chapters?: boolean;
   chapter_template?: string;

ui/src/app/services/downloads.service.ts

@@ -107,8 +107,8 @@ export class DownloadsService {
     return of({status: 'error', msg: msg})
   }
 
-  public add(url: string, quality: string, format: string, folder: string, customNamePrefix: string, playlistStrictMode: boolean, playlistItemLimit: number, autoStart: boolean, splitByChapters: boolean, chapterTemplate: string) {
-    return this.http.post<Status>('add', { url: url, quality: quality, format: format, folder: folder, custom_name_prefix: customNamePrefix, playlist_strict_mode: playlistStrictMode, playlist_item_limit: playlistItemLimit, auto_start: autoStart, split_by_chapters: splitByChapters, chapter_template: chapterTemplate }).pipe(
+  public add(url: string, quality: string, format: string, folder: string, customNamePrefix: string, playlistItemLimit: number, autoStart: boolean, splitByChapters: boolean, chapterTemplate: string) {
+    return this.http.post<Status>('add', { url: url, quality: quality, format: format, folder: folder, custom_name_prefix: customNamePrefix, playlist_item_limit: playlistItemLimit, auto_start: autoStart, split_by_chapters: splitByChapters, chapter_template: chapterTemplate }).pipe(
       catchError(this.handleHTTPError)
     );
   }
@@ -118,12 +118,15 @@ export class DownloadsService {
   }
 
   public delById(where: State, ids: string[]) {
-    ids.forEach(id => {
-      const obj = this[where].get(id)
-      if (obj) {
-        obj.deleting = true
+    const map = this[where];
+    if (map) {
+      for (const id of ids) {
+        const obj = map.get(id);
+        if (obj) {
+          obj.deleting = true;
+        }
       }
-  });
+    }
     return this.http.post('delete', {where: where, ids: ids});
   }
 
@@ -147,14 +150,13 @@ export class DownloadsService {
     const defaultFormat = 'mp4';
     const defaultFolder = ''; 
     const defaultCustomNamePrefix = '';
-    const defaultPlaylistStrictMode = false;
     const defaultPlaylistItemLimit = 0;
     const defaultAutoStart = true;
     const defaultSplitByChapters = false;
     const defaultChapterTemplate = this.configuration['OUTPUT_TEMPLATE_CHAPTER'];
 
     return new Promise((resolve, reject) => {
-      this.add(url, defaultQuality, defaultFormat, defaultFolder, defaultCustomNamePrefix, defaultPlaylistStrictMode, defaultPlaylistItemLimit, defaultAutoStart, defaultSplitByChapters, defaultChapterTemplate)
+      this.add(url, defaultQuality, defaultFormat, defaultFolder, defaultCustomNamePrefix, defaultPlaylistItemLimit, defaultAutoStart, defaultSplitByChapters, defaultChapterTemplate)
         .subscribe({
           next: (response) => resolve(response),
           error: (error) => reject(error)