add file command to the docker image (fixes #870)

feature/repair-persistent-queues

.github/dependabot.yml

@@ -0,0 +1,13 @@
+# Keep GitHub Actions up to date with GitHub's Dependabot...
+# https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot
+# https://docs.github.com/en/code-security/dependabot/working-with-dependabot/dependabot-options-reference#package-ecosystem
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: /
+    groups:
+      github-actions:
+        patterns:
+          - "*"  # Group all Actions updates into a single larger pull request
+    schedule:
+      interval: weekly

.github/workflows/main.yml

@@ -15,7 +15,7 @@ jobs:
         run: echo "::set-output name=date::$(date +'%Y.%m.%d')"
       -
         name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
       -
         name: Set up QEMU
         uses: docker/setup-qemu-action@v3
@@ -37,7 +37,7 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
       -
         name: Build and push
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: .
           platforms: linux/amd64,linux/arm64
@@ -74,7 +74,7 @@ jobs:
         id: date
         run: echo "date=$(date +'%Y.%m.%d')" >> $GITHUB_OUTPUT
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
         with:
           fetch-depth: 0
       - name: Get commits since last release
@@ -167,7 +167,7 @@ jobs:
             git push origin ":refs/tags/$TAG_NAME" || true
           fi
       - name: Create GitHub Release
-        uses: softprops/action-gh-release@v1
+        uses: softprops/action-gh-release@v2
         with:
           tag_name: ${{ steps.date.outputs.date }}
           name: Release ${{ steps.date.outputs.date }}

.github/workflows/update-yt-dlp.yml

@@ -10,17 +10,17 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
         with:
           token: ${{ secrets.AUTOUPDATE_PAT }}
       -
         name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v6
         with:
           python-version: '3.13'
       -
         name: Install uv
-        uses: astral-sh/setup-uv@v6
+        uses: astral-sh/setup-uv@v7
       -
         name: Update yt-dlp
         run: |

Dockerfile

@@ -16,7 +16,7 @@ COPY pyproject.toml uv.lock docker-entrypoint.sh ./
 # Install dependencies
 RUN sed -i 's/\r$//g' docker-entrypoint.sh && \
     chmod +x docker-entrypoint.sh && \
-    apk add --update ffmpeg aria2 coreutils shadow su-exec curl tini deno && \
+    apk add --update ffmpeg aria2 coreutils shadow su-exec curl tini deno gdbm-tools sqlite file && \
     apk add --update --virtual .build-deps gcc g++ musl-dev uv && \
     UV_PROJECT_ENVIRONMENT=/usr/local uv sync --frozen --no-dev --compile-bytecode && \
     apk del .build-deps && \

README.md

@@ -270,8 +270,9 @@ MeTube development relies on code contributions by the community. The program as
 Make sure you have Node.js 22+ and Python 3.13 installed.
 
 ```bash
-cd metube/ui
 # install Angular and build the UI
+cd ui
+curl -fsSL https://get.pnpm.io/install.sh | sh -
 pnpm install
 pnpm run build
 # install python dependencies

app/ytdl.py

@@ -1,4 +1,5 @@
 import os
+import shutil
 import yt_dlp
 from collections import OrderedDict
 import shelve
@@ -8,6 +9,8 @@ import multiprocessing
 import logging
 import re
 import types
+import dbm
+import subprocess
 
 import yt_dlp.networking.impersonate
 from dl_formats import get_format, get_opts, AUDIO_FORMATS
@@ -234,13 +237,16 @@ class Download:
             await self.notifier.updated(self.info)
 
 class PersistentQueue:
-    def __init__(self, path):
+    def __init__(self, name, path):
+        self.identifier = name
         pdir = os.path.dirname(path)
         if not os.path.isdir(pdir):
             os.mkdir(pdir)
         with shelve.open(path, 'c'):
             pass
+
         self.path = path
+        self.repair()
         self.dict = OrderedDict()
 
     def load(self):
@@ -279,13 +285,84 @@ class PersistentQueue:
     def empty(self):
         return not bool(self.dict)
 
+    def repair(self):
+        # check DB format
+        type_check = subprocess.run(
+            ["file", self.path],
+            capture_output=True,
+            text=True
+        )
+        db_type = type_check.stdout.lower()
+
+        # create backup (<queue>.old)
+        try:
+            shutil.copy2(self.path, f"{self.path}.old")
+        except Exception as e:
+            # if we cannot backup then its not safe to attempt a repair
+            #  since it could be due to a filesystem error
+            log.debug(f"PersistentQueue:{self.identifier} backup failed, skipping repair")
+            return
+
+        if "gnu dbm" in db_type:
+            # perform gdbm repair
+            log_prefix = f"PersistentQueue:{self.identifier} repair (dbm/file)"
+            log.debug(f"{log_prefix} started")
+            try:
+                result = subprocess.run(
+                    ["gdbmtool", self.path],
+                    input="recover verbose summary\n",
+                    text=True,
+                    capture_output=True,
+                    timeout=60
+                )
+                log.debug(f"{log_prefix} {result.stdout}")
+                if result.stderr:
+                    log.debug(f"{log_prefix} failed: {result.stderr}")
+            except FileNotFoundError:
+                log.debug(f"{log_prefix} failed: 'gdbmtool' was not found")
+
+            # perform null key cleanup
+            log_prefix = f"PersistentQueue:{self.identifier} repair (null keys)"
+            log.debug(f"{log_prefix} started")
+            deleted = 0
+            try:
+                with dbm.open(self.path, "w") as db:
+                    for key in list(db.keys()):
+                        if len(key) > 0 and all(b == 0x00 for b in key):
+                            log.debug(f"{log_prefix} deleting key of length {len(key)} (all NUL bytes)")
+                            del db[key]
+                            deleted += 1
+                log.debug(f"{log_prefix} done - deleted {deleted} key(s)")
+            except dbm.error:
+                log.debug(f"{log_prefix} failed: db type is dbm.gnu, but the module is not available (dbm.error; module support may be missing or the file may be corrupted)")
+
+        elif "sqlite" in db_type:
+            # perform sqlite3 recovery
+            log_prefix = f"PersistentQueue:{self.identifier} repair (sqlite3/file)"
+            log.debug(f"{log_prefix} started")
+            try:
+                result = subprocess.run(
+                    f"sqlite3 {self.path} '.recover' | sqlite3 {self.path}.tmp",
+                    capture_output=True,
+                    text=True,
+                    shell=True,
+                    timeout=60
+                )
+                if result.stderr:
+                    log.debug(f"{log_prefix} failed: {result.stderr}")
+                else:
+                    shutil.move(f"{self.path}.tmp", self.path)
+                    log.debug(f"{log_prefix}{result.stdout or " was successful, no output"}")
+            except FileNotFoundError:
+                log.debug(f"{log_prefix} failed: 'sqlite3' was not found")
+                
 class DownloadQueue:
     def __init__(self, config, notifier):
         self.config = config
         self.notifier = notifier
-        self.queue = PersistentQueue(self.config.STATE_DIR + '/queue')
+        self.queue = PersistentQueue("queue", self.config.STATE_DIR + '/queue')
-        self.done = PersistentQueue(self.config.STATE_DIR + '/completed')
+        self.done = PersistentQueue("completed", self.config.STATE_DIR + '/completed')
-        self.pending = PersistentQueue(self.config.STATE_DIR + '/pending')
+        self.pending = PersistentQueue("pending", self.config.STATE_DIR + '/pending')
         self.active_downloads = set()
         self.semaphore = None
         # For sequential mode, use an asyncio lock to ensure one-at-a-time execution.