Compare commits
21 Commits
2025.12.27
...
2026.01.09
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9be0781c7f | ||
|
|
e378179e05 | ||
|
|
5a7dd8769b | ||
|
|
e601ce99f5 | ||
|
|
a74b201ed8 | ||
|
|
191f17ee38 | ||
|
|
a002af9bf2 | ||
|
|
37aaa29efb | ||
|
|
d10f2a0358 | ||
|
|
c7008763d7 | ||
|
|
351058e9f4 | ||
|
|
d799a4a8eb | ||
|
|
df87a1aa2b | ||
|
|
02480afddf | ||
|
|
d51f2ce628 | ||
|
|
962929d42d | ||
|
|
179452b4f4 | ||
|
|
4fce74d1ed | ||
|
|
09a2e95515 | ||
|
|
d947876a71 | ||
|
|
6ba681a3cd |
13
.github/dependabot.yml
vendored
Normal file
13
.github/dependabot.yml
vendored
Normal file
@@ -0,0 +1,13 @@
|
||||
# Keep GitHub Actions up to date with GitHub's Dependabot...
|
||||
# https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot
|
||||
# https://docs.github.com/en/code-security/dependabot/working-with-dependabot/dependabot-options-reference#package-ecosystem
|
||||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: github-actions
|
||||
directory: /
|
||||
groups:
|
||||
github-actions:
|
||||
patterns:
|
||||
- "*" # Group all Actions updates into a single larger pull request
|
||||
schedule:
|
||||
interval: weekly
|
||||
8
.github/workflows/main.yml
vendored
8
.github/workflows/main.yml
vendored
@@ -15,7 +15,7 @@ jobs:
|
||||
run: echo "::set-output name=date::$(date +'%Y.%m.%d')"
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v6
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
@@ -37,7 +37,7 @@ jobs:
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
-
|
||||
name: Build and push
|
||||
uses: docker/build-push-action@v5
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
platforms: linux/amd64,linux/arm64
|
||||
@@ -74,7 +74,7 @@ jobs:
|
||||
id: date
|
||||
run: echo "date=$(date +'%Y.%m.%d')" >> $GITHUB_OUTPUT
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Get commits since last release
|
||||
@@ -167,7 +167,7 @@ jobs:
|
||||
git push origin ":refs/tags/$TAG_NAME" || true
|
||||
fi
|
||||
- name: Create GitHub Release
|
||||
uses: softprops/action-gh-release@v1
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
tag_name: ${{ steps.date.outputs.date }}
|
||||
name: Release ${{ steps.date.outputs.date }}
|
||||
|
||||
6
.github/workflows/update-yt-dlp.yml
vendored
6
.github/workflows/update-yt-dlp.yml
vendored
@@ -10,17 +10,17 @@ jobs:
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
token: ${{ secrets.AUTOUPDATE_PAT }}
|
||||
-
|
||||
name: Set up Python
|
||||
uses: actions/setup-python@v4
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: '3.13'
|
||||
-
|
||||
name: Install uv
|
||||
uses: astral-sh/setup-uv@v6
|
||||
uses: astral-sh/setup-uv@v7
|
||||
-
|
||||
name: Update yt-dlp
|
||||
run: |
|
||||
|
||||
@@ -16,7 +16,7 @@ COPY pyproject.toml uv.lock docker-entrypoint.sh ./
|
||||
# Install dependencies
|
||||
RUN sed -i 's/\r$//g' docker-entrypoint.sh && \
|
||||
chmod +x docker-entrypoint.sh && \
|
||||
apk add --update ffmpeg aria2 coreutils shadow su-exec curl tini deno && \
|
||||
apk add --update ffmpeg aria2 coreutils shadow su-exec curl tini deno gdbm-tools sqlite file && \
|
||||
apk add --update --virtual .build-deps gcc g++ musl-dev uv && \
|
||||
UV_PROJECT_ENVIRONMENT=/usr/local uv sync --frozen --no-dev --compile-bytecode && \
|
||||
apk del .build-deps && \
|
||||
|
||||
13
README.md
13
README.md
@@ -33,14 +33,8 @@ Certain values can be set via environment variables, using the `-e` parameter on
|
||||
|
||||
### ⬇️ Download Behavior
|
||||
|
||||
* __DOWNLOAD_MODE__: This flag controls how downloads are scheduled and executed. Options are `sequential`, `concurrent`, and `limited`. Defaults to `limited`:
|
||||
* `sequential`: Downloads are processed one at a time. A new download won't start until the previous one has finished. This mode is useful for conserving system resources or ensuring downloads occur in strict order.
|
||||
* `concurrent`: Downloads are started immediately as they are added, with no built-in limit on how many run simultaneously. This mode may overwhelm your system if too many downloads start at once.
|
||||
* `limited`: Downloads are started concurrently but are capped by a concurrency limit. In this mode, a semaphore is used so that at most a fixed number of downloads run at any given time.
|
||||
* __MAX_CONCURRENT_DOWNLOADS__: This flag is used only when `DOWNLOAD_MODE` is set to `limited`.
|
||||
It specifies the maximum number of simultaneous downloads allowed. For example, if set to `5`, then at most five downloads will run concurrently, and any additional downloads will wait until one of the active downloads completes. Defaults to `3`.
|
||||
* __MAX_CONCURRENT_DOWNLOADS__: Maximum number of simultaneous downloads allowed. For example, if set to `5`, then at most five downloads will run concurrently, and any additional downloads will wait until one of the active downloads completes. Defaults to `3`.
|
||||
* __DELETE_FILE_ON_TRASHCAN__: if `true`, downloaded files are deleted on the server, when they are trashed from the "Completed" section of the UI. Defaults to `false`.
|
||||
* __DEFAULT_OPTION_PLAYLIST_STRICT_MODE__: if `true`, the "Strict Playlist mode" switch will be enabled by default. In this mode the playlists will be downloaded only if the URL strictly points to a playlist. URLs to videos inside a playlist will be treated same as direct video URL. Defaults to `false` .
|
||||
* __DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT__: Maximum number of playlist items that can be downloaded. Defaults to `0` (no limit).
|
||||
|
||||
### 📁 Storage & Directories
|
||||
@@ -66,6 +60,8 @@ Certain values can be set via environment variables, using the `-e` parameter on
|
||||
|
||||
### 🌐 Web Server & URLs
|
||||
|
||||
* __HOST__: The host address the web server will bind to. Defaults to `0.0.0.0` (all interfaces).
|
||||
* __PORT__: The port number the web server will listen on. Defaults to `8081`.
|
||||
* __URL_PREFIX__: Base path for the web server (for use when hosting behind a reverse proxy). Defaults to `/`.
|
||||
* __PUBLIC_HOST_URL__: Base URL for the download links shown in the UI for completed files. By default, MeTube serves them under its own URL. If your download directory is accessible on another URL and you want the download links to be based there, use this variable to set it.
|
||||
* __PUBLIC_HOST_AUDIO_URL__: Same as PUBLIC_HOST_URL but for audio downloads.
|
||||
@@ -270,8 +266,9 @@ MeTube development relies on code contributions by the community. The program as
|
||||
Make sure you have Node.js 22+ and Python 3.13 installed.
|
||||
|
||||
```bash
|
||||
cd metube/ui
|
||||
# install Angular and build the UI
|
||||
cd ui
|
||||
curl -fsSL https://get.pnpm.io/install.sh | sh -
|
||||
pnpm install
|
||||
pnpm run build
|
||||
# install python dependencies
|
||||
|
||||
58
app/main.py
58
app/main.py
@@ -21,6 +21,26 @@ from yt_dlp.version import __version__ as yt_dlp_version
|
||||
|
||||
log = logging.getLogger('main')
|
||||
|
||||
def parseLogLevel(logLevel):
|
||||
match logLevel:
|
||||
case 'DEBUG':
|
||||
return logging.DEBUG
|
||||
case 'INFO':
|
||||
return logging.INFO
|
||||
case 'WARNING':
|
||||
return logging.WARNING
|
||||
case 'ERROR':
|
||||
return logging.ERROR
|
||||
case 'CRITICAL':
|
||||
return logging.CRITICAL
|
||||
case _:
|
||||
return None
|
||||
|
||||
# Configure logging before Config() uses it so early messages are not dropped.
|
||||
# Only configure if no handlers are set (avoid clobbering hosting app settings).
|
||||
if not logging.getLogger().hasHandlers():
|
||||
logging.basicConfig(level=parseLogLevel(os.environ.get('LOGLEVEL', 'INFO')) or logging.INFO)
|
||||
|
||||
class Config:
|
||||
_DEFAULTS = {
|
||||
'DOWNLOAD_DIR': '.',
|
||||
@@ -36,9 +56,8 @@ class Config:
|
||||
'PUBLIC_HOST_URL': 'download/',
|
||||
'PUBLIC_HOST_AUDIO_URL': 'audio_download/',
|
||||
'OUTPUT_TEMPLATE': '%(title)s.%(ext)s',
|
||||
'OUTPUT_TEMPLATE_CHAPTER': '%(title)s - %(section_number)s %(section_title)s.%(ext)s',
|
||||
'OUTPUT_TEMPLATE_CHAPTER': '%(title)s - %(section_number)02d - %(section_title)s.%(ext)s',
|
||||
'OUTPUT_TEMPLATE_PLAYLIST': '%(playlist_title)s/%(title)s.%(ext)s',
|
||||
'DEFAULT_OPTION_PLAYLIST_STRICT_MODE' : 'false',
|
||||
'DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT' : '0',
|
||||
'YTDL_OPTIONS': '{}',
|
||||
'YTDL_OPTIONS_FILE': '',
|
||||
@@ -50,13 +69,12 @@ class Config:
|
||||
'KEYFILE': '',
|
||||
'BASE_DIR': '',
|
||||
'DEFAULT_THEME': 'auto',
|
||||
'DOWNLOAD_MODE': 'limited',
|
||||
'MAX_CONCURRENT_DOWNLOADS': 3,
|
||||
'LOGLEVEL': 'INFO',
|
||||
'ENABLE_ACCESSLOG': 'false',
|
||||
}
|
||||
|
||||
_BOOLEAN = ('DOWNLOAD_DIRS_INDEXABLE', 'CUSTOM_DIRS', 'CREATE_CUSTOM_DIRS', 'DELETE_FILE_ON_TRASHCAN', 'DEFAULT_OPTION_PLAYLIST_STRICT_MODE', 'HTTPS', 'ENABLE_ACCESSLOG')
|
||||
_BOOLEAN = ('DOWNLOAD_DIRS_INDEXABLE', 'CUSTOM_DIRS', 'CREATE_CUSTOM_DIRS', 'DELETE_FILE_ON_TRASHCAN', 'HTTPS', 'ENABLE_ACCESSLOG')
|
||||
|
||||
def __init__(self):
|
||||
for k, v in self._DEFAULTS.items():
|
||||
@@ -112,6 +130,10 @@ class Config:
|
||||
return (True, '')
|
||||
|
||||
config = Config()
|
||||
# Align root logger level with Config (keeps a single source of truth).
|
||||
# This re-applies the log level after Config loads, in case LOGLEVEL was
|
||||
# overridden by config file settings or differs from the environment variable.
|
||||
logging.getLogger().setLevel(parseLogLevel(str(config.LOGLEVEL)) or logging.INFO)
|
||||
|
||||
class ObjectSerializer(json.JSONEncoder):
|
||||
def default(self, obj):
|
||||
@@ -220,22 +242,25 @@ async def add(request):
|
||||
format = post.get('format')
|
||||
folder = post.get('folder')
|
||||
custom_name_prefix = post.get('custom_name_prefix')
|
||||
playlist_strict_mode = post.get('playlist_strict_mode')
|
||||
playlist_item_limit = post.get('playlist_item_limit')
|
||||
auto_start = post.get('auto_start')
|
||||
split_by_chapters = post.get('split_by_chapters')
|
||||
chapter_template = post.get('chapter_template')
|
||||
|
||||
if custom_name_prefix is None:
|
||||
custom_name_prefix = ''
|
||||
if auto_start is None:
|
||||
auto_start = True
|
||||
if playlist_strict_mode is None:
|
||||
playlist_strict_mode = config.DEFAULT_OPTION_PLAYLIST_STRICT_MODE
|
||||
if playlist_item_limit is None:
|
||||
playlist_item_limit = config.DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT
|
||||
if split_by_chapters is None:
|
||||
split_by_chapters = False
|
||||
if chapter_template is None:
|
||||
chapter_template = config.OUTPUT_TEMPLATE_CHAPTER
|
||||
|
||||
playlist_item_limit = int(playlist_item_limit)
|
||||
|
||||
status = await dqueue.add(url, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start)
|
||||
status = await dqueue.add(url, quality, format, folder, custom_name_prefix, playlist_item_limit, auto_start, split_by_chapters, chapter_template)
|
||||
return web.Response(text=serializer.encode(status))
|
||||
|
||||
@routes.post(config.URL_PREFIX + 'delete')
|
||||
@@ -386,21 +411,6 @@ def supports_reuse_port():
|
||||
except (AttributeError, OSError):
|
||||
return False
|
||||
|
||||
def parseLogLevel(logLevel):
|
||||
match logLevel:
|
||||
case 'DEBUG':
|
||||
return logging.DEBUG
|
||||
case 'INFO':
|
||||
return logging.INFO
|
||||
case 'WARNING':
|
||||
return logging.WARNING
|
||||
case 'ERROR':
|
||||
return logging.ERROR
|
||||
case 'CRITICAL':
|
||||
return logging.CRITICAL
|
||||
case _:
|
||||
return None
|
||||
|
||||
def isAccessLogEnabled():
|
||||
if config.ENABLE_ACCESSLOG:
|
||||
return access_logger
|
||||
@@ -408,7 +418,7 @@ def isAccessLogEnabled():
|
||||
return None
|
||||
|
||||
if __name__ == '__main__':
|
||||
logging.basicConfig(level=parseLogLevel(config.LOGLEVEL))
|
||||
logging.getLogger().setLevel(parseLogLevel(config.LOGLEVEL) or logging.INFO)
|
||||
log.info(f"Listening on {config.HOST}:{config.PORT}")
|
||||
|
||||
if config.HTTPS:
|
||||
|
||||
187
app/ytdl.py
187
app/ytdl.py
@@ -1,4 +1,5 @@
|
||||
import os
|
||||
import shutil
|
||||
import yt_dlp
|
||||
from collections import OrderedDict
|
||||
import shelve
|
||||
@@ -8,6 +9,8 @@ import multiprocessing
|
||||
import logging
|
||||
import re
|
||||
import types
|
||||
import dbm
|
||||
import subprocess
|
||||
|
||||
import yt_dlp.networking.impersonate
|
||||
from dl_formats import get_format, get_opts, AUDIO_FORMATS
|
||||
@@ -43,7 +46,7 @@ class DownloadQueueNotifier:
|
||||
raise NotImplementedError
|
||||
|
||||
class DownloadInfo:
|
||||
def __init__(self, id, title, url, quality, format, folder, custom_name_prefix, error, entry, playlist_item_limit):
|
||||
def __init__(self, id, title, url, quality, format, folder, custom_name_prefix, error, entry, playlist_item_limit, split_by_chapters, chapter_template):
|
||||
self.id = id if len(custom_name_prefix) == 0 else f'{custom_name_prefix}.{id}'
|
||||
self.title = title if len(custom_name_prefix) == 0 else f'{custom_name_prefix}.{title}'
|
||||
self.url = url
|
||||
@@ -59,6 +62,8 @@ class DownloadInfo:
|
||||
# Convert generators to lists to make entry pickleable
|
||||
self.entry = _convert_generators_to_lists(entry) if entry is not None else None
|
||||
self.playlist_item_limit = playlist_item_limit
|
||||
self.split_by_chapters = split_by_chapters
|
||||
self.chapter_template = chapter_template
|
||||
|
||||
class Download:
|
||||
manager = None
|
||||
@@ -83,6 +88,7 @@ class Download:
|
||||
def _download(self):
|
||||
log.info(f"Starting download for: {self.info.title} ({self.info.url})")
|
||||
try:
|
||||
debug_logging = logging.getLogger().isEnabledFor(logging.DEBUG)
|
||||
def put_status(st):
|
||||
self.status_queue.put({k: v for k, v in st.items() if k in (
|
||||
'tmpfilename',
|
||||
@@ -104,8 +110,20 @@ class Download:
|
||||
filename = d['info_dict']['filepath']
|
||||
self.status_queue.put({'status': 'finished', 'filename': filename})
|
||||
|
||||
ret = yt_dlp.YoutubeDL(params={
|
||||
'quiet': True,
|
||||
# Capture all chapter files when SplitChapters finishes
|
||||
elif d.get('postprocessor') == 'SplitChapters' and d.get('status') == 'finished':
|
||||
chapters = d.get('info_dict', {}).get('chapters', [])
|
||||
if chapters:
|
||||
for chapter in chapters:
|
||||
if isinstance(chapter, dict) and 'filepath' in chapter:
|
||||
log.info(f"Captured chapter file: {chapter['filepath']}")
|
||||
self.status_queue.put({'chapter_file': chapter['filepath']})
|
||||
else:
|
||||
log.warning("SplitChapters finished but no chapter files found in info_dict")
|
||||
|
||||
ytdl_params = {
|
||||
'quiet': not debug_logging,
|
||||
'verbose': debug_logging,
|
||||
'no_color': True,
|
||||
'paths': {"home": self.download_dir, "temp": self.temp_dir},
|
||||
'outtmpl': { "default": self.output_template, "chapter": self.output_template_chapter },
|
||||
@@ -115,7 +133,19 @@ class Download:
|
||||
'progress_hooks': [put_status],
|
||||
'postprocessor_hooks': [put_status_postprocessor],
|
||||
**self.ytdl_opts,
|
||||
}).download([self.info.url])
|
||||
}
|
||||
|
||||
# Add chapter splitting options if enabled
|
||||
if self.info.split_by_chapters:
|
||||
ytdl_params['outtmpl']['chapter'] = self.info.chapter_template
|
||||
if 'postprocessors' not in ytdl_params:
|
||||
ytdl_params['postprocessors'] = []
|
||||
ytdl_params['postprocessors'].append({
|
||||
'key': 'FFmpegSplitChapters',
|
||||
'force_keyframes': False
|
||||
})
|
||||
|
||||
ret = yt_dlp.YoutubeDL(params=ytdl_params).download([self.info.url])
|
||||
self.status_queue.put({'status': 'finished' if ret == 0 else 'error'})
|
||||
log.info(f"Finished download for: {self.info.title}")
|
||||
except yt_dlp.utils.YoutubeDLError as exc:
|
||||
@@ -179,6 +209,22 @@ class Download:
|
||||
self.info.size = os.path.getsize(fileName) if os.path.exists(fileName) else None
|
||||
if self.info.format == 'thumbnail':
|
||||
self.info.filename = re.sub(r'\.webm$', '.jpg', self.info.filename)
|
||||
|
||||
# Handle chapter files
|
||||
log.debug(f"Update status for {self.info.title}: {status}")
|
||||
if 'chapter_file' in status:
|
||||
chapter_file = status.get('chapter_file')
|
||||
if not hasattr(self.info, 'chapter_files'):
|
||||
self.info.chapter_files = []
|
||||
rel_path = os.path.relpath(chapter_file, self.download_dir)
|
||||
file_size = os.path.getsize(chapter_file) if os.path.exists(chapter_file) else None
|
||||
#Postprocessor hook called multiple times with chapters. Only insert if not already present.
|
||||
existing = next((cf for cf in self.info.chapter_files if cf['filename'] == rel_path), None)
|
||||
if not existing:
|
||||
self.info.chapter_files.append({'filename': rel_path, 'size': file_size})
|
||||
# Skip the rest of status processing for chapter files
|
||||
continue
|
||||
|
||||
self.info.status = status['status']
|
||||
self.info.msg = status.get('msg')
|
||||
if 'downloaded_bytes' in status:
|
||||
@@ -191,13 +237,16 @@ class Download:
|
||||
await self.notifier.updated(self.info)
|
||||
|
||||
class PersistentQueue:
|
||||
def __init__(self, path):
|
||||
def __init__(self, name, path):
|
||||
self.identifier = name
|
||||
pdir = os.path.dirname(path)
|
||||
if not os.path.isdir(pdir):
|
||||
os.mkdir(pdir)
|
||||
with shelve.open(path, 'c'):
|
||||
pass
|
||||
|
||||
self.path = path
|
||||
self.repair()
|
||||
self.dict = OrderedDict()
|
||||
|
||||
def load(self):
|
||||
@@ -236,21 +285,86 @@ class PersistentQueue:
|
||||
def empty(self):
|
||||
return not bool(self.dict)
|
||||
|
||||
def repair(self):
|
||||
# check DB format
|
||||
type_check = subprocess.run(
|
||||
["file", self.path],
|
||||
capture_output=True,
|
||||
text=True
|
||||
)
|
||||
db_type = type_check.stdout.lower()
|
||||
|
||||
# create backup (<queue>.old)
|
||||
try:
|
||||
shutil.copy2(self.path, f"{self.path}.old")
|
||||
except Exception as e:
|
||||
# if we cannot backup then its not safe to attempt a repair
|
||||
# since it could be due to a filesystem error
|
||||
log.debug(f"PersistentQueue:{self.identifier} backup failed, skipping repair")
|
||||
return
|
||||
|
||||
if "gnu dbm" in db_type:
|
||||
# perform gdbm repair
|
||||
log_prefix = f"PersistentQueue:{self.identifier} repair (dbm/file)"
|
||||
log.debug(f"{log_prefix} started")
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["gdbmtool", self.path],
|
||||
input="recover verbose summary\n",
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=60
|
||||
)
|
||||
log.debug(f"{log_prefix} {result.stdout}")
|
||||
if result.stderr:
|
||||
log.debug(f"{log_prefix} failed: {result.stderr}")
|
||||
except FileNotFoundError:
|
||||
log.debug(f"{log_prefix} failed: 'gdbmtool' was not found")
|
||||
|
||||
# perform null key cleanup
|
||||
log_prefix = f"PersistentQueue:{self.identifier} repair (null keys)"
|
||||
log.debug(f"{log_prefix} started")
|
||||
deleted = 0
|
||||
try:
|
||||
with dbm.open(self.path, "w") as db:
|
||||
for key in list(db.keys()):
|
||||
if len(key) > 0 and all(b == 0x00 for b in key):
|
||||
log.debug(f"{log_prefix} deleting key of length {len(key)} (all NUL bytes)")
|
||||
del db[key]
|
||||
deleted += 1
|
||||
log.debug(f"{log_prefix} done - deleted {deleted} key(s)")
|
||||
except dbm.error:
|
||||
log.debug(f"{log_prefix} failed: db type is dbm.gnu, but the module is not available (dbm.error; module support may be missing or the file may be corrupted)")
|
||||
|
||||
elif "sqlite" in db_type:
|
||||
# perform sqlite3 recovery
|
||||
log_prefix = f"PersistentQueue:{self.identifier} repair (sqlite3/file)"
|
||||
log.debug(f"{log_prefix} started")
|
||||
try:
|
||||
result = subprocess.run(
|
||||
f"sqlite3 {self.path} '.recover' | sqlite3 {self.path}.tmp",
|
||||
capture_output=True,
|
||||
text=True,
|
||||
shell=True,
|
||||
timeout=60
|
||||
)
|
||||
if result.stderr:
|
||||
log.debug(f"{log_prefix} failed: {result.stderr}")
|
||||
else:
|
||||
shutil.move(f"{self.path}.tmp", self.path)
|
||||
log.debug(f"{log_prefix}{result.stdout or " was successful, no output"}")
|
||||
except FileNotFoundError:
|
||||
log.debug(f"{log_prefix} failed: 'sqlite3' was not found")
|
||||
|
||||
class DownloadQueue:
|
||||
def __init__(self, config, notifier):
|
||||
self.config = config
|
||||
self.notifier = notifier
|
||||
self.queue = PersistentQueue(self.config.STATE_DIR + '/queue')
|
||||
self.done = PersistentQueue(self.config.STATE_DIR + '/completed')
|
||||
self.pending = PersistentQueue(self.config.STATE_DIR + '/pending')
|
||||
self.queue = PersistentQueue("queue", self.config.STATE_DIR + '/queue')
|
||||
self.done = PersistentQueue("completed", self.config.STATE_DIR + '/completed')
|
||||
self.pending = PersistentQueue("pending", self.config.STATE_DIR + '/pending')
|
||||
self.active_downloads = set()
|
||||
self.semaphore = None
|
||||
# For sequential mode, use an asyncio lock to ensure one-at-a-time execution.
|
||||
if self.config.DOWNLOAD_MODE == 'sequential':
|
||||
self.seq_lock = asyncio.Lock()
|
||||
elif self.config.DOWNLOAD_MODE == 'limited':
|
||||
self.semaphore = asyncio.Semaphore(int(self.config.MAX_CONCURRENT_DOWNLOADS))
|
||||
|
||||
self.done.load()
|
||||
|
||||
async def __import_queue(self):
|
||||
@@ -270,28 +384,9 @@ class DownloadQueue:
|
||||
if download.canceled:
|
||||
log.info(f"Download {download.info.title} was canceled, skipping start.")
|
||||
return
|
||||
if self.config.DOWNLOAD_MODE == 'sequential':
|
||||
async with self.seq_lock:
|
||||
log.info("Starting sequential download.")
|
||||
await download.start(self.notifier)
|
||||
self._post_download_cleanup(download)
|
||||
elif self.config.DOWNLOAD_MODE == 'limited' and self.semaphore is not None:
|
||||
await self.__limited_concurrent_download(download)
|
||||
else:
|
||||
await self.__concurrent_download(download)
|
||||
|
||||
async def __concurrent_download(self, download):
|
||||
log.info("Starting concurrent download without limits.")
|
||||
asyncio.create_task(self._run_download(download))
|
||||
|
||||
async def __limited_concurrent_download(self, download):
|
||||
log.info("Starting limited concurrent download.")
|
||||
async with self.semaphore:
|
||||
await self._run_download(download)
|
||||
|
||||
async def _run_download(self, download):
|
||||
if download.canceled:
|
||||
log.info(f"Download {download.info.title} is canceled; skipping start.")
|
||||
log.info(f"Download {download.info.title} was canceled, skipping start.")
|
||||
return
|
||||
await download.start(self.notifier)
|
||||
self._post_download_cleanup(download)
|
||||
@@ -313,13 +408,15 @@ class DownloadQueue:
|
||||
self.done.put(download)
|
||||
asyncio.create_task(self.notifier.completed(download.info))
|
||||
|
||||
def __extract_info(self, url, playlist_strict_mode):
|
||||
def __extract_info(self, url):
|
||||
debug_logging = logging.getLogger().isEnabledFor(logging.DEBUG)
|
||||
return yt_dlp.YoutubeDL(params={
|
||||
'quiet': True,
|
||||
'quiet': not debug_logging,
|
||||
'verbose': debug_logging,
|
||||
'no_color': True,
|
||||
'extract_flat': True,
|
||||
'ignore_no_formats_error': True,
|
||||
'noplaylist': playlist_strict_mode,
|
||||
'noplaylist': True,
|
||||
'paths': {"home": self.config.DOWNLOAD_DIR, "temp": self.config.TEMP_DIR},
|
||||
**self.config.YTDL_OPTIONS,
|
||||
**({'impersonate': yt_dlp.networking.impersonate.ImpersonateTarget.from_str(self.config.YTDL_OPTIONS['impersonate'])} if 'impersonate' in self.config.YTDL_OPTIONS else {}),
|
||||
@@ -368,7 +465,7 @@ class DownloadQueue:
|
||||
self.pending.put(download)
|
||||
await self.notifier.added(dl)
|
||||
|
||||
async def __add_entry(self, entry, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, already):
|
||||
async def __add_entry(self, entry, quality, format, folder, custom_name_prefix, playlist_item_limit, auto_start, split_by_chapters, chapter_template, already):
|
||||
if not entry:
|
||||
return {'status': 'error', 'msg': "Invalid/empty data was given."}
|
||||
|
||||
@@ -384,7 +481,7 @@ class DownloadQueue:
|
||||
|
||||
if etype.startswith('url'):
|
||||
log.debug('Processing as an url')
|
||||
return await self.add(entry['url'], quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, already)
|
||||
return await self.add(entry['url'], quality, format, folder, custom_name_prefix, playlist_item_limit, auto_start, split_by_chapters, chapter_template, already)
|
||||
elif etype == 'playlist':
|
||||
log.debug('Processing as a playlist')
|
||||
entries = entry['entries']
|
||||
@@ -404,7 +501,7 @@ class DownloadQueue:
|
||||
for property in ("id", "title", "uploader", "uploader_id"):
|
||||
if property in entry:
|
||||
etr[f"playlist_{property}"] = entry[property]
|
||||
results.append(await self.__add_entry(etr, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, already))
|
||||
results.append(await self.__add_entry(etr, quality, format, folder, custom_name_prefix, playlist_item_limit, auto_start, split_by_chapters, chapter_template, already))
|
||||
if any(res['status'] == 'error' for res in results):
|
||||
return {'status': 'error', 'msg': ', '.join(res['msg'] for res in results if res['status'] == 'error' and 'msg' in res)}
|
||||
return {'status': 'ok'}
|
||||
@@ -412,13 +509,13 @@ class DownloadQueue:
|
||||
log.debug('Processing as a video')
|
||||
key = entry.get('webpage_url') or entry['url']
|
||||
if not self.queue.exists(key):
|
||||
dl = DownloadInfo(entry['id'], entry.get('title') or entry['id'], key, quality, format, folder, custom_name_prefix, error, entry, playlist_item_limit)
|
||||
dl = DownloadInfo(entry['id'], entry.get('title') or entry['id'], key, quality, format, folder, custom_name_prefix, error, entry, playlist_item_limit, split_by_chapters, chapter_template)
|
||||
await self.__add_download(dl, auto_start)
|
||||
return {'status': 'ok'}
|
||||
return {'status': 'error', 'msg': f'Unsupported resource "{etype}"'}
|
||||
|
||||
async def add(self, url, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start=True, already=None):
|
||||
log.info(f'adding {url}: {quality=} {format=} {already=} {folder=} {custom_name_prefix=} {playlist_strict_mode=} {playlist_item_limit=} {auto_start=}')
|
||||
async def add(self, url, quality, format, folder, custom_name_prefix, playlist_item_limit, auto_start=True, split_by_chapters=False, chapter_template=None, already=None):
|
||||
log.info(f'adding {url}: {quality=} {format=} {already=} {folder=} {custom_name_prefix=} {playlist_item_limit=} {auto_start=} {split_by_chapters=} {chapter_template=}')
|
||||
already = set() if already is None else already
|
||||
if url in already:
|
||||
log.info('recursion detected, skipping')
|
||||
@@ -426,10 +523,10 @@ class DownloadQueue:
|
||||
else:
|
||||
already.add(url)
|
||||
try:
|
||||
entry = await asyncio.get_running_loop().run_in_executor(None, self.__extract_info, url, playlist_strict_mode)
|
||||
entry = await asyncio.get_running_loop().run_in_executor(None, self.__extract_info, url)
|
||||
except yt_dlp.utils.YoutubeDLError as exc:
|
||||
return {'status': 'error', 'msg': str(exc)}
|
||||
return await self.__add_entry(entry, quality, format, folder, custom_name_prefix, playlist_strict_mode, playlist_item_limit, auto_start, already)
|
||||
return await self.__add_entry(entry, quality, format, folder, custom_name_prefix, playlist_item_limit, auto_start, split_by_chapters, chapter_template, already)
|
||||
|
||||
async def start_pending(self, ids):
|
||||
for id in ids:
|
||||
|
||||
@@ -219,16 +219,26 @@
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-12">
|
||||
<div class="row g-2 align-items-center">
|
||||
<div class="col-auto">
|
||||
<div class="form-check form-switch">
|
||||
<input class="form-check-input"
|
||||
type="checkbox"
|
||||
role="switch"
|
||||
id="checkbox-strict-mode"
|
||||
name="playlistStrictMode"
|
||||
[(ngModel)]="playlistStrictMode"
|
||||
<input class="form-check-input" type="checkbox" role="switch" id="checkbox-split-chapters"
|
||||
name="splitByChapters" [(ngModel)]="splitByChapters" (change)="splitByChaptersChanged()"
|
||||
[disabled]="addInProgress || downloads.loading"
|
||||
ngbTooltip="Only download playlists when URL explicitly points to a playlist">
|
||||
<label class="form-check-label" for="checkbox-strict-mode">Strict Playlist Mode</label>
|
||||
ngbTooltip="Split video into separate files by chapters">
|
||||
<label class="form-check-label" for="checkbox-split-chapters">Split by chapters</label>
|
||||
</div>
|
||||
</div>
|
||||
@if (splitByChapters) {
|
||||
<div class="col">
|
||||
<div class="input-group">
|
||||
<span class="input-group-text">Template</span>
|
||||
<input type="text" class="form-control" name="chapterTemplate" [(ngModel)]="chapterTemplate"
|
||||
(change)="chapterTemplateChanged()" [disabled]="addInProgress || downloads.loading"
|
||||
ngbTooltip="Output template for chapter files">
|
||||
</div>
|
||||
</div>
|
||||
}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -425,6 +435,31 @@
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
@if (download.value.chapter_files && download.value.chapter_files.length > 0) {
|
||||
@for (chapterFile of download.value.chapter_files; track chapterFile.filename) {
|
||||
<tr [class.disabled]='download.value.deleting'>
|
||||
<td></td>
|
||||
<td>
|
||||
<div style="padding-left: 2rem;">
|
||||
<fa-icon [icon]="faCheckCircle" class="text-success me-2" />
|
||||
<a href="{{buildChapterDownloadLink(download.value, chapterFile.filename)}}" target="_blank">{{
|
||||
getChapterFileName(chapterFile.filename) }}</a>
|
||||
</div>
|
||||
</td>
|
||||
<td>
|
||||
@if (chapterFile.size) {
|
||||
<span>{{ chapterFile.size | fileSize }}</span>
|
||||
}
|
||||
</td>
|
||||
<td>
|
||||
<div class="d-flex">
|
||||
<a href="{{buildChapterDownloadLink(download.value, chapterFile.filename)}}" download
|
||||
class="btn btn-link"><fa-icon [icon]="faDownload" /></a>
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
}
|
||||
}
|
||||
}
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
@@ -46,8 +46,9 @@ export class App implements AfterViewInit, OnInit {
|
||||
folder!: string;
|
||||
customNamePrefix!: string;
|
||||
autoStart: boolean;
|
||||
playlistStrictMode!: boolean;
|
||||
playlistItemLimit!: number;
|
||||
splitByChapters: boolean;
|
||||
chapterTemplate: string;
|
||||
addInProgress = false;
|
||||
themes: Theme[] = Themes;
|
||||
activeTheme: Theme | undefined;
|
||||
@@ -103,6 +104,9 @@ export class App implements AfterViewInit, OnInit {
|
||||
this.setQualities()
|
||||
this.quality = this.cookieService.get('metube_quality') || 'best';
|
||||
this.autoStart = this.cookieService.get('metube_auto_start') !== 'false';
|
||||
this.splitByChapters = this.cookieService.get('metube_split_chapters') === 'true';
|
||||
// Will be set from backend configuration, use empty string as placeholder
|
||||
this.chapterTemplate = this.cookieService.get('metube_chapter_template') || '';
|
||||
|
||||
this.activeTheme = this.getPreferredTheme(this.cookieService);
|
||||
|
||||
@@ -216,11 +220,14 @@ export class App implements AfterViewInit, OnInit {
|
||||
this.downloads.configurationChanged.subscribe({
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
next: (config: any) => {
|
||||
this.playlistStrictMode = config['DEFAULT_OPTION_PLAYLIST_STRICT_MODE'];
|
||||
const playlistItemLimit = config['DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT'];
|
||||
if (playlistItemLimit !== '0') {
|
||||
this.playlistItemLimit = playlistItemLimit;
|
||||
}
|
||||
// Set chapter template from backend config if not already set by cookie
|
||||
if (!this.chapterTemplate) {
|
||||
this.chapterTemplate = config['OUTPUT_TEMPLATE_CHAPTER'];
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -260,6 +267,18 @@ export class App implements AfterViewInit, OnInit {
|
||||
this.cookieService.set('metube_auto_start', this.autoStart ? 'true' : 'false', { expires: 3650 });
|
||||
}
|
||||
|
||||
splitByChaptersChanged() {
|
||||
this.cookieService.set('metube_split_chapters', this.splitByChapters ? 'true' : 'false', { expires: 3650 });
|
||||
}
|
||||
|
||||
chapterTemplateChanged() {
|
||||
// Restore default if template is cleared - get from configuration
|
||||
if (!this.chapterTemplate || this.chapterTemplate.trim() === '') {
|
||||
this.chapterTemplate = this.downloads.configuration['OUTPUT_TEMPLATE_CHAPTER'];
|
||||
}
|
||||
this.cookieService.set('metube_chapter_template', this.chapterTemplate, { expires: 3650 });
|
||||
}
|
||||
|
||||
queueSelectionChanged(checked: number) {
|
||||
this.queueDelSelected().nativeElement.disabled = checked == 0;
|
||||
this.queueDownloadSelected().nativeElement.disabled = checked == 0;
|
||||
@@ -280,19 +299,26 @@ export class App implements AfterViewInit, OnInit {
|
||||
}
|
||||
}
|
||||
|
||||
addDownload(url?: string, quality?: string, format?: string, folder?: string, customNamePrefix?: string, playlistStrictMode?: boolean, playlistItemLimit?: number, autoStart?: boolean) {
|
||||
addDownload(url?: string, quality?: string, format?: string, folder?: string, customNamePrefix?: string, playlistItemLimit?: number, autoStart?: boolean, splitByChapters?: boolean, chapterTemplate?: string) {
|
||||
url = url ?? this.addUrl
|
||||
quality = quality ?? this.quality
|
||||
format = format ?? this.format
|
||||
folder = folder ?? this.folder
|
||||
customNamePrefix = customNamePrefix ?? this.customNamePrefix
|
||||
playlistStrictMode = playlistStrictMode ?? this.playlistStrictMode
|
||||
playlistItemLimit = playlistItemLimit ?? this.playlistItemLimit
|
||||
autoStart = autoStart ?? this.autoStart
|
||||
splitByChapters = splitByChapters ?? this.splitByChapters
|
||||
chapterTemplate = chapterTemplate ?? this.chapterTemplate
|
||||
|
||||
console.debug('Downloading: url='+url+' quality='+quality+' format='+format+' folder='+folder+' customNamePrefix='+customNamePrefix+' playlistStrictMode='+playlistStrictMode+' playlistItemLimit='+playlistItemLimit+' autoStart='+autoStart);
|
||||
// Validate chapter template if chapter splitting is enabled
|
||||
if (splitByChapters && !chapterTemplate.includes('%(section_number)')) {
|
||||
alert('Chapter template must include %(section_number)');
|
||||
return;
|
||||
}
|
||||
|
||||
console.debug('Downloading: url=' + url + ' quality=' + quality + ' format=' + format + ' folder=' + folder + ' customNamePrefix=' + customNamePrefix + ' playlistItemLimit=' + playlistItemLimit + ' autoStart=' + autoStart + ' splitByChapters=' + splitByChapters + ' chapterTemplate=' + chapterTemplate);
|
||||
this.addInProgress = true;
|
||||
this.downloads.add(url, quality, format, folder, customNamePrefix, playlistStrictMode, playlistItemLimit, autoStart).subscribe((status: Status) => {
|
||||
this.downloads.add(url, quality, format, folder, customNamePrefix, playlistItemLimit, autoStart, splitByChapters, chapterTemplate).subscribe((status: Status) => {
|
||||
if (status.status === 'error') {
|
||||
alert(`Error adding URL: ${status.msg}`);
|
||||
} else {
|
||||
@@ -307,7 +333,7 @@ export class App implements AfterViewInit, OnInit {
|
||||
}
|
||||
|
||||
retryDownload(key: string, download: Download) {
|
||||
this.addDownload(download.url, download.quality, download.format, download.folder, download.custom_name_prefix, download.playlist_strict_mode, download.playlist_item_limit, true);
|
||||
this.addDownload(download.url, download.quality, download.format, download.folder, download.custom_name_prefix, download.playlist_item_limit, true, download.split_by_chapters, download.chapter_template);
|
||||
this.downloads.delById('done', [key]).subscribe();
|
||||
}
|
||||
|
||||
@@ -378,6 +404,25 @@ export class App implements AfterViewInit, OnInit {
|
||||
return parts.join(' | ');
|
||||
}
|
||||
|
||||
buildChapterDownloadLink(download: Download, chapterFilename: string) {
|
||||
let baseDir = this.downloads.configuration["PUBLIC_HOST_URL"];
|
||||
if (download.quality == 'audio' || chapterFilename.endsWith('.mp3')) {
|
||||
baseDir = this.downloads.configuration["PUBLIC_HOST_AUDIO_URL"];
|
||||
}
|
||||
|
||||
if (download.folder) {
|
||||
baseDir += download.folder + '/';
|
||||
}
|
||||
|
||||
return baseDir + encodeURIComponent(chapterFilename);
|
||||
}
|
||||
|
||||
getChapterFileName(filepath: string) {
|
||||
// Extract just the filename from the path
|
||||
const parts = filepath.split('/');
|
||||
return parts[parts.length - 1];
|
||||
}
|
||||
|
||||
isNumber(event: KeyboardEvent) {
|
||||
const charCode = +event.code || event.keyCode;
|
||||
if (charCode > 31 && (charCode < 48 || charCode > 57)) {
|
||||
@@ -434,7 +479,7 @@ export class App implements AfterViewInit, OnInit {
|
||||
this.batchImportStatus = `Importing URL ${index + 1} of ${urls.length}: ${url}`;
|
||||
// Now pass the selected quality, format, folder, etc. to the add() method
|
||||
this.downloads.add(url, this.quality, this.format, this.folder, this.customNamePrefix,
|
||||
this.playlistStrictMode, this.playlistItemLimit, this.autoStart)
|
||||
this.playlistItemLimit, this.autoStart, this.splitByChapters, this.chapterTemplate)
|
||||
.subscribe({
|
||||
next: (status: Status) => {
|
||||
if (status.status === 'error') {
|
||||
|
||||
@@ -7,8 +7,9 @@ export interface Download {
|
||||
format: string;
|
||||
folder: string;
|
||||
custom_name_prefix: string;
|
||||
playlist_strict_mode: boolean;
|
||||
playlist_item_limit: number;
|
||||
split_by_chapters?: boolean;
|
||||
chapter_template?: string;
|
||||
status: string;
|
||||
msg: string;
|
||||
percent: number;
|
||||
@@ -19,4 +20,5 @@ export interface Download {
|
||||
size?: number;
|
||||
error?: string;
|
||||
deleting?: boolean;
|
||||
chapter_files?: Array<{ filename: string, size: number }>;
|
||||
}
|
||||
|
||||
@@ -107,8 +107,8 @@ export class DownloadsService {
|
||||
return of({status: 'error', msg: msg})
|
||||
}
|
||||
|
||||
public add(url: string, quality: string, format: string, folder: string, customNamePrefix: string, playlistStrictMode: boolean, playlistItemLimit: number, autoStart: boolean) {
|
||||
return this.http.post<Status>('add', {url: url, quality: quality, format: format, folder: folder, custom_name_prefix: customNamePrefix, playlist_strict_mode: playlistStrictMode, playlist_item_limit: playlistItemLimit, auto_start: autoStart}).pipe(
|
||||
public add(url: string, quality: string, format: string, folder: string, customNamePrefix: string, playlistItemLimit: number, autoStart: boolean, splitByChapters: boolean, chapterTemplate: string) {
|
||||
return this.http.post<Status>('add', { url: url, quality: quality, format: format, folder: folder, custom_name_prefix: customNamePrefix, playlist_item_limit: playlistItemLimit, auto_start: autoStart, split_by_chapters: splitByChapters, chapter_template: chapterTemplate }).pipe(
|
||||
catchError(this.handleHTTPError)
|
||||
);
|
||||
}
|
||||
@@ -147,12 +147,13 @@ export class DownloadsService {
|
||||
const defaultFormat = 'mp4';
|
||||
const defaultFolder = '';
|
||||
const defaultCustomNamePrefix = '';
|
||||
const defaultPlaylistStrictMode = false;
|
||||
const defaultPlaylistItemLimit = 0;
|
||||
const defaultAutoStart = true;
|
||||
const defaultSplitByChapters = false;
|
||||
const defaultChapterTemplate = this.configuration['OUTPUT_TEMPLATE_CHAPTER'];
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
this.add(url, defaultQuality, defaultFormat, defaultFolder, defaultCustomNamePrefix, defaultPlaylistStrictMode, defaultPlaylistItemLimit, defaultAutoStart)
|
||||
this.add(url, defaultQuality, defaultFormat, defaultFolder, defaultCustomNamePrefix, defaultPlaylistItemLimit, defaultAutoStart, defaultSplitByChapters, defaultChapterTemplate)
|
||||
.subscribe({
|
||||
next: (response) => resolve(response),
|
||||
error: (error) => reject(error)
|
||||
|
||||
Reference in New Issue
Block a user