thomas/metube
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into master

Browse Source
This commit is contained in:
Nikolay G
2023-02-17 20:51:50 +03:00
committed by GitHub
parent cdfdf1742f a61607c372
commit f30bd109f7
16 changed files with 3399 additions and 2527 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
app/dl_formats.py
View File

@@ -19,6 +19,10 @@ def get_format(format: str, quality: str) -> str:
if format.startswith("custom:"):
return format[7:]
if format == "thumbnail":
# Quality is irrelevant in this case since we skip the download
return "bestaudio/best"
if format == "mp3":
# Audio quality needs to be set post-download, set in opts
return "bestaudio/best"
@@ -63,7 +67,13 @@ def get_opts(format: str, quality: str, ytdl_opts: dict) -> dict:
"preferredquality": 0 if quality == "best" else quality,
})
opts["writethumbnail"] = True
opts["postprocessors"].append({"key": "FFmpegThumbnailsConvertor", "format": "jpg", "when": "before_dl"})
opts["postprocessors"].append({"key": "FFmpegMetadata"})
opts["postprocessors"].append({"key": "EmbedThumbnail"})
if format == "thumbnail":
opts["skip_download"] = True
opts["writethumbnail"] = True
opts["postprocessors"].append({"key": "FFmpegThumbnailsConvertor", "format": "jpg", "when": "before_dl"})
return opts

56
app/main.py
View File

@@ -7,6 +7,7 @@ from aiohttp import web
import socketio
import logging
import json
import pathlib
from ytdl import DownloadQueueNotifier, DownloadQueue
@@ -16,6 +17,8 @@ class Config:
_DEFAULTS = {
'DOWNLOAD_DIR': '.',
'AUDIO_DOWNLOAD_DIR': '%%DOWNLOAD_DIR',
'CUSTOM_DIRS': 'true',
'CREATE_CUSTOM_DIRS': 'true',
'STATE_DIR': '.',
'URL_PREFIX': '',
'OUTPUT_TEMPLATE': '%(title)s.%(ext)s',
@@ -26,12 +29,19 @@ class Config:
'BASE_DIR': ''
}
_BOOLEAN = ('CUSTOM_DIRS', 'CREATE_CUSTOM_DIRS')
def __init__(self):
for k, v in self._DEFAULTS.items():
setattr(self, k, os.environ[k] if k in os.environ else v)
for k, v in self.__dict__.items():
if v.startswith('%%'):
setattr(self, k, getattr(self, v[2:]))
if k in self._BOOLEAN:
if v not in ('true', 'false', 'True', 'False', 'on', 'off', '1', '0'):
log.error(f'Environment variable "{k}" is set to a non-boolean value "{v}"')
sys.exit(1)
setattr(self, k, v in ('true', 'True', 'on', '1'))
if not self.URL_PREFIX.endswith('/'):
self.URL_PREFIX += '/'
try:
@@ -83,7 +93,8 @@ async def add(request):
if not url or not quality:
raise web.HTTPBadRequest()
format = post.get('format')
status = await dqueue.add(url, quality, format)
folder = post.get('folder')
status = await dqueue.add(url, quality, format, folder)
return web.Response(text=serializer.encode(status))
@routes.post(config.URL_PREFIX + 'delete')
@@ -99,6 +110,40 @@ async def delete(request):
@sio.event
async def connect(sid, environ):
await sio.emit('all', serializer.encode(dqueue.get()), to=sid)
await sio.emit('configuration', serializer.encode(config), to=sid)
if config.CUSTOM_DIRS:
await sio.emit('custom_dirs', serializer.encode(get_custom_dirs()), to=sid)
def get_custom_dirs():
def recursive_dirs(base):
path = pathlib.Path(base)
# Converts PosixPath object to string, and remove base/ prefix
def convert(p):
s = str(p)
if s.startswith(base):
s = s[len(base):]
if s.startswith('/'):
s = s[1:]
return s
# Recursively lists all subdirectories of DOWNLOAD_DIR
dirs = list(filter(None, map(convert, path.glob('**'))))
return dirs
download_dir = recursive_dirs(config.DOWNLOAD_DIR)
audio_download_dir = download_dir
if config.DOWNLOAD_DIR != config.AUDIO_DOWNLOAD_DIR:
audio_download_dir = recursive_dirs(config.AUDIO_DOWNLOAD_DIR)
return {
"download_dir": download_dir,
"audio_download_dir": audio_download_dir
}
@routes.get(config.URL_PREFIX)
def index(request):
@@ -115,9 +160,14 @@ if config.URL_PREFIX != '/':
routes.static(config.URL_PREFIX + 'favicon/', os.path.join(config.BASE_DIR, 'favicon'))
routes.static(config.URL_PREFIX + 'download/', config.DOWNLOAD_DIR)
routes.static(config.URL_PREFIX + 'audio_download/', config.AUDIO_DOWNLOAD_DIR)
routes.static(config.URL_PREFIX, os.path.join(config.BASE_DIR, 'ui/dist/metube'))
app.add_routes(routes)
try:
app.add_routes(routes)
except ValueError as e:
if 'ui/dist/metube' in str(e):
raise RuntimeError('Could not find the frontend UI static assets. Please run `node_modules/.bin/ng build` inside the ui folder') from e
raise e
# https://github.com/aio-libs/aiohttp/pull/4615 waiting for release
# @routes.options(config.URL_PREFIX + 'add')

42
app/ytdl.py
View File

@@ -6,6 +6,7 @@ import time
import asyncio
import multiprocessing
import logging
import re
from dl_formats import get_format, get_opts
log = logging.getLogger('ytdl')
@@ -27,10 +28,11 @@ class DownloadQueueNotifier:
raise NotImplementedError
class DownloadInfo:
def __init__(self, id, title, url, quality, format):
def __init__(self, id, title, url, quality, format, folder):
self.id, self.title, self.url = id, title, url
self.quality = quality
self.format = format
self.folder = folder
self.status = self.msg = self.percent = self.speed = self.eta = None
self.filename = None
self.timestamp = time.time_ns()
@@ -126,6 +128,10 @@ class Download:
self.tmpfilename = status.get('tmpfilename')
if 'filename' in status:
self.info.filename = os.path.relpath(status.get('filename'), self.download_dir)
# Set correct file extension for thumbnails
if(self.info.format == 'thumbnail'):
self.info.filename = re.sub(r'\.webm$', '.jpg', self.info.filename)
self.info.status = status['status']
self.info.msg = status.get('msg')
if 'downloaded_bytes' in status:
@@ -164,7 +170,7 @@ class PersistentQueue:
return sorted(shelf.items(), key=lambda item: item[1].timestamp)
def put(self, value):
key = value.info.id
key = value.info.url
self.dict[key] = value
with shelve.open(self.path, 'w') as shelf:
shelf[key] = value.info
@@ -192,7 +198,7 @@ class DownloadQueue:
async def __import_queue(self):
for k, v in self.queue.saved_items():
await self.add(v.url, v.quality, v.format)
await self.add(v.url, v.quality, v.format, folder=v.folder)
async def initialize(self):
self.event = asyncio.Event()
@@ -207,7 +213,7 @@ class DownloadQueue:
**self.config.YTDL_OPTIONS,
}).extract_info(url, download=False)
async def __add_entry(self, entry, quality, format, already):
async def __add_entry(self, entry, quality, format, folder, already):
etype = entry.get('_type') or 'video'
if etype == 'playlist':
entries = entry['entries']
@@ -220,14 +226,28 @@ class DownloadQueue:
for property in ("id", "title", "uploader", "uploader_id"):
if property in entry:
etr[f"playlist_{property}"] = entry[property]
results.append(await self.__add_entry(etr, quality, format, already))
results.append(await self.__add_entry(etr, quality, format, folder, already))
if any(res['status'] == 'error' for res in results):
return {'status': 'error', 'msg': ', '.join(res['msg'] for res in results if res['status'] == 'error' and 'msg' in res)}
return {'status': 'ok'}
elif etype == 'video' or etype.startswith('url') and 'id' in entry and 'title' in entry:
if not self.queue.exists(entry['id']):
dl = DownloadInfo(entry['id'], entry['title'], entry.get('webpage_url') or entry['url'], quality, format)
dldirectory = self.config.DOWNLOAD_DIR if (quality != 'audio' and format != 'mp3') else self.config.AUDIO_DOWNLOAD_DIR
dl = DownloadInfo(entry['id'], entry['title'], entry.get('webpage_url') or entry['url'], quality, format, folder)
# Keep consistent with frontend
base_directory = self.config.DOWNLOAD_DIR if (quality != 'audio' and format != 'mp3') else self.config.AUDIO_DOWNLOAD_DIR
if folder:
if not self.config.CUSTOM_DIRS:
return {'status': 'error', 'msg': f'A folder for the download was specified but CUSTOM_DIRS is not true in the configuration.'}
dldirectory = os.path.realpath(os.path.join(base_directory, folder))
real_base_directory = os.path.realpath(base_directory)
if not dldirectory.startswith(real_base_directory):
return {'status': 'error', 'msg': f'Folder "{folder}" must resolve inside the base download directory "{real_base_directory}"'}
if not os.path.isdir(dldirectory):
if not self.config.CREATE_CUSTOM_DIRS:
return {'status': 'error', 'msg': f'Folder "{folder}" for download does not exist inside base directory "{real_base_directory}", and CREATE_CUSTOM_DIRS is not true in the configuration.'}
os.makedirs(dldirectory, exist_ok=True)
else:
dldirectory = base_directory
output = self.config.OUTPUT_TEMPLATE
output_chapter = self.config.OUTPUT_TEMPLATE_CHAPTER
for property, value in entry.items():
@@ -238,11 +258,11 @@ class DownloadQueue:
await self.notifier.added(dl)
return {'status': 'ok'}
elif etype.startswith('url'):
return await self.add(entry['url'], quality, format, already)
return await self.add(entry['url'], quality, format, folder, already)
return {'status': 'error', 'msg': f'Unsupported resource "{etype}"'}
async def add(self, url, quality, format, already=None):
log.info(f'adding {url}')
async def add(self, url, quality, format, folder, already=None):
log.info(f'adding {url}: {quality=} {format=} {already=} {folder=}')
already = set() if already is None else already
if url in already:
log.info('recursion detected, skipping')
@@ -253,7 +273,7 @@ class DownloadQueue:
entry = await asyncio.get_running_loop().run_in_executor(None, self.__extract_info, url)
except yt_dlp.utils.YoutubeDLError as exc:
return {'status': 'error', 'msg': str(exc)}
return await self.__add_entry(entry, quality, format, already)
return await self.__add_entry(entry, quality, format, folder, already)
async def cancel(self, ids):
for id in ids: