Add config options for concurrent fragment downloads and extractor thread pool.

Author: itsTheFae

musicbot/config.py

@@ -968,6 +968,32 @@ def __init__(self, config_file: pathlib.Path) -> None:
                 "To allow either IPv4 or v6, set this to:  *"
             ),
         )
+        self.ytdlp_concurrent_frags: int = self.register.init_option(
+            section="MusicBot",
+            option="YtdlpConcurrentFrags",
+            dest="ytdlp_concurrent_frags",
+            default=ConfigDefaults.ytdlp_concurrent_frags,
+            getter="getint",
+            comment=_Dd(
+                "Use this many simultaneous threads to download each track.\n"
+                "Same as ytdlp -N or --concurrent-fragments options.\n"
+                "This option may speed up downloads, but has no effect on streams.\n"
+                "Should not be higher than the available number of CPU cores."
+            ),
+        )
+
+        self.downloader_threads_max: int = self.register.init_option(
+            section="MusicBot",
+            option="DownloaderTheadsMax",
+            dest="downloader_threads_max",
+            default=ConfigDefaults.downloader_threads_max,
+            getter="getint",
+            comment=_Dd(
+                "MusicBot may use up-to this many threads for each separate extraction.\n"
+                "Basically, the max number of simultaneous downloads or extractions.\n"
+                "MusicBot will spawn threads as they are needed, not right away.\n"
+            ),
+        )
 
         self.user_blocklist_enabled: bool = self.register.init_option(
             section="MusicBot",
@@ -1551,6 +1577,8 @@ class ConfigDefaults:
     ytdlp_proxy: str = ""
     ytdlp_user_agent: str = ""
     ytdlp_source_address: str = "*"
+    ytdlp_concurrent_frags: int = 1
+    downloader_threads_max: int = 2
 
     pre_download_next_song: bool = True
     default_search_service: str = "ytsearch"

musicbot/constants.py

@@ -121,8 +121,6 @@
 # Each retry increases the timeout by multiplying attempts by the above timeout.
 VOICE_CLIENT_MAX_RETRY_CONNECT: int = 5
 
-# Maximum number of threads MusicBot will use for downloading and extracting info.
-DEFAULT_MAX_INFO_DL_THREADS: int = 5
 # Maximum number of seconds to wait for HEAD request on media files.
 DEFAULT_MAX_INFO_REQUEST_TIMEOUT: int = 10
 

musicbot/downloader.py

@@ -20,7 +20,7 @@
 from yt_dlp.utils import DownloadError  # type: ignore[import-untyped]
 from yt_dlp.utils import UnsupportedError
 
-from .constants import DEFAULT_MAX_INFO_DL_THREADS, DEFAULT_MAX_INFO_REQUEST_TIMEOUT
+from .constants import DEFAULT_MAX_INFO_REQUEST_TIMEOUT
 from .exceptions import ExtractionError, MusicbotException
 from .i18n import _L
 from .spotify import Spotify
@@ -123,7 +123,7 @@ def __init__(self, bot: "MusicBot") -> None:
         self.download_folder: pathlib.Path = bot.config.audio_cache_path
         # NOTE: this executor may not be good for long-running downloads...
         self.thread_pool = ThreadPoolExecutor(
-            max_workers=DEFAULT_MAX_INFO_DL_THREADS,
+            max_workers=bot.config.downloader_threads_max,
             thread_name_prefix="MB_Downloader",
         )
         self._supported_search = [
@@ -147,6 +147,12 @@ def __init__(self, bot: "MusicBot") -> None:
         ytdl_format_options = ytdl_format_options_immutable.copy()
         ytdl_format_options["http_headers"] = self.http_req_headers
 
+        # add concurrent-fragments option if it is needed.
+        if bot.config.ytdlp_concurrent_frags > 1:
+            ytdl_format_options["concurrent_fragment_downloads"] = (
+                bot.config.ytdlp_concurrent_frags
+            )
+
         # apply source address settings.
         if bot.config.ytdlp_source_address != "*":
             ytdl_format_options["source_address"] = bot.config.ytdlp_source_address