diff --git a/streamrip/clients.py b/streamrip/clients.py index 4e0f9ad..6cf05a0 100644 --- a/streamrip/clients.py +++ b/streamrip/clients.py @@ -26,7 +26,7 @@ from .exceptions import ( InvalidQuality, ) from .spoofbuz import Spoofer -from .utils import get_quality +from .utils import gen_threadsafe_session, get_quality urllib3.disable_warnings() requests.adapters.DEFAULT_RETRIES = 5 @@ -149,12 +149,8 @@ class QobuzClient(ClientInterface): self.app_id = str(kwargs["app_id"]) # Ensure it is a string self.secrets = kwargs["secrets"] - self.session = requests.Session() - self.session.headers.update( - { - "User-Agent": AGENT, - "X-App-Id": self.app_id, - } + self.session = gen_threadsafe_session( + headers={"User-Agent": AGENT, "X-App-Id": self.app_id} ) self._api_login(email, pwd) @@ -373,7 +369,9 @@ class DeezerClient(ClientInterface): max_quality = 2 def __init__(self): - self.session = requests.Session() + self.session = gen_threadsafe_session() + + # no login required self.logged_in = True def search(self, query: str, media_type: str = "album", limit: int = 200) -> dict: @@ -389,9 +387,6 @@ class DeezerClient(ClientInterface): # TODO: more robust url sanitize query = query.replace(" ", "+") - if media_type.endswith("s"): - media_type = media_type[:-1] - # TODO: use limit parameter response = self.session.get(f"{DEEZER_BASE}/search/{media_type}?q={query}") response.raise_for_status() @@ -447,6 +442,8 @@ class TidalClient(ClientInterface): self.refresh_token = None self.expiry = None + self.session = gen_threadsafe_session() + def login( self, user_id=None, @@ -492,7 +489,7 @@ class TidalClient(ClientInterface): try: manifest = json.loads(base64.b64decode(resp["manifest"]).decode("utf-8")) except KeyError: - raise Exception("You must have a TIDAL Hi-Fi account to download tracks.") + raise Exception(resp['userMessage']) logger.debug(manifest) return { @@ -588,7 +585,9 @@ class TidalClient(ClientInterface): headers = { "authorization": f"Bearer {token}", } - r = requests.get("https://api.tidal.com/v1/sessions", headers=headers).json() + r = self.session.get( + "https://api.tidal.com/v1/sessions", headers=headers + ).json() if r.status != 200: raise Exception("Login failed") @@ -614,10 +613,13 @@ class TidalClient(ClientInterface): self.country_code = resp["user"]["countryCode"] self.access_token = resp["access_token"] self.token_expiry = resp["expires_in"] + time.time() + self._update_authorization() def _login_by_access_token(self, token, user_id=None): - headers = {"authorization": f"Bearer {token}"} - resp = requests.get("https://api.tidal.com/v1/sessions", headers=headers).json() + headers = {"authorization": f"Bearer {token}"} # temporary + resp = self.session.get( + "https://api.tidal.com/v1/sessions", headers=headers + ).json() if resp.get("status", 200) != 200: raise Exception(f"Login failed {resp}") @@ -627,6 +629,7 @@ class TidalClient(ClientInterface): self.user_id = resp["userId"] self.country_code = resp["countryCode"] self.access_token = token + self._update_authorization() def _api_get(self, item_id: str, media_type: str) -> dict: url = f"{media_type}s/{item_id}" @@ -654,22 +657,27 @@ class TidalClient(ClientInterface): if params is None: params = {} - headers = {"authorization": f"Bearer {self.access_token}"} params["countryCode"] = self.country_code params["limit"] = 100 - r = requests.get(f"{TIDAL_BASE}/{path}", headers=headers, params=params).json() + r = self.session.get(f"{TIDAL_BASE}/{path}", params=params).json() return r def _api_post(self, url, data, auth=None): - r = requests.post(url, data=data, auth=auth, verify=False).json() + r = self.session.post(url, data=data, auth=auth, verify=False).json() return r + def _update_authorization(self): + self.session.headers.update({"authorization": f"Bearer {self.access_token}"}) + class SoundCloudClient(ClientInterface): source = "soundcloud" max_quality = 0 logged_in = True + def __init__(self): + self.session = gen_threadsafe_session(headers={"User-Agent": AGENT}) + def login(self): raise NotImplementedError @@ -721,7 +729,7 @@ class SoundCloudClient(ClientInterface): url = f"{SOUNDCLOUD_BASE}/{path}" logger.debug(f"Fetching url {url}") - r = requests.get(url, params=params) + r = self.session.get(url, params=params) if resp_obj: return r diff --git a/streamrip/config.py b/streamrip/config.py index f0d3ec4..4750ffe 100644 --- a/streamrip/config.py +++ b/streamrip/config.py @@ -27,14 +27,12 @@ class Config: Usage: >>> config = Config('test_config.yaml') + >>> config.defaults['qobuz']['quality'] + 3 If test_config was already initialized with values, this will load them into `config`. Otherwise, a new config file is created with the default values. - - >>> config.update_from_cli(**args) - - This will update the config values based on command line args. """ defaults = { @@ -42,7 +40,7 @@ class Config: "quality": 3, "email": None, "password": None, - "app_id": "", # Avoid NoneType error + "app_id": "", "secrets": [], }, "tidal": { @@ -82,10 +80,12 @@ class Config: }, "metadata": { "set_playlist_to_album": False, + "new_playlist_tracknumbers": True, }, "path_format": {"folder": FOLDER_FORMAT, "track": TRACK_FORMAT}, "check_for_updates": True, "lastfm": {"source": "qobuz"}, + "concurrent_downloads": False, } def __init__(self, path: str = None): diff --git a/streamrip/converter.py b/streamrip/converter.py index 08bb184..ff1a8d1 100644 --- a/streamrip/converter.py +++ b/streamrip/converter.py @@ -5,12 +5,12 @@ import subprocess from tempfile import gettempdir from typing import Optional -from mutagen.flac import FLAC as FLAC_META - from .exceptions import ConversionError logger = logging.getLogger(__name__) +SAMPLING_RATES = (44100, 48000, 88200, 96000, 176400, 192000) + class Converter: """Base class for audio codecs.""" @@ -111,9 +111,11 @@ class Converter: if self.lossless: if isinstance(self.sampling_rate, int): - audio = FLAC_META(self.filename) - old_sr = audio.info.sample_rate - command.extend(["-ar", str(min(old_sr, self.sampling_rate))]) + sampling_rates = "|".join( + str(rate) for rate in SAMPLING_RATES if rate <= self.sampling_rate + ) + command.extend(["-af", f"aformat=sample_rates={sampling_rates}"]) + elif self.sampling_rate is not None: raise TypeError( f"Sampling rate must be int, not {type(self.sampling_rate)}" @@ -129,6 +131,7 @@ class Converter: elif self.bit_depth is not None: raise TypeError(f"Bit depth must be int, not {type(self.bit_depth)}") + # automatically overwrite command.extend(["-y", self.tempfile]) return command diff --git a/streamrip/core.py b/streamrip/core.py index 067983f..c4af261 100644 --- a/streamrip/core.py +++ b/streamrip/core.py @@ -2,6 +2,7 @@ import logging import os import re import sys +import threading from getpass import getpass from hashlib import md5 from string import Formatter @@ -23,7 +24,12 @@ from .constants import ( ) from .db import MusicDB from .downloader import Album, Artist, Label, Playlist, Track, Tracklist -from .exceptions import AuthenticationError, NoResultsFound, ParsingError, NonStreamable +from .exceptions import ( + AuthenticationError, + NonStreamable, + NoResultsFound, + ParsingError, +) from .utils import capitalize logger = logging.getLogger(__name__) @@ -153,21 +159,25 @@ class MusicDL(list): "parent_folder": self.config.session["downloads"]["folder"], "folder_format": self.config.session["path_format"]["folder"], "track_format": self.config.session["path_format"]["track"], - "embed_cover": self.config.session["artwork"]["embed"], "embed_cover_size": self.config.session["artwork"]["size"], - "keep_hires_cover": self.config.session['artwork']['keep_hires_cover'], - + "keep_hires_cover": self.config.session["artwork"]["keep_hires_cover"], "set_playlist_to_album": self.config.session["metadata"][ "set_playlist_to_album" ], "stay_temp": self.config.session["conversion"]["enabled"], + "conversion": self.config.session["conversion"], + "concurrent_downloads": self.config.session["concurrent_downloads"], + "new_tracknumbers": self.config.session['metadata']['new_playlist_tracknumbers'] } logger.debug("Arguments from config: %s", arguments) + + source_subdirs = self.config.session["downloads"]["source_subdirectories"] for item in self: - if self.config.session["downloads"]["source_subdirectories"]: - arguments["parent_folder"] = os.path.join( - arguments["parent_folder"], capitalize(item.client.source) + + if source_subdirs: + arguments["parent_folder"] = self.__get_source_subdir( + item.client.source ) arguments["quality"] = self.config.session[item.client.source]["quality"] @@ -182,7 +192,7 @@ class MusicDL(list): try: item.load_meta() except NonStreamable: - click.secho(f"{item!s} is not available, skipping.", fg='red') + click.secho(f"{item!s} is not available, skipping.", fg="red") continue if isinstance(item, Track): @@ -194,8 +204,8 @@ class MusicDL(list): if self.db != [] and hasattr(item, "id"): self.db.add(item.id) - if self.config.session["conversion"]["enabled"]: - item.convert(**self.config.session["conversion"]) + # if self.config.session["conversion"]["enabled"]: + # item.convert(**self.config.session["conversion"]) def get_client(self, source: str): client = self.clients[source] @@ -261,24 +271,36 @@ class MusicDL(list): def handle_lastfm_urls(self, urls): lastfm_urls = self.lastfm_url_parse.findall(urls) lastfm_source = self.config.session["lastfm"]["source"] + tracks_not_found = 0 + + def search_query(query: str, playlist: Playlist): + global tracks_not_found + try: + track = next(self.search(lastfm_source, query, media_type="track")) + playlist.append(track) + except NoResultsFound: + tracks_not_found += 1 + return + for purl in lastfm_urls: click.secho(f"Fetching playlist at {purl}", fg="blue") title, queries = self.get_lastfm_playlist(purl) - pl = Playlist(client=self.clients[lastfm_source], name=title) - tracks_not_found = 0 - for title, artist in tqdm(queries, unit="tracks", desc="Searching"): + pl = Playlist(client=self.get_client(lastfm_source), name=title) + processes = [] + + for title, artist in queries: query = f"{title} {artist}" + proc = threading.Thread( + target=search_query, args=(query, pl), daemon=True + ) + proc.start() + processes.append(proc) - try: - track = next(self.search(lastfm_source, query, media_type="track")) - except NoResultsFound: - tracks_not_found += 1 - continue - - pl.append(track) - pl.loaded = True + for proc in tqdm(processes, unit="tracks", desc="Searching"): + proc.join() + pl.loaded = True click.secho(f"{tracks_not_found} tracks not found.", fg="yellow") self.append(pl) @@ -463,3 +485,7 @@ class MusicDL(list): remaining_tracks -= 50 return playlist_title, info + + def __get_source_subdir(self, source: str) -> str: + path = self.config.session["downloads"]["folder"] + return os.path.join(path, capitalize(source)) diff --git a/streamrip/downloader.py b/streamrip/downloader.py index 4d6c2e3..b5c3664 100644 --- a/streamrip/downloader.py +++ b/streamrip/downloader.py @@ -7,15 +7,17 @@ import os import re import shutil import subprocess +import threading from pprint import pformat from tempfile import gettempdir -from typing import Any, Callable, Optional, Tuple, Union +from typing import Any, Generator, Iterable, Union import click from mutagen.flac import FLAC, Picture from mutagen.id3 import APIC, ID3, ID3NoHeaderError from mutagen.mp4 import MP4, MP4Cover from pathvalidate import sanitize_filename, sanitize_filepath +from requests.packages import urllib3 from . import converter from .clients import ClientInterface @@ -44,6 +46,7 @@ from .utils import ( ) logger = logging.getLogger(__name__) +urllib3.disable_warnings() TIDAL_Q_MAP = { "LOW": 0, @@ -145,7 +148,7 @@ class Track: self.cover_url = None @staticmethod - def _get_tracklist(resp, source): + def _get_tracklist(resp, source) -> list: if source == "qobuz": return resp["tracks"]["items"] if source in ("tidal", "deezer"): @@ -161,7 +164,7 @@ class Track: database: MusicDB = None, tag: bool = False, **kwargs, - ): + ) -> bool: """ Download the track. @@ -211,7 +214,11 @@ class Track: else: url_id = self.id - dl_info = self.client.get_file_url(url_id, self.quality) + try: + dl_info = self.client.get_file_url(url_id, self.quality) + except Exception as e: + click.secho(f"Unable to download track. {e}", fg='red') + return False self.path = os.path.join(gettempdir(), f"{hash(self.id)}_{self.quality}.tmp") logger.debug("Temporary file path: %s", self.path) @@ -227,15 +234,13 @@ class Track: self.sampling_rate = dl_info.get("sampling_rate") self.bit_depth = dl_info.get("bit_depth") - click.secho(f"\nDownloading {self!s}", fg="blue") - # --------- Download Track ---------- if self.client.source in ("qobuz", "tidal"): logger.debug("Downloadable URL found: %s", dl_info.get("url")) - tqdm_download(dl_info["url"], self.path) # downloads file + tqdm_download(dl_info["url"], self.path, desc=self._progress_desc) # downloads file elif self.client.source == "deezer": # Deezer - logger.debug("Downloadable URL found: %s", dl_info) + logger.debug("Downloadable URL found: %s", dl_info, desc=self._progress_desc) try: tqdm_download(dl_info, self.path) # downloads file except NonStreamable: @@ -300,7 +305,7 @@ class Track: ] ) elif dl_info["type"] == "original": - tqdm_download(dl_info["url"], self.path) + tqdm_download(dl_info["url"], self.path, desc=self._progress_desc) # if a wav is returned, convert to flac engine = converter.FLAC(self.path) @@ -310,6 +315,10 @@ class Track: self.final_path = self.final_path.replace(".mp3", ".flac") self.quality = 2 + @property + def _progress_desc(self): + return click.style(f"Track {int(self.meta.tracknumber):02}", fg='blue') + def download_cover(self): """Downloads the cover art, if cover_url is given.""" @@ -317,10 +326,10 @@ class Track: self.cover_path = os.path.join(self.folder, f"cover{hash(self.cover_url)}.jpg") logger.debug(f"Downloading cover from {self.cover_url}") - click.secho(f"\nDownloading cover art for {self!s}", fg="blue") + # click.secho(f"\nDownloading cover art for {self!s}", fg="blue") if not os.path.exists(self.cover_path): - tqdm_download(self.cover_url, self.cover_path) + tqdm_download(self.cover_url, self.cover_path, desc=click.style('Cover', fg='cyan')) else: logger.debug("Cover already exists, skipping download") @@ -515,16 +524,18 @@ class Track: sampling_rate=kwargs.get("sampling_rate"), remove_source=kwargs.get("remove_source", True), ) - click.secho(f"Converting {self!s}", fg="blue") + # click.secho(f"Converting {self!s}", fg="blue") engine.convert() self.path = engine.final_fn - self.final_path = self.final_path.replace(ext(self.quality, self.client.source), f".{engine.container}") + self.final_path = self.final_path.replace( + ext(self.quality, self.client.source), f".{engine.container}" + ) if not kwargs.get("stay_temp", False): self.move(self.final_path) @property - def title(self): + def title(self) -> str: if hasattr(self, "meta"): _title = self.meta.title if self.meta.explicit: @@ -533,7 +544,7 @@ class Track: else: raise Exception("Track must be loaded before accessing title") - def get(self, *keys, default=None): + def get(self, *keys, default=None) -> Any: """Safe get method that allows for layered access. :param keys: @@ -550,14 +561,14 @@ class Track: """ self.__setitem__(key, val) - def __getitem__(self, key): + def __getitem__(self, key: str) -> Any: """Dict-like interface for Track metadata. :param key: """ return getattr(self.meta, key) - def __setitem__(self, key, val): + def __setitem__(self, key: str, val: Any): """Dict-like interface for Track metadata. :param key: @@ -588,20 +599,56 @@ class Tracklist(list): subclass is subscripted with [s: str], it will return an attribute s. If it is subscripted with [i: int] it will return the i'th track in the tracklist. - - >>> tlist = Tracklist() - >>> tlist.tracklistname = 'my tracklist' - >>> tlist.append('first track') - >>> tlist[0] - 'first track' - >>> tlist['tracklistname'] - 'my tracklist' - >>> tlist[2] - IndexError """ essence_regex = re.compile(r"([^\(]+)(?:\s*[\(\[][^\)][\)\]])*") + def download(self, **kwargs): + self._prepare_download(**kwargs) + if kwargs.get("conversion", False): + has_conversion = kwargs["conversion"]["enabled"] + else: + has_conversion = False + kwargs["stay_temp"] = False + + if has_conversion: + target = self._download_and_convert_item + else: + target = self._download_item + + if kwargs.get("concurrent_downloads", True): + processes = [] + for item in self: + proc = threading.Thread( + target=target, args=(item,), kwargs=kwargs, daemon=True + ) + proc.start() + processes.append(proc) + + try: + for proc in processes: + proc.join() + except (KeyboardInterrupt, SystemExit): + click.echo("Aborted!") + exit() + + else: + for item in self: + click.secho(f'\nDownloading "{item!s}"', fg="blue") + target(item, **kwargs) + + self.downloaded = True + + def _download_and_convert_item(self, item, **kwargs): + if self._download_item(item, **kwargs): + item.convert(**kwargs["conversion"]) + + def _download_item(item, **kwargs): + raise NotImplementedError + + def _prepare_download(**kwargs): + raise NotImplementedError + def get(self, key: Union[str, int], default=None): if isinstance(key, str): if hasattr(self, key): @@ -696,16 +743,13 @@ class Tracklist(list): def download_message(self): click.secho( - f"\nDownloading {self.title} ({self.__class__.__name__})\n", + f"\n\nDownloading {self.title} ({self.__class__.__name__})\n", fg="blue", ) @staticmethod def _parse_get_resp(item, client): - pass - - def download(self, **kwargs): - pass + raise NotImplementedError @staticmethod def essence(album: str) -> str: @@ -763,13 +807,15 @@ class Album(Tracklist): setattr(self, k, v) # to improve from_api method speed - if kwargs.get("load_on_init"): + if kwargs.get("load_on_init", False): self.load_meta() self.loaded = False self.downloaded = False def load_meta(self): + """Load detailed metadata from API using the id.""" + assert hasattr(self, "id"), "id must be set to load metadata" self.meta = self.client.get(self.id, media_type="album") @@ -784,13 +830,83 @@ class Album(Tracklist): self.loaded = True @classmethod - def from_api(cls, resp, client): + def from_api(cls, resp: dict, client: ClientInterface): if client.source == "soundcloud": return Playlist.from_api(resp, client) info = cls._parse_get_resp(resp, client) return cls(client, **info) + def _prepare_download(self, **kwargs): + self.folder_format = kwargs.get("folder_format", FOLDER_FORMAT) + self.quality = min(kwargs.get("quality", 3), self.client.max_quality) + self.folder = self._get_formatted_folder( + kwargs.get("parent_folder", "StreamripDownloads"), self.quality + ) + os.makedirs(self.folder, exist_ok=True) + + self.download_message() + + # choose optimal cover size and download it + click.secho("Downloading cover art", fg="magenta") + cover_path = os.path.join(gettempdir(), f"cover_{hash(self)}.jpg") + embed_cover_size = kwargs.get("embed_cover_size", "large") + + assert ( + embed_cover_size in self.cover_urls + ), f"Invalid cover size. Must be in {self.cover_urls.keys()}" + + embed_cover_url = self.cover_urls[embed_cover_size] + if embed_cover_url is not None: + tqdm_download(embed_cover_url, cover_path) + else: # sometimes happens with Deezer + tqdm_download(self.cover_urls["small"], cover_path) + + if kwargs.get("keep_hires_cover", True): + tqdm_download( + self.cover_urls["original"], os.path.join(self.folder, "cover.jpg") + ) + + cover_size = os.path.getsize(cover_path) + if cover_size > FLAC_MAX_BLOCKSIZE: # 16.77 MB + click.secho( + "Downgrading embedded cover size, too large ({cover_size}).", + fg="bright_yellow", + ) + # large is about 600x600px which is guaranteed < 16.7 MB + tqdm_download(self.cover_urls["large"], cover_path) + + embed_cover = kwargs.get("embed_cover", True) # embed by default + if self.client.source != "deezer" and embed_cover: + self.cover_obj = self.get_cover_obj( + cover_path, self.quality, self.client.source + ) + else: + self.cover_obj = None + + def _download_item( + self, + track: Track, + quality: int = 3, + database: MusicDB = None, + **kwargs, + ) -> bool: + logger.debug("Downloading track to %s", self.folder) + if self.disctotal > 1: + disc_folder = os.path.join(self.folder, f"Disc {track.meta.discnumber}") + kwargs["parent_folder"] = disc_folder + else: + kwargs["parent_folder"] = self.folder + + if not track.download(quality=quality, database=database, **kwargs): + return False + + # deezer tracks come tagged + if kwargs.get("tag_tracks", True) and self.client.source != "deezer": + track.tag(cover=self.cover_obj, embed_cover=kwargs.get("embed_cover", True)) + + return True + @staticmethod def _parse_get_resp(resp: dict, client: ClientInterface) -> dict: """Parse information from a client.get(query, 'album') call. @@ -903,110 +1019,6 @@ class Album(Tracklist): Track.from_album_meta(album=self.meta, pos=i, client=self.client) ) - @property - def title(self) -> str: - """Return the title of the album. - - It is formatted so that "version" keys are included. - - :rtype: str - """ - album_title = self._title - if hasattr(self, "version") and isinstance(self.version, str): - if self.version.lower() not in album_title.lower(): - album_title = f"{album_title} ({self.version})" - - if self.get("explicit", False): - album_title = f"{album_title} (Explicit)" - - return album_title - - @title.setter - def title(self, val): - """Sets the internal _title attribute to the given value. - - :param val: title to set - """ - self._title = val - - def download( - self, - quality: int = 3, - parent_folder: Union[str, os.PathLike] = "StreamripDownloads", - database: MusicDB = None, - **kwargs, - ): - """Download all of the tracks in the album. - - :param quality: (0, 1, 2, 3, 4) - :type quality: int - :param parent_folder: the folder to download the album to - :type parent_folder: Union[str, os.PathLike] - :param progress_bar: turn on/off a tqdm progress bar - :type progress_bar: bool - :param large_cover: Download the large cover. This may fail when - embedding covers. - :param tag_tracks: Tag the tracks after downloading, True by default - :param keep_cover: Keep the cover art image after downloading. - True by default. - """ - self.folder_format = kwargs.get("folder_format", FOLDER_FORMAT) - quality = min(quality, self.client.max_quality) - folder = self._get_formatted_folder(parent_folder, quality) - - # choose optimal cover size and download it - self.download_message() - - click.secho("Downloading cover art", fg="magenta") - cover_path = os.path.join(gettempdir(), f"cover_{hash(self)}.jpg") - embed_cover_size = kwargs.get("embed_cover_size", "large") - - assert ( - embed_cover_size in self.cover_urls - ), f"Invalid cover size. Must be in {self.cover_urls.keys()}" - - tqdm_download(self.cover_urls[embed_cover_size], cover_path) - - if kwargs.get("keep_hires_cover", True): - tqdm_download(self.cover_urls['original'], os.path.join(folder, 'cover.jpg')) - - cover_size = os.path.getsize(cover_path) - if cover_size > FLAC_MAX_BLOCKSIZE: # 16.77 MB - click.secho( - "Downgrading embedded cover size, too large ({cover_size}).", - fg="bright_yellow", - ) - # large is about 600x600px which is guaranteed < 16.7 MB - tqdm_download(self.cover_urls["large"], cover_path) - - embed_cover = kwargs.get("embed_cover", True) # embed by default - if self.client.source != "deezer" and embed_cover: - cover = self.get_cover_obj(cover_path, quality, self.client.source) - - download_args = { - "quality": quality, - "parent_folder": folder, - "progress_bar": kwargs.get("progress_bar", True), - "database": database, - "track_format": kwargs.get("track_format", TRACK_FORMAT), - "stay_temp": kwargs.get("stay_temp") - } - for track in self: - logger.debug("Downloading track to %s", folder) - if self.disctotal > 1: - disc_folder = os.path.join(folder, f"Disc {track.meta.discnumber}") - download_args["parent_folder"] = disc_folder - - track.download(quality=quality, parent_folder=folder, database=database, **kwargs) - - # deezer tracks come tagged - if kwargs.get("tag_tracks", True) and self.client.source != "deezer": - track.tag(cover=cover, embed_cover=embed_cover) - - os.remove(cover_path) - - self.downloaded = True - def _get_formatter(self) -> dict: fmt = dict() for key in ALBUM_KEYS: @@ -1035,9 +1047,34 @@ class Album(Tracklist): return os.path.join(parent_folder, formatted_folder) + @property + def title(self) -> str: + """Return the title of the album. + + It is formatted so that "version" keys are included. + + :rtype: str + """ + album_title = self._title + if hasattr(self, "version") and isinstance(self.version, str): + if self.version.lower() not in album_title.lower(): + album_title = f"{album_title} ({self.version})" + + if self.get("explicit", False): + album_title = f"{album_title} (Explicit)" + + return album_title + + @title.setter + def title(self, val): + """Sets the internal _title attribute to the given value. + + :param val: title to set + """ + self._title = val + def __repr__(self) -> str: """Return a string representation of this Album object. - Useful for pprint and json.dumps. :rtype: str """ @@ -1122,6 +1159,7 @@ class Playlist(Tracklist): :param new_tracknumbers: replace tracknumber tag with playlist position :type new_tracknumbers: bool """ + # TODO: redundant parsing with _parse_get_pres if self.client.source == "qobuz": self.name = self.meta["name"] self.image = self.meta["images"] @@ -1198,50 +1236,36 @@ class Playlist(Tracklist): logger.debug(f"Loaded {len(self)} tracks from playlist {self.name}") - def download( - self, - parent_folder: str = "StreamripDownloads", - quality: int = 3, - filters: Callable = None, - database: MusicDB = None, - **kwargs, - ): - """Download and tag all of the tracks. - - :param parent_folder: - :type parent_folder: str - :param quality: - :type quality: int - :param filters: - :type filters: Callable - """ - folder = sanitize_filename(self.name) - folder = os.path.join(parent_folder, folder) - logger.debug(f"Parent folder {folder}") + def _prepare_download(self, parent_folder: str = "StreamripDownloads", **kwargs): + fname = sanitize_filename(self.name) + self.folder = os.path.join(parent_folder, fname) + self.__download_index = 1 self.download_message() - set_playlist_to_album = kwargs.get("set_playlist_to_album", False) - for i, track in enumerate(self): - if self.client.source == "soundcloud": - track.load_meta() + def _download_item(self, item: Track, **kwargs): + if self.client.source == "soundcloud": + item.load_meta() - if set_playlist_to_album and hasattr(self, "image"): - track["album"] = self.name - track["albumartist"] = self.creator + if kwargs.get("set_playlist_to_album", False): + item["album"] = self.name + item["albumartist"] = self.creator - if kwargs.get("new_tracknumbers", True): - track.meta["tracknumber"] = str(i + 1) + if kwargs.get("new_tracknumbers", True): + item["tracknumber"] = self.__download_index + item['discnumber'] = 1 - if ( - track.download(parent_folder=folder, quality=quality, database=database, **kwargs) - and self.client.source != "deezer" - ): + self.__download_index += 1 - track.tag(embed_cover=kwargs.get("embed_cover", True)) + self.downloaded = item.download(**kwargs) + + if self.downloaded and self.client.source != "deezer": + item.tag(embed_cover=kwargs.get("embed_cover", True)) + + return self.downloaded @staticmethod - def _parse_get_resp(item: dict, client: ClientInterface): + def _parse_get_resp(item: dict, client: ClientInterface) -> dict: """Parses information from a search result returned by a client.search call. @@ -1277,12 +1301,11 @@ class Playlist(Tracklist): raise InvalidSourceError(client.source) @property - def title(self): + def title(self) -> str: return self.name def __repr__(self) -> str: """Return a string representation of this Playlist object. - Useful for pprint and json.dumps. :rtype: str """ @@ -1332,6 +1355,12 @@ class Artist(Tracklist): self._load_albums() self.loaded = True + # override + def download(self, **kwargs): + iterator = self._prepare_download(**kwargs) + for item in iterator: + self._download_item(item, **kwargs) + def _load_albums(self): """From the discography returned by client.get(query, 'artist'), generate album objects and append them to self. @@ -1355,25 +1384,9 @@ class Artist(Tracklist): logger.debug("Appending album: %s", album.get("title")) self.append(Album.from_api(album, self.client)) - def download( - self, - parent_folder: str = "StreamripDownloads", - filters: Optional[Tuple] = None, - no_repeats: bool = False, - quality: int = 6, - database: MusicDB = None, - **kwargs, - ): - """Download all albums in the discography. - - :param filters: Filters to apply to discography, see options below. - These only work for Qobuz. - :type filters: Optional[Tuple] - :param no_repeats: Remove repeats - :type no_repeats: bool - :param quality: in (0, 1, 2, 3, 4) - :type quality: int - """ + def _prepare_download( + self, parent_folder: str = "StreamripDownloads", filters: tuple = (), **kwargs + ) -> Iterable: folder = sanitize_filename(self.name) folder = os.path.join(parent_folder, folder) @@ -1393,21 +1406,33 @@ class Artist(Tracklist): final = filter(func, final) self.download_message() - for album in final: - try: - album.load_meta() - except NonStreamable: - logger.info("Skipping album, not available to stream.") - continue - album.download( - parent_folder=folder, - quality=quality, - database=database, - **kwargs, - ) + return final + + def _download_item( + self, + item, + parent_folder: str = "StreamripDownloads", + quality: int = 3, + database: MusicDB = None, + **kwargs, + ) -> bool: + try: + item.load_meta() + except NonStreamable: + logger.info("Skipping album, not available to stream.") + return + + # always an Album + status = item.download( + parent_folder=parent_folder, + quality=quality, + database=database, + **kwargs, + ) + return status @property - def title(self): + def title(self) -> str: return self.name @classmethod @@ -1427,7 +1452,7 @@ class Artist(Tracklist): return cls(client=client, **info) @staticmethod - def _parse_get_resp(item: dict, client: ClientInterface): + def _parse_get_resp(item: dict, client: ClientInterface) -> dict: """Parse a result from a client.search call. :param item: the item to parse @@ -1452,7 +1477,7 @@ class Artist(Tracklist): # ----------- Filters -------------- - def _remove_repeats(self, bit_depth=max, sampling_rate=max): + def _remove_repeats(self, bit_depth=max, sampling_rate=max) -> Generator: """Remove the repeated albums from self. May remove different versions of the same album. @@ -1489,7 +1514,7 @@ class Artist(Tracklist): and TYPE_REGEXES["extra"].search(album.title) is None ) - def _features(self, album): + def _features(self, album: Album) -> bool: """Passed as a parameter by the user. This will download only albums where the requested @@ -1502,7 +1527,7 @@ class Artist(Tracklist): """ return self["name"] == album["albumartist"] - def _extras(self, album): + def _extras(self, album: Album) -> bool: """Passed as a parameter by the user. This will skip any extras. @@ -1514,7 +1539,7 @@ class Artist(Tracklist): """ return TYPE_REGEXES["extra"].search(album.title) is None - def _non_remasters(self, album): + def _non_remasters(self, album: Album) -> bool: """Passed as a parameter by the user. This will download only remasterd albums. @@ -1526,7 +1551,7 @@ class Artist(Tracklist): """ return TYPE_REGEXES["remaster"].search(album.title) is not None - def _non_albums(self, album): + def _non_albums(self, album: Album) -> bool: """This will ignore non-album releases. :param artist: usually self @@ -1541,7 +1566,6 @@ class Artist(Tracklist): def __repr__(self) -> str: """Return a string representation of this Artist object. - Useful for pprint and json.dumps. :rtype: str """ diff --git a/streamrip/metadata.py b/streamrip/metadata.py index 339fcfe..9590a82 100644 --- a/streamrip/metadata.py +++ b/streamrip/metadata.py @@ -361,6 +361,9 @@ class TrackMetadata: for k, v in FLAC_KEY.items(): tag = getattr(self, k) if tag: + if k in ('tracknumber', 'discnumber', 'tracktotal', 'disctotal'): + tag = f"{int(tag):02}" + logger.debug("Adding tag %s: %s", v, tag) yield (v, str(tag)) diff --git a/streamrip/utils.py b/streamrip/utils.py index 062cdad..1706990 100644 --- a/streamrip/utils.py +++ b/streamrip/utils.py @@ -8,11 +8,13 @@ import requests from Crypto.Cipher import AES from Crypto.Util import Counter from pathvalidate import sanitize_filename +from requests.packages import urllib3 from tqdm import tqdm from .constants import LOG_DIR, TIDAL_COVER_URL from .exceptions import InvalidSourceError, NonStreamable +urllib3.disable_warnings() logger = logging.getLogger(__name__) @@ -95,7 +97,7 @@ def get_quality_id(bit_depth: Optional[int], sampling_rate: Optional[int]): return 4 -def tqdm_download(url: str, filepath: str, params: dict = None): +def tqdm_download(url: str, filepath: str, params: dict = None, desc: str = None): """Downloads a file with a progress bar. :param url: url to direct download @@ -107,7 +109,8 @@ def tqdm_download(url: str, filepath: str, params: dict = None): if params is None: params = {} - r = requests.get(url, allow_redirects=True, stream=True, params=params) + session = gen_threadsafe_session() + r = session.get(url, allow_redirects=True, stream=True, params=params) total = int(r.headers.get("content-length", 0)) logger.debug(f"File size = {total}") if total < 1000 and not url.endswith("jpg") and not url.endswith("png"): @@ -115,7 +118,7 @@ def tqdm_download(url: str, filepath: str, params: dict = None): try: with open(filepath, "wb") as file, tqdm( - total=total, unit="iB", unit_scale=True, unit_divisor=1024 + total=total, unit="iB", unit_scale=True, unit_divisor=1024, desc=desc ) as bar: for data in r.iter_content(chunk_size=1024): size = file.write(data) @@ -141,8 +144,10 @@ def clean_format(formatter: str, format_info): clean_dict = dict() for key in fmt_keys: - if isinstance(format_info.get(key), (str, int, float)): # int for track numbers + if isinstance(format_info.get(key), (str, float)): clean_dict[key] = sanitize_filename(str(format_info[key])) + elif isinstance(format_info.get(key), int): # track/discnumber + clean_dict[key] = f"{format_info[key]:02}" else: clean_dict[key] = "Unknown" @@ -214,3 +219,16 @@ def ext(quality: int, source: str): return ".mp3" else: return ".flac" + + +def gen_threadsafe_session( + headers: dict = None, pool_connections: int = 100, pool_maxsize: int = 100 +) -> requests.Session: + if headers is None: + headers = {} + + session = requests.Session() + adapter = requests.adapters.HTTPAdapter(pool_connections=100, pool_maxsize=100) + session.mount("https://", adapter) + session.headers.update(headers) + return session