Merge branch 'async_conversion'

streamrip/clients.py

@@ -26,7 +26,7 @@ from .exceptions import (
     InvalidQuality,
 )
 from .spoofbuz import Spoofer
-from .utils import get_quality
+from .utils import gen_threadsafe_session, get_quality
 
 urllib3.disable_warnings()
 requests.adapters.DEFAULT_RETRIES = 5
@@ -149,12 +149,8 @@ class QobuzClient(ClientInterface):
         self.app_id = str(kwargs["app_id"])  # Ensure it is a string
         self.secrets = kwargs["secrets"]
 
-        self.session = requests.Session()
+        self.session = gen_threadsafe_session(
-        self.session.headers.update(
+            headers={"User-Agent": AGENT, "X-App-Id": self.app_id}
-            {
-                "User-Agent": AGENT,
-                "X-App-Id": self.app_id,
-            }
         )
 
         self._api_login(email, pwd)
@@ -373,7 +369,9 @@ class DeezerClient(ClientInterface):
     max_quality = 2
 
     def __init__(self):
-        self.session = requests.Session()
+        self.session = gen_threadsafe_session()
+
+        # no login required
         self.logged_in = True
 
     def search(self, query: str, media_type: str = "album", limit: int = 200) -> dict:
@@ -389,9 +387,6 @@ class DeezerClient(ClientInterface):
         # TODO: more robust url sanitize
         query = query.replace(" ", "+")
 
-        if media_type.endswith("s"):
-            media_type = media_type[:-1]
-
         # TODO: use limit parameter
         response = self.session.get(f"{DEEZER_BASE}/search/{media_type}?q={query}")
         response.raise_for_status()
@@ -447,6 +442,8 @@ class TidalClient(ClientInterface):
         self.refresh_token = None
         self.expiry = None
 
+        self.session = gen_threadsafe_session()
+
     def login(
         self,
         user_id=None,
@@ -492,7 +489,7 @@ class TidalClient(ClientInterface):
         try:
             manifest = json.loads(base64.b64decode(resp["manifest"]).decode("utf-8"))
         except KeyError:
-            raise Exception("You must have a TIDAL Hi-Fi account to download tracks.")
+            raise Exception(resp['userMessage'])
 
         logger.debug(manifest)
         return {
@@ -588,7 +585,9 @@ class TidalClient(ClientInterface):
         headers = {
             "authorization": f"Bearer {token}",
         }
-        r = requests.get("https://api.tidal.com/v1/sessions", headers=headers).json()
+        r = self.session.get(
+            "https://api.tidal.com/v1/sessions", headers=headers
+        ).json()
         if r.status != 200:
             raise Exception("Login failed")
 
@@ -614,10 +613,13 @@ class TidalClient(ClientInterface):
         self.country_code = resp["user"]["countryCode"]
         self.access_token = resp["access_token"]
         self.token_expiry = resp["expires_in"] + time.time()
+        self._update_authorization()
 
     def _login_by_access_token(self, token, user_id=None):
-        headers = {"authorization": f"Bearer {token}"}
+        headers = {"authorization": f"Bearer {token}"}  # temporary
-        resp = requests.get("https://api.tidal.com/v1/sessions", headers=headers).json()
+        resp = self.session.get(
+            "https://api.tidal.com/v1/sessions", headers=headers
+        ).json()
         if resp.get("status", 200) != 200:
             raise Exception(f"Login failed {resp}")
 
@@ -627,6 +629,7 @@ class TidalClient(ClientInterface):
         self.user_id = resp["userId"]
         self.country_code = resp["countryCode"]
         self.access_token = token
+        self._update_authorization()
 
     def _api_get(self, item_id: str, media_type: str) -> dict:
         url = f"{media_type}s/{item_id}"
@@ -654,22 +657,27 @@ class TidalClient(ClientInterface):
         if params is None:
             params = {}
 
-        headers = {"authorization": f"Bearer {self.access_token}"}
         params["countryCode"] = self.country_code
         params["limit"] = 100
-        r = requests.get(f"{TIDAL_BASE}/{path}", headers=headers, params=params).json()
+        r = self.session.get(f"{TIDAL_BASE}/{path}", params=params).json()
         return r
 
     def _api_post(self, url, data, auth=None):
-        r = requests.post(url, data=data, auth=auth, verify=False).json()
+        r = self.session.post(url, data=data, auth=auth, verify=False).json()
         return r
 
+    def _update_authorization(self):
+        self.session.headers.update({"authorization": f"Bearer {self.access_token}"})
+
 
 class SoundCloudClient(ClientInterface):
     source = "soundcloud"
     max_quality = 0
     logged_in = True
 
+    def __init__(self):
+        self.session = gen_threadsafe_session(headers={"User-Agent": AGENT})
+
     def login(self):
         raise NotImplementedError
 
@@ -721,7 +729,7 @@ class SoundCloudClient(ClientInterface):
             url = f"{SOUNDCLOUD_BASE}/{path}"
 
         logger.debug(f"Fetching url {url}")
-        r = requests.get(url, params=params)
+        r = self.session.get(url, params=params)
         if resp_obj:
             return r
 

streamrip/config.py

@@ -27,14 +27,12 @@ class Config:
 
     Usage:
     >>> config = Config('test_config.yaml')
+    >>> config.defaults['qobuz']['quality']
+    3
 
     If test_config was already initialized with values, this will load them
     into `config`. Otherwise, a new config file is created with the default
     values.
-
-    >>> config.update_from_cli(**args)
-
-    This will update the config values based on command line args.
     """
 
     defaults = {
@@ -42,7 +40,7 @@ class Config:
             "quality": 3,
             "email": None,
             "password": None,
-            "app_id": "",  # Avoid NoneType error
+            "app_id": "",
             "secrets": [],
         },
         "tidal": {
@@ -82,10 +80,12 @@ class Config:
         },
         "metadata": {
             "set_playlist_to_album": False,
+            "new_playlist_tracknumbers": True,
         },
         "path_format": {"folder": FOLDER_FORMAT, "track": TRACK_FORMAT},
         "check_for_updates": True,
         "lastfm": {"source": "qobuz"},
+        "concurrent_downloads": False,
     }
 
     def __init__(self, path: str = None):

streamrip/converter.py

@@ -5,12 +5,12 @@ import subprocess
 from tempfile import gettempdir
 from typing import Optional
 
-from mutagen.flac import FLAC as FLAC_META
-
 from .exceptions import ConversionError
 
 logger = logging.getLogger(__name__)
 
+SAMPLING_RATES = (44100, 48000, 88200, 96000, 176400, 192000)
+
 
 class Converter:
     """Base class for audio codecs."""
@@ -111,9 +111,11 @@ class Converter:
 
         if self.lossless:
             if isinstance(self.sampling_rate, int):
-                audio = FLAC_META(self.filename)
+                sampling_rates = "|".join(
-                old_sr = audio.info.sample_rate
+                    str(rate) for rate in SAMPLING_RATES if rate <= self.sampling_rate
-                command.extend(["-ar", str(min(old_sr, self.sampling_rate))])
+                )
+                command.extend(["-af", f"aformat=sample_rates={sampling_rates}"])
+
             elif self.sampling_rate is not None:
                 raise TypeError(
                     f"Sampling rate must be int, not {type(self.sampling_rate)}"
@@ -129,6 +131,7 @@ class Converter:
             elif self.bit_depth is not None:
                 raise TypeError(f"Bit depth must be int, not {type(self.bit_depth)}")
 
+        # automatically overwrite
         command.extend(["-y", self.tempfile])
 
         return command

streamrip/core.py

@@ -2,6 +2,7 @@ import logging
 import os
 import re
 import sys
+import threading
 from getpass import getpass
 from hashlib import md5
 from string import Formatter
@@ -23,7 +24,12 @@ from .constants import (
 )
 from .db import MusicDB
 from .downloader import Album, Artist, Label, Playlist, Track, Tracklist
-from .exceptions import AuthenticationError, NoResultsFound, ParsingError, NonStreamable
+from .exceptions import (
+    AuthenticationError,
+    NonStreamable,
+    NoResultsFound,
+    ParsingError,
+)
 from .utils import capitalize
 
 logger = logging.getLogger(__name__)
@@ -153,21 +159,25 @@ class MusicDL(list):
             "parent_folder": self.config.session["downloads"]["folder"],
             "folder_format": self.config.session["path_format"]["folder"],
             "track_format": self.config.session["path_format"]["track"],
-
             "embed_cover": self.config.session["artwork"]["embed"],
             "embed_cover_size": self.config.session["artwork"]["size"],
-            "keep_hires_cover": self.config.session['artwork']['keep_hires_cover'],
+            "keep_hires_cover": self.config.session["artwork"]["keep_hires_cover"],
-
             "set_playlist_to_album": self.config.session["metadata"][
                 "set_playlist_to_album"
             ],
             "stay_temp": self.config.session["conversion"]["enabled"],
+            "conversion": self.config.session["conversion"],
+            "concurrent_downloads": self.config.session["concurrent_downloads"],
+            "new_tracknumbers": self.config.session['metadata']['new_playlist_tracknumbers']
         }
         logger.debug("Arguments from config: %s", arguments)
+
+        source_subdirs = self.config.session["downloads"]["source_subdirectories"]
         for item in self:
-            if self.config.session["downloads"]["source_subdirectories"]:
+
-                arguments["parent_folder"] = os.path.join(
+            if source_subdirs:
-                    arguments["parent_folder"], capitalize(item.client.source)
+                arguments["parent_folder"] = self.__get_source_subdir(
+                    item.client.source
                 )
 
             arguments["quality"] = self.config.session[item.client.source]["quality"]
@@ -182,7 +192,7 @@ class MusicDL(list):
                 try:
                     item.load_meta()
                 except NonStreamable:
-                    click.secho(f"{item!s} is not available, skipping.", fg='red')
+                    click.secho(f"{item!s} is not available, skipping.", fg="red")
                     continue
 
             if isinstance(item, Track):
@@ -194,8 +204,8 @@ class MusicDL(list):
             if self.db != [] and hasattr(item, "id"):
                 self.db.add(item.id)
 
-            if self.config.session["conversion"]["enabled"]:
+            # if self.config.session["conversion"]["enabled"]:
-                item.convert(**self.config.session["conversion"])
+            # item.convert(**self.config.session["conversion"])
 
     def get_client(self, source: str):
         client = self.clients[source]
@@ -261,24 +271,36 @@ class MusicDL(list):
     def handle_lastfm_urls(self, urls):
         lastfm_urls = self.lastfm_url_parse.findall(urls)
         lastfm_source = self.config.session["lastfm"]["source"]
+        tracks_not_found = 0
+
+        def search_query(query: str, playlist: Playlist):
+            global tracks_not_found
+            try:
+                track = next(self.search(lastfm_source, query, media_type="track"))
+                playlist.append(track)
+            except NoResultsFound:
+                tracks_not_found += 1
+                return
+
         for purl in lastfm_urls:
             click.secho(f"Fetching playlist at {purl}", fg="blue")
             title, queries = self.get_lastfm_playlist(purl)
 
-            pl = Playlist(client=self.clients[lastfm_source], name=title)
+            pl = Playlist(client=self.get_client(lastfm_source), name=title)
-            tracks_not_found = 0
+            processes = []
-            for title, artist in tqdm(queries, unit="tracks", desc="Searching"):
+
+            for title, artist in queries:
                 query = f"{title} {artist}"
+                proc = threading.Thread(
+                    target=search_query, args=(query, pl), daemon=True
+                )
+                proc.start()
+                processes.append(proc)
 
-                try:
+            for proc in tqdm(processes, unit="tracks", desc="Searching"):
-                    track = next(self.search(lastfm_source, query, media_type="track"))
+                proc.join()
-                except NoResultsFound:
-                    tracks_not_found += 1
-                    continue
 
-                pl.append(track)
             pl.loaded = True
-
             click.secho(f"{tracks_not_found} tracks not found.", fg="yellow")
             self.append(pl)
 
@@ -463,3 +485,7 @@ class MusicDL(list):
             remaining_tracks -= 50
 
         return playlist_title, info
+
+    def __get_source_subdir(self, source: str) -> str:
+        path = self.config.session["downloads"]["folder"]
+        return os.path.join(path, capitalize(source))

streamrip/downloader.py

@@ -7,15 +7,17 @@ import os
 import re
 import shutil
 import subprocess
+import threading
 from pprint import pformat
 from tempfile import gettempdir
-from typing import Any, Callable, Optional, Tuple, Union
+from typing import Any, Generator, Iterable, Union
 
 import click
 from mutagen.flac import FLAC, Picture
 from mutagen.id3 import APIC, ID3, ID3NoHeaderError
 from mutagen.mp4 import MP4, MP4Cover
 from pathvalidate import sanitize_filename, sanitize_filepath
+from requests.packages import urllib3
 
 from . import converter
 from .clients import ClientInterface
@@ -44,6 +46,7 @@ from .utils import (
 )
 
 logger = logging.getLogger(__name__)
+urllib3.disable_warnings()
 
 TIDAL_Q_MAP = {
     "LOW": 0,
@@ -145,7 +148,7 @@ class Track:
             self.cover_url = None
 
     @staticmethod
-    def _get_tracklist(resp, source):
+    def _get_tracklist(resp, source) -> list:
         if source == "qobuz":
             return resp["tracks"]["items"]
         if source in ("tidal", "deezer"):
@@ -161,7 +164,7 @@ class Track:
         database: MusicDB = None,
         tag: bool = False,
         **kwargs,
-    ):
+    ) -> bool:
         """
         Download the track.
 
@@ -211,7 +214,11 @@ class Track:
         else:
             url_id = self.id
 
+        try:
             dl_info = self.client.get_file_url(url_id, self.quality)
+        except Exception as e:
+            click.secho(f"Unable to download track. {e}", fg='red')
+            return False
 
         self.path = os.path.join(gettempdir(), f"{hash(self.id)}_{self.quality}.tmp")
         logger.debug("Temporary file path: %s", self.path)
@@ -227,15 +234,13 @@ class Track:
             self.sampling_rate = dl_info.get("sampling_rate")
             self.bit_depth = dl_info.get("bit_depth")
 
-        click.secho(f"\nDownloading {self!s}", fg="blue")
-
         # --------- Download Track ----------
         if self.client.source in ("qobuz", "tidal"):
             logger.debug("Downloadable URL found: %s", dl_info.get("url"))
-            tqdm_download(dl_info["url"], self.path)  # downloads file
+            tqdm_download(dl_info["url"], self.path, desc=self._progress_desc)  # downloads file
 
         elif self.client.source == "deezer":  # Deezer
-            logger.debug("Downloadable URL found: %s", dl_info)
+            logger.debug("Downloadable URL found: %s", dl_info, desc=self._progress_desc)
             try:
                 tqdm_download(dl_info, self.path)  # downloads file
             except NonStreamable:
@@ -300,7 +305,7 @@ class Track:
                 ]
             )
         elif dl_info["type"] == "original":
-            tqdm_download(dl_info["url"], self.path)
+            tqdm_download(dl_info["url"], self.path, desc=self._progress_desc)
 
             # if a wav is returned, convert to flac
             engine = converter.FLAC(self.path)
@@ -310,6 +315,10 @@ class Track:
             self.final_path = self.final_path.replace(".mp3", ".flac")
             self.quality = 2
 
+    @property
+    def _progress_desc(self):
+        return click.style(f"Track {int(self.meta.tracknumber):02}", fg='blue')
+
     def download_cover(self):
         """Downloads the cover art, if cover_url is given."""
 
@@ -317,10 +326,10 @@ class Track:
 
         self.cover_path = os.path.join(self.folder, f"cover{hash(self.cover_url)}.jpg")
         logger.debug(f"Downloading cover from {self.cover_url}")
-        click.secho(f"\nDownloading cover art for {self!s}", fg="blue")
+        # click.secho(f"\nDownloading cover art for {self!s}", fg="blue")
 
         if not os.path.exists(self.cover_path):
-            tqdm_download(self.cover_url, self.cover_path)
+            tqdm_download(self.cover_url, self.cover_path, desc=click.style('Cover', fg='cyan'))
         else:
             logger.debug("Cover already exists, skipping download")
 
@@ -515,16 +524,18 @@ class Track:
             sampling_rate=kwargs.get("sampling_rate"),
             remove_source=kwargs.get("remove_source", True),
         )
-        click.secho(f"Converting {self!s}", fg="blue")
+        # click.secho(f"Converting {self!s}", fg="blue")
         engine.convert()
         self.path = engine.final_fn
-        self.final_path = self.final_path.replace(ext(self.quality, self.client.source), f".{engine.container}")
+        self.final_path = self.final_path.replace(
+            ext(self.quality, self.client.source), f".{engine.container}"
+        )
 
         if not kwargs.get("stay_temp", False):
             self.move(self.final_path)
 
     @property
-    def title(self):
+    def title(self) -> str:
         if hasattr(self, "meta"):
             _title = self.meta.title
             if self.meta.explicit:
@@ -533,7 +544,7 @@ class Track:
         else:
             raise Exception("Track must be loaded before accessing title")
 
-    def get(self, *keys, default=None):
+    def get(self, *keys, default=None) -> Any:
         """Safe get method that allows for layered access.
 
         :param keys:
@@ -550,14 +561,14 @@ class Track:
         """
         self.__setitem__(key, val)
 
-    def __getitem__(self, key):
+    def __getitem__(self, key: str) -> Any:
         """Dict-like interface for Track metadata.
 
         :param key:
         """
         return getattr(self.meta, key)
 
-    def __setitem__(self, key, val):
+    def __setitem__(self, key: str, val: Any):
         """Dict-like interface for Track metadata.
 
         :param key:
@@ -588,20 +599,56 @@ class Tracklist(list):
     subclass is subscripted with [s: str], it will return an attribute s.
     If it is subscripted with [i: int] it will return the i'th track in
     the tracklist.
-
-    >>> tlist = Tracklist()
-    >>> tlist.tracklistname = 'my tracklist'
-    >>> tlist.append('first track')
-    >>> tlist[0]
-    'first track'
-    >>> tlist['tracklistname']
-    'my tracklist'
-    >>> tlist[2]
-    IndexError
     """
 
     essence_regex = re.compile(r"([^\(]+)(?:\s*[\(\[][^\)][\)\]])*")
 
+    def download(self, **kwargs):
+        self._prepare_download(**kwargs)
+        if kwargs.get("conversion", False):
+            has_conversion = kwargs["conversion"]["enabled"]
+        else:
+            has_conversion = False
+            kwargs["stay_temp"] = False
+
+        if has_conversion:
+            target = self._download_and_convert_item
+        else:
+            target = self._download_item
+
+        if kwargs.get("concurrent_downloads", True):
+            processes = []
+            for item in self:
+                proc = threading.Thread(
+                    target=target, args=(item,), kwargs=kwargs, daemon=True
+                )
+                proc.start()
+                processes.append(proc)
+
+            try:
+                for proc in processes:
+                    proc.join()
+            except (KeyboardInterrupt, SystemExit):
+                click.echo("Aborted!")
+                exit()
+
+        else:
+            for item in self:
+                click.secho(f'\nDownloading "{item!s}"', fg="blue")
+                target(item, **kwargs)
+
+        self.downloaded = True
+
+    def _download_and_convert_item(self, item, **kwargs):
+        if self._download_item(item, **kwargs):
+            item.convert(**kwargs["conversion"])
+
+    def _download_item(item, **kwargs):
+        raise NotImplementedError
+
+    def _prepare_download(**kwargs):
+        raise NotImplementedError
+
     def get(self, key: Union[str, int], default=None):
         if isinstance(key, str):
             if hasattr(self, key):
@@ -696,16 +743,13 @@ class Tracklist(list):
 
     def download_message(self):
         click.secho(
-            f"\nDownloading {self.title} ({self.__class__.__name__})\n",
+            f"\n\nDownloading {self.title} ({self.__class__.__name__})\n",
             fg="blue",
         )
 
     @staticmethod
     def _parse_get_resp(item, client):
-        pass
+        raise NotImplementedError
-
-    def download(self, **kwargs):
-        pass
 
     @staticmethod
     def essence(album: str) -> str:
@@ -763,13 +807,15 @@ class Album(Tracklist):
             setattr(self, k, v)
 
         # to improve from_api method speed
-        if kwargs.get("load_on_init"):
+        if kwargs.get("load_on_init", False):
             self.load_meta()
 
         self.loaded = False
         self.downloaded = False
 
     def load_meta(self):
+        """Load detailed metadata from API using the id."""
+
         assert hasattr(self, "id"), "id must be set to load metadata"
         self.meta = self.client.get(self.id, media_type="album")
 
@@ -784,13 +830,83 @@ class Album(Tracklist):
         self.loaded = True
 
     @classmethod
-    def from_api(cls, resp, client):
+    def from_api(cls, resp: dict, client: ClientInterface):
         if client.source == "soundcloud":
             return Playlist.from_api(resp, client)
 
         info = cls._parse_get_resp(resp, client)
         return cls(client, **info)
 
+    def _prepare_download(self, **kwargs):
+        self.folder_format = kwargs.get("folder_format", FOLDER_FORMAT)
+        self.quality = min(kwargs.get("quality", 3), self.client.max_quality)
+        self.folder = self._get_formatted_folder(
+            kwargs.get("parent_folder", "StreamripDownloads"), self.quality
+        )
+        os.makedirs(self.folder, exist_ok=True)
+
+        self.download_message()
+
+        # choose optimal cover size and download it
+        click.secho("Downloading cover art", fg="magenta")
+        cover_path = os.path.join(gettempdir(), f"cover_{hash(self)}.jpg")
+        embed_cover_size = kwargs.get("embed_cover_size", "large")
+
+        assert (
+            embed_cover_size in self.cover_urls
+        ), f"Invalid cover size. Must be in {self.cover_urls.keys()}"
+
+        embed_cover_url = self.cover_urls[embed_cover_size]
+        if embed_cover_url is not None:
+            tqdm_download(embed_cover_url, cover_path)
+        else:  # sometimes happens with Deezer
+            tqdm_download(self.cover_urls["small"], cover_path)
+
+        if kwargs.get("keep_hires_cover", True):
+            tqdm_download(
+                self.cover_urls["original"], os.path.join(self.folder, "cover.jpg")
+            )
+
+        cover_size = os.path.getsize(cover_path)
+        if cover_size > FLAC_MAX_BLOCKSIZE:  # 16.77 MB
+            click.secho(
+                "Downgrading embedded cover size, too large ({cover_size}).",
+                fg="bright_yellow",
+            )
+            # large is about 600x600px which is guaranteed < 16.7 MB
+            tqdm_download(self.cover_urls["large"], cover_path)
+
+        embed_cover = kwargs.get("embed_cover", True)  # embed by default
+        if self.client.source != "deezer" and embed_cover:
+            self.cover_obj = self.get_cover_obj(
+                cover_path, self.quality, self.client.source
+            )
+        else:
+            self.cover_obj = None
+
+    def _download_item(
+        self,
+        track: Track,
+        quality: int = 3,
+        database: MusicDB = None,
+        **kwargs,
+    ) -> bool:
+        logger.debug("Downloading track to %s", self.folder)
+        if self.disctotal > 1:
+            disc_folder = os.path.join(self.folder, f"Disc {track.meta.discnumber}")
+            kwargs["parent_folder"] = disc_folder
+        else:
+            kwargs["parent_folder"] = self.folder
+
+        if not track.download(quality=quality, database=database, **kwargs):
+            return False
+
+        # deezer tracks come tagged
+        if kwargs.get("tag_tracks", True) and self.client.source != "deezer":
+            track.tag(cover=self.cover_obj, embed_cover=kwargs.get("embed_cover", True))
+
+        return True
+
     @staticmethod
     def _parse_get_resp(resp: dict, client: ClientInterface) -> dict:
         """Parse information from a client.get(query, 'album') call.
@@ -903,110 +1019,6 @@ class Album(Tracklist):
                 Track.from_album_meta(album=self.meta, pos=i, client=self.client)
             )
 
-    @property
-    def title(self) -> str:
-        """Return the title of the album.
-
-        It is formatted so that "version" keys are included.
-
-        :rtype: str
-        """
-        album_title = self._title
-        if hasattr(self, "version") and isinstance(self.version, str):
-            if self.version.lower() not in album_title.lower():
-                album_title = f"{album_title} ({self.version})"
-
-        if self.get("explicit", False):
-            album_title = f"{album_title} (Explicit)"
-
-        return album_title
-
-    @title.setter
-    def title(self, val):
-        """Sets the internal _title attribute to the given value.
-
-        :param val: title to set
-        """
-        self._title = val
-
-    def download(
-        self,
-        quality: int = 3,
-        parent_folder: Union[str, os.PathLike] = "StreamripDownloads",
-        database: MusicDB = None,
-        **kwargs,
-    ):
-        """Download all of the tracks in the album.
-
-        :param quality: (0, 1, 2, 3, 4)
-        :type quality: int
-        :param parent_folder: the folder to download the album to
-        :type parent_folder: Union[str, os.PathLike]
-        :param progress_bar: turn on/off a tqdm progress bar
-        :type progress_bar: bool
-        :param large_cover: Download the large cover. This may fail when
-        embedding covers.
-        :param tag_tracks: Tag the tracks after downloading, True by default
-        :param keep_cover: Keep the cover art image after downloading.
-        True by default.
-        """
-        self.folder_format = kwargs.get("folder_format", FOLDER_FORMAT)
-        quality = min(quality, self.client.max_quality)
-        folder = self._get_formatted_folder(parent_folder, quality)
-
-        # choose optimal cover size and download it
-        self.download_message()
-
-        click.secho("Downloading cover art", fg="magenta")
-        cover_path = os.path.join(gettempdir(), f"cover_{hash(self)}.jpg")
-        embed_cover_size = kwargs.get("embed_cover_size", "large")
-
-        assert (
-            embed_cover_size in self.cover_urls
-        ), f"Invalid cover size. Must be in {self.cover_urls.keys()}"
-
-        tqdm_download(self.cover_urls[embed_cover_size], cover_path)
-
-        if kwargs.get("keep_hires_cover", True):
-            tqdm_download(self.cover_urls['original'], os.path.join(folder, 'cover.jpg'))
-
-        cover_size = os.path.getsize(cover_path)
-        if cover_size > FLAC_MAX_BLOCKSIZE:  # 16.77 MB
-            click.secho(
-                "Downgrading embedded cover size, too large ({cover_size}).",
-                fg="bright_yellow",
-            )
-            # large is about 600x600px which is guaranteed < 16.7 MB
-            tqdm_download(self.cover_urls["large"], cover_path)
-
-        embed_cover = kwargs.get("embed_cover", True)  # embed by default
-        if self.client.source != "deezer" and embed_cover:
-            cover = self.get_cover_obj(cover_path, quality, self.client.source)
-
-        download_args = {
-            "quality": quality,
-            "parent_folder": folder,
-            "progress_bar": kwargs.get("progress_bar", True),
-            "database": database,
-            "track_format": kwargs.get("track_format", TRACK_FORMAT),
-            "stay_temp": kwargs.get("stay_temp")
-        }
-        for track in self:
-            logger.debug("Downloading track to %s", folder)
-            if self.disctotal > 1:
-                disc_folder = os.path.join(folder, f"Disc {track.meta.discnumber}")
-                download_args["parent_folder"] = disc_folder
-
-            track.download(quality=quality, parent_folder=folder, database=database, **kwargs)
-
-            # deezer tracks come tagged
-            if kwargs.get("tag_tracks", True) and self.client.source != "deezer":
-                track.tag(cover=cover, embed_cover=embed_cover)
-
-        os.remove(cover_path)
-
-        self.downloaded = True
-
     def _get_formatter(self) -> dict:
         fmt = dict()
         for key in ALBUM_KEYS:
@@ -1035,9 +1047,34 @@ class Album(Tracklist):
 
         return os.path.join(parent_folder, formatted_folder)
 
+    @property
+    def title(self) -> str:
+        """Return the title of the album.
+
+        It is formatted so that "version" keys are included.
+
+        :rtype: str
+        """
+        album_title = self._title
+        if hasattr(self, "version") and isinstance(self.version, str):
+            if self.version.lower() not in album_title.lower():
+                album_title = f"{album_title} ({self.version})"
+
+        if self.get("explicit", False):
+            album_title = f"{album_title} (Explicit)"
+
+        return album_title
+
+    @title.setter
+    def title(self, val):
+        """Sets the internal _title attribute to the given value.
+
+        :param val: title to set
+        """
+        self._title = val
+
     def __repr__(self) -> str:
         """Return a string representation of this Album object.
-        Useful for pprint and json.dumps.
 
         :rtype: str
         """
@@ -1122,6 +1159,7 @@ class Playlist(Tracklist):
         :param new_tracknumbers: replace tracknumber tag with playlist position
         :type new_tracknumbers: bool
         """
+        # TODO: redundant parsing with _parse_get_pres
         if self.client.source == "qobuz":
             self.name = self.meta["name"]
             self.image = self.meta["images"]
@@ -1198,50 +1236,36 @@ class Playlist(Tracklist):
 
         logger.debug(f"Loaded {len(self)} tracks from playlist {self.name}")
 
-    def download(
+    def _prepare_download(self, parent_folder: str = "StreamripDownloads", **kwargs):
-        self,
+        fname = sanitize_filename(self.name)
-        parent_folder: str = "StreamripDownloads",
+        self.folder = os.path.join(parent_folder, fname)
-        quality: int = 3,
-        filters: Callable = None,
-        database: MusicDB = None,
-        **kwargs,
-    ):
-        """Download and tag all of the tracks.
-
-        :param parent_folder:
-        :type parent_folder: str
-        :param quality:
-        :type quality: int
-        :param filters:
-        :type filters: Callable
-        """
-        folder = sanitize_filename(self.name)
-        folder = os.path.join(parent_folder, folder)
-        logger.debug(f"Parent folder {folder}")
 
+        self.__download_index = 1
         self.download_message()
-        set_playlist_to_album = kwargs.get("set_playlist_to_album", False)
-        for i, track in enumerate(self):
 
+    def _download_item(self, item: Track, **kwargs):
         if self.client.source == "soundcloud":
-                track.load_meta()
+            item.load_meta()
 
-            if set_playlist_to_album and hasattr(self, "image"):
+        if kwargs.get("set_playlist_to_album", False):
-                track["album"] = self.name
+            item["album"] = self.name
-                track["albumartist"] = self.creator
+            item["albumartist"] = self.creator
 
         if kwargs.get("new_tracknumbers", True):
-                track.meta["tracknumber"] = str(i + 1)
+            item["tracknumber"] = self.__download_index
+            item['discnumber'] = 1
 
-            if (
+            self.__download_index += 1
-                track.download(parent_folder=folder, quality=quality, database=database, **kwargs)
-                and self.client.source != "deezer"
-            ):
 
-                track.tag(embed_cover=kwargs.get("embed_cover", True))
+        self.downloaded = item.download(**kwargs)
+
+        if self.downloaded and self.client.source != "deezer":
+            item.tag(embed_cover=kwargs.get("embed_cover", True))
+
+        return self.downloaded
 
     @staticmethod
-    def _parse_get_resp(item: dict, client: ClientInterface):
+    def _parse_get_resp(item: dict, client: ClientInterface) -> dict:
         """Parses information from a search result returned
         by a client.search call.
 
@@ -1277,12 +1301,11 @@ class Playlist(Tracklist):
         raise InvalidSourceError(client.source)
 
     @property
-    def title(self):
+    def title(self) -> str:
         return self.name
 
     def __repr__(self) -> str:
         """Return a string representation of this Playlist object.
-        Useful for pprint and json.dumps.
 
         :rtype: str
         """
@@ -1332,6 +1355,12 @@ class Artist(Tracklist):
         self._load_albums()
         self.loaded = True
 
+    # override
+    def download(self, **kwargs):
+        iterator = self._prepare_download(**kwargs)
+        for item in iterator:
+            self._download_item(item, **kwargs)
+
     def _load_albums(self):
         """From the discography returned by client.get(query, 'artist'),
         generate album objects and append them to self.
@@ -1355,25 +1384,9 @@ class Artist(Tracklist):
             logger.debug("Appending album: %s", album.get("title"))
             self.append(Album.from_api(album, self.client))
 
-    def download(
+    def _prepare_download(
-        self,
+        self, parent_folder: str = "StreamripDownloads", filters: tuple = (), **kwargs
-        parent_folder: str = "StreamripDownloads",
+    ) -> Iterable:
-        filters: Optional[Tuple] = None,
-        no_repeats: bool = False,
-        quality: int = 6,
-        database: MusicDB = None,
-        **kwargs,
-    ):
-        """Download all albums in the discography.
-
-        :param filters: Filters to apply to discography, see options below.
-        These only work for Qobuz.
-        :type filters: Optional[Tuple]
-        :param no_repeats: Remove repeats
-        :type no_repeats: bool
-        :param quality: in (0, 1, 2, 3, 4)
-        :type quality: int
-        """
         folder = sanitize_filename(self.name)
         folder = os.path.join(parent_folder, folder)
 
@@ -1393,21 +1406,33 @@ class Artist(Tracklist):
                 final = filter(func, final)
 
         self.download_message()
-        for album in final:
+        return final
+
+    def _download_item(
+        self,
+        item,
+        parent_folder: str = "StreamripDownloads",
+        quality: int = 3,
+        database: MusicDB = None,
+        **kwargs,
+    ) -> bool:
         try:
-                album.load_meta()
+            item.load_meta()
         except NonStreamable:
             logger.info("Skipping album, not available to stream.")
-                continue
+            return
-            album.download(
+
-                parent_folder=folder,
+        # always an Album
+        status = item.download(
+            parent_folder=parent_folder,
             quality=quality,
             database=database,
             **kwargs,
         )
+        return status
 
     @property
-    def title(self):
+    def title(self) -> str:
         return self.name
 
     @classmethod
@@ -1427,7 +1452,7 @@ class Artist(Tracklist):
         return cls(client=client, **info)
 
     @staticmethod
-    def _parse_get_resp(item: dict, client: ClientInterface):
+    def _parse_get_resp(item: dict, client: ClientInterface) -> dict:
         """Parse a result from a client.search call.
 
         :param item: the item to parse
@@ -1452,7 +1477,7 @@ class Artist(Tracklist):
 
     # ----------- Filters --------------
 
-    def _remove_repeats(self, bit_depth=max, sampling_rate=max):
+    def _remove_repeats(self, bit_depth=max, sampling_rate=max) -> Generator:
         """Remove the repeated albums from self. May remove different
         versions of the same album.
 
@@ -1489,7 +1514,7 @@ class Artist(Tracklist):
             and TYPE_REGEXES["extra"].search(album.title) is None
         )
 
-    def _features(self, album):
+    def _features(self, album: Album) -> bool:
         """Passed as a parameter by the user.
 
         This will download only albums where the requested
@@ -1502,7 +1527,7 @@ class Artist(Tracklist):
         """
         return self["name"] == album["albumartist"]
 
-    def _extras(self, album):
+    def _extras(self, album: Album) -> bool:
         """Passed as a parameter by the user.
 
         This will skip any extras.
@@ -1514,7 +1539,7 @@ class Artist(Tracklist):
         """
         return TYPE_REGEXES["extra"].search(album.title) is None
 
-    def _non_remasters(self, album):
+    def _non_remasters(self, album: Album) -> bool:
         """Passed as a parameter by the user.
 
         This will download only remasterd albums.
@@ -1526,7 +1551,7 @@ class Artist(Tracklist):
         """
         return TYPE_REGEXES["remaster"].search(album.title) is not None
 
-    def _non_albums(self, album):
+    def _non_albums(self, album: Album) -> bool:
         """This will ignore non-album releases.
 
         :param artist: usually self
@@ -1541,7 +1566,6 @@ class Artist(Tracklist):
 
     def __repr__(self) -> str:
         """Return a string representation of this Artist object.
-        Useful for pprint and json.dumps.
 
         :rtype: str
         """

streamrip/metadata.py

@@ -361,6 +361,9 @@ class TrackMetadata:
         for k, v in FLAC_KEY.items():
             tag = getattr(self, k)
             if tag:
+                if k in ('tracknumber', 'discnumber', 'tracktotal', 'disctotal'):
+                    tag = f"{int(tag):02}"
+
                 logger.debug("Adding tag %s: %s", v, tag)
                 yield (v, str(tag))
 

streamrip/utils.py

@@ -8,11 +8,13 @@ import requests
 from Crypto.Cipher import AES
 from Crypto.Util import Counter
 from pathvalidate import sanitize_filename
+from requests.packages import urllib3
 from tqdm import tqdm
 
 from .constants import LOG_DIR, TIDAL_COVER_URL
 from .exceptions import InvalidSourceError, NonStreamable
 
+urllib3.disable_warnings()
 logger = logging.getLogger(__name__)
 
 
@@ -95,7 +97,7 @@ def get_quality_id(bit_depth: Optional[int], sampling_rate: Optional[int]):
         return 4
 
 
-def tqdm_download(url: str, filepath: str, params: dict = None):
+def tqdm_download(url: str, filepath: str, params: dict = None, desc: str = None):
     """Downloads a file with a progress bar.
 
     :param url: url to direct download
@@ -107,7 +109,8 @@ def tqdm_download(url: str, filepath: str, params: dict = None):
     if params is None:
         params = {}
 
-    r = requests.get(url, allow_redirects=True, stream=True, params=params)
+    session = gen_threadsafe_session()
+    r = session.get(url, allow_redirects=True, stream=True, params=params)
     total = int(r.headers.get("content-length", 0))
     logger.debug(f"File size = {total}")
     if total < 1000 and not url.endswith("jpg") and not url.endswith("png"):
@@ -115,7 +118,7 @@ def tqdm_download(url: str, filepath: str, params: dict = None):
 
     try:
         with open(filepath, "wb") as file, tqdm(
-            total=total, unit="iB", unit_scale=True, unit_divisor=1024
+            total=total, unit="iB", unit_scale=True, unit_divisor=1024, desc=desc
         ) as bar:
             for data in r.iter_content(chunk_size=1024):
                 size = file.write(data)
@@ -141,8 +144,10 @@ def clean_format(formatter: str, format_info):
 
     clean_dict = dict()
     for key in fmt_keys:
-        if isinstance(format_info.get(key), (str, int, float)):  # int for track numbers
+        if isinstance(format_info.get(key), (str, float)):
             clean_dict[key] = sanitize_filename(str(format_info[key]))
+        elif isinstance(format_info.get(key), int):  # track/discnumber
+            clean_dict[key] = f"{format_info[key]:02}"
         else:
             clean_dict[key] = "Unknown"
 
@@ -214,3 +219,16 @@ def ext(quality: int, source: str):
             return ".mp3"
     else:
         return ".flac"
+
+
+def gen_threadsafe_session(
+    headers: dict = None, pool_connections: int = 100, pool_maxsize: int = 100
+) -> requests.Session:
+    if headers is None:
+        headers = {}
+
+    session = requests.Session()
+    adapter = requests.adapters.HTTPAdapter(pool_connections=100, pool_maxsize=100)
+    session.mount("https://", adapter)
+    session.headers.update(headers)
+    return session