mirror of
https://github.com/nathom/streamrip.git
synced 2024-09-19 19:28:46 -04:00
Merge branch 'async_conversion'
This commit is contained in:
nathom
commit
bee2adbcaa
7 changed files with 351 additions and 269 deletions
|
|
@ -26,7 +26,7 @@ from .exceptions import (
|
|||
InvalidQuality,
|
||||
)
|
||||
from .spoofbuz import Spoofer
|
||||
from .utils import get_quality
|
||||
from .utils import gen_threadsafe_session, get_quality
|
||||
|
||||
urllib3.disable_warnings()
|
||||
requests.adapters.DEFAULT_RETRIES = 5
|
||||
|
|
@ -149,12 +149,8 @@ class QobuzClient(ClientInterface):
|
|||
self.app_id = str(kwargs["app_id"]) # Ensure it is a string
|
||||
self.secrets = kwargs["secrets"]
|
||||
|
||||
self.session = requests.Session()
|
||||
self.session.headers.update(
|
||||
{
|
||||
"User-Agent": AGENT,
|
||||
"X-App-Id": self.app_id,
|
||||
}
|
||||
self.session = gen_threadsafe_session(
|
||||
headers={"User-Agent": AGENT, "X-App-Id": self.app_id}
|
||||
)
|
||||
|
||||
self._api_login(email, pwd)
|
||||
|
|
@ -373,7 +369,9 @@ class DeezerClient(ClientInterface):
|
|||
max_quality = 2
|
||||
|
||||
def __init__(self):
|
||||
self.session = requests.Session()
|
||||
self.session = gen_threadsafe_session()
|
||||
|
||||
# no login required
|
||||
self.logged_in = True
|
||||
|
||||
def search(self, query: str, media_type: str = "album", limit: int = 200) -> dict:
|
||||
|
|
@ -389,9 +387,6 @@ class DeezerClient(ClientInterface):
|
|||
# TODO: more robust url sanitize
|
||||
query = query.replace(" ", "+")
|
||||
|
||||
if media_type.endswith("s"):
|
||||
media_type = media_type[:-1]
|
||||
|
||||
# TODO: use limit parameter
|
||||
response = self.session.get(f"{DEEZER_BASE}/search/{media_type}?q={query}")
|
||||
response.raise_for_status()
|
||||
|
|
@ -447,6 +442,8 @@ class TidalClient(ClientInterface):
|
|||
self.refresh_token = None
|
||||
self.expiry = None
|
||||
|
||||
self.session = gen_threadsafe_session()
|
||||
|
||||
def login(
|
||||
self,
|
||||
user_id=None,
|
||||
|
|
@ -492,7 +489,7 @@ class TidalClient(ClientInterface):
|
|||
try:
|
||||
manifest = json.loads(base64.b64decode(resp["manifest"]).decode("utf-8"))
|
||||
except KeyError:
|
||||
raise Exception("You must have a TIDAL Hi-Fi account to download tracks.")
|
||||
raise Exception(resp['userMessage'])
|
||||
|
||||
logger.debug(manifest)
|
||||
return {
|
||||
|
|
@ -588,7 +585,9 @@ class TidalClient(ClientInterface):
|
|||
headers = {
|
||||
"authorization": f"Bearer {token}",
|
||||
}
|
||||
r = requests.get("https://api.tidal.com/v1/sessions", headers=headers).json()
|
||||
r = self.session.get(
|
||||
"https://api.tidal.com/v1/sessions", headers=headers
|
||||
).json()
|
||||
if r.status != 200:
|
||||
raise Exception("Login failed")
|
||||
|
||||
|
|
@ -614,10 +613,13 @@ class TidalClient(ClientInterface):
|
|||
self.country_code = resp["user"]["countryCode"]
|
||||
self.access_token = resp["access_token"]
|
||||
self.token_expiry = resp["expires_in"] + time.time()
|
||||
self._update_authorization()
|
||||
|
||||
def _login_by_access_token(self, token, user_id=None):
|
||||
headers = {"authorization": f"Bearer {token}"}
|
||||
resp = requests.get("https://api.tidal.com/v1/sessions", headers=headers).json()
|
||||
headers = {"authorization": f"Bearer {token}"} # temporary
|
||||
resp = self.session.get(
|
||||
"https://api.tidal.com/v1/sessions", headers=headers
|
||||
).json()
|
||||
if resp.get("status", 200) != 200:
|
||||
raise Exception(f"Login failed {resp}")
|
||||
|
||||
|
|
@ -627,6 +629,7 @@ class TidalClient(ClientInterface):
|
|||
self.user_id = resp["userId"]
|
||||
self.country_code = resp["countryCode"]
|
||||
self.access_token = token
|
||||
self._update_authorization()
|
||||
|
||||
def _api_get(self, item_id: str, media_type: str) -> dict:
|
||||
url = f"{media_type}s/{item_id}"
|
||||
|
|
@ -654,22 +657,27 @@ class TidalClient(ClientInterface):
|
|||
if params is None:
|
||||
params = {}
|
||||
|
||||
headers = {"authorization": f"Bearer {self.access_token}"}
|
||||
params["countryCode"] = self.country_code
|
||||
params["limit"] = 100
|
||||
r = requests.get(f"{TIDAL_BASE}/{path}", headers=headers, params=params).json()
|
||||
r = self.session.get(f"{TIDAL_BASE}/{path}", params=params).json()
|
||||
return r
|
||||
|
||||
def _api_post(self, url, data, auth=None):
|
||||
r = requests.post(url, data=data, auth=auth, verify=False).json()
|
||||
r = self.session.post(url, data=data, auth=auth, verify=False).json()
|
||||
return r
|
||||
|
||||
def _update_authorization(self):
|
||||
self.session.headers.update({"authorization": f"Bearer {self.access_token}"})
|
||||
|
||||
|
||||
class SoundCloudClient(ClientInterface):
|
||||
source = "soundcloud"
|
||||
max_quality = 0
|
||||
logged_in = True
|
||||
|
||||
def __init__(self):
|
||||
self.session = gen_threadsafe_session(headers={"User-Agent": AGENT})
|
||||
|
||||
def login(self):
|
||||
raise NotImplementedError
|
||||
|
||||
|
|
@ -721,7 +729,7 @@ class SoundCloudClient(ClientInterface):
|
|||
url = f"{SOUNDCLOUD_BASE}/{path}"
|
||||
|
||||
logger.debug(f"Fetching url {url}")
|
||||
r = requests.get(url, params=params)
|
||||
r = self.session.get(url, params=params)
|
||||
if resp_obj:
|
||||
return r
|
||||
|
||||
|
|
|
|||
|
|
@ -27,14 +27,12 @@ class Config:
|
|||
|
||||
Usage:
|
||||
>>> config = Config('test_config.yaml')
|
||||
>>> config.defaults['qobuz']['quality']
|
||||
3
|
||||
|
||||
If test_config was already initialized with values, this will load them
|
||||
into `config`. Otherwise, a new config file is created with the default
|
||||
values.
|
||||
|
||||
>>> config.update_from_cli(**args)
|
||||
|
||||
This will update the config values based on command line args.
|
||||
"""
|
||||
|
||||
defaults = {
|
||||
|
|
@ -42,7 +40,7 @@ class Config:
|
|||
"quality": 3,
|
||||
"email": None,
|
||||
"password": None,
|
||||
"app_id": "", # Avoid NoneType error
|
||||
"app_id": "",
|
||||
"secrets": [],
|
||||
},
|
||||
"tidal": {
|
||||
|
|
@ -82,10 +80,12 @@ class Config:
|
|||
},
|
||||
"metadata": {
|
||||
"set_playlist_to_album": False,
|
||||
"new_playlist_tracknumbers": True,
|
||||
},
|
||||
"path_format": {"folder": FOLDER_FORMAT, "track": TRACK_FORMAT},
|
||||
"check_for_updates": True,
|
||||
"lastfm": {"source": "qobuz"},
|
||||
"concurrent_downloads": False,
|
||||
}
|
||||
|
||||
def __init__(self, path: str = None):
|
||||
|
|
|
|||
|
|
@ -5,12 +5,12 @@ import subprocess
|
|||
from tempfile import gettempdir
|
||||
from typing import Optional
|
||||
|
||||
from mutagen.flac import FLAC as FLAC_META
|
||||
|
||||
from .exceptions import ConversionError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SAMPLING_RATES = (44100, 48000, 88200, 96000, 176400, 192000)
|
||||
|
||||
|
||||
class Converter:
|
||||
"""Base class for audio codecs."""
|
||||
|
|
@ -111,9 +111,11 @@ class Converter:
|
|||
|
||||
if self.lossless:
|
||||
if isinstance(self.sampling_rate, int):
|
||||
audio = FLAC_META(self.filename)
|
||||
old_sr = audio.info.sample_rate
|
||||
command.extend(["-ar", str(min(old_sr, self.sampling_rate))])
|
||||
sampling_rates = "|".join(
|
||||
str(rate) for rate in SAMPLING_RATES if rate <= self.sampling_rate
|
||||
)
|
||||
command.extend(["-af", f"aformat=sample_rates={sampling_rates}"])
|
||||
|
||||
elif self.sampling_rate is not None:
|
||||
raise TypeError(
|
||||
f"Sampling rate must be int, not {type(self.sampling_rate)}"
|
||||
|
|
@ -129,6 +131,7 @@ class Converter:
|
|||
elif self.bit_depth is not None:
|
||||
raise TypeError(f"Bit depth must be int, not {type(self.bit_depth)}")
|
||||
|
||||
# automatically overwrite
|
||||
command.extend(["-y", self.tempfile])
|
||||
|
||||
return command
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import logging
|
|||
import os
|
||||
import re
|
||||
import sys
|
||||
import threading
|
||||
from getpass import getpass
|
||||
from hashlib import md5
|
||||
from string import Formatter
|
||||
|
|
@ -23,7 +24,12 @@ from .constants import (
|
|||
)
|
||||
from .db import MusicDB
|
||||
from .downloader import Album, Artist, Label, Playlist, Track, Tracklist
|
||||
from .exceptions import AuthenticationError, NoResultsFound, ParsingError, NonStreamable
|
||||
from .exceptions import (
|
||||
AuthenticationError,
|
||||
NonStreamable,
|
||||
NoResultsFound,
|
||||
ParsingError,
|
||||
)
|
||||
from .utils import capitalize
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -153,21 +159,25 @@ class MusicDL(list):
|
|||
"parent_folder": self.config.session["downloads"]["folder"],
|
||||
"folder_format": self.config.session["path_format"]["folder"],
|
||||
"track_format": self.config.session["path_format"]["track"],
|
||||
|
||||
"embed_cover": self.config.session["artwork"]["embed"],
|
||||
"embed_cover_size": self.config.session["artwork"]["size"],
|
||||
"keep_hires_cover": self.config.session['artwork']['keep_hires_cover'],
|
||||
|
||||
"keep_hires_cover": self.config.session["artwork"]["keep_hires_cover"],
|
||||
"set_playlist_to_album": self.config.session["metadata"][
|
||||
"set_playlist_to_album"
|
||||
],
|
||||
"stay_temp": self.config.session["conversion"]["enabled"],
|
||||
"conversion": self.config.session["conversion"],
|
||||
"concurrent_downloads": self.config.session["concurrent_downloads"],
|
||||
"new_tracknumbers": self.config.session['metadata']['new_playlist_tracknumbers']
|
||||
}
|
||||
logger.debug("Arguments from config: %s", arguments)
|
||||
|
||||
source_subdirs = self.config.session["downloads"]["source_subdirectories"]
|
||||
for item in self:
|
||||
if self.config.session["downloads"]["source_subdirectories"]:
|
||||
arguments["parent_folder"] = os.path.join(
|
||||
arguments["parent_folder"], capitalize(item.client.source)
|
||||
|
||||
if source_subdirs:
|
||||
arguments["parent_folder"] = self.__get_source_subdir(
|
||||
item.client.source
|
||||
)
|
||||
|
||||
arguments["quality"] = self.config.session[item.client.source]["quality"]
|
||||
|
|
@ -182,7 +192,7 @@ class MusicDL(list):
|
|||
try:
|
||||
item.load_meta()
|
||||
except NonStreamable:
|
||||
click.secho(f"{item!s} is not available, skipping.", fg='red')
|
||||
click.secho(f"{item!s} is not available, skipping.", fg="red")
|
||||
continue
|
||||
|
||||
if isinstance(item, Track):
|
||||
|
|
@ -194,8 +204,8 @@ class MusicDL(list):
|
|||
if self.db != [] and hasattr(item, "id"):
|
||||
self.db.add(item.id)
|
||||
|
||||
if self.config.session["conversion"]["enabled"]:
|
||||
item.convert(**self.config.session["conversion"])
|
||||
# if self.config.session["conversion"]["enabled"]:
|
||||
# item.convert(**self.config.session["conversion"])
|
||||
|
||||
def get_client(self, source: str):
|
||||
client = self.clients[source]
|
||||
|
|
@ -261,24 +271,36 @@ class MusicDL(list):
|
|||
def handle_lastfm_urls(self, urls):
|
||||
lastfm_urls = self.lastfm_url_parse.findall(urls)
|
||||
lastfm_source = self.config.session["lastfm"]["source"]
|
||||
tracks_not_found = 0
|
||||
|
||||
def search_query(query: str, playlist: Playlist):
|
||||
global tracks_not_found
|
||||
try:
|
||||
track = next(self.search(lastfm_source, query, media_type="track"))
|
||||
playlist.append(track)
|
||||
except NoResultsFound:
|
||||
tracks_not_found += 1
|
||||
return
|
||||
|
||||
for purl in lastfm_urls:
|
||||
click.secho(f"Fetching playlist at {purl}", fg="blue")
|
||||
title, queries = self.get_lastfm_playlist(purl)
|
||||
|
||||
pl = Playlist(client=self.clients[lastfm_source], name=title)
|
||||
tracks_not_found = 0
|
||||
for title, artist in tqdm(queries, unit="tracks", desc="Searching"):
|
||||
pl = Playlist(client=self.get_client(lastfm_source), name=title)
|
||||
processes = []
|
||||
|
||||
for title, artist in queries:
|
||||
query = f"{title} {artist}"
|
||||
proc = threading.Thread(
|
||||
target=search_query, args=(query, pl), daemon=True
|
||||
)
|
||||
proc.start()
|
||||
processes.append(proc)
|
||||
|
||||
try:
|
||||
track = next(self.search(lastfm_source, query, media_type="track"))
|
||||
except NoResultsFound:
|
||||
tracks_not_found += 1
|
||||
continue
|
||||
for proc in tqdm(processes, unit="tracks", desc="Searching"):
|
||||
proc.join()
|
||||
|
||||
pl.append(track)
|
||||
pl.loaded = True
|
||||
|
||||
click.secho(f"{tracks_not_found} tracks not found.", fg="yellow")
|
||||
self.append(pl)
|
||||
|
||||
|
|
@ -463,3 +485,7 @@ class MusicDL(list):
|
|||
remaining_tracks -= 50
|
||||
|
||||
return playlist_title, info
|
||||
|
||||
def __get_source_subdir(self, source: str) -> str:
|
||||
path = self.config.session["downloads"]["folder"]
|
||||
return os.path.join(path, capitalize(source))
|
||||
|
|
|
|||
|
|
@ -7,15 +7,17 @@ import os
|
|||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import threading
|
||||
from pprint import pformat
|
||||
from tempfile import gettempdir
|
||||
from typing import Any, Callable, Optional, Tuple, Union
|
||||
from typing import Any, Generator, Iterable, Union
|
||||
|
||||
import click
|
||||
from mutagen.flac import FLAC, Picture
|
||||
from mutagen.id3 import APIC, ID3, ID3NoHeaderError
|
||||
from mutagen.mp4 import MP4, MP4Cover
|
||||
from pathvalidate import sanitize_filename, sanitize_filepath
|
||||
from requests.packages import urllib3
|
||||
|
||||
from . import converter
|
||||
from .clients import ClientInterface
|
||||
|
|
@ -44,6 +46,7 @@ from .utils import (
|
|||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
urllib3.disable_warnings()
|
||||
|
||||
TIDAL_Q_MAP = {
|
||||
"LOW": 0,
|
||||
|
|
@ -145,7 +148,7 @@ class Track:
|
|||
self.cover_url = None
|
||||
|
||||
@staticmethod
|
||||
def _get_tracklist(resp, source):
|
||||
def _get_tracklist(resp, source) -> list:
|
||||
if source == "qobuz":
|
||||
return resp["tracks"]["items"]
|
||||
if source in ("tidal", "deezer"):
|
||||
|
|
@ -161,7 +164,7 @@ class Track:
|
|||
database: MusicDB = None,
|
||||
tag: bool = False,
|
||||
**kwargs,
|
||||
):
|
||||
) -> bool:
|
||||
"""
|
||||
Download the track.
|
||||
|
||||
|
|
@ -211,7 +214,11 @@ class Track:
|
|||
else:
|
||||
url_id = self.id
|
||||
|
||||
try:
|
||||
dl_info = self.client.get_file_url(url_id, self.quality)
|
||||
except Exception as e:
|
||||
click.secho(f"Unable to download track. {e}", fg='red')
|
||||
return False
|
||||
|
||||
self.path = os.path.join(gettempdir(), f"{hash(self.id)}_{self.quality}.tmp")
|
||||
logger.debug("Temporary file path: %s", self.path)
|
||||
|
|
@ -227,15 +234,13 @@ class Track:
|
|||
self.sampling_rate = dl_info.get("sampling_rate")
|
||||
self.bit_depth = dl_info.get("bit_depth")
|
||||
|
||||
click.secho(f"\nDownloading {self!s}", fg="blue")
|
||||
|
||||
# --------- Download Track ----------
|
||||
if self.client.source in ("qobuz", "tidal"):
|
||||
logger.debug("Downloadable URL found: %s", dl_info.get("url"))
|
||||
tqdm_download(dl_info["url"], self.path) # downloads file
|
||||
tqdm_download(dl_info["url"], self.path, desc=self._progress_desc) # downloads file
|
||||
|
||||
elif self.client.source == "deezer": # Deezer
|
||||
logger.debug("Downloadable URL found: %s", dl_info)
|
||||
logger.debug("Downloadable URL found: %s", dl_info, desc=self._progress_desc)
|
||||
try:
|
||||
tqdm_download(dl_info, self.path) # downloads file
|
||||
except NonStreamable:
|
||||
|
|
@ -300,7 +305,7 @@ class Track:
|
|||
]
|
||||
)
|
||||
elif dl_info["type"] == "original":
|
||||
tqdm_download(dl_info["url"], self.path)
|
||||
tqdm_download(dl_info["url"], self.path, desc=self._progress_desc)
|
||||
|
||||
# if a wav is returned, convert to flac
|
||||
engine = converter.FLAC(self.path)
|
||||
|
|
@ -310,6 +315,10 @@ class Track:
|
|||
self.final_path = self.final_path.replace(".mp3", ".flac")
|
||||
self.quality = 2
|
||||
|
||||
@property
|
||||
def _progress_desc(self):
|
||||
return click.style(f"Track {int(self.meta.tracknumber):02}", fg='blue')
|
||||
|
||||
def download_cover(self):
|
||||
"""Downloads the cover art, if cover_url is given."""
|
||||
|
||||
|
|
@ -317,10 +326,10 @@ class Track:
|
|||
|
||||
self.cover_path = os.path.join(self.folder, f"cover{hash(self.cover_url)}.jpg")
|
||||
logger.debug(f"Downloading cover from {self.cover_url}")
|
||||
click.secho(f"\nDownloading cover art for {self!s}", fg="blue")
|
||||
# click.secho(f"\nDownloading cover art for {self!s}", fg="blue")
|
||||
|
||||
if not os.path.exists(self.cover_path):
|
||||
tqdm_download(self.cover_url, self.cover_path)
|
||||
tqdm_download(self.cover_url, self.cover_path, desc=click.style('Cover', fg='cyan'))
|
||||
else:
|
||||
logger.debug("Cover already exists, skipping download")
|
||||
|
||||
|
|
@ -515,16 +524,18 @@ class Track:
|
|||
sampling_rate=kwargs.get("sampling_rate"),
|
||||
remove_source=kwargs.get("remove_source", True),
|
||||
)
|
||||
click.secho(f"Converting {self!s}", fg="blue")
|
||||
# click.secho(f"Converting {self!s}", fg="blue")
|
||||
engine.convert()
|
||||
self.path = engine.final_fn
|
||||
self.final_path = self.final_path.replace(ext(self.quality, self.client.source), f".{engine.container}")
|
||||
self.final_path = self.final_path.replace(
|
||||
ext(self.quality, self.client.source), f".{engine.container}"
|
||||
)
|
||||
|
||||
if not kwargs.get("stay_temp", False):
|
||||
self.move(self.final_path)
|
||||
|
||||
@property
|
||||
def title(self):
|
||||
def title(self) -> str:
|
||||
if hasattr(self, "meta"):
|
||||
_title = self.meta.title
|
||||
if self.meta.explicit:
|
||||
|
|
@ -533,7 +544,7 @@ class Track:
|
|||
else:
|
||||
raise Exception("Track must be loaded before accessing title")
|
||||
|
||||
def get(self, *keys, default=None):
|
||||
def get(self, *keys, default=None) -> Any:
|
||||
"""Safe get method that allows for layered access.
|
||||
|
||||
:param keys:
|
||||
|
|
@ -550,14 +561,14 @@ class Track:
|
|||
"""
|
||||
self.__setitem__(key, val)
|
||||
|
||||
def __getitem__(self, key):
|
||||
def __getitem__(self, key: str) -> Any:
|
||||
"""Dict-like interface for Track metadata.
|
||||
|
||||
:param key:
|
||||
"""
|
||||
return getattr(self.meta, key)
|
||||
|
||||
def __setitem__(self, key, val):
|
||||
def __setitem__(self, key: str, val: Any):
|
||||
"""Dict-like interface for Track metadata.
|
||||
|
||||
:param key:
|
||||
|
|
@ -588,20 +599,56 @@ class Tracklist(list):
|
|||
subclass is subscripted with [s: str], it will return an attribute s.
|
||||
If it is subscripted with [i: int] it will return the i'th track in
|
||||
the tracklist.
|
||||
|
||||
>>> tlist = Tracklist()
|
||||
>>> tlist.tracklistname = 'my tracklist'
|
||||
>>> tlist.append('first track')
|
||||
>>> tlist[0]
|
||||
'first track'
|
||||
>>> tlist['tracklistname']
|
||||
'my tracklist'
|
||||
>>> tlist[2]
|
||||
IndexError
|
||||
"""
|
||||
|
||||
essence_regex = re.compile(r"([^\(]+)(?:\s*[\(\[][^\)][\)\]])*")
|
||||
|
||||
def download(self, **kwargs):
|
||||
self._prepare_download(**kwargs)
|
||||
if kwargs.get("conversion", False):
|
||||
has_conversion = kwargs["conversion"]["enabled"]
|
||||
else:
|
||||
has_conversion = False
|
||||
kwargs["stay_temp"] = False
|
||||
|
||||
if has_conversion:
|
||||
target = self._download_and_convert_item
|
||||
else:
|
||||
target = self._download_item
|
||||
|
||||
if kwargs.get("concurrent_downloads", True):
|
||||
processes = []
|
||||
for item in self:
|
||||
proc = threading.Thread(
|
||||
target=target, args=(item,), kwargs=kwargs, daemon=True
|
||||
)
|
||||
proc.start()
|
||||
processes.append(proc)
|
||||
|
||||
try:
|
||||
for proc in processes:
|
||||
proc.join()
|
||||
except (KeyboardInterrupt, SystemExit):
|
||||
click.echo("Aborted!")
|
||||
exit()
|
||||
|
||||
else:
|
||||
for item in self:
|
||||
click.secho(f'\nDownloading "{item!s}"', fg="blue")
|
||||
target(item, **kwargs)
|
||||
|
||||
self.downloaded = True
|
||||
|
||||
def _download_and_convert_item(self, item, **kwargs):
|
||||
if self._download_item(item, **kwargs):
|
||||
item.convert(**kwargs["conversion"])
|
||||
|
||||
def _download_item(item, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
def _prepare_download(**kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
def get(self, key: Union[str, int], default=None):
|
||||
if isinstance(key, str):
|
||||
if hasattr(self, key):
|
||||
|
|
@ -696,16 +743,13 @@ class Tracklist(list):
|
|||
|
||||
def download_message(self):
|
||||
click.secho(
|
||||
f"\nDownloading {self.title} ({self.__class__.__name__})\n",
|
||||
f"\n\nDownloading {self.title} ({self.__class__.__name__})\n",
|
||||
fg="blue",
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _parse_get_resp(item, client):
|
||||
pass
|
||||
|
||||
def download(self, **kwargs):
|
||||
pass
|
||||
raise NotImplementedError
|
||||
|
||||
@staticmethod
|
||||
def essence(album: str) -> str:
|
||||
|
|
@ -763,13 +807,15 @@ class Album(Tracklist):
|
|||
setattr(self, k, v)
|
||||
|
||||
# to improve from_api method speed
|
||||
if kwargs.get("load_on_init"):
|
||||
if kwargs.get("load_on_init", False):
|
||||
self.load_meta()
|
||||
|
||||
self.loaded = False
|
||||
self.downloaded = False
|
||||
|
||||
def load_meta(self):
|
||||
"""Load detailed metadata from API using the id."""
|
||||
|
||||
assert hasattr(self, "id"), "id must be set to load metadata"
|
||||
self.meta = self.client.get(self.id, media_type="album")
|
||||
|
||||
|
|
@ -784,13 +830,83 @@ class Album(Tracklist):
|
|||
self.loaded = True
|
||||
|
||||
@classmethod
|
||||
def from_api(cls, resp, client):
|
||||
def from_api(cls, resp: dict, client: ClientInterface):
|
||||
if client.source == "soundcloud":
|
||||
return Playlist.from_api(resp, client)
|
||||
|
||||
info = cls._parse_get_resp(resp, client)
|
||||
return cls(client, **info)
|
||||
|
||||
def _prepare_download(self, **kwargs):
|
||||
self.folder_format = kwargs.get("folder_format", FOLDER_FORMAT)
|
||||
self.quality = min(kwargs.get("quality", 3), self.client.max_quality)
|
||||
self.folder = self._get_formatted_folder(
|
||||
kwargs.get("parent_folder", "StreamripDownloads"), self.quality
|
||||
)
|
||||
os.makedirs(self.folder, exist_ok=True)
|
||||
|
||||
self.download_message()
|
||||
|
||||
# choose optimal cover size and download it
|
||||
click.secho("Downloading cover art", fg="magenta")
|
||||
cover_path = os.path.join(gettempdir(), f"cover_{hash(self)}.jpg")
|
||||
embed_cover_size = kwargs.get("embed_cover_size", "large")
|
||||
|
||||
assert (
|
||||
embed_cover_size in self.cover_urls
|
||||
), f"Invalid cover size. Must be in {self.cover_urls.keys()}"
|
||||
|
||||
embed_cover_url = self.cover_urls[embed_cover_size]
|
||||
if embed_cover_url is not None:
|
||||
tqdm_download(embed_cover_url, cover_path)
|
||||
else: # sometimes happens with Deezer
|
||||
tqdm_download(self.cover_urls["small"], cover_path)
|
||||
|
||||
if kwargs.get("keep_hires_cover", True):
|
||||
tqdm_download(
|
||||
self.cover_urls["original"], os.path.join(self.folder, "cover.jpg")
|
||||
)
|
||||
|
||||
cover_size = os.path.getsize(cover_path)
|
||||
if cover_size > FLAC_MAX_BLOCKSIZE: # 16.77 MB
|
||||
click.secho(
|
||||
"Downgrading embedded cover size, too large ({cover_size}).",
|
||||
fg="bright_yellow",
|
||||
)
|
||||
# large is about 600x600px which is guaranteed < 16.7 MB
|
||||
tqdm_download(self.cover_urls["large"], cover_path)
|
||||
|
||||
embed_cover = kwargs.get("embed_cover", True) # embed by default
|
||||
if self.client.source != "deezer" and embed_cover:
|
||||
self.cover_obj = self.get_cover_obj(
|
||||
cover_path, self.quality, self.client.source
|
||||
)
|
||||
else:
|
||||
self.cover_obj = None
|
||||
|
||||
def _download_item(
|
||||
self,
|
||||
track: Track,
|
||||
quality: int = 3,
|
||||
database: MusicDB = None,
|
||||
**kwargs,
|
||||
) -> bool:
|
||||
logger.debug("Downloading track to %s", self.folder)
|
||||
if self.disctotal > 1:
|
||||
disc_folder = os.path.join(self.folder, f"Disc {track.meta.discnumber}")
|
||||
kwargs["parent_folder"] = disc_folder
|
||||
else:
|
||||
kwargs["parent_folder"] = self.folder
|
||||
|
||||
if not track.download(quality=quality, database=database, **kwargs):
|
||||
return False
|
||||
|
||||
# deezer tracks come tagged
|
||||
if kwargs.get("tag_tracks", True) and self.client.source != "deezer":
|
||||
track.tag(cover=self.cover_obj, embed_cover=kwargs.get("embed_cover", True))
|
||||
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _parse_get_resp(resp: dict, client: ClientInterface) -> dict:
|
||||
"""Parse information from a client.get(query, 'album') call.
|
||||
|
|
@ -903,110 +1019,6 @@ class Album(Tracklist):
|
|||
Track.from_album_meta(album=self.meta, pos=i, client=self.client)
|
||||
)
|
||||
|
||||
@property
|
||||
def title(self) -> str:
|
||||
"""Return the title of the album.
|
||||
|
||||
It is formatted so that "version" keys are included.
|
||||
|
||||
:rtype: str
|
||||
"""
|
||||
album_title = self._title
|
||||
if hasattr(self, "version") and isinstance(self.version, str):
|
||||
if self.version.lower() not in album_title.lower():
|
||||
album_title = f"{album_title} ({self.version})"
|
||||
|
||||
if self.get("explicit", False):
|
||||
album_title = f"{album_title} (Explicit)"
|
||||
|
||||
return album_title
|
||||
|
||||
@title.setter
|
||||
def title(self, val):
|
||||
"""Sets the internal _title attribute to the given value.
|
||||
|
||||
:param val: title to set
|
||||
"""
|
||||
self._title = val
|
||||
|
||||
def download(
|
||||
self,
|
||||
quality: int = 3,
|
||||
parent_folder: Union[str, os.PathLike] = "StreamripDownloads",
|
||||
database: MusicDB = None,
|
||||
**kwargs,
|
||||
):
|
||||
"""Download all of the tracks in the album.
|
||||
|
||||
:param quality: (0, 1, 2, 3, 4)
|
||||
:type quality: int
|
||||
:param parent_folder: the folder to download the album to
|
||||
:type parent_folder: Union[str, os.PathLike]
|
||||
:param progress_bar: turn on/off a tqdm progress bar
|
||||
:type progress_bar: bool
|
||||
:param large_cover: Download the large cover. This may fail when
|
||||
embedding covers.
|
||||
:param tag_tracks: Tag the tracks after downloading, True by default
|
||||
:param keep_cover: Keep the cover art image after downloading.
|
||||
True by default.
|
||||
"""
|
||||
self.folder_format = kwargs.get("folder_format", FOLDER_FORMAT)
|
||||
quality = min(quality, self.client.max_quality)
|
||||
folder = self._get_formatted_folder(parent_folder, quality)
|
||||
|
||||
# choose optimal cover size and download it
|
||||
self.download_message()
|
||||
|
||||
click.secho("Downloading cover art", fg="magenta")
|
||||
cover_path = os.path.join(gettempdir(), f"cover_{hash(self)}.jpg")
|
||||
embed_cover_size = kwargs.get("embed_cover_size", "large")
|
||||
|
||||
assert (
|
||||
embed_cover_size in self.cover_urls
|
||||
), f"Invalid cover size. Must be in {self.cover_urls.keys()}"
|
||||
|
||||
tqdm_download(self.cover_urls[embed_cover_size], cover_path)
|
||||
|
||||
if kwargs.get("keep_hires_cover", True):
|
||||
tqdm_download(self.cover_urls['original'], os.path.join(folder, 'cover.jpg'))
|
||||
|
||||
cover_size = os.path.getsize(cover_path)
|
||||
if cover_size > FLAC_MAX_BLOCKSIZE: # 16.77 MB
|
||||
click.secho(
|
||||
"Downgrading embedded cover size, too large ({cover_size}).",
|
||||
fg="bright_yellow",
|
||||
)
|
||||
# large is about 600x600px which is guaranteed < 16.7 MB
|
||||
tqdm_download(self.cover_urls["large"], cover_path)
|
||||
|
||||
embed_cover = kwargs.get("embed_cover", True) # embed by default
|
||||
if self.client.source != "deezer" and embed_cover:
|
||||
cover = self.get_cover_obj(cover_path, quality, self.client.source)
|
||||
|
||||
download_args = {
|
||||
"quality": quality,
|
||||
"parent_folder": folder,
|
||||
"progress_bar": kwargs.get("progress_bar", True),
|
||||
"database": database,
|
||||
"track_format": kwargs.get("track_format", TRACK_FORMAT),
|
||||
"stay_temp": kwargs.get("stay_temp")
|
||||
}
|
||||
for track in self:
|
||||
logger.debug("Downloading track to %s", folder)
|
||||
if self.disctotal > 1:
|
||||
disc_folder = os.path.join(folder, f"Disc {track.meta.discnumber}")
|
||||
download_args["parent_folder"] = disc_folder
|
||||
|
||||
track.download(quality=quality, parent_folder=folder, database=database, **kwargs)
|
||||
|
||||
# deezer tracks come tagged
|
||||
if kwargs.get("tag_tracks", True) and self.client.source != "deezer":
|
||||
track.tag(cover=cover, embed_cover=embed_cover)
|
||||
|
||||
os.remove(cover_path)
|
||||
|
||||
self.downloaded = True
|
||||
|
||||
def _get_formatter(self) -> dict:
|
||||
fmt = dict()
|
||||
for key in ALBUM_KEYS:
|
||||
|
|
@ -1035,9 +1047,34 @@ class Album(Tracklist):
|
|||
|
||||
return os.path.join(parent_folder, formatted_folder)
|
||||
|
||||
@property
|
||||
def title(self) -> str:
|
||||
"""Return the title of the album.
|
||||
|
||||
It is formatted so that "version" keys are included.
|
||||
|
||||
:rtype: str
|
||||
"""
|
||||
album_title = self._title
|
||||
if hasattr(self, "version") and isinstance(self.version, str):
|
||||
if self.version.lower() not in album_title.lower():
|
||||
album_title = f"{album_title} ({self.version})"
|
||||
|
||||
if self.get("explicit", False):
|
||||
album_title = f"{album_title} (Explicit)"
|
||||
|
||||
return album_title
|
||||
|
||||
@title.setter
|
||||
def title(self, val):
|
||||
"""Sets the internal _title attribute to the given value.
|
||||
|
||||
:param val: title to set
|
||||
"""
|
||||
self._title = val
|
||||
|
||||
def __repr__(self) -> str:
|
||||
"""Return a string representation of this Album object.
|
||||
Useful for pprint and json.dumps.
|
||||
|
||||
:rtype: str
|
||||
"""
|
||||
|
|
@ -1122,6 +1159,7 @@ class Playlist(Tracklist):
|
|||
:param new_tracknumbers: replace tracknumber tag with playlist position
|
||||
:type new_tracknumbers: bool
|
||||
"""
|
||||
# TODO: redundant parsing with _parse_get_pres
|
||||
if self.client.source == "qobuz":
|
||||
self.name = self.meta["name"]
|
||||
self.image = self.meta["images"]
|
||||
|
|
@ -1198,50 +1236,36 @@ class Playlist(Tracklist):
|
|||
|
||||
logger.debug(f"Loaded {len(self)} tracks from playlist {self.name}")
|
||||
|
||||
def download(
|
||||
self,
|
||||
parent_folder: str = "StreamripDownloads",
|
||||
quality: int = 3,
|
||||
filters: Callable = None,
|
||||
database: MusicDB = None,
|
||||
**kwargs,
|
||||
):
|
||||
"""Download and tag all of the tracks.
|
||||
|
||||
:param parent_folder:
|
||||
:type parent_folder: str
|
||||
:param quality:
|
||||
:type quality: int
|
||||
:param filters:
|
||||
:type filters: Callable
|
||||
"""
|
||||
folder = sanitize_filename(self.name)
|
||||
folder = os.path.join(parent_folder, folder)
|
||||
logger.debug(f"Parent folder {folder}")
|
||||
def _prepare_download(self, parent_folder: str = "StreamripDownloads", **kwargs):
|
||||
fname = sanitize_filename(self.name)
|
||||
self.folder = os.path.join(parent_folder, fname)
|
||||
|
||||
self.__download_index = 1
|
||||
self.download_message()
|
||||
set_playlist_to_album = kwargs.get("set_playlist_to_album", False)
|
||||
for i, track in enumerate(self):
|
||||
|
||||
def _download_item(self, item: Track, **kwargs):
|
||||
if self.client.source == "soundcloud":
|
||||
track.load_meta()
|
||||
item.load_meta()
|
||||
|
||||
if set_playlist_to_album and hasattr(self, "image"):
|
||||
track["album"] = self.name
|
||||
track["albumartist"] = self.creator
|
||||
if kwargs.get("set_playlist_to_album", False):
|
||||
item["album"] = self.name
|
||||
item["albumartist"] = self.creator
|
||||
|
||||
if kwargs.get("new_tracknumbers", True):
|
||||
track.meta["tracknumber"] = str(i + 1)
|
||||
item["tracknumber"] = self.__download_index
|
||||
item['discnumber'] = 1
|
||||
|
||||
if (
|
||||
track.download(parent_folder=folder, quality=quality, database=database, **kwargs)
|
||||
and self.client.source != "deezer"
|
||||
):
|
||||
self.__download_index += 1
|
||||
|
||||
track.tag(embed_cover=kwargs.get("embed_cover", True))
|
||||
self.downloaded = item.download(**kwargs)
|
||||
|
||||
if self.downloaded and self.client.source != "deezer":
|
||||
item.tag(embed_cover=kwargs.get("embed_cover", True))
|
||||
|
||||
return self.downloaded
|
||||
|
||||
@staticmethod
|
||||
def _parse_get_resp(item: dict, client: ClientInterface):
|
||||
def _parse_get_resp(item: dict, client: ClientInterface) -> dict:
|
||||
"""Parses information from a search result returned
|
||||
by a client.search call.
|
||||
|
||||
|
|
@ -1277,12 +1301,11 @@ class Playlist(Tracklist):
|
|||
raise InvalidSourceError(client.source)
|
||||
|
||||
@property
|
||||
def title(self):
|
||||
def title(self) -> str:
|
||||
return self.name
|
||||
|
||||
def __repr__(self) -> str:
|
||||
"""Return a string representation of this Playlist object.
|
||||
Useful for pprint and json.dumps.
|
||||
|
||||
:rtype: str
|
||||
"""
|
||||
|
|
@ -1332,6 +1355,12 @@ class Artist(Tracklist):
|
|||
self._load_albums()
|
||||
self.loaded = True
|
||||
|
||||
# override
|
||||
def download(self, **kwargs):
|
||||
iterator = self._prepare_download(**kwargs)
|
||||
for item in iterator:
|
||||
self._download_item(item, **kwargs)
|
||||
|
||||
def _load_albums(self):
|
||||
"""From the discography returned by client.get(query, 'artist'),
|
||||
generate album objects and append them to self.
|
||||
|
|
@ -1355,25 +1384,9 @@ class Artist(Tracklist):
|
|||
logger.debug("Appending album: %s", album.get("title"))
|
||||
self.append(Album.from_api(album, self.client))
|
||||
|
||||
def download(
|
||||
self,
|
||||
parent_folder: str = "StreamripDownloads",
|
||||
filters: Optional[Tuple] = None,
|
||||
no_repeats: bool = False,
|
||||
quality: int = 6,
|
||||
database: MusicDB = None,
|
||||
**kwargs,
|
||||
):
|
||||
"""Download all albums in the discography.
|
||||
|
||||
:param filters: Filters to apply to discography, see options below.
|
||||
These only work for Qobuz.
|
||||
:type filters: Optional[Tuple]
|
||||
:param no_repeats: Remove repeats
|
||||
:type no_repeats: bool
|
||||
:param quality: in (0, 1, 2, 3, 4)
|
||||
:type quality: int
|
||||
"""
|
||||
def _prepare_download(
|
||||
self, parent_folder: str = "StreamripDownloads", filters: tuple = (), **kwargs
|
||||
) -> Iterable:
|
||||
folder = sanitize_filename(self.name)
|
||||
folder = os.path.join(parent_folder, folder)
|
||||
|
||||
|
|
@ -1393,21 +1406,33 @@ class Artist(Tracklist):
|
|||
final = filter(func, final)
|
||||
|
||||
self.download_message()
|
||||
for album in final:
|
||||
return final
|
||||
|
||||
def _download_item(
|
||||
self,
|
||||
item,
|
||||
parent_folder: str = "StreamripDownloads",
|
||||
quality: int = 3,
|
||||
database: MusicDB = None,
|
||||
**kwargs,
|
||||
) -> bool:
|
||||
try:
|
||||
album.load_meta()
|
||||
item.load_meta()
|
||||
except NonStreamable:
|
||||
logger.info("Skipping album, not available to stream.")
|
||||
continue
|
||||
album.download(
|
||||
parent_folder=folder,
|
||||
return
|
||||
|
||||
# always an Album
|
||||
status = item.download(
|
||||
parent_folder=parent_folder,
|
||||
quality=quality,
|
||||
database=database,
|
||||
**kwargs,
|
||||
)
|
||||
return status
|
||||
|
||||
@property
|
||||
def title(self):
|
||||
def title(self) -> str:
|
||||
return self.name
|
||||
|
||||
@classmethod
|
||||
|
|
@ -1427,7 +1452,7 @@ class Artist(Tracklist):
|
|||
return cls(client=client, **info)
|
||||
|
||||
@staticmethod
|
||||
def _parse_get_resp(item: dict, client: ClientInterface):
|
||||
def _parse_get_resp(item: dict, client: ClientInterface) -> dict:
|
||||
"""Parse a result from a client.search call.
|
||||
|
||||
:param item: the item to parse
|
||||
|
|
@ -1452,7 +1477,7 @@ class Artist(Tracklist):
|
|||
|
||||
# ----------- Filters --------------
|
||||
|
||||
def _remove_repeats(self, bit_depth=max, sampling_rate=max):
|
||||
def _remove_repeats(self, bit_depth=max, sampling_rate=max) -> Generator:
|
||||
"""Remove the repeated albums from self. May remove different
|
||||
versions of the same album.
|
||||
|
||||
|
|
@ -1489,7 +1514,7 @@ class Artist(Tracklist):
|
|||
and TYPE_REGEXES["extra"].search(album.title) is None
|
||||
)
|
||||
|
||||
def _features(self, album):
|
||||
def _features(self, album: Album) -> bool:
|
||||
"""Passed as a parameter by the user.
|
||||
|
||||
This will download only albums where the requested
|
||||
|
|
@ -1502,7 +1527,7 @@ class Artist(Tracklist):
|
|||
"""
|
||||
return self["name"] == album["albumartist"]
|
||||
|
||||
def _extras(self, album):
|
||||
def _extras(self, album: Album) -> bool:
|
||||
"""Passed as a parameter by the user.
|
||||
|
||||
This will skip any extras.
|
||||
|
|
@ -1514,7 +1539,7 @@ class Artist(Tracklist):
|
|||
"""
|
||||
return TYPE_REGEXES["extra"].search(album.title) is None
|
||||
|
||||
def _non_remasters(self, album):
|
||||
def _non_remasters(self, album: Album) -> bool:
|
||||
"""Passed as a parameter by the user.
|
||||
|
||||
This will download only remasterd albums.
|
||||
|
|
@ -1526,7 +1551,7 @@ class Artist(Tracklist):
|
|||
"""
|
||||
return TYPE_REGEXES["remaster"].search(album.title) is not None
|
||||
|
||||
def _non_albums(self, album):
|
||||
def _non_albums(self, album: Album) -> bool:
|
||||
"""This will ignore non-album releases.
|
||||
|
||||
:param artist: usually self
|
||||
|
|
@ -1541,7 +1566,6 @@ class Artist(Tracklist):
|
|||
|
||||
def __repr__(self) -> str:
|
||||
"""Return a string representation of this Artist object.
|
||||
Useful for pprint and json.dumps.
|
||||
|
||||
:rtype: str
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -361,6 +361,9 @@ class TrackMetadata:
|
|||
for k, v in FLAC_KEY.items():
|
||||
tag = getattr(self, k)
|
||||
if tag:
|
||||
if k in ('tracknumber', 'discnumber', 'tracktotal', 'disctotal'):
|
||||
tag = f"{int(tag):02}"
|
||||
|
||||
logger.debug("Adding tag %s: %s", v, tag)
|
||||
yield (v, str(tag))
|
||||
|
||||
|
|
|
|||
|
|
@ -8,11 +8,13 @@ import requests
|
|||
from Crypto.Cipher import AES
|
||||
from Crypto.Util import Counter
|
||||
from pathvalidate import sanitize_filename
|
||||
from requests.packages import urllib3
|
||||
from tqdm import tqdm
|
||||
|
||||
from .constants import LOG_DIR, TIDAL_COVER_URL
|
||||
from .exceptions import InvalidSourceError, NonStreamable
|
||||
|
||||
urllib3.disable_warnings()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
|
|
@ -95,7 +97,7 @@ def get_quality_id(bit_depth: Optional[int], sampling_rate: Optional[int]):
|
|||
return 4
|
||||
|
||||
|
||||
def tqdm_download(url: str, filepath: str, params: dict = None):
|
||||
def tqdm_download(url: str, filepath: str, params: dict = None, desc: str = None):
|
||||
"""Downloads a file with a progress bar.
|
||||
|
||||
:param url: url to direct download
|
||||
|
|
@ -107,7 +109,8 @@ def tqdm_download(url: str, filepath: str, params: dict = None):
|
|||
if params is None:
|
||||
params = {}
|
||||
|
||||
r = requests.get(url, allow_redirects=True, stream=True, params=params)
|
||||
session = gen_threadsafe_session()
|
||||
r = session.get(url, allow_redirects=True, stream=True, params=params)
|
||||
total = int(r.headers.get("content-length", 0))
|
||||
logger.debug(f"File size = {total}")
|
||||
if total < 1000 and not url.endswith("jpg") and not url.endswith("png"):
|
||||
|
|
@ -115,7 +118,7 @@ def tqdm_download(url: str, filepath: str, params: dict = None):
|
|||
|
||||
try:
|
||||
with open(filepath, "wb") as file, tqdm(
|
||||
total=total, unit="iB", unit_scale=True, unit_divisor=1024
|
||||
total=total, unit="iB", unit_scale=True, unit_divisor=1024, desc=desc
|
||||
) as bar:
|
||||
for data in r.iter_content(chunk_size=1024):
|
||||
size = file.write(data)
|
||||
|
|
@ -141,8 +144,10 @@ def clean_format(formatter: str, format_info):
|
|||
|
||||
clean_dict = dict()
|
||||
for key in fmt_keys:
|
||||
if isinstance(format_info.get(key), (str, int, float)): # int for track numbers
|
||||
if isinstance(format_info.get(key), (str, float)):
|
||||
clean_dict[key] = sanitize_filename(str(format_info[key]))
|
||||
elif isinstance(format_info.get(key), int): # track/discnumber
|
||||
clean_dict[key] = f"{format_info[key]:02}"
|
||||
else:
|
||||
clean_dict[key] = "Unknown"
|
||||
|
||||
|
|
@ -214,3 +219,16 @@ def ext(quality: int, source: str):
|
|||
return ".mp3"
|
||||
else:
|
||||
return ".flac"
|
||||
|
||||
|
||||
def gen_threadsafe_session(
|
||||
headers: dict = None, pool_connections: int = 100, pool_maxsize: int = 100
|
||||
) -> requests.Session:
|
||||
if headers is None:
|
||||
headers = {}
|
||||
|
||||
session = requests.Session()
|
||||
adapter = requests.adapters.HTTPAdapter(pool_connections=100, pool_maxsize=100)
|
||||
session.mount("https://", adapter)
|
||||
session.headers.update(headers)
|
||||
return session
|
||||
|
|
|
|||
Loading…
Reference in a new issue