mirrors/streamrip
1
0
Fork 0
mirror of https://github.com/nathom/streamrip.git synced 2024-09-19 19:28:46 -04:00
Code Issues Projects Releases Packages Wiki Activity

Multithreaded album working

Browse source
This commit is contained in:
nathom 2021-04-12 08:42:29 -07:00
parent c31d334ae7
commit 8f9414685f
5 changed files with 254 additions and 122 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -11,3 +11,4 @@ test.py
*.mp3
StreamripDownloads
*.wav
*.log

8
streamrip/converter.py
View file

@ -6,6 +6,7 @@ from tempfile import gettempdir
from typing import Optional
from mutagen.flac import FLAC as FLAC_META
from mutagen.mp4 import MP4 as M4A_META
from .exceptions import ConversionError
@ -111,9 +112,14 @@ class Converter:
if self.lossless:
if isinstance(self.sampling_rate, int):
audio = FLAC_META(self.filename)
meta_objects = {
'flac': FLAC_META,
'alac': M4A_META,
}
audio = meta_objects[self.container](self.filename)
old_sr = audio.info.sample_rate
command.extend(["-ar", str(min(old_sr, self.sampling_rate))])
elif self.sampling_rate is not None:
raise TypeError(
f"Sampling rate must be int, not {type(self.sampling_rate)}"

19
streamrip/core.py
View file

@ -1,3 +1,4 @@
import asyncio
import logging
import os
import re
@ -23,7 +24,12 @@ from .constants import (
)
from .db import MusicDB
from .downloader import Album, Artist, Label, Playlist, Track, Tracklist
from .exceptions import AuthenticationError, NoResultsFound, ParsingError, NonStreamable
from .exceptions import (
AuthenticationError,
NonStreamable,
NoResultsFound,
ParsingError,
)
from .utils import capitalize
logger = logging.getLogger(__name__)
@ -153,15 +159,14 @@ class MusicDL(list):
"parent_folder": self.config.session["downloads"]["folder"],
"folder_format": self.config.session["path_format"]["folder"],
"track_format": self.config.session["path_format"]["track"],
"embed_cover": self.config.session["artwork"]["embed"],
"embed_cover_size": self.config.session["artwork"]["size"],
"keep_hires_cover": self.config.session['artwork']['keep_hires_cover'],
"keep_hires_cover": self.config.session["artwork"]["keep_hires_cover"],
"set_playlist_to_album": self.config.session["metadata"][
"set_playlist_to_album"
],
"stay_temp": self.config.session["conversion"]["enabled"],
"conversion": self.config.session["conversion"],
}
logger.debug("Arguments from config: %s", arguments)
for item in self:
@ -182,7 +187,7 @@ class MusicDL(list):
try:
item.load_meta()
except NonStreamable:
click.secho(f"{item!s} is not available, skipping.", fg='red')
click.secho(f"{item!s} is not available, skipping.", fg="red")
continue
if isinstance(item, Track):
@ -194,8 +199,8 @@ class MusicDL(list):
if self.db != [] and hasattr(item, "id"):
self.db.add(item.id)
if self.config.session["conversion"]["enabled"]:
item.convert(**self.config.session["conversion"])
# if self.config.session["conversion"]["enabled"]:
# item.convert(**self.config.session["conversion"])
def get_client(self, source: str):
client = self.clients[source]

346
streamrip/downloader.py
View file

@ -7,15 +7,17 @@ import os
import re
import shutil
import subprocess
import threading
from pprint import pformat
from tempfile import gettempdir
from typing import Any, Callable, Optional, Tuple, Union
import click
from pathvalidate import sanitize_filepath, sanitize_filename
from mutagen.flac import FLAC, Picture
from mutagen.id3 import APIC, ID3, ID3NoHeaderError
from mutagen.mp4 import MP4, MP4Cover
from pathvalidate import sanitize_filename, sanitize_filepath
from requests.packages import urllib3
from . import converter
from .clients import ClientInterface
@ -44,6 +46,7 @@ from .utils import (
)
logger = logging.getLogger(__name__)
urllib3.disable_warnings()
TIDAL_Q_MAP = {
"LOW": 0,
@ -227,7 +230,7 @@ class Track:
self.sampling_rate = dl_info.get("sampling_rate")
self.bit_depth = dl_info.get("bit_depth")
click.secho(f"\nDownloading {self!s}", fg="blue")
# click.secho(f"\nDownloading {self!s}", fg="blue")
# --------- Download Track ----------
if self.client.source in ("qobuz", "tidal"):
@ -515,10 +518,12 @@ class Track:
sampling_rate=kwargs.get("sampling_rate"),
remove_source=kwargs.get("remove_source", True),
)
click.secho(f"Converting {self!s}", fg="blue")
# click.secho(f"Converting {self!s}", fg="blue")
engine.convert()
self.path = engine.final_fn
self.final_path = self.final_path.replace(ext(self.quality, self.client.source), f".{engine.container}")
self.final_path = self.final_path.replace(
ext(self.quality, self.client.source), f".{engine.container}"
)
if not kwargs.get("stay_temp", False):
self.move(self.final_path)
@ -602,6 +607,34 @@ class Tracklist(list):
essence_regex = re.compile(r"([^\(]+)(?:\s*[\(\[][^\)][\)\]])*")
def download(self, **kwargs):
self._prepare_download(**kwargs)
has_conversion = "conversion" in kwargs.get("conversion", {})
if has_conversion:
target = self._download_and_convert_item
else:
target = self._download_item
processes = []
for item in self:
proc = threading.Thread(target=target, kwargs={"item": item, **kwargs})
proc.start()
processes.append(proc)
for proc in processes:
proc.join()
def _download_and_convert_item(self, item, **kwargs):
self._download_item(item, **kwargs)
item.convert(**kwargs["conversion"])
def _download_item(item, **kwargs):
raise NotImplementedError
def _prepare_download(**kwargs):
raise NotImplementedError
def get(self, key: Union[str, int], default=None):
if isinstance(key, str):
if hasattr(self, key):
@ -702,10 +735,7 @@ class Tracklist(list):
@staticmethod
def _parse_get_resp(item, client):
pass
def download(self, **kwargs):
pass
raise NotImplementedError
@staticmethod
def essence(album: str) -> str:
@ -763,13 +793,15 @@ class Album(Tracklist):
setattr(self, k, v)
# to improve from_api method speed
if kwargs.get("load_on_init"):
if kwargs.get("load_on_init", False):
self.load_meta()
self.loaded = False
self.downloaded = False
def load_meta(self):
"""Load detailed metadata from API using the id."""
assert hasattr(self, "id"), "id must be set to load metadata"
self.meta = self.client.get(self.id, media_type="album")
@ -783,6 +815,102 @@ class Album(Tracklist):
self._load_tracks()
self.loaded = True
def download(
self,
quality: int = 3,
parent_folder: Union[str, os.PathLike] = "StreamripDownloads",
database: MusicDB = None,
**kwargs,
):
"""Download all of the tracks in the album.
:param quality: (0, 1, 2, 3, 4)
:type quality: int
:param parent_folder: the folder to download the album to
:type parent_folder: Union[str, os.PathLike]
:param progress_bar: turn on/off a tqdm progress bar
:type progress_bar: bool
:param large_cover: Download the large cover. This may fail when
embedding covers.
:param tag_tracks: Tag the tracks after downloading, True by default
:param keep_cover: Keep the cover art image after downloading.
True by default.
"""
self.folder_format = kwargs.get("folder_format", FOLDER_FORMAT)
quality = min(quality, self.client.max_quality)
folder = self._get_formatted_folder(parent_folder, quality)
# choose optimal cover size and download it
self.download_message()
click.secho("Downloading cover art", fg="magenta")
cover_path = os.path.join(gettempdir(), f"cover_{hash(self)}.jpg")
embed_cover_size = kwargs.get("embed_cover_size", "large")
assert (
embed_cover_size in self.cover_urls
), f"Invalid cover size. Must be in {self.cover_urls.keys()}"
tqdm_download(self.cover_urls[embed_cover_size], cover_path)
if kwargs.get("keep_hires_cover", True):
tqdm_download(
self.cover_urls["original"], os.path.join(folder, "cover.jpg")
)
cover_size = os.path.getsize(cover_path)
if cover_size > FLAC_MAX_BLOCKSIZE: # 16.77 MB
click.secho(
"Downgrading embedded cover size, too large ({cover_size}).",
fg="bright_yellow",
)
# large is about 600x600px which is guaranteed < 16.7 MB
tqdm_download(self.cover_urls["large"], cover_path)
embed_cover = kwargs.get("embed_cover", True) # embed by default
if self.client.source != "deezer" and embed_cover:
cover = self.get_cover_obj(cover_path, quality, self.client.source)
download_args = {
"quality": quality,
"parent_folder": folder,
"progress_bar": kwargs.get("progress_bar", True),
"database": database,
"track_format": kwargs.get("track_format", TRACK_FORMAT),
"stay_temp": kwargs.get("stay_temp"),
}
def _download_track(track):
logger.debug("Downloading track to %s", folder)
if self.disctotal > 1:
disc_folder = os.path.join(folder, f"Disc {track.meta.discnumber}")
download_args["parent_folder"] = disc_folder
track.download(
quality=quality, parent_folder=folder, database=database, **kwargs
)
# deezer tracks come tagged
if kwargs.get("tag_tracks", True) and self.client.source != "deezer":
track.tag(cover=cover, embed_cover=embed_cover)
if kwargs.get("conversion", False):
track.convert(**kwargs["conversion"])
click.echo()
processes = []
for track in self:
proc = threading.Thread(target=_download_track, args=(track,))
proc.start()
processes.append(proc)
for proc in processes:
proc.join()
os.remove(cover_path)
self.downloaded = True
@classmethod
def from_api(cls, resp, client):
if client.source == "soundcloud":
@ -791,6 +919,72 @@ class Album(Tracklist):
info = cls._parse_get_resp(resp, client)
return cls(client, **info)
def _prepare_download(self, **kwargs):
self.folder_format = kwargs.get("folder_format", FOLDER_FORMAT)
self.quality = min(kwargs.get("quality", 3), self.client.max_quality)
self.folder = self._get_formatted_folder(
kwargs.get("parent_folder", "StreamripDownloads"), self.quality
)
self.download_message()
# choose optimal cover size and download it
click.secho("Downloading cover art", fg="magenta")
cover_path = os.path.join(gettempdir(), f"cover_{hash(self)}.jpg")
embed_cover_size = kwargs.get("embed_cover_size", "large")
assert (
embed_cover_size in self.cover_urls
), f"Invalid cover size. Must be in {self.cover_urls.keys()}"
tqdm_download(self.cover_urls[embed_cover_size], cover_path)
if kwargs.get("keep_hires_cover", True):
tqdm_download(
self.cover_urls["original"], os.path.join(self.folder, "cover.jpg")
)
cover_size = os.path.getsize(cover_path)
if cover_size > FLAC_MAX_BLOCKSIZE: # 16.77 MB
click.secho(
"Downgrading embedded cover size, too large ({cover_size}).",
fg="bright_yellow",
)
# large is about 600x600px which is guaranteed < 16.7 MB
tqdm_download(self.cover_urls["large"], cover_path)
embed_cover = kwargs.get("embed_cover", True) # embed by default
if self.client.source != "deezer" and embed_cover:
self.cover_obj = self.get_cover_obj(
cover_path, self.quality, self.client.source
)
def _download_item(
self,
track,
quality: int = 3,
database: MusicDB = None,
**kwargs,
):
logger.debug("Downloading track to %s", self.folder)
if self.disctotal > 1:
disc_folder = os.path.join(self.folder, f"Disc {track.meta.discnumber}")
kwargs["parent_folder"] = disc_folder
track.download(
quality=quality, parent_folder=self.folder, database=database, **kwargs
)
# deezer tracks come tagged
if kwargs.get("tag_tracks", True) and self.client.source != "deezer":
track.tag(cover=self.cover_obj, embed_cover=kwargs.get("embed_cover", True))
if kwargs.get("conversion", False):
track.convert(**kwargs["conversion"])
if isinstance(database, MusicDB):
database.add(track.id)
@staticmethod
def _parse_get_resp(resp: dict, client: ClientInterface) -> dict:
"""Parse information from a client.get(query, 'album') call.
@ -903,110 +1097,6 @@ class Album(Tracklist):
Track.from_album_meta(album=self.meta, pos=i, client=self.client)
)
@property
def title(self) -> str:
"""Return the title of the album.
It is formatted so that "version" keys are included.
:rtype: str
"""
album_title = self._title
if hasattr(self, "version") and isinstance(self.version, str):
if self.version.lower() not in album_title.lower():
album_title = f"{album_title} ({self.version})"
if self.get("explicit", False):
album_title = f"{album_title} (Explicit)"
return album_title
@title.setter
def title(self, val):
"""Sets the internal _title attribute to the given value.
:param val: title to set
"""
self._title = val
def download(
self,
quality: int = 3,
parent_folder: Union[str, os.PathLike] = "StreamripDownloads",
database: MusicDB = None,
**kwargs,
):
"""Download all of the tracks in the album.
:param quality: (0, 1, 2, 3, 4)
:type quality: int
:param parent_folder: the folder to download the album to
:type parent_folder: Union[str, os.PathLike]
:param progress_bar: turn on/off a tqdm progress bar
:type progress_bar: bool
:param large_cover: Download the large cover. This may fail when
embedding covers.
:param tag_tracks: Tag the tracks after downloading, True by default
:param keep_cover: Keep the cover art image after downloading.
True by default.
"""
self.folder_format = kwargs.get("folder_format", FOLDER_FORMAT)
quality = min(quality, self.client.max_quality)
folder = self._get_formatted_folder(parent_folder, quality)
# choose optimal cover size and download it
self.download_message()
click.secho("Downloading cover art", fg="magenta")
cover_path = os.path.join(gettempdir(), f"cover_{hash(self)}.jpg")
embed_cover_size = kwargs.get("embed_cover_size", "large")
assert (
embed_cover_size in self.cover_urls
), f"Invalid cover size. Must be in {self.cover_urls.keys()}"
tqdm_download(self.cover_urls[embed_cover_size], cover_path)
if kwargs.get("keep_hires_cover", True):
tqdm_download(self.cover_urls['original'], os.path.join(folder, 'cover.jpg'))
cover_size = os.path.getsize(cover_path)
if cover_size > FLAC_MAX_BLOCKSIZE: # 16.77 MB
click.secho(
"Downgrading embedded cover size, too large ({cover_size}).",
fg="bright_yellow",
)
# large is about 600x600px which is guaranteed < 16.7 MB
tqdm_download(self.cover_urls["large"], cover_path)
embed_cover = kwargs.get("embed_cover", True) # embed by default
if self.client.source != "deezer" and embed_cover:
cover = self.get_cover_obj(cover_path, quality, self.client.source)
download_args = {
"quality": quality,
"parent_folder": folder,
"progress_bar": kwargs.get("progress_bar", True),
"database": database,
"track_format": kwargs.get("track_format", TRACK_FORMAT),
"stay_temp": kwargs.get("stay_temp")
}
for track in self:
logger.debug("Downloading track to %s", folder)
if self.disctotal > 1:
disc_folder = os.path.join(folder, f"Disc {track.meta.discnumber}")
download_args["parent_folder"] = disc_folder
track.download(quality=quality, parent_folder=folder, database=database, **kwargs)
# deezer tracks come tagged
if kwargs.get("tag_tracks", True) and self.client.source != "deezer":
track.tag(cover=cover, embed_cover=embed_cover)
os.remove(cover_path)
self.downloaded = True
def _get_formatter(self) -> dict:
fmt = dict()
for key in ALBUM_KEYS:
@ -1035,6 +1125,32 @@ class Album(Tracklist):
return os.path.join(parent_folder, formatted_folder)
@property
def title(self) -> str:
"""Return the title of the album.
It is formatted so that "version" keys are included.
:rtype: str
"""
album_title = self._title
if hasattr(self, "version") and isinstance(self.version, str):
if self.version.lower() not in album_title.lower():
album_title = f"{album_title} ({self.version})"
if self.get("explicit", False):
album_title = f"{album_title} (Explicit)"
return album_title
@title.setter
def title(self, val):
"""Sets the internal _title attribute to the given value.
:param val: title to set
"""
self._title = val
def __repr__(self) -> str:
"""Return a string representation of this Album object.
Useful for pprint and json.dumps.
@ -1234,7 +1350,9 @@ class Playlist(Tracklist):
track.meta["tracknumber"] = str(i + 1)
if (
track.download(parent_folder=folder, quality=quality, database=database, **kwargs)
track.download(
parent_folder=folder, quality=quality, database=database, **kwargs
)
and self.client.source != "deezer"
):

2
streamrip/utils.py
View file

@ -8,11 +8,13 @@ import requests
from Crypto.Cipher import AES
from Crypto.Util import Counter
from pathvalidate import sanitize_filename
from requests.packages import urllib3
from tqdm import tqdm
from .constants import LOG_DIR, TIDAL_COVER_URL
from .exceptions import InvalidSourceError, NonStreamable
urllib3.disable_warnings()
logger = logging.getLogger(__name__)