From ad73a01a03f44b5cdd65087c144df0292fde62fd Mon Sep 17 00:00:00 2001 From: Nathan Thomas Date: Tue, 14 May 2024 15:45:46 -0700 Subject: [PATCH] Preserve previous config data after update (#680) * Add config updating mechanism * Update tests * Fix version not updating --- streamrip/config.py | 89 +++++++++++++++++++++-- streamrip/rip/cli.py | 7 +- tests/silence.flac | Bin 33300 -> 50902 bytes tests/test_config.py | 96 +++++++++++++++++++++++++ tests/test_config_old.toml | 142 +++++++++++++++++++++++++++++++++++++ 5 files changed, 329 insertions(+), 5 deletions(-) create mode 100644 tests/test_config_old.toml diff --git a/streamrip/config.py b/streamrip/config.py index d8ee2cb..9705049 100644 --- a/streamrip/config.py +++ b/streamrip/config.py @@ -1,6 +1,7 @@ -"""A config class that manages arguments between the config file and CLI.""" +"""Classes and functions that manage config state.""" import copy +import functools import logging import os import shutil @@ -19,6 +20,10 @@ DEFAULT_CONFIG_PATH = os.path.join(APP_DIR, "config.toml") CURRENT_CONFIG_VERSION = "2.0.6" +class OutdatedConfigError(Exception): + pass + + @dataclass(slots=True) class QobuzConfig: use_auth_token: bool @@ -262,7 +267,7 @@ class ConfigData: # TODO: handle the mistake where Windows people forget to escape backslash toml = parse(toml_str) if (v := toml["misc"]["version"]) != CURRENT_CONFIG_VERSION: # type: ignore - raise Exception( + raise OutdatedConfigError( f"Need to update config from {v} to {CURRENT_CONFIG_VERSION}", ) @@ -367,6 +372,26 @@ class Config: self.file.update_toml() toml_file.write(dumps(self.file.toml)) + @staticmethod + def _update_file(old_path: str, new_path: str): + """Updates the current config based on a newer config `new_toml`.""" + with open(new_path) as new_conf: + new_toml = parse(new_conf.read()) + + toml_set_user_defaults(new_toml) + + with open(old_path) as old_conf: + old_toml = parse(old_conf.read()) + + update_config(old_toml, new_toml) + + with open(old_path, "w") as f: + f.write(dumps(new_toml)) + + @classmethod + def update_file(cls, path: str): + cls._update_file(path, BLANK_CONFIG_PATH) + @classmethod def defaults(cls): return cls(BLANK_CONFIG_PATH) @@ -384,9 +409,65 @@ def set_user_defaults(path: str, /): with open(path) as f: toml = parse(f.read()) + + toml_set_user_defaults(toml) + + with open(path, "w") as f: + f.write(dumps(toml)) + + +def toml_set_user_defaults(toml: TOMLDocument): toml["downloads"]["folder"] = DEFAULT_DOWNLOADS_FOLDER # type: ignore toml["database"]["downloads_path"] = DEFAULT_DOWNLOADS_DB_PATH # type: ignore toml["database"]["failed_downloads_path"] = DEFAULT_FAILED_DOWNLOADS_DB_PATH # type: ignore toml["youtube"]["video_downloads_folder"] = DEFAULT_YOUTUBE_VIDEO_DOWNLOADS_FOLDER # type: ignore - with open(path, "w") as f: - f.write(dumps(toml)) + + +def _get_dict_keys_r(d: dict) -> set[tuple]: + """Get all possible key combinations in nested dicts. + + See tests/test_config.py for example. + """ + keys = d.keys() + ret = set() + for cur in keys: + val = d[cur] + if isinstance(val, dict): + ret.update((cur, *remaining) for remaining in _get_dict_keys_r(val)) + else: + ret.add((cur,)) + return ret + + +def _nested_get(dictionary, *keys, default=None): + return functools.reduce( + lambda d, key: d.get(key, default) if isinstance(d, dict) else default, + keys, + dictionary, + ) + + +def _nested_set(dictionary, *keys, val): + """Nested set. Throws exception if keys are invalid.""" + assert len(keys) > 0 + final = functools.reduce(lambda d, key: d.get(key), keys[:-1], dictionary) + final[keys[-1]] = val + + +def update_config(old_with_data: dict, new_without_data: dict): + """Used to update config when a new config version is detected. + + All data associated with keys that are shared between the old and + new configs are copied from old to new. The remaining keep their default value. + + Assumes that new_without_data contains default config values of the + latest version. + """ + old_keys = _get_dict_keys_r(old_with_data) + new_keys = _get_dict_keys_r(new_without_data) + common = old_keys.intersection(new_keys) + common.discard(("misc", "version")) + + for k in common: + old_val = _nested_get(old_with_data, *k) + _nested_set(new_without_data, *k, val=old_val) diff --git a/streamrip/rip/cli.py b/streamrip/rip/cli.py index 0e0e517..2864ab7 100644 --- a/streamrip/rip/cli.py +++ b/streamrip/rip/cli.py @@ -17,7 +17,7 @@ from rich.prompt import Confirm from rich.traceback import install from .. import __version__, db -from ..config import DEFAULT_CONFIG_PATH, Config, set_user_defaults +from ..config import DEFAULT_CONFIG_PATH, Config, OutdatedConfigError, set_user_defaults from ..console import console from .main import Main @@ -116,6 +116,11 @@ def rip(ctx, config_path, folder, no_db, quality, codec, no_progress, verbose): try: c = Config(config_path) + except OutdatedConfigError as e: + console.print(e) + console.print("Auto-updating config file...") + Config.update_file(config_path) + c = Config(config_path) except Exception as e: console.print( f"Error loading config from [bold cyan]{config_path}[/bold cyan]: {e}\n" diff --git a/tests/silence.flac b/tests/silence.flac index 164e81a0a3777c9036c15249dfc3b58fa057019d..7db44ce17764e5e2e9401d9669557c2006f9d3ef 100644 GIT binary patch delta 24 ecmbQz!gQ^ddBeTr%^NiJtboK6uFdz7&Ex=%Sqj(y delta 14 WcmccC%RHrpX~Vtb%?nnT$pHX03I@Rd diff --git a/tests/test_config.py b/tests/test_config.py index c8a9da2..f4911ef 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,10 +1,14 @@ +import os import shutil import pytest +import tomlkit from streamrip.config import * +from streamrip.config import _get_dict_keys_r, _nested_set SAMPLE_CONFIG = "tests/test_config.toml" +OLD_CONFIG = "tests/test_config_old.toml" # Define a fixture to create a sample ConfigData instance for testing @@ -26,6 +30,98 @@ def sample_config() -> Config: return config +def test_get_keys_r(): + d = { + "key1": { + "key2": { + "key3": 1, + "key4": 1, + }, + "key6": [1, 2], + 5: 1, + } + } + res = _get_dict_keys_r(d) + print(res) + assert res == { + ("key1", "key2", "key3"), + ("key1", "key2", "key4"), + ("key1", "key6"), + ("key1", 5), + } + + +def test_safe_set(): + d = { + "key1": { + "key2": { + "key3": 1, + "key4": 1, + }, + "key6": [1, 2], + 5: 1, + } + } + _nested_set(d, "key1", "key2", "key3", val=5) + assert d == { + "key1": { + "key2": { + "key3": 5, + "key4": 1, + }, + "key6": [1, 2], + 5: 1, + } + } + + +def test_config_update(): + old = { + "downloads": {"folder": "some_path", "use_service": True}, + "qobuz": {"email": "asdf@gmail.com", "password": "test"}, + "legacy_conf": {"something": 1, "other": 2}, + } + new = { + "downloads": {"folder": "", "use_service": False, "keep_artwork": True}, + "qobuz": {"email": "", "password": ""}, + "tidal": {"email": "", "password": ""}, + } + update_config(old, new) + assert new == { + "downloads": {"folder": "some_path", "use_service": True, "keep_artwork": True}, + "qobuz": {"email": "asdf@gmail.com", "password": "test"}, + "tidal": {"email": "", "password": ""}, + } + + +def test_config_throws_outdated(): + with pytest.raises(Exception, match="update"): + _ = Config(OLD_CONFIG) + + +def test_config_file_update(): + tmp_conf = "tests/test_config_old2.toml" + shutil.copy("tests/test_config_old.toml", tmp_conf) + Config._update_file(tmp_conf, SAMPLE_CONFIG) + + with open(tmp_conf) as f: + s = f.read() + toml = tomlkit.parse(s) # type: ignore + + assert toml["downloads"]["folder"] == "old_value" # type: ignore + assert toml["downloads"]["source_subdirectories"] is True # type: ignore + assert toml["downloads"]["concurrency"] is True # type: ignore + assert toml["downloads"]["max_connections"] == 6 # type: ignore + assert toml["downloads"]["requests_per_minute"] == 60 # type: ignore + assert toml["cli"]["text_output"] is True # type: ignore + assert toml["cli"]["progress_bars"] is True # type: ignore + assert toml["cli"]["max_search_results"] == 100 # type: ignore + assert toml["misc"]["version"] == "2.0.6" # type: ignore + assert "YouTubeVideos" in str(toml["youtube"]["video_downloads_folder"]) + # type: ignore + os.remove("tests/test_config_old2.toml") + + def test_sample_config_data_properties(sample_config_data): # Test the properties of ConfigData assert sample_config_data.modified is False # Ensure initial state is not modified diff --git a/tests/test_config_old.toml b/tests/test_config_old.toml new file mode 100644 index 0000000..018c5ea --- /dev/null +++ b/tests/test_config_old.toml @@ -0,0 +1,142 @@ +[downloads] +# Folder where tracks are downloaded to +folder = "old_value" +# Put Qobuz albums in a 'Qobuz' folder, Tidal albums in 'Tidal' etc. +source_subdirectories = true + +[qobuz] +# 1: 320kbps MP3, 2: 16/44.1, 3: 24/<=96, 4: 24/>=96 +quality = 3 + +# Authenticate to Qobuz using auth token? Value can be true/false only +use_auth_token = false +# Enter your userid if the above use_auth_token is set to true, else enter your email +email_or_userid = "old_test@gmail.com" +# Enter your auth token if the above use_auth_token is set to true, else enter the md5 hash of your plaintext password +password_or_token = "old_test_pwd" +# Do not change +app_id = "old_12345" +# Do not change +secrets = ['old_secret1', 'old_secret2'] + +[tidal] +# 0: 256kbps AAC, 1: 320kbps AAC, 2: 16/44.1 "HiFi" FLAC, 3: 24/44.1 "MQA" FLAC +quality = 3 +# This will download videos included in Video Albums. +download_videos = true + +# Do not change any of the fields below +user_id = "old_userid" +country_code = "old_countrycode" +access_token = "old_accesstoken" +refresh_token = "old_refreshtoken" +# Tokens last 1 week after refresh. This is the Unix timestamp of the expiration +# time. If you haven't used streamrip in more than a week, you may have to log +# in again using `rip config --tidal` +token_expiry = "old_tokenexpiry" + +[deezer] +# 0, 1, or 2 +# This only applies to paid Deezer subscriptions. Those using deezloader +# are automatically limited to quality = 1 +quality = 2 +# An authentication cookie that allows streamrip to use your Deezer account +# See https://github.com/nathom/streamrip/wiki/Finding-Your-Deezer-ARL-Cookie +# for instructions on how to find this +arl = "old_testarl" +# This allows for free 320kbps MP3 downloads from Deezer +# If an arl is provided, deezloader is never used +use_deezloader = true +# This warns you when the paid deezer account is not logged in and rip falls +# back to deezloader, which is unreliable +deezloader_warnings = true + +[soundcloud] +# Only 0 is available for now +quality = 0 +# This changes periodically, so it needs to be updated +client_id = "old_clientid" +app_version = "old_appversion" + +[youtube] +# Only 0 is available for now +quality = 0 +# Download the video along with the audio +download_videos = false + +[database] +# Create a database that contains all the track IDs downloaded so far +# Any time a track logged in the database is requested, it is skipped +# This can be disabled temporarily with the --no-db flag +downloads_enabled = true +# Path to the downloads database +downloads_path = "old_downloadspath" +# If a download fails, the item ID is stored here. Then, `rip repair` can be +# called to retry the downloads +failed_downloads_enabled = true +failed_downloads_path = "old_faileddownloadspath" + +# Convert tracks to a codec after downloading them. +[conversion] +enabled = false +# FLAC, ALAC, OPUS, MP3, VORBIS, or AAC +codec = "old_ALAC" +# In Hz. Tracks are downsampled if their sampling rate is greater than this. +# Value of 48000 is recommended to maximize quality and minimize space +sampling_rate = 48000 +# Only 16 and 24 are available. It is only applied when the bit depth is higher +# than this value. +bit_depth = 24 +# Only applicable for lossy codecs +lossy_bitrate = 320 + +# Filter a Qobuz artist's discography. Set to 'true' to turn on a filter. +[qobuz_filters] +# Remove Collectors Editions, live recordings, etc. +extras = false +# Picks the highest quality out of albums with identical titles. +repeats = false +# Remove EPs and Singles +non_albums = false +# Remove albums whose artist is not the one requested +features = false +# Skip non studio albums +non_studio_albums = false +# Only download remastered albums +non_remaster = false + +[artwork] +# Write the image to the audio file +embed = true +# The size of the artwork to embed. Options: thumbnail, small, large, original. +# "original" images can be up to 30MB, and may fail embedding. +# Using "large" is recommended. +embed_size = "old_large" + + +[metadata] +# Sets the value of the 'ALBUM' field in the metadata to the playlist's name. +# This is useful if your music library software organizes tracks based on album name. +set_playlist_to_album = true +# If part of a playlist, sets the `tracknumber` field in the metadata to the track's +# position in the playlist instead of its position in its album +renumber_playlist_tracks = true +# The following metadata tags won't be applied +# See https://github.com/nathom/streamrip/wiki/Metadata-Tag-Names for more info +exclude = [] + +# Changes the folder and file names generated by streamrip. +[filepaths] +# Create folders for single tracks within the downloads directory using the folder_format +# template +add_singles_to_folder = false +# Available keys: "albumartist", "title", "year", "bit_depth", "sampling_rate", +# "id", and "albumcomposer" +folder_format = "old_{albumartist} - {title} ({year}) [{container}] [{bit_depth}B-{sampling_rate}kHz]" +# Available keys: "tracknumber", "artist", "albumartist", "composer", "title", +# and "albumcomposer", "explicit" + +[misc] +# Metadata to identify this config file. Do not change. +version = "0.0.1" +check_for_updates = true