jerry/OpenBB
1
0
Fork 0
mirror of https://github.com/OpenBB-finance/OpenBB.git synced 2026-11-04 16:32:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

[BugFix] Exclude default values when model dumping (#6516)

Browse Source 
* fix: exclude defaults when model dumping

* fix: rename method in system service

* fix: v3 to v4 keys mapping

* feat: settings json templates

* fix: unit test

* fix: unit test
This commit is contained in:
montezdesousa
2024-06-20 19:54:16 +01:00
committed by GitHub
parent ea61e6d5e7
commit 604b69ccd8
12 changed files with 77 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
openbb_platform/core/openbb_core/api/router/commands.py
View File

@@ -192,7 +192,7 @@ def build_api_wrapper(
user_settings: UserSettings = UserSettings.model_validate(
kwargs.pop(
"__authenticated_user_settings",
UserService.read_default_user_settings(),
UserService.read_from_file(),
)
)
execute = partial(command_runner.run, path, user_settings)

2
openbb_platform/core/openbb_core/app/command_runner.py
View File

@@ -410,7 +410,7 @@ class CommandRunner:
"""Initialize the command runner."""
self._command_map = command_map or CommandMap()
self._system_settings = system_settings or SystemService().system_settings
self._user_settings = user_settings or UserService.read_default_user_settings()
self._user_settings = user_settings or UserService.read_from_file()
def init_logging_service(self) -> None:
"""Initialize the logging service."""

8
openbb_platform/core/openbb_core/app/model/hub/hub_user_settings.py
View File

@@ -2,7 +2,7 @@
from typing import Any, Dict, Optional
from pydantic import BaseModel, ConfigDict, Field
from pydantic import BaseModel, ConfigDict, Field, field_validator
class HubUserSettings(BaseModel):
@@ -14,3 +14,9 @@ class HubUserSettings(BaseModel):
# features_terminal_style: Dict[str, Union[str, Dict[str, str]]]
model_config = ConfigDict(validate_assignment=True)
@field_validator("features_keys", mode="before", check_fields=False)
@classmethod
def to_lower(cls, d: dict) -> dict:
"""Convert dict keys to lowercase."""
return {k.lower(): v for k, v in d.items()}

15
openbb_platform/core/openbb_core/app/model/system_settings.py
View File

@@ -66,9 +66,9 @@ class SystemSettings(Tagged):
)
@staticmethod
def create_empty_json(path: Path) -> None:
def create_json(path: Path, template: Optional[dict] = None) -> None:
"""Create an empty JSON file."""
path.write_text(json.dumps({}), encoding="utf-8")
path.write_text(json.dumps(obj=template or {}, indent=4), encoding="utf-8")
# TODO: Figure out why this works only opposite to what the docs say
# https://docs.pydantic.dev/latest/concepts/validators/#model-validators
@@ -82,9 +82,14 @@ class SystemSettings(Tagged):
system_settings = Path(values.system_settings_path).resolve()
obb_dir.mkdir(parents=True, exist_ok=True)
for path in [user_settings, system_settings]:
if not path.exists():
cls.create_empty_json(path)
if not user_settings.exists():
cls.create_json(
user_settings,
{"credentials": {}, "preferences": {}, "defaults": {"commands": {}}},
)
if not system_settings.exists():
cls.create_json(system_settings, {})
return values

36
openbb_platform/core/openbb_core/app/service/hub_service.py
View File

@@ -22,14 +22,14 @@ class HubService:
TIMEOUT = 10
# Mapping of V3 keys to V4 keys for backward compatibility
V3TOV4 = {
"API_KEY_ALPHAVANTAGE": "alpha_vantage_api_key",
"API_BIZTOC_TOKEN": "biztoc_api_key",
"API_FRED_KEY": "fred_api_key",
"API_KEY_FINANCIALMODELINGPREP": "fmp_api_key",
"API_INTRINIO_KEY": "intrinio_api_key",
"API_POLYGON_KEY": "polygon_api_key",
"API_KEY_QUANDL": "nasdaq_api_key",
"API_TRADIER_TOKEN": "tradier_api_key",
"api_key_alphavantage": "alpha_vantage_api_key",
"api_biztoc_token": "biztoc_api_key",
"api_fred_key": "fred_api_key",
"api_key_financialmodelingprep": "fmp_api_key",
"api_intrinio_key": "intrinio_api_key",
"api_polygon_key": "polygon_api_key",
"api_key_quandl": "nasdaq_api_key",
"api_tradier_token": "tradier_api_key",
}
V4TOV3 = {v: k for k, v in V3TOV4.items()}
@@ -218,7 +218,7 @@ class HubService:
response = put(
url=self._base_url + "/user",
headers={"Authorization": authorization},
json=settings.model_dump(),
json=settings.model_dump(exclude_defaults=True),
timeout=self.TIMEOUT,
)
@@ -230,13 +230,13 @@ class HubService:
def hub2platform(self, settings: HubUserSettings) -> Tuple[Credentials, Defaults]:
"""Convert Hub user settings to Platform models."""
if any(k in settings.features_keys for k in self.V3TOV4):
deprecated = {
k: v for k, v in self.V3TOV4.items() if k in settings.features_keys
}
deprecated = {
k: v for k, v in self.V3TOV4.items() if k in settings.features_keys
}
if deprecated:
msg = ""
for k, v in deprecated.items():
msg += f"\n'{k}' -> '{v.upper()}', "
msg += f"\n'{k.upper()}' -> '{v.upper()}', "
msg = msg.strip(", ")
warn(
message=f"\nDeprecated v3 credentials found.\n{msg}"
@@ -255,14 +255,18 @@ class HubService:
) -> HubUserSettings:
"""Convert Platform models to Hub user settings."""
# Dump mode json ensures SecretStr values are serialized as strings
credentials = credentials.model_dump(mode="json", exclude_none=True)
credentials = credentials.model_dump(
mode="json", exclude_none=True, exclude_defaults=True
)
settings = self._hub_user_settings or HubUserSettings()
for v4_k, v in sorted(credentials.items()):
v3_k = self.V4TOV3.get(v4_k, None)
# If v3 key was in the hub already, we keep it
k = v3_k if v3_k in settings.features_keys else v4_k
settings.features_keys[k] = v
defaults_ = defaults.model_dump(mode="json", exclude_none=True)
defaults_ = defaults.model_dump(
mode="json", exclude_none=True, exclude_defaults=True
)
settings.features_settings.update({"defaults": defaults_})
return settings

14
openbb_platform/core/openbb_core/app/service/system_service.py
View File

@@ -31,7 +31,7 @@ class SystemService(metaclass=SingletonMeta):
**kwargs,
):
"""Initialize system service."""
self._system_settings = self._read_default_system_settings(
self._system_settings = self._read_from_file(
path=self.SYSTEM_SETTINGS_PATH, **kwargs
)
@@ -46,9 +46,7 @@ class SystemService(metaclass=SingletonMeta):
return hashed_input == existing_hash
@classmethod
def _read_default_system_settings(
cls, path: Optional[Path] = None, **kwargs
) -> SystemSettings:
def _read_from_file(cls, path: Optional[Path] = None, **kwargs) -> SystemSettings:
"""Read default system settings."""
path = path or cls.SYSTEM_SETTINGS_PATH
@@ -77,7 +75,7 @@ class SystemService(metaclass=SingletonMeta):
return system_settings
@classmethod
def write_default_system_settings(
def write_to_file(
cls,
system_settings: SystemSettings,
path: Optional[Path] = None,
@@ -86,7 +84,9 @@ class SystemService(metaclass=SingletonMeta):
path = path or cls.SYSTEM_SETTINGS_PATH
system_settings_json = system_settings.model_dump_json(
include=cls.SYSTEM_SETTINGS_ALLOWED_FIELD_SET, indent=4
indent=4,
include=cls.SYSTEM_SETTINGS_ALLOWED_FIELD_SET,
exclude_defaults=True,
)
with path.open(mode="w") as file:
file.write(system_settings_json)
@@ -103,6 +103,6 @@ class SystemService(metaclass=SingletonMeta):
def refresh_system_settings(self) -> SystemSettings:
"""Refresh system settings."""
self._system_settings = self._read_default_system_settings()
self._system_settings = self._read_from_file()
return self._system_settings

15
openbb_platform/core/openbb_core/app/service/user_service.py
View File

@@ -21,13 +21,11 @@ class UserService(metaclass=SingletonMeta):
default_user_settings: Optional[UserSettings] = None,
):
"""Initialize user service."""
self._default_user_settings = (
default_user_settings or self.read_default_user_settings()
)
self._default_user_settings = default_user_settings or self.read_from_file()
@classmethod
def read_default_user_settings(cls, path: Optional[Path] = None) -> UserSettings:
"""Read default user settings."""
def read_from_file(cls, path: Optional[Path] = None) -> UserSettings:
"""Read user settings from json into UserSettings."""
path = path or cls.USER_SETTINGS_PATH
return (
@@ -37,16 +35,15 @@ class UserService(metaclass=SingletonMeta):
)
@classmethod
def write_default_user_settings(
def write_to_file(
cls,
user_settings: UserSettings,
path: Optional[Path] = None,
) -> None:
"""Write default user settings."""
"""Write user settings to json."""
path = path or cls.USER_SETTINGS_PATH
user_settings_json = user_settings.model_dump_json(
include=cls.USER_SETTINGS_ALLOWED_FIELD_SET, indent=4
indent=4, include=cls.USER_SETTINGS_ALLOWED_FIELD_SET, exclude_defaults=True
)
path.write_text(user_settings_json, encoding="utf-8")

12
openbb_platform/core/openbb_core/app/static/account.py
View File

@@ -156,9 +156,7 @@ class Account: # noqa: D205, D400
User settings: profile, credentials, preferences
"""
if not self._hub_service:
UserService.write_default_user_settings(
self._base_app._command_runner.user_settings
)
UserService.write_to_file(self._base_app._command_runner.user_settings)
else:
self._hub_service.push(self._base_app._command_runner.user_settings)
@@ -181,9 +179,7 @@ class Account: # noqa: D205, D400
User settings: profile, credentials, preferences
"""
if not self._hub_service:
self._base_app._command_runner.user_settings = (
UserService.read_default_user_settings()
)
self._base_app._command_runner.user_settings = UserService.read_from_file()
else:
incoming = self._hub_service.pull()
self._base_app.user.profile = incoming.profile
@@ -216,9 +212,7 @@ class Account: # noqa: D205, D400
if session_file.exists():
session_file.unlink()
self._base_app._command_runner.user_settings = (
UserService.read_default_user_settings()
)
self._base_app._command_runner.user_settings = UserService.read_from_file()
if return_settings:
return self._base_app._command_runner.user_settings

20
openbb_platform/core/tests/app/service/test_hub_service.py
View File

@@ -256,9 +256,9 @@ def test_hub2platform_v3_only():
"""Test hub2platform."""
mock_user_settings = MagicMock(spec=HubUserSettings)
mock_user_settings.features_keys = {
"API_KEY_FINANCIALMODELINGPREP": "abc",
"API_POLYGON_KEY": "def",
"API_FRED_KEY": "ghi",
"api_key_financialmodelingprep": "abc",
"api_polygon_key": "def",
"api_fred_key": "ghi",
}
mock_user_settings.features_settings = {}
@@ -273,10 +273,10 @@ def test_hub2platform_v3v4():
"""Test hub2platform."""
mock_user_settings = MagicMock(spec=HubUserSettings)
mock_user_settings.features_keys = {
"API_KEY_FINANCIALMODELINGPREP": "abc",
"api_key_financialmodelingprep": "abc",
"fmp_api_key": "other_key",
"API_POLYGON_KEY": "def",
"API_FRED_KEY": "ghi",
"api_polygon_key": "def",
"api_fred_key": "ghi",
}
mock_user_settings.features_settings = {}
@@ -291,9 +291,9 @@ def test_platform2hub():
"""Test platform2hub."""
mock_user_settings = MagicMock(spec=HubUserSettings)
mock_user_settings.features_keys = { # Received from Hub
"API_KEY_FINANCIALMODELINGPREP": "abc",
"api_key_financialmodelingprep": "abc",
"fmp_api_key": "other_key",
"API_FRED_KEY": "ghi",
"api_fred_key": "ghi",
}
mock_user_settings.features_settings = {}
mock_hub_service = HubService()
@@ -309,10 +309,10 @@ def test_platform2hub():
user_settings = mock_hub_service.platform2hub(mock_credentials, mock_defaults)
assert isinstance(user_settings, HubUserSettings)
assert user_settings.features_keys["API_KEY_FINANCIALMODELINGPREP"] == "fmp"
assert user_settings.features_keys["api_key_financialmodelingprep"] == "fmp"
assert user_settings.features_keys["fmp_api_key"] == "other_key"
assert user_settings.features_keys["polygon_api_key"] == "polygon"
assert user_settings.features_keys["API_FRED_KEY"] == "fred"
assert user_settings.features_keys["api_fred_key"] == "fred"
assert user_settings.features_keys["benzinga_api_key"] == "benzinga"
assert "some_api_key" not in user_settings.features_keys
assert "defaults" in user_settings.features_settings

12
openbb_platform/core/tests/app/service/test_system_service.py
View File

@@ -17,17 +17,19 @@ def test_system_service_init(system_service):
assert system_service
def test_read_default_system_settings(system_service):
def test_read_from_file(system_service):
"""Test read default system settings."""
system_settings = system_service._read_default_system_settings()
# pylint: disable=protected-access
system_settings = system_service._read_from_file()
assert system_settings
def test_write_default_system_settings(system_service):
def test_write_to_file(system_service):
"""Test write default system settings."""
system_settings = system_service._read_default_system_settings()
system_service.write_default_system_settings(system_settings=system_settings)
# pylint: disable=protected-access
system_settings = system_service._read_from_file()
system_service.write_to_file(system_settings=system_settings)
assert system_service

8
openbb_platform/core/tests/app/service/test_user_service.py
View File

@@ -10,15 +10,15 @@ from openbb_core.app.service.user_service import (
)
def test_read_default_user_settings_file_exists():
def test_read_from_file_file_exists():
"""Test read default user settings."""
result = UserService.read_default_user_settings(path=Path("some_path"))
result = UserService.read_from_file(path=Path("some_path"))
assert result
assert isinstance(result, UserSettings)
def test_write_default_user_settings():
def test_write_to_file():
"""Test write default user settings."""
# Create a temporary file for this test
with tempfile.NamedTemporaryFile(delete=False) as temp_file:
@@ -31,7 +31,7 @@ def test_write_default_user_settings():
user_settings.defaults = {"language": "en"} # type: ignore[assignment]
# Write the user settings to the temporary file
UserService.write_default_user_settings(user_settings, temp_path)
UserService.write_to_file(user_settings, temp_path)
# Read the file and verify its contents
with open(temp_path, encoding="utf-8") as file:

2
openbb_platform/core/tests/app/static/test_container.py
View File

@@ -89,7 +89,7 @@ def test_container__check_credentials(container):
OpenBBError,
escape(
"Provider fallback failed."
"\n[Providers]\n * 'x' -> not found\n * 'y' -> not found\n * 'z' -> not found"
"\n[Providers]\n * 'x' -> not installed, please install openbb-x\n * 'y' -> not installed, please install openbb-y\n * 'z' -> not installed, please install openbb-z"
),
),
],