From d197acf220cd19e6e5eb8d2ddca9b743797616c0 Mon Sep 17 00:00:00 2001 From: montezdesousa Date: Sat, 8 Jul 2023 18:05:56 +0100 Subject: [PATCH] add support for default providers by route --- .../openbb_sdk_core/api/rest_api.py | 5 +- .../openbb_sdk_core/api/router/account.py | 3 +- .../openbb_sdk_core/app/command_runner.py | 37 +++++++++++++- .../openbb_sdk_core/app/model/defaults.py | 11 ++--- .../openbb_sdk_core/app/provider_interface.py | 3 +- .../openbb_sdk_core/app/router.py | 49 ++++++++++++++----- .../app/service/user_service.py | 32 +++++++++++- .../app/static/package_builder.py | 6 +-- .../openbb_sdk_core/app/static/parsers.py | 23 +++++++++ .../openbb_sdk_core/app/static/utils.py | 45 ----------------- .../settings/user_settings.json.tpl | 13 +++++ 11 files changed, 156 insertions(+), 71 deletions(-) create mode 100644 openbb_sdk/openbb_sdk_core/openbb_sdk_core/app/static/parsers.py delete mode 100644 openbb_sdk/openbb_sdk_core/openbb_sdk_core/app/static/utils.py diff --git a/openbb_sdk/openbb_sdk_core/openbb_sdk_core/api/rest_api.py b/openbb_sdk/openbb_sdk_core/openbb_sdk_core/api/rest_api.py index 2766f51f0f7..3a9463f81bf 100644 --- a/openbb_sdk/openbb_sdk_core/openbb_sdk_core/api/rest_api.py +++ b/openbb_sdk/openbb_sdk_core/openbb_sdk_core/api/rest_api.py @@ -69,7 +69,10 @@ async def setup_default_user_settings(): user_settings_repository.update(model=user_settings) else: default_user_settings = UserSettings( - profile=default_profile, credentials=default_user_settings.credentials + profile=default_profile, + credentials=default_user_settings.credentials, + preferences=default_user_settings.preferences, + defaults=default_user_settings.defaults, ) user_settings_repository.create(model=default_user_settings) diff --git a/openbb_sdk/openbb_sdk_core/openbb_sdk_core/api/router/account.py b/openbb_sdk/openbb_sdk_core/openbb_sdk_core/api/router/account.py index 5e67f363abf..2300dbc67ac 100644 --- a/openbb_sdk/openbb_sdk_core/openbb_sdk_core/api/router/account.py +++ b/openbb_sdk/openbb_sdk_core/openbb_sdk_core/api/router/account.py @@ -36,7 +36,8 @@ async def login_for_access_token( if user_settings: user_settings.profile.username = form_data.username user_settings.profile.password_hash = get_password_hash(form_data.password) - user_service.user_settings_repository.create(user_settings) + updated_user_settings = UserService.update_default(user_settings) + user_service.user_settings_repository.create(updated_user_settings) if not user_settings: raise HTTPException( diff --git a/openbb_sdk/openbb_sdk_core/openbb_sdk_core/app/command_runner.py b/openbb_sdk/openbb_sdk_core/openbb_sdk_core/app/command_runner.py index 988f1848875..618e890ff78 100644 --- a/openbb_sdk/openbb_sdk_core/openbb_sdk_core/app/command_runner.py +++ b/openbb_sdk/openbb_sdk_core/openbb_sdk_core/app/command_runner.py @@ -134,6 +134,30 @@ class ParametersBuilder: return kwargs + @staticmethod + def update_provider_choices( + route: str, + kwargs: Dict[str, Any], + user_settings: UserSettings, + ) -> Dict[str, Any]: + provider_choices = kwargs.get("provider_choices", None) + if provider_choices and isinstance(provider_choices, dict): + provider = provider_choices.get("provider", None) + if provider is None: + route_defaults = user_settings.defaults.routes.get(route, None) + if route_defaults is None: + raise ValueError( + f"'provider' not found: route '{route}' not found in settings.defaults." + ) + + provider = route_defaults.get("provider", None) + if provider is None: + raise ValueError( + f"'provider' not found: default 'provider' not found in settings.defaults for '{route}'." + ) + kwargs["provider_choices"] = {"provider": provider} + return kwargs + @classmethod def validate_kwargs( cls, @@ -166,6 +190,7 @@ class ParametersBuilder: args: Tuple[Any], execution_context: ExecutionContext, func: Callable, + route: str, kwargs: Dict[str, Any], ) -> Dict[str, Any]: func = cls.get_polished_func(func=func) @@ -190,6 +215,11 @@ class ParametersBuilder: system_settings=system_settings, user_settings=user_settings, ) + kwargs = cls.update_provider_choices( + route=route, + kwargs=kwargs, + user_settings=user_settings, + ) kwargs = cls.validate_kwargs(func=func, kwargs=kwargs) return kwargs @@ -221,6 +251,7 @@ class StaticCommandRunner: args=args, execution_context=execution_context, func=func, + route=route, kwargs=kwargs, ) @@ -384,7 +415,11 @@ class CommandRunnerSession: ) -> None: self._command_runner = command_runner or CommandRunner() self._journal = journal or Journal() - self._user_settings = user_settings or UserService.read_default_user_settings() + self._user_settings = ( + UserService.update_default(user_settings) + if user_settings + else UserService.read_default_user_settings() + ) @property def command_runner(self) -> CommandRunner: diff --git a/openbb_sdk/openbb_sdk_core/openbb_sdk_core/app/model/defaults.py b/openbb_sdk/openbb_sdk_core/openbb_sdk_core/app/model/defaults.py index b18ec454223..8170a1a31c7 100644 --- a/openbb_sdk/openbb_sdk_core/openbb_sdk_core/app/model/defaults.py +++ b/openbb_sdk/openbb_sdk_core/openbb_sdk_core/app/model/defaults.py @@ -1,16 +1,15 @@ from typing import Dict -from pydantic import BaseModel +from pydantic import BaseModel, Field class Defaults(BaseModel): """Defaults.""" - # TODO: Understand if this is good and if it should be in the hub or just locally - endpoints: Dict[str, Dict[str, str]] = { - "/stocks/load": {"provider": "polygon"}, - "/stocks/news": {"provider": "fmp"}, - } + class Config: + validate_assignment = True + + routes: Dict[str, Dict[str, str]] = Field(default_factory=dict) def __repr__(self) -> str: return ( diff --git a/openbb_sdk/openbb_sdk_core/openbb_sdk_core/app/provider_interface.py b/openbb_sdk/openbb_sdk_core/openbb_sdk_core/app/provider_interface.py index 0b8eb5708de..f928088eb95 100644 --- a/openbb_sdk/openbb_sdk_core/openbb_sdk_core/app/provider_interface.py +++ b/openbb_sdk/openbb_sdk_core/openbb_sdk_core/app/provider_interface.py @@ -92,10 +92,11 @@ class ProviderInterface: return Literal[tuple(providers)] # type: ignore @property - def provider_choices(self) -> type: + def provider_choices(self) -> ProviderChoices: return make_dataclass( cls_name="ProviderChoices", fields=[("provider", self.providers_literal)], + bases=(ProviderChoices,), ) @property diff --git a/openbb_sdk/openbb_sdk_core/openbb_sdk_core/app/router.py b/openbb_sdk/openbb_sdk_core/openbb_sdk_core/app/router.py index de8921b1042..76bc2ba1e14 100644 --- a/openbb_sdk/openbb_sdk_core/openbb_sdk_core/app/router.py +++ b/openbb_sdk/openbb_sdk_core/openbb_sdk_core/app/router.py @@ -1,5 +1,7 @@ import sys -from inspect import signature +from functools import partial +from inspect import Parameter, Signature, signature +from types import MappingProxyType from typing import ( Annotated, Any, @@ -9,6 +11,7 @@ from typing import ( Optional, Type, get_args, + get_origin, get_type_hints, overload, ) @@ -23,6 +26,7 @@ from pydantic.validators import find_validators from openbb_sdk_core.app.model.command_context import CommandContext from openbb_sdk_core.app.model.command_output import CommandOutput from openbb_sdk_core.app.provider_interface import ( + ExtraParams, ProviderChoices, StandardParams, get_provider_interface, @@ -66,21 +70,44 @@ class CommandValidator: value_type=value_type ) or cls.is_valid_pydantic_model_type(model_type=value_type) + @staticmethod + def is_annotated_dc(annotation) -> bool: + return get_origin(annotation) == Annotated and hasattr( + annotation.__args__[0], "__dataclass_fields__" + ) + + @staticmethod + def check_reserved_param( + name: str, + expected_annot: Any, + parameter_map: MappingProxyType[str, Parameter], + func: Callable, + sig: Signature, + ): + if name in parameter_map: + annotation = getattr(parameter_map[name], "annotation", None) + if annotation is not None and CommandValidator.is_annotated_dc(annotation): + annotation = annotation.__args__[0].__bases__[0] + if not annotation == expected_annot: + raise TypeError( + f"The parameter `{name}` must be a {expected_annot}.\n" + f"module = {func.__module__}\n" + f"function = {func.__name__}\n" + f"signature = {sig}\n" + ) + @classmethod def check_parameters(cls, func: Callable): sig = signature(func) parameter_map = sig.parameters - if ( - "cc" in parameter_map - and not parameter_map["cc"].annotation == CommandContext - ): - raise TypeError( - "The parameter `cc` must be a CommandContext.\n" - f"module = {func.__module__}\n" - f"function = {func.__name__}\n" - f"signature = {sig}\n" - ) + check_res = partial( + cls.check_reserved_param, parameter_map=parameter_map, func=func, sig=sig + ) + check_res("cc", CommandContext) + check_res("provider_choices", ProviderChoices) + check_res("standard_params", StandardParams) + check_res("extra_params", ExtraParams) for parameter in parameter_map.values(): if not cls.is_serializable_value_type(value_type=parameter.annotation): diff --git a/openbb_sdk/openbb_sdk_core/openbb_sdk_core/app/service/user_service.py b/openbb_sdk/openbb_sdk_core/openbb_sdk_core/app/service/user_service.py index 96d6a7aebcd..b99c42619ae 100644 --- a/openbb_sdk/openbb_sdk_core/openbb_sdk_core/app/service/user_service.py +++ b/openbb_sdk/openbb_sdk_core/openbb_sdk_core/app/service/user_service.py @@ -1,5 +1,6 @@ +from functools import reduce from pathlib import Path -from typing import Optional +from typing import Any, Dict, List, MutableMapping, Optional from openbb_sdk_core.app.model.user_settings import UserSettings from openbb_sdk_core.app.repository.abstract.access_token_repository import ( @@ -26,7 +27,7 @@ from pymongo.mongo_client import MongoClient class UserService: REPOSITORY_DIRECTORY = Path(__file__).parent.parent.parent.parent USER_SETTINGS_PATH = Path(REPOSITORY_DIRECTORY, "settings", "user_settings.json") - SYSTEM_SETTINGS_ALLOWED_FIELD_SET = {"credentials", "profile"} + SYSTEM_SETTINGS_ALLOWED_FIELD_SET = {"credentials", "preferences", "defaults"} @staticmethod def build_token_repository( @@ -103,6 +104,33 @@ class UserService: with path.open(mode="w") as file: file.write(user_settings_json) + @classmethod + def update_default(cls, user_settings: UserSettings) -> UserSettings: + d1 = cls.read_default_user_settings().dict() + d2 = user_settings.dict() if user_settings else {} + updated = cls.merge_dicts([d1, d2]) + + return UserSettings.parse_obj(updated) + + @staticmethod + def merge_dicts(list_of_dicts: List[Dict[str, Any]]) -> Dict[str, Any]: + """Merge a list of dictionaries.""" + + def recursive_merge(d1: Dict, d2: Dict) -> Dict: + """Recursively merge dict d2 into dict d1 if d2 is value is not None.""" + for k, v in d1.items(): + if k in d2 and all(isinstance(e, MutableMapping) for e in (v, d2[k])): + d2[k] = recursive_merge(v, d2[k]) + + d3 = d1.copy() + d3.update((k, v) for k, v in d2.items() if v is not None) + return d3 + + result: Dict[str, Any] = {} + for d in list_of_dicts: + result = reduce(recursive_merge, (result, d)) + return result + def __init__( self, mongodb_client: Optional[MongoClient] = None, diff --git a/openbb_sdk/openbb_sdk_core/openbb_sdk_core/app/static/package_builder.py b/openbb_sdk/openbb_sdk_core/openbb_sdk_core/app/static/package_builder.py index 0117199ea0a..83277a3b12a 100644 --- a/openbb_sdk/openbb_sdk_core/openbb_sdk_core/app/static/package_builder.py +++ b/openbb_sdk/openbb_sdk_core/openbb_sdk_core/app/static/package_builder.py @@ -173,7 +173,7 @@ class ImportDefinition: code += "\nfrom inspect import Parameter" code += "\nfrom typing import List, Dict, Union, Optional, Literal" code += "\nfrom openbb_sdk_core.app.utils import df_to_basemodel" - code += "\nfrom openbb_sdk_core.app.static.utils import parse_command_input, parse_command_output\n" + code += "\nfrom openbb_sdk_core.app.static.parsers import parse_command_inputs, parse_command_output\n" module_list = [hint_type.__module__ for hint_type in hint_type_list] module_list = list(set(module_list)) @@ -377,7 +377,7 @@ class MethodDefinition: parameter_map = dict(sig.parameters) parameter_map.pop("cc", None) - code = " inputs = parse_command_input(\n" + code = " inputs = parse_command_inputs(\n" for name, param in parameter_map.items(): if name == "extra_params": code += f" {name}=kwargs,\n" @@ -390,7 +390,7 @@ class MethodDefinition: code += "},\n" else: code += f" {name}={name},\n" - code += " )\n" + code += " )\n\n" code += " o = self._command_runner_session.run(\n" code += f""" "{path}",\n""" code += " **inputs,\n" diff --git a/openbb_sdk/openbb_sdk_core/openbb_sdk_core/app/static/parsers.py b/openbb_sdk/openbb_sdk_core/openbb_sdk_core/app/static/parsers.py new file mode 100644 index 00000000000..84ebdf20a4e --- /dev/null +++ b/openbb_sdk/openbb_sdk_core/openbb_sdk_core/app/static/parsers.py @@ -0,0 +1,23 @@ +import builtins +import warnings + +from openbb_sdk_core.app.model.abstract.warning import OpenBBWarning +from openbb_sdk_core.app.model.command_output import CommandOutput + + +def parse_command_inputs(**kwargs) -> dict: + """Parse and edit the inputs before running the command""" + return kwargs + + +def parse_command_output(command_output: CommandOutput) -> CommandOutput: + """Parse command output""" + if command_output.warnings: + for w in command_output.warnings: + category = getattr(builtins, w.category, OpenBBWarning) + warnings.warn(w.message, category) + + if command_output.error: + raise Exception(command_output.error.message) + + return command_output diff --git a/openbb_sdk/openbb_sdk_core/openbb_sdk_core/app/static/utils.py b/openbb_sdk/openbb_sdk_core/openbb_sdk_core/app/static/utils.py deleted file mode 100644 index e25c78e2106..00000000000 --- a/openbb_sdk/openbb_sdk_core/openbb_sdk_core/app/static/utils.py +++ /dev/null @@ -1,45 +0,0 @@ -import builtins -import warnings - -from openbb_sdk_core.app.model.abstract.warning import OpenBBWarning -from openbb_sdk_core.app.model.command_output import CommandOutput - - -def parse_command_input(**kwargs) -> dict: - # Here we can inject provider defaults - # Covert data from pandas.DataFrame to BaseModel - # And other input parsing stuff - if "provider_choices" in kwargs: - if kwargs["provider_choices"].get("provider", None) is None: - raise NotImplementedError( - "'provider' cannot be None, defaults are not implemented yet." - ) - return kwargs - - -# POC -# if "data" in parameter_map: -# code += " if isinstance(data, pandas.DataFrame):\n" -# code += " data = df_to_basemodel(data, data.index.name is not None)\n" -# code += "\n" - -# if "provider_choices" in parameter_map: -# code += " if provider is None:\n" -# code += " defaults = self._command_runner_session.user_settings.defaults.endpoints.get(\n" -# code += f' "{path}",\n' -# code += " None,\n" -# code += " )\n" -# code += ' provider = defaults.get("provider", None) if defaults else Parameter.empty\n' -# code += "\n" - - -def parse_command_output(command_output: CommandOutput) -> CommandOutput: - if command_output.warnings: - for w in command_output.warnings: - category = getattr(builtins, w.category, OpenBBWarning) - warnings.warn(w.message, category) - - if command_output.error: - raise Exception(command_output.error.message) - - return command_output diff --git a/openbb_sdk/openbb_sdk_core/settings/user_settings.json.tpl b/openbb_sdk/openbb_sdk_core/settings/user_settings.json.tpl index 9ba78001e00..7efaf5353f5 100644 --- a/openbb_sdk/openbb_sdk_core/settings/user_settings.json.tpl +++ b/openbb_sdk/openbb_sdk_core/settings/user_settings.json.tpl @@ -3,5 +3,18 @@ "benzinga_api_key": null, "fmp_api_key": null, "polygon_api_key": null + }, + "defaults": { + "routes": { + "/stocks/fa/balance": { + "provider": "polygon" + }, + "/stocks/load": { + "provider": "fmp" + }, + "/stocks/news": { + "provider": "benzinga" + } + } } }