From fa172be157df4f070f5afa684171064084ec1dce Mon Sep 17 00:00:00 2001 From: Igor Radovanovic <74266147+IgorWounds@users.noreply.github.com> Date: Tue, 11 Jul 2023 06:11:38 -0700 Subject: [PATCH] Feature/docstring generation (#5209) * Docstring poc * Linting * Improve docstrings * linting * Docstring improve * Docstring cleanup * Docstring cleanup * Cleanup * Docstring improve * Docstring improve * Cleanup * Improvements * Improvements * Fix bug * Improvements --- .../openbb_sdk_core/api/rest_api.py | 2 +- .../openbb_sdk_core/app/provider_interface.py | 2 - .../openbb_sdk_core/app/static/coverage.py | 1 - .../app/static/package_builder.py | 176 +++++++++++++++++- 4 files changed, 175 insertions(+), 6 deletions(-) diff --git a/openbb_sdk/openbb_sdk_core/openbb_sdk_core/api/rest_api.py b/openbb_sdk/openbb_sdk_core/openbb_sdk_core/api/rest_api.py index 9238c1651cc..91b52e2807b 100644 --- a/openbb_sdk/openbb_sdk_core/openbb_sdk_core/api/rest_api.py +++ b/openbb_sdk/openbb_sdk_core/openbb_sdk_core/api/rest_api.py @@ -4,9 +4,9 @@ from openbb_sdk_core.api.dependency.system import get_system_settings from openbb_sdk_core.api.dependency.user import get_user_service from openbb_sdk_core.api.router.account import router as router_account from openbb_sdk_core.api.router.commands import router as router_commands +from openbb_sdk_core.api.router.coverage import router as router_coverage from openbb_sdk_core.api.router.settings import router as router_settings from openbb_sdk_core.api.router.system import router as router_system -from openbb_sdk_core.api.router.coverage import router as router_coverage from openbb_sdk_core.app.model.profile import Profile from openbb_sdk_core.app.model.user_settings import UserSettings diff --git a/openbb_sdk/openbb_sdk_core/openbb_sdk_core/app/provider_interface.py b/openbb_sdk/openbb_sdk_core/openbb_sdk_core/app/provider_interface.py index b549faf19f2..e248164b131 100644 --- a/openbb_sdk/openbb_sdk_core/openbb_sdk_core/app/provider_interface.py +++ b/openbb_sdk/openbb_sdk_core/openbb_sdk_core/app/provider_interface.py @@ -158,7 +158,6 @@ class ProviderInterface: cls, providers: Any, ) -> Tuple[Dict[str, Tuple[str, Any, Any]], Dict[str, Tuple[str, Any, Any]]]: - standard: Dict[str, Tuple[str, Any, Any]] = {} extra: Dict[str, Tuple[str, Any, Any]] = {} @@ -206,7 +205,6 @@ class ProviderInterface: cls, providers: Any, ) -> Tuple[Dict[str, Tuple[str, Any, Any]], Dict[str, Tuple[str, Any, Any]]]: - standard: Dict[str, Tuple[str, Any, Any]] = {} extra: Dict[str, Tuple[str, Any, Any]] = {} diff --git a/openbb_sdk/openbb_sdk_core/openbb_sdk_core/app/static/coverage.py b/openbb_sdk/openbb_sdk_core/openbb_sdk_core/app/static/coverage.py index 912e75771eb..1d1ee19ba89 100644 --- a/openbb_sdk/openbb_sdk_core/openbb_sdk_core/app/static/coverage.py +++ b/openbb_sdk/openbb_sdk_core/openbb_sdk_core/app/static/coverage.py @@ -2,7 +2,6 @@ from openbb_sdk_core.app.router import CommandMap class Coverage: - __command_map = CommandMap() @property diff --git a/openbb_sdk/openbb_sdk_core/openbb_sdk_core/app/static/package_builder.py b/openbb_sdk/openbb_sdk_core/openbb_sdk_core/app/static/package_builder.py index 6a220ef1588..6fd68b38f3d 100644 --- a/openbb_sdk/openbb_sdk_core/openbb_sdk_core/app/static/package_builder.py +++ b/openbb_sdk/openbb_sdk_core/openbb_sdk_core/app/static/package_builder.py @@ -25,6 +25,7 @@ from uuid import NAMESPACE_DNS, uuid5 import pandas as pd from starlette.routing import BaseRoute +from openbb_sdk_core.app.provider_interface import get_provider_interface from openbb_sdk_core.app.router import RouterLoader @@ -201,13 +202,176 @@ class ClassDefinition: for child_path in child_path_list: route = PathHandler.get_route(path=child_path, route_map=route_map) if route: - code += MethodDefinition.build_command_method(path=route.path, func=route.endpoint) # type: ignore + code += MethodDefinition.build_command_method( + path=route.path, + func=route.endpoint, + query_name=route.openapi_extra.get("query", None) + if route.openapi_extra + else None, + ) # type: ignore else: code += MethodDefinition.build_class_loader_method(path=child_path) return code +class DocstringGenerator: + """Generate docstrings for the commands dynamically.""" + + @staticmethod + def get_docstrings(query_mapping: dict) -> dict: + """Get docstrings from the query mapping. + + Parameter + --------- + query_mapping (dict): The query mapping. + + Returns + ------- + dict: A dictionary with the docstrings. + """ + mapping = query_mapping.copy() + for _, provider_mapping in mapping.items(): + for _, query_params_mapping in provider_mapping.items(): + query_params_mapping.pop("fields", None) + return mapping + + @staticmethod + def clean_provider_docstring( + section_name: str, docstring: str, query_name: str + ) -> str: + """Clean the provider docstring from standard fields. + + Parameter + --------- + section_name (str): The section name. Should be either "QueryParams" or "Data". + docstring (str): The docstring. + query_name (str): The query name. + + Returns + ------- + str: The cleaned docstring. + """ + provider_interface = get_provider_interface() + if section_name == "QueryParams": + standard_fields = provider_interface.params[query_name][ + "standard" + ].__dataclass_fields__.keys() + elif section_name == "Data": + standard_fields = provider_interface.data[query_name][ + "standard" + ].__dataclass_fields__.keys() + else: + return docstring + + doc_lines = docstring.split("\n") + skip_next_line = False + cleaned_lines = [] + + for line in doc_lines: + if any(word in line for word in standard_fields) or skip_next_line: + skip_next_line = not skip_next_line + continue + + cleaned_lines.append(line) + + for i, line in enumerate(cleaned_lines): + try: + if line == " ---------" and cleaned_lines[i + 1] == " ": + cleaned_lines[i] = "---------" + cleaned_lines[i + 1] = " All fields are standardized.\n" + break + except IndexError: + cleaned_lines.append(" All fields are standardized.\n") + + return "\n".join(cleaned_lines) + + @classmethod + def generate_provider_docstrings( + cls, docstring: str, docstring_mapping: dict, query_name: str + ) -> str: + """Generate the docstring for the provider. + + Parameter + ---------- + docstring (str): The docstring. + docstring_mapping (dict): The docstring mapping. + + Returns + ------- + str: The final docstring. + """ + for provider, provider_mapping in docstring_mapping.items(): + docstring += f"\n{provider}" + docstring += f"\n{'=' * len(provider)}" + for section_name, section_docstring in provider_mapping.items(): + section_docstring = ( + section_docstring["docstring"] + if section_docstring["docstring"] + else "\n Returns\n-------\n Documentation not available.\n\n" + ) + + # clean the docstring from its original indentation + if ( + "\n Returns\n-------\n Documentation not available.\n\n" # noqa: SIM300 + != section_docstring + ): + section_docstring = "\n".join( + line[4:] for line in section_docstring.split("\n")[1:] + ) + section_docstring = "\n".join( + f" {line}" for line in section_docstring.split("\n") + ) + + if provider != "Standard": + section_docstring = cls.clean_provider_docstring( + section_name, + docstring=section_docstring, + query_name=query_name, + ) + + docstring += f"\n{section_docstring}" + + return docstring + + @classmethod + def generate_command_docstring(cls, func, query_name: str): + """Generate the docstring for the command. + + Parameter + ---------- + func (function): The command function. + query_name (str): The query name. + + Returns + ------- + function: The function with the updated docstring. + """ + provider_interface_mapping = get_provider_interface().map + query_mapping = provider_interface_mapping.get(query_name, None) + if query_mapping: + docstring_mapping = cls.get_docstrings(query_mapping) + + docstring = func.__doc__ or "" + + available_providers = ", ".join(docstring_mapping.keys()) + available_providers = available_providers.replace("openbb, ", "") + + docstring += f"\n\nAvailable providers: {available_providers}\n" + + docstring_mapping_ordered = { + "Standard": docstring_mapping.pop("openbb", None), # type: ignore + **docstring_mapping, + } + + docstring = cls.generate_provider_docstrings( + docstring, docstring_mapping_ordered, query_name + ) + + func.__doc__ = docstring + return func + + class MethodDefinition: @staticmethod def build_class_loader_method(path: str) -> str: @@ -402,7 +566,9 @@ class MethodDefinition: return code @classmethod - def build_command_method(cls, path: str, func: Callable) -> str: + def build_command_method( + cls, path: str, func: Callable, query_name: Optional[str] + ) -> str: # Name func_name = func.__name__ @@ -410,6 +576,12 @@ class MethodDefinition: sig = signature(func) parameter_map = dict(sig.parameters) + # Docstring + if query_name: + func = DocstringGenerator.generate_command_docstring( + func=func, query_name=query_name + ) + code = cls.build_command_method_signature( func_name=func_name, parameter_map=parameter_map,