diff --git a/openbb_platform/core/openbb_core/app/static/package_builder/docstring_generator.py b/openbb_platform/core/openbb_core/app/static/package_builder/docstring_generator.py index 448c3882ada..b7d2f61d3c5 100644 --- a/openbb_platform/core/openbb_core/app/static/package_builder/docstring_generator.py +++ b/openbb_platform/core/openbb_core/app/static/package_builder/docstring_generator.py @@ -5,6 +5,7 @@ import re from collections import OrderedDict from collections.abc import Callable from inspect import Parameter, isclass +from textwrap import indent from types import UnionType from typing import ( TYPE_CHECKING, @@ -112,7 +113,10 @@ class DocstringGenerator: continue if get_origin(arg) is Literal: continue - type_name = str(arg) + if hasattr(arg, "__forward_arg__"): + type_name = arg.__forward_arg__ + else: + type_name = str(arg) if hasattr(arg, "__name__") and not get_args(arg): type_name = arg.__name__ type_name = ( @@ -121,6 +125,9 @@ class DocstringGenerator: .replace("datetime.datetime", "datetime") .replace("datetime.date", "date") ) + # Unwrap any ForwardRef('X') left inside a container repr, + # e.g. ``list[ForwardRef('DataFrame')]`` -> ``list[DataFrame]``. + type_name = re.sub(r"ForwardRef\('([^']+)'\)", r"\1", type_name) if "openbb_" in type_name: # Strip the dotted module path but preserve any # generic-container prefix like ``list[`` so @@ -160,16 +167,18 @@ class DocstringGenerator: + str(_type).rsplit(".", maxsplit=1)[-1] ) + # Render optionality as ``X | None`` (PEP 604), not ``Optional[X]``. _type = ( - f"Optional[{_type}]" + f"{_type} | None" if is_optional and "Optional" not in str(_type) - and " | " not in str(_type) + and " | None" not in str(_type) else _type ) if target == "website": _type = re.sub(r"Optional\[(.*)\]", r"\1", _type) + _type = re.sub(r"\s*\|\s*None$", "", _type) return _type @@ -887,6 +896,16 @@ class DocstringGenerator: doc_has_examples = bool( re.search(r"^\s*Examples\s*\n[-=~`]{3,}", doc, re.MULTILINE) ) + + if doc_has_returns: + doc = re.sub( + r"\n[ \t]*Returns[ \t]*\n[-=~`]{3,}[ \t]*\n.*?" + r"(?=\n[ \t]*[A-Z][A-Za-z ]*[ \t]*\n[-=~`]{3,}|\Z)", + "\n", + doc, + flags=re.DOTALL, + ) + doc_has_returns = False result_doc = doc.strip("\n") if result_doc: @@ -924,10 +943,14 @@ class DocstringGenerator: type_str = cls.get_field_type( p_type, param.default is Parameter.empty ) - param_section += f"{create_indent(1)}{param_name} : {type_str}\n" + # Emit at column 0; a single blanket indent at the end of + # this branch shifts the whole docstring to the method's + # docstring column. Param name aligns with the header, + # description sits one level in. + param_section += f"{param_name} : {type_str}\n" if description and description.strip() != '""': - param_section += f"{create_indent(2)}{description}\n" + param_section += f"{create_indent(1)}{description}\n" result_doc += param_section + "\n" @@ -952,6 +975,10 @@ class DocstringGenerator: .replace("'>", "") .replace("OBBject[T]", "OBBject") ) + # Collapse ``OBBject[list[XxxData]]`` to ``OBBject`` — the + # envelope fields below already name the ``results`` type, + # and the data-model section names the row type in full. + type_name = re.sub(r"OBBject\[.*\]", "OBBject", type_name) returns_section += f"{type_name}\n" is_primitive = type_name.lower() in primitive_types @@ -975,12 +1002,12 @@ class DocstringGenerator: if field_name != "id": returns_section += "\n" - returns_section += f"{create_indent(2)}{field_name.strip()} : {field_type}" + returns_section += f"{create_indent(1)}{field_name.strip()} : {field_type}" else: - returns_section += f"{create_indent(2)}{field_name} : {field_type}\n" + returns_section += f"{create_indent(1)}{field_name} : {field_type}\n" if description: returns_section += ( - f"\n{create_indent(3)}{description}" + f"\n{create_indent(2)}{description}" ) except (AttributeError, TypeError): @@ -989,47 +1016,50 @@ class DocstringGenerator: returns_section += "Any\n" result_doc += returns_section + "\n" - result_doc = result_doc.replace("\n ", f"\n{create_indent(2)}") doc = result_doc.rstrip() - # Check response type for OBBject types to extract inner type - # Expand the docstring with the schema fields like in model-based commands + # Append a data-model section documenting every field of the row + # type carried in ``OBBject.results``. The model class is taken + # straight from the return annotation (``OBBject[list[XxxData]]`` + # or ``OBBject[XxxData]``) so it works for non-provider, + # data-processing extensions that have no standard-model registry. if type_name and "OBBject" in type_name: - type_str = str(return_annotation).replace("[T]", "") - match = re.search(r"OBBject\[(.*)\]", type_str) - inner = match.group(1) if match else "" - # Extract from list[Type] or dict[str, Type] - type_match = re.search(r"\[([^\[\]]+)\]$", inner) - extracted_type = type_match.group(1) if type_match else inner + result_model: type | None = None + results_field = getattr(return_annotation, "model_fields", {}).get( + "results" + ) + if results_field is not None: + _pending = [results_field.annotation] + while _pending: + node = _pending.pop() + if isclass(node) and hasattr(node, "model_fields"): + result_model = node + break + _pending.extend(get_args(node)) - if extracted_type and extracted_type.lower() not in primitive_types: - from openbb_core.app.static.package_builder.path_handler import ( - PathHandler, - ) - from openbb_core.app.static.package_builder.reference_generator import ( - ReferenceGenerator, - ) + if result_model is not None: + model_label = result_model.__name__ + if doc and not doc.endswith("\n\n"): + doc += "\n\n" + doc += f"{model_label}\n" + doc += f"{'-' * len(model_label)}\n" - route_map = PathHandler.build_route_map() - paths = ReferenceGenerator.get_paths(route_map) - route_path = paths.get(path, {}).get("data", {}).get("standard", []) + for field_name, field in result_model.model_fields.items(): + field_type = cls.get_field_type( + field.annotation, field.is_required + ) + field_description = field.description or "" + doc += f"{field_name} : {field_type}\n" + if field_description: + doc += f"{create_indent(1)}{field_description}\n" - if route_path: - if doc and not doc.endswith("\n\n"): - doc += "\n\n" - doc += f"{extracted_type}\n" - doc += f"{'-' * len(extracted_type)}\n" + doc += "\n" - for field in route_path: - field_name = field.get("name", "") - field_type = field.get("type", "Any") - field_description = field.get("description", "") - doc += f"{create_indent(2)}{field_name} : {field_type}\n" - if field_description: - doc += f"{create_indent(3)}{field_description}\n" - - doc += "\n" + doc_lines = doc.split("\n") + if len(doc_lines) > 1: + indented_body = indent("\n".join(doc_lines[1:]), create_indent(2)) + doc = doc_lines[0] + "\n" + indented_body if "examples" in sections and not doc_has_examples: if doc and not doc.endswith("\n\n"): diff --git a/openbb_platform/core/openbb_core/app/static/package_builder/import_definition.py b/openbb_platform/core/openbb_core/app/static/package_builder/import_definition.py index e6ad008e789..48634bc3e86 100644 --- a/openbb_platform/core/openbb_core/app/static/package_builder/import_definition.py +++ b/openbb_platform/core/openbb_core/app/static/package_builder/import_definition.py @@ -215,6 +215,7 @@ class ImportDefinition: else return_type ) hint_type_list.append(hint_type) + hint_type_list.extend(_unwrap_generic_args(return_type)) hint_type_list = cls.filter_hint_type_list(hint_type_list) diff --git a/openbb_platform/core/openbb_core/app/static/package_builder/method_definition.py b/openbb_platform/core/openbb_core/app/static/package_builder/method_definition.py index 94de12f758f..1c94148385e 100644 --- a/openbb_platform/core/openbb_core/app/static/package_builder/method_definition.py +++ b/openbb_platform/core/openbb_core/app/static/package_builder/method_definition.py @@ -31,6 +31,7 @@ from openbb_core.app.model.example import Example from openbb_core.app.model.field import OpenBBField from openbb_core.app.model.obbject import OBBject from openbb_core.app.provider_interface import ProviderInterface +from openbb_core.provider.abstract.query_params import QueryParams if TYPE_CHECKING: from numpy import ndarray # noqa @@ -306,6 +307,27 @@ class MethodDefinition: annotation.__args__[0], "__dataclass_fields__" ) + @staticmethod + def query_params_model(annotation) -> type[QueryParams] | None: + """Return the ``QueryParams`` class if ``annotation`` is one (or wraps one). + + Endpoints can declare a single ``params: XxxQueryParams`` argument + instead of listing every field; the builder flattens that model into + individual kwargs so the generated SDK method, docstring, and CLI all + expose the fields directly. ``Annotated[XxxQueryParams, ...]`` is + unwrapped first. + """ + candidate = annotation + if isinstance(candidate, _AnnotatedAlias): + candidate = candidate.__args__[0] + if ( + isclass(candidate) + and issubclass(candidate, QueryParams) + and candidate is not QueryParams + ): + return candidate + return None + @staticmethod def is_data_processing_function(path: str) -> bool: """Check if the function is a data processing function.""" @@ -499,7 +521,7 @@ class MethodDefinition: return True @staticmethod - def format_params( + def format_params( # noqa: PLR0912 path: str, parameter_map: dict[str, Parameter], func: Callable | None = None, @@ -585,6 +607,45 @@ class MethodDefinition: ) continue + # Case 3: a ``QueryParams`` model parameter — flatten its fields + # into individual kwargs so the generated method exposes each + # field directly (``method(field1=, field2=, ...)``) rather than a + # single opaque ``params`` argument. The method body re-packages + # them into the model the route expects. + if ( + qp_model := MethodDefinition.query_params_model(param.annotation) + ) is not None: + for field_name, field in qp_model.model_fields.items(): + field_anno = field.annotation + new_type = MethodDefinition.get_expanded_type( + field_name, None, field_anno + ) + if hasattr(new_type, "__constraints__"): + updated_type = Union[ # noqa + new_type.__constraints__ + (field_anno,) # type: ignore + ] + elif new_type is ...: + updated_type = field_anno + else: + updated_type = Union[field_anno, new_type] # noqa + + if field.is_required(): + field_default: Any = Parameter.empty + elif field.default_factory is not None: + field_default = field.default_factory() + else: + field_default = field.default + formatted[field_name] = Parameter( + name=field_name, + kind=Parameter.POSITIONAL_OR_KEYWORD, + annotation=Annotated[ + updated_type, + OpenBBField(description=field.description or ""), + ], + default=field_default, + ) + continue + if name == "extra_params": formatted[name] = Parameter(name="kwargs", kind=Parameter.VAR_KEYWORD) var_kw.append(name) @@ -651,13 +712,16 @@ class MethodDefinition: if has_depends: continue - # If not a dependency, process it as a normal parameter + # If not a dependency, process it as a normal parameter. + inner_type = param.annotation.__origin__ new_type = MethodDefinition.get_expanded_type(name) - updated_type = ( - param.annotation - if new_type is ... - else Union[param.annotation, new_type] # noqa - ) + if hasattr(new_type, "__constraints__"): + types = new_type.__constraints__ + (inner_type,) # type: ignore + updated_type = Union[types] # noqa + else: + updated_type = ( + inner_type if new_type is ... else Union[inner_type, new_type] # noqa + ) metadata = getattr(param.annotation, "__metadata__", []) description = ( @@ -1155,6 +1219,16 @@ class MethodDefinition: if extra := MethodDefinition.get_extra(fields[k]): info[k] = extra code += " },\n" + elif ( + qp_model := MethodDefinition.query_params_model(param.annotation) + ) is not None: + # Re-pack the flattened kwargs into the ``QueryParams`` model + # the route declares — the inverse of the flattening done in + # ``format_params``. + code += f" {name}={{\n" + for field_name in qp_model.model_fields: + code += f' "{field_name}": {field_name},\n' + code += " },\n" elif ( isinstance(param.annotation, _AnnotatedAlias) and ( diff --git a/openbb_platform/core/openbb_core/app/static/package_builder/reference_generator.py b/openbb_platform/core/openbb_core/app/static/package_builder/reference_generator.py index 3aeaa1eb6b8..8e4cd7fd604 100644 --- a/openbb_platform/core/openbb_core/app/static/package_builder/reference_generator.py +++ b/openbb_platform/core/openbb_core/app/static/package_builder/reference_generator.py @@ -559,6 +559,9 @@ class ReferenceGenerator: description = "" choices = None default = param.default if param.default is not Parameter.empty else None + + if default is PydanticUndefined: + default = None json_extra: dict = {} # Check if type is optional diff --git a/openbb_platform/core/tests/app/static/package_builder/test_docstring_generator_helpers.py b/openbb_platform/core/tests/app/static/package_builder/test_docstring_generator_helpers.py index f4aba01f77c..e2cca4d4afd 100644 --- a/openbb_platform/core/tests/app/static/package_builder/test_docstring_generator_helpers.py +++ b/openbb_platform/core/tests/app/static/package_builder/test_docstring_generator_helpers.py @@ -24,7 +24,7 @@ def test_get_field_type_simple_required(): def test_get_field_type_simple_optional_wraps_in_optional(): out = DocstringGenerator.get_field_type(int, is_required=False) - assert out == "Optional[int]" + assert out == "int | None" def test_get_field_type_optional_target_website_strips_optional_wrapper(): @@ -414,30 +414,11 @@ def test_generate_returns_section_non_obbject_model_fields(): assert "_RetModel" in out or "RetModel" in out -def test_generate_obbject_inner_type_extraction(monkeypatch): - """Drive lines ~971-996 by ensuring ReferenceGenerator.get_paths returns data - for the function's path, so the schema fields are appended to the docstring.""" - - from openbb_core.app.static.package_builder import ( - path_handler as ph_mod, - reference_generator as rg_mod, - ) - - fake_field = { - "name": "symbol", - "type": "str", - "description": "Sym field.", - } - - def fake_paths(_route_map): - return {"/test/inner": {"data": {"standard": [fake_field]}}} - - monkeypatch.setattr(ph_mod.PathHandler, "build_route_map", lambda: {}) - monkeypatch.setattr( - rg_mod.ReferenceGenerator, - "get_paths", - classmethod(lambda cls, _rm: fake_paths(_rm)), - ) +def test_generate_obbject_inner_type_extraction(): + """A ``OBBject[list[XxxData]]`` return annotation has its row model + introspected directly: the data-model field section is built from the + model class carried in ``OBBject.results``, not from ReferenceGenerator. + """ async def _func_obbject_typed() -> OBBject[list[_RetModel]]: """Typed OBBject.""" @@ -449,8 +430,12 @@ def test_generate_obbject_inner_type_extraction(monkeypatch): formatted_params=OrderedDict(), model_name=None, ) - assert "symbol" in out - assert "Sym field." in out + # Row model fields are documented from ``_RetModel`` itself. + assert "_RetModel" in out + assert "foo : str" in out + assert "The foo field." in out + assert "bar : int | None" in out + assert "A bar number." in out def test_generate_model_docstring_format_type_union_literal(): diff --git a/openbb_platform/core/tests/app/static/package_builder/test_method_definition_helpers.py b/openbb_platform/core/tests/app/static/package_builder/test_method_definition_helpers.py index 55516fbd11e..e8a06ece2d4 100644 --- a/openbb_platform/core/tests/app/static/package_builder/test_method_definition_helpers.py +++ b/openbb_platform/core/tests/app/static/package_builder/test_method_definition_helpers.py @@ -1261,6 +1261,30 @@ def test_format_params_annotated_non_depends_new_type_ellipsis(monkeypatch): assert "x" in out +def test_format_params_annotated_non_depends_constrained_type(monkeypatch): + """An annotated, non-Depends param whose expanded type is a constrained + TypeVar unrolls ``__constraints__`` into a ``Union`` with the inner type. + """ + from openbb_core.app.static.package_builder.path_handler import PathHandler + + class _Constrained: + __constraints__ = (str, bytes) + + p = Parameter( + name="x", + kind=Parameter.POSITIONAL_OR_KEYWORD, + annotation=Annotated[int, OpenBBField(description="")], + ) + monkeypatch.setattr(PathHandler, "build_route_map", staticmethod(lambda: {})) + monkeypatch.setattr( + MethodDefinition, + "get_expanded_type", + classmethod(lambda cls, *_a, **_k: _Constrained), + ) + out = MethodDefinition.format_params("/x/y", {"x": p}) + assert "x" in out + + def test_format_params_annotated_with_depends_continue(monkeypatch): from openbb_core.app.static.package_builder.path_handler import PathHandler diff --git a/openbb_platform/core/tests/app/static/package_builder/test_reference_generator_helpers.py b/openbb_platform/core/tests/app/static/package_builder/test_reference_generator_helpers.py index 4504e0d24e1..6250de37a8a 100644 --- a/openbb_platform/core/tests/app/static/package_builder/test_reference_generator_helpers.py +++ b/openbb_platform/core/tests/app/static/package_builder/test_reference_generator_helpers.py @@ -257,6 +257,19 @@ def test_get_function_signature_info_with_query_default(): assert out[0]["default"] == "AAPL" +def test_get_function_signature_info_pydantic_undefined_default_becomes_none(): + """A parameter whose default is ``PydanticUndefined`` (e.g. a required + model field surfaced into the signature) is normalised to ``None``.""" + from pydantic_core import PydanticUndefined + + def f(x: int = PydanticUndefined) -> None: # type: ignore[assignment] + return None + + out = ReferenceGenerator._get_function_signature_info(f) + assert out[0]["name"] == "x" + assert out[0]["default"] is None + + def test_get_function_signature_info_with_annotated_metadata(): def f(x: Annotated[int, Field(description="x val")]) -> None: return None diff --git a/openbb_platform/core/tests/app/static/test_package_builder.py b/openbb_platform/core/tests/app/static/test_package_builder.py index f359d6589cb..e0e0f77cadf 100644 --- a/openbb_platform/core/tests/app/static/test_package_builder.py +++ b/openbb_platform/core/tests/app/static/test_package_builder.py @@ -62,6 +62,14 @@ def test_save_modules(package_builder): package_builder._save_modules() +def test_save_modules_empty_path_list(tmp_openbb_dir): + """Test that ``_save_modules`` short-circuits when there is nothing to write.""" + builder = PackageBuilder(tmp_openbb_dir) + builder.path_list = [] + # Should return early without raising; nothing to write. + builder._save_modules() + + def test_save_package(package_builder): """Test save package.""" package_builder._save_package()