jerry/OpenBB
1
0
Fork 0
mirror of https://github.com/OpenBB-finance/OpenBB.git synced 2026-11-08 02:11:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

[Enhancement] ReferenceGenerator class in package_builder.py (#6179)

Browse Source 
* fix docstrings in quantitative,technical,econometrics routers

* add reference generator class to create reference.json file in openbb/assets folder

* modify platform markdown generator script to not generate the reference.json file

* get properly formatted examples for the website

* modify get_field_type function

* remove TODO comment

* path change / to .

* remove extra '-' below 'Args' in function docstring

* get obbject extensions in extension_map.json

* skew in stats_router had extra . rip

* black

* extra space (sigh)

* modify get_provider_parameter_info func to use model_provider from provider interface

* update lock files

* black

* Revert "update lock files"

This reverts commit 13e7d8b280.

* add function to check all extensions are installed before the markdow generator runs

* better comment

* fix docstring

fix mypy

obbject description function

* black

* 'static

* remove duplicate standard params & data from reference data

add standard params & data fields while generating markdown files
This commit is contained in:
Pratyush Shukla
2024-03-13 23:00:10 +05:30
committed by GitHub
parent d984637844
commit 540f578852
10 changed files with 41786 additions and 734 deletions
Download Patch File Download Diff File Expand all files Collapse all files

451
openbb_platform/core/openbb_core/app/static/package_builder.py
View File

@@ -3,13 +3,15 @@
# pylint: disable=too-many-lines
import builtins
import inspect
import re
import shutil
import sys
from dataclasses import Field
from inspect import Parameter, _empty, isclass, signature
from json import dumps, load
from json import dump, dumps, load
from pathlib import Path
from typing import (
Any,
Callable,
Dict,
List,
@@ -112,9 +114,9 @@ class PackageBuilder:
self._clean(modules)
ext_map = self._get_extension_map()
self._save_extension_map(ext_map)
self._save_module_map()
self._save_modules(modules, ext_map)
self._save_package()
self._save_reference_file()
if self.lint:
self._run_linters()
@@ -147,17 +149,6 @@ class PackageBuilder:
self.console.log("Writing extension map...")
self._write(code=code, name="extension_map", extension="json", folder="assets")
def _save_module_map(self):
"""Save the module map."""
route_map = PathHandler.build_route_map()
path_list = PathHandler.build_path_list(route_map=route_map)
module_map = {
PathHandler.build_module_name(path=path): path for path in path_list
}
code = dumps(obj=dict(sorted(module_map.items())), indent=4)
self.console.log("\nWriting module map...")
self._write(code=code, name="module_map", extension="json", folder="assets")
def _save_modules(
self,
modules: Optional[Union[str, List[str]]] = None,
@@ -194,6 +185,16 @@ class PackageBuilder:
code = "### THIS FILE IS AUTO-GENERATED. DO NOT EDIT. ###\n"
self._write(code=code, name="__init__")
def _save_reference_file(self):
"""Save the reference.json file."""
self.console.log("\nWriting reference file...")
data = ReferenceGenerator.get_reference_data()
file_path = self.directory / "assets" / "reference.json"
# Dumping the reference dictionary as a JSON file
self.console.log(str(file_path))
with open(file_path, "w", encoding="utf-8") as f:
dump(data, f, indent=4)
def _run_linters(self):
"""Run the linters."""
self.console.log("\nRunning linters...")
@@ -886,12 +887,14 @@ class DocstringGenerator:
@staticmethod
def get_field_type(
field: FieldInfo, target: Literal["docstring", "website"] = "docstring"
field_type: Any,
is_required: bool,
target: Literal["docstring", "website"] = "docstring",
) -> str:
"""Get the implicit data type of a defined Pydantic field.
Args
----
Parameters
----------
field (FieldInfo): Pydantic field object containing field information.
target (Literal["docstring", "website"], optional): Target to return type for. Defaults to "docstring".
@@ -899,10 +902,10 @@ class DocstringGenerator:
-------
str: String representation of the field type.
"""
is_optional = not field.is_required() if target == "docstring" else False
is_optional = not is_required
try:
_type = field.annotation
_type = field_type
if "BeforeValidator" in str(_type):
_type = "Optional[int]" if is_optional else "int" # type: ignore
@@ -918,38 +921,47 @@ class DocstringGenerator:
.replace("NoneType", "None")
.replace(", None", "")
)
field_type = (
f"Optional[{field_type}]"
if is_optional and "Optional" not in str(_type)
else field_type
)
if target == "website":
field_type = re.sub(r"Optional\[(.*)\]", r"\1", field_type)
field_type = re.sub(r"Annotated\[(.*)\]", r"\1", field_type)
return field_type
except TypeError:
# Fallback to the annotation if the repr fails
field_type = field.annotation # type: ignore
return field_type
return field_type # type: ignore
@staticmethod
def get_OBBject_description(
results_type: str,
providers: Optional[str],
target: Literal["docstring", "website"] = "docstring",
) -> str:
"""Get the command output description."""
available_providers = providers or "Optional[str]"
indent = 2 if target == "docstring" else 0
obbject_description = (
f"{create_indent(2)}OBBject\n"
f"{create_indent(3)}results : {results_type}\n"
f"{create_indent(4)}Serializable results.\n"
f"{create_indent(3)}provider : {available_providers}\n"
f"{create_indent(4)}Provider name.\n"
f"{create_indent(3)}warnings : Optional[List[Warning_]]\n"
f"{create_indent(4)}List of warnings.\n"
f"{create_indent(3)}chart : Optional[Chart]\n"
f"{create_indent(4)}Chart object.\n"
f"{create_indent(3)}extra : Dict[str, Any]\n"
f"{create_indent(4)}Extra info.\n"
f"{create_indent(indent)}OBBject\n"
f"{create_indent(indent+1)}results : {results_type}\n"
f"{create_indent(indent+2)}Serializable results.\n"
f"{create_indent(indent+1)}provider : {available_providers}\n"
f"{create_indent(indent+2)}Provider name.\n"
f"{create_indent(indent+1)}warnings : Optional[List[Warning_]]\n"
f"{create_indent(indent+2)}List of warnings.\n"
f"{create_indent(indent+1)}chart : Optional[Chart]\n"
f"{create_indent(indent+2)}Chart object.\n"
f"{create_indent(indent+1)}extra : Dict[str, Any]\n"
f"{create_indent(indent+2)}Extra info.\n"
)
obbject_description = obbject_description.replace("NoneType", "None")
return obbject_description
@@ -1066,7 +1078,7 @@ class DocstringGenerator:
docstring += f"{create_indent(2)}{underline}\n"
for name, field in returns.items():
field_type = cls.get_field_type(field)
field_type = cls.get_field_type(field.annotation, field.is_required())
description = getattr(field, "description", "")
docstring += f"{create_indent(2)}{field.alias or name} : {field_type}\n"
docstring += f"{create_indent(3)}{format_description(description)}\n"
@@ -1190,3 +1202,376 @@ class PathHandler:
if not path:
return "Extensions"
return f"ROUTER_{cls.clean_path(path=path)}"
class ReferenceGenerator:
"""Generate the reference for the Platform."""
REFERENCE_FIELDS = [
"deprecated",
"description",
"examples",
"parameters",
"returns",
"data",
]
# pylint: disable=protected-access
pi = DocstringGenerator.provider_interface
@classmethod
def get_endpoint_examples(
cls,
path: str,
func: Callable,
examples: Optional[List[Example]],
) -> str:
"""Get the examples for the given standard model or function.
For a given standard model or function, the examples are fetched from the
list of Example objects and formatted into a string.
Parameters
----------
path (str):
Path of the router.
func (Callable):
Router endpoint function.
examples (Optional[List[Example]]):
List of Examples (APIEx or PythonEx type)
for the endpoint.
Returns
-------
str:
Formatted string containing the examples for the endpoint.
"""
sig = signature(func)
parameter_map = dict(sig.parameters)
formatted_params = MethodDefinition.format_params(
path=path, parameter_map=parameter_map
)
explicit_params = dict(formatted_params)
explicit_params.pop("extra_params", None)
param_types = {k: v.annotation for k, v in explicit_params.items()}
return DocstringGenerator.build_examples(
path.replace("/", "."),
param_types,
examples,
"website",
)
@classmethod
def get_provider_parameter_info(cls, model: str) -> Dict[str, str]:
"""Get the name, type, description, default value and optionality information for the provider parameter.
Parameters
----------
model (str):
Standard model to access the model providers.
Returns
-------
Dict[str, str]:
Dictionary of the provider parameter information
"""
pi_model_provider = cls.pi.model_providers[model]
provider_params_field = pi_model_provider.__dataclass_fields__["provider"]
name = provider_params_field.name
field_type = DocstringGenerator.get_field_type(
provider_params_field.type, False, "website"
)
default = provider_params_field.type.__args__[0]
description = (
"The provider to use for the query, by default None. "
"If None, the provider specified in defaults is selected "
f"or '{default}' if there is no default."
)
provider_parameter_info = {
"name": name,
"type": field_type,
"description": description,
"default": default,
"optional": True,
}
return provider_parameter_info
@classmethod
def get_provider_field_params(
cls,
model: str,
params_type: str,
provider: str = "openbb",
) -> List[Dict[str, Any]]:
"""Get the fields of the given parameter type for the given provider of the standard_model.
Parameters
----------
model (str):
Model name to access the provider interface
params_type (str):
Parameters to fetch data for (QueryParams or Data)
provider (str, optional):
Provider name. Defaults to "openbb".
Returns
-------
List[Dict[str, str]]:
List of dictionaries containing the field name, type, description, default,
optional flag and standard flag for each provider.
"""
provider_field_params = []
expanded_types = MethodDefinition.TYPE_EXPANSION
model_map = cls.pi._map[model] # pylint: disable=protected-access
for field, field_info in model_map[provider][params_type]["fields"].items():
# Determine the field type, expanding it if necessary and if params_type is "Parameters"
field_type = field_info.annotation
is_required = field_info.is_required()
field_type = DocstringGenerator.get_field_type(
field_type, is_required, "website"
)
if params_type == "QueryParams" and field in expanded_types:
expanded_type = DocstringGenerator.get_field_type(
expanded_types[field], is_required, "website"
)
field_type = f"Union[{field_type}, {expanded_type}]"
cleaned_description = (
str(field_info.description)
.strip().replace("\n", " ").replace(" ", " ").replace('"', "'")
) # fmt: skip
# Add information for the providers supporting multiple symbols
if params_type == "QueryParams" and field_info.json_schema_extra:
multiple_items_list = field_info.json_schema_extra.get(
"multiple_items_allowed", None
)
if multiple_items_list:
multiple_items = ", ".join(multiple_items_list)
cleaned_description += (
f" Multiple items allowed for provider(s): {multiple_items}."
)
# Manually setting to List[<field_type>] for multiple items
# Should be removed if TYPE_EXPANSION is updated to include this
field_type = f"Union[{field_type}, List[{field_type}]]"
default_value = "" if field_info.default is PydanticUndefined else str(field_info.default) # fmt: skip
provider_field_params.append(
{
"name": field,
"type": field_type,
"description": cleaned_description,
"default": default_value,
"optional": not is_required,
}
)
return provider_field_params
@staticmethod
def get_post_method_parameters_info(
docstring: str,
) -> List[Dict[str, Union[bool, str]]]:
"""Get the parameters for the POST method endpoints.
Parameters
----------
docstring (str):
Router endpoint function's docstring
Returns
-------
List[Dict[str, str]]:
List of dictionaries containing the name,type, description, default
and optionality of each parameter.
"""
# Define a regex pattern to match parameter blocks
# This pattern looks for a parameter name followed by " : ", then captures the type and description
pattern = re.compile(
r"\n\s*(?P<name>\w+)\s*:\s*(?P<type>[^\n]+?)(?:\s*=\s*(?P<default>[^\n]+))?\n\s*(?P<description>[^\n]+)"
)
# Find all matches in the docstring
matches = pattern.finditer(docstring)
# Initialize an empty list to store parameter dictionaries
parameters_list = []
# Iterate over the matches to extract details
for match in matches:
# Extract named groups as a dictionary
param_info = match.groupdict()
# Determine if the parameter is optional
is_optional = "Optional" in param_info["type"]
# If no default value is captured, set it to an empty string
default_value = (
param_info["default"] if param_info["default"] is not None else ""
)
# Create a new dictionary with fields in the desired order
param_dict = {
"name": param_info["name"],
"type": param_info["type"],
"description": param_info["description"],
"default": default_value,
"optional": is_optional,
}
# Append the dictionary to the list
parameters_list.append(param_dict)
return parameters_list
@staticmethod
def get_post_method_returns_info(docstring: str) -> str:
"""Get the returns information for the POST method endpoints.
Parameters
----------
docstring (str):
Router endpoint function's docstring
Returns
-------
Dict[str, str]:
Dictionary containing the name, type, description of the return value
"""
# Define a regex pattern to match the Returns section
# This pattern captures the model name inside "OBBject[]" and its description
match = re.search(r"Returns\n\s*-------\n\s*([^\n]+)\n\s*([^\n]+)", docstring)
return_type = match.group(1).strip() # type: ignore
# Remove newlines and indentation from the description
description = match.group(2).strip().replace("\n", "").replace(" ", "") # type: ignore
# Adjust regex to correctly capture content inside brackets, including nested brackets
content_inside_brackets = re.search(
r"OBBject\[\s*((?:[^\[\]]|\[[^\[\]]*\])*)\s*\]", return_type
)
return_type_content = content_inside_brackets.group(1) # type: ignore
return_info = (
f"OBBject\n"
f"{create_indent(1)}results : {return_type_content}\n"
f"{create_indent(2)}{description}"
)
return return_info
@classmethod
def get_reference_data(cls) -> Dict[str, Dict[str, Any]]:
"""Get the reference data for the Platform.
The reference data is a dictionary containing the description, parameters,
returns and examples for each endpoint. This is currently useful for
automating the creation of the website documentation files.
Returns
-------
Dict[str, Dict[str, Any]]:
Dictionary containing the description, parameters, returns and
examples for each endpoint.
"""
reference: Dict[str, Dict] = {}
route_map = PathHandler.build_route_map()
for path, route in route_map.items():
# Initialize the reference fields as empty dictionaries
reference[path] = {field: {} for field in cls.REFERENCE_FIELDS}
# Route method is used to distinguish between GET and POST methods
route_method = getattr(route, "methods", None)
# Route endpoint is the callable function
route_func = getattr(route, "endpoint", None)
# Attribute contains the model and examples info for the endpoint
openapi_extra = getattr(route, "openapi_extra", {})
# Standard model is used as the key for the ProviderInterface Map dictionary
standard_model = openapi_extra.get("model", "")
# Add endpoint model for GET methods
reference[path]["model"] = standard_model
# Add endpoint deprecation details
reference[path]["deprecated"] = {
"flag": MethodDefinition.is_deprecated_function(path),
"message": MethodDefinition.get_deprecation_message(path),
}
# Add endpoint examples
examples = openapi_extra.get("examples", [])
reference[path]["examples"] = cls.get_endpoint_examples(
path, route_func, examples # type: ignore
)
# Add data for the endpoints having a standard model
if route_method == {"GET"}:
reference[path]["description"] = getattr(
route, "description", "No description available."
)
# Access model map from the ProviderInterface
model_map = cls.pi._map[
standard_model
] # pylint: disable=protected-access
for provider in model_map:
if provider == "openbb":
# openbb provider is always present hence its the standard field
reference[path]["parameters"]["standard"] = (
cls.get_provider_field_params(standard_model, "QueryParams")
)
# Add `provider` parameter fields to the openbb provider
provider_parameter_fields = cls.get_provider_parameter_info(
standard_model
)
reference[path]["parameters"]["standard"].append(
provider_parameter_fields
)
# Add endpoint data fields for standard provider
reference[path]["data"]["standard"] = (
cls.get_provider_field_params(standard_model, "Data")
)
continue
# Adds provider specific parameter fields to the reference
reference[path]["parameters"][provider] = (
cls.get_provider_field_params(
standard_model, "QueryParams", provider
)
)
# Adds provider specific data fields to the reference
reference[path]["data"][provider] = cls.get_provider_field_params(
standard_model, "Data", provider
)
# Add endpoint returns data
# Currently only OBBject object is returned
providers = provider_parameter_fields["type"]
reference[path]["returns"]["OBBject"] = (
DocstringGenerator.get_OBBject_description(
standard_model, providers, "website"
)
)
# Add data for the endpoints without a standard model (data processing endpoints)
elif route_method == {"POST"}:
# POST method router `description` attribute is unreliable as it may or
# may not contain the "Parameters" and "Returns" sections. Hence, the
# endpoint function docstring is used instead.
description = route_func.__doc__.split("Parameters")[0].strip() # type: ignore
# Remove extra spaces in between the string
reference[path]["description"] = re.sub(" +", " ", description)
# Add endpoint parameters fields for POST methods
reference[path]["parameters"][
"standard"
] = ReferenceGenerator.get_post_method_parameters_info(
route_func.__doc__ # type: ignore
)
# Add endpoint returns data
# Currently only OBBject object is returned
reference[path]["returns"][
"OBBject"
] = cls.get_post_method_returns_info(
route_func.__doc__ # type: ignore
)
return reference

5
openbb_platform/core/tests/app/static/test_package_builder.py
View File

@@ -46,11 +46,6 @@ def test_package_builder_build(package_builder):
package_builder.build()
def test_save_module_map(package_builder):
"""Test save module map."""
package_builder._save_module_map()
def test_save_modules(package_builder):
"""Test save module."""
package_builder._save_modules()

28
openbb_platform/extensions/econometrics/openbb_econometrics/econometrics_router.py
View File

@@ -58,7 +58,7 @@ def correlation_matrix(data: List[Data]) -> OBBject[List[Data]]:
Returns
-------
OBBject[List[Data]]:
OBBject[List[Data]]
Correlation matrix.
"""
df = basemodel_to_df(data)
@@ -120,7 +120,7 @@ def ols_regression(
Returns
-------
OBBject[Dict]:
OBBject[Dict]
OBBject with the results being model and results objects.
"""
X = sm.add_constant(get_target_columns(basemodel_to_df(data), x_columns))
@@ -169,7 +169,7 @@ def ols_regression_summary(
Returns
-------
OBBject[Data]:
OBBject[Data]
OBBject with the results being summary object.
"""
X = sm.add_constant(get_target_columns(basemodel_to_df(data), x_columns))
@@ -260,7 +260,7 @@ def autocorrelation(
Returns
-------
OBBject[Dict]:
OBBject[Dict]
OBBject with the results being the score from the test.
"""
X = sm.add_constant(get_target_columns(basemodel_to_df(data), x_columns))
@@ -317,7 +317,7 @@ def residual_autocorrelation(
Returns
-------
OBBject[Data]:
OBBject[Data]
OBBject with the results being the score from the test.
"""
X = sm.add_constant(get_target_columns(basemodel_to_df(data), x_columns))
@@ -374,7 +374,7 @@ def cointegration(
Returns
-------
OBBject[Data]:
OBBject[Data]
OBBject with the results being the score from the test.
"""
pairs = list(combinations(columns, 2))
@@ -450,7 +450,7 @@ def causality(
Returns
-------
OBBject[Data]:
OBBject[Data]
OBBject with the results being the score from the test.
"""
X = get_target_column(basemodel_to_df(data), x_column)
@@ -518,7 +518,7 @@ def unit_root(
Returns
-------
OBBject[Data]:
OBBject[Data]
OBBject with the results being the score from the test.
"""
dataset = get_target_column(basemodel_to_df(data), column)
@@ -568,7 +568,7 @@ def panel_random_effects(
Returns
-------
OBBject[Dict]:
OBBject[Dict]
OBBject with the fit model returned
"""
X = get_target_columns(basemodel_to_df(data), x_columns)
@@ -615,7 +615,7 @@ def panel_between(
Returns
-------
OBBject[Dict]:
OBBject[Dict]
OBBject with the fit model returned
"""
X = get_target_columns(basemodel_to_df(data), x_columns)
@@ -661,7 +661,7 @@ def panel_pooled(
Returns
-------
OBBject[Dict]:
OBBject[Dict]
OBBject with the fit model returned
"""
X = get_target_columns(basemodel_to_df(data), x_columns)
@@ -706,7 +706,7 @@ def panel_fixed(
Returns
-------
OBBject[Dict]:
OBBject[Dict]
OBBject with the fit model returned
"""
X = get_target_columns(basemodel_to_df(data), x_columns)
@@ -751,7 +751,7 @@ def panel_first_difference(
Returns
-------
OBBject[Dict]:
OBBject[Dict]
OBBject with the fit model returned
"""
X = get_target_columns(basemodel_to_df(data), x_columns)
@@ -797,7 +797,7 @@ def panel_fmac(
Returns
-------
OBBject[Dict]:
OBBject[Dict]
OBBject with the fit model returned
"""
X = get_target_columns(basemodel_to_df(data), x_columns)

24
openbb_platform/extensions/quantitative/openbb_quantitative/rolling/rolling_router.py
View File

@@ -76,7 +76,6 @@ def skew(
Rolling skew.
"""
df = basemodel_to_df(data, index=index)
series_target = get_target_column(df, target)
series_target.name = f"rolling_skew_{window}"
@@ -132,7 +131,7 @@ def variance(
index: str, optional
The name of the index column, default is "date".
Returns:
Returns
-------
OBBject[List[Data]]
An object containing the rolling variance values.
@@ -177,8 +176,8 @@ def stdev(
Calculate the rolling standard deviation of a target column within a given window size.
Standard deviation is a measure of the amount of variation or dispersion of a set of values.
It is widely used to assess the risk and volatility of financial returns or other time series data
over a specified rolling window. It is the square root of the variance.
It is widely used to assess the risk and volatility of financial returns or other time series data
over a specified rolling window. It is the square root of the variance.
Parameters
----------
@@ -191,12 +190,11 @@ def stdev(
index: str, optional
The name of the index column, default is "date".
Returns:
Returns
-------
OBBject[List[Data]]
An object containing the rolling standard deviation values.
"""
df = basemodel_to_df(data, index=index)
series_target = get_target_column(df, target)
series_target.name = f"rolling_stdev_{window}"
@@ -255,12 +253,11 @@ def kurtosis(
index: str, optional
The name of the index column, default is "date".
Returns:
Returns
-------
OBBject[List[Data]]
An object containing the rolling kurtosis values.
"""
df = basemodel_to_df(data, index=index)
series_target = get_target_column(df, target)
series_target.name = f"rolling_kurtosis_{window}"
@@ -324,12 +321,11 @@ def quantile(
index: str, optional
The name of the index column, default is "date".
Returns:
Returns
-------
OBBject[List[Data]]
An object containing the rolling quantile values with the median.
"""
df = basemodel_to_df(data, index=index)
series_target = get_target_column(df, target)
validate_window(series_target, window)
@@ -398,11 +394,11 @@ def mean(
index: str, optional
The name of the index column, default is "date".
Returns:
OBBject[List[Data]]
An object containing the rolling mean values.
Returns
-------
OBBject[List[Data]]
An object containing the rolling mean values.
"""
df = basemodel_to_df(data, index=index)
series_target = get_target_column(df, target)
series_target.name = f"rolling_mean_{window}"

9
openbb_platform/extensions/quantitative/openbb_quantitative/stats/stats_router.py
View File

@@ -50,7 +50,7 @@ def skew(
data: List[Data],
target: str,
) -> OBBject[List[Data]]:
"""Get the skew. of the data set
"""Get the skew of the data set.
Skew is a statistical measure that reveals the degree of asymmetry of a distribution around its mean.
Positive skewness indicates a distribution with an extended tail to the right, while negative skewness shows a tail
@@ -70,7 +70,6 @@ def skew(
OBBject[List[Data]]
Rolling skew.
"""
df = basemodel_to_df(data)
series_target = get_target_column(df, target)
results = pd.DataFrame([skew_(series_target)], columns=["skew"])
@@ -170,7 +169,6 @@ def stdev(data: List[Data], target: str) -> OBBject[List[Data]]:
OBBject[List[Data]]
An object containing the rolling standard deviation values.
"""
df = basemodel_to_df(data)
series_target = get_target_column(df, target)
results = pd.DataFrame([std_dev_(series_target)], columns=["stdev"])
@@ -218,11 +216,10 @@ def kurtosis(data: List[Data], target: str) -> OBBject[List[Data]]:
The name of the column for which to calculate kurtosis.
Returns
------
-------
OBBject[List[Data]]
An object containing the kurtosis value
"""
df = basemodel_to_df(data)
series_target = get_target_column(df, target)
results = pd.DataFrame([kurtosis_(series_target)], columns=["kurtosis"])
@@ -278,7 +275,6 @@ def quantile(
OBBject[List[Data]]
An object containing the rolling quantile values with the median.
"""
df = basemodel_to_df(
data,
)
@@ -335,7 +331,6 @@ def mean(
OBBject[List[Data]]
An object containing the mean value.
"""
df = basemodel_to_df(data)
series_target = get_target_column(df, target)
results = pd.DataFrame([mean_(series_target)], columns=["mean"])

2
openbb_platform/extensions/technical/openbb_technical/technical_router.py
View File

@@ -315,6 +315,7 @@ def adosc(
Returns
-------
OBBject[List[Data]]
The calculated data.
"""
validate_data(data, [fast, slow])
df = basemodel_to_df(data, index=index)
@@ -1093,6 +1094,7 @@ def ad(data: List[Data], index: str = "date", offset: int = 0) -> OBBject[List[D
Returns
-------
OBBject[List[Data]]
The calculated data.
"""
df = basemodel_to_df(data, index=index)
df_target = get_target_columns(df, ["high", "low", "close", "volume"])

1
openbb_platform/openbb/assets/extension_map.json vendored
View File

@@ -12,6 +12,7 @@
"news@1.1.3",
"regulators@1.1.3"
],
"openbb_obbject_extension": [],
"openbb_provider_extension": [
"benzinga@1.1.3",
"federal_reserve@1.1.3",

152
openbb_platform/openbb/assets/module_map.json vendored
View File

@@ -1,152 +0,0 @@
{
"__extensions__": "",
"crypto": "/crypto",
"crypto_price": "/crypto/price",
"crypto_price_historical": "/crypto/price/historical",
"crypto_search": "/crypto/search",
"currency": "/currency",
"currency_price": "/currency/price",
"currency_price_historical": "/currency/price/historical",
"currency_search": "/currency/search",
"currency_snapshots": "/currency/snapshots",
"derivatives": "/derivatives",
"derivatives_futures": "/derivatives/futures",
"derivatives_futures_curve": "/derivatives/futures/curve",
"derivatives_futures_historical": "/derivatives/futures/historical",
"derivatives_options": "/derivatives/options",
"derivatives_options_chains": "/derivatives/options/chains",
"derivatives_options_unusual": "/derivatives/options/unusual",
"economy": "/economy",
"economy_calendar": "/economy/calendar",
"economy_composite_leading_indicator": "/economy/composite_leading_indicator",
"economy_cpi": "/economy/cpi",
"economy_fred_regional": "/economy/fred_regional",
"economy_fred_search": "/economy/fred_search",
"economy_fred_series": "/economy/fred_series",
"economy_gdp": "/economy/gdp",
"economy_gdp_forecast": "/economy/gdp/forecast",
"economy_gdp_nominal": "/economy/gdp/nominal",
"economy_gdp_real": "/economy/gdp/real",
"economy_long_term_interest_rate": "/economy/long_term_interest_rate",
"economy_money_measures": "/economy/money_measures",
"economy_risk_premium": "/economy/risk_premium",
"economy_short_term_interest_rate": "/economy/short_term_interest_rate",
"economy_unemployment": "/economy/unemployment",
"equity": "/equity",
"equity_calendar": "/equity/calendar",
"equity_calendar_dividend": "/equity/calendar/dividend",
"equity_calendar_earnings": "/equity/calendar/earnings",
"equity_calendar_ipo": "/equity/calendar/ipo",
"equity_calendar_splits": "/equity/calendar/splits",
"equity_compare": "/equity/compare",
"equity_compare_peers": "/equity/compare/peers",
"equity_discovery": "/equity/discovery",
"equity_discovery_active": "/equity/discovery/active",
"equity_discovery_aggressive_small_caps": "/equity/discovery/aggressive_small_caps",
"equity_discovery_filings": "/equity/discovery/filings",
"equity_discovery_gainers": "/equity/discovery/gainers",
"equity_discovery_growth_tech": "/equity/discovery/growth_tech",
"equity_discovery_losers": "/equity/discovery/losers",
"equity_discovery_undervalued_growth": "/equity/discovery/undervalued_growth",
"equity_discovery_undervalued_large_caps": "/equity/discovery/undervalued_large_caps",
"equity_estimates": "/equity/estimates",
"equity_estimates_analyst_search": "/equity/estimates/analyst_search",
"equity_estimates_consensus": "/equity/estimates/consensus",
"equity_estimates_historical": "/equity/estimates/historical",
"equity_estimates_price_target": "/equity/estimates/price_target",
"equity_fundamental": "/equity/fundamental",
"equity_fundamental_balance": "/equity/fundamental/balance",
"equity_fundamental_balance_growth": "/equity/fundamental/balance_growth",
"equity_fundamental_cash": "/equity/fundamental/cash",
"equity_fundamental_cash_growth": "/equity/fundamental/cash_growth",
"equity_fundamental_dividends": "/equity/fundamental/dividends",
"equity_fundamental_employee_count": "/equity/fundamental/employee_count",
"equity_fundamental_filings": "/equity/fundamental/filings",
"equity_fundamental_historical_attributes": "/equity/fundamental/historical_attributes",
"equity_fundamental_historical_eps": "/equity/fundamental/historical_eps",
"equity_fundamental_historical_splits": "/equity/fundamental/historical_splits",
"equity_fundamental_income": "/equity/fundamental/income",
"equity_fundamental_income_growth": "/equity/fundamental/income_growth",
"equity_fundamental_latest_attributes": "/equity/fundamental/latest_attributes",
"equity_fundamental_management": "/equity/fundamental/management",
"equity_fundamental_management_compensation": "/equity/fundamental/management_compensation",
"equity_fundamental_metrics": "/equity/fundamental/metrics",
"equity_fundamental_multiples": "/equity/fundamental/multiples",
"equity_fundamental_overview": "/equity/fundamental/overview",
"equity_fundamental_ratios": "/equity/fundamental/ratios",
"equity_fundamental_reported_financials": "/equity/fundamental/reported_financials",
"equity_fundamental_revenue_per_geography": "/equity/fundamental/revenue_per_geography",
"equity_fundamental_revenue_per_segment": "/equity/fundamental/revenue_per_segment",
"equity_fundamental_search_attributes": "/equity/fundamental/search_attributes",
"equity_fundamental_trailing_dividend_yield": "/equity/fundamental/trailing_dividend_yield",
"equity_fundamental_transcript": "/equity/fundamental/transcript",
"equity_market_snapshots": "/equity/market_snapshots",
"equity_ownership": "/equity/ownership",
"equity_ownership_form_13f": "/equity/ownership/form_13f",
"equity_ownership_insider_trading": "/equity/ownership/insider_trading",
"equity_ownership_institutional": "/equity/ownership/institutional",
"equity_ownership_major_holders": "/equity/ownership/major_holders",
"equity_ownership_share_statistics": "/equity/ownership/share_statistics",
"equity_price": "/equity/price",
"equity_price_historical": "/equity/price/historical",
"equity_price_nbbo": "/equity/price/nbbo",
"equity_price_performance": "/equity/price/performance",
"equity_price_quote": "/equity/price/quote",
"equity_profile": "/equity/profile",
"equity_screener": "/equity/screener",
"equity_search": "/equity/search",
"equity_shorts": "/equity/shorts",
"equity_shorts_fails_to_deliver": "/equity/shorts/fails_to_deliver",
"etf": "/etf",
"etf_countries": "/etf/countries",
"etf_equity_exposure": "/etf/equity_exposure",
"etf_historical": "/etf/historical",
"etf_holdings": "/etf/holdings",
"etf_holdings_date": "/etf/holdings_date",
"etf_holdings_performance": "/etf/holdings_performance",
"etf_info": "/etf/info",
"etf_price_performance": "/etf/price_performance",
"etf_search": "/etf/search",
"etf_sectors": "/etf/sectors",
"fixedincome": "/fixedincome",
"fixedincome_corporate": "/fixedincome/corporate",
"fixedincome_corporate_commercial_paper": "/fixedincome/corporate/commercial_paper",
"fixedincome_corporate_hqm": "/fixedincome/corporate/hqm",
"fixedincome_corporate_ice_bofa": "/fixedincome/corporate/ice_bofa",
"fixedincome_corporate_moody": "/fixedincome/corporate/moody",
"fixedincome_corporate_spot_rates": "/fixedincome/corporate/spot_rates",
"fixedincome_government": "/fixedincome/government",
"fixedincome_government_treasury_rates": "/fixedincome/government/treasury_rates",
"fixedincome_government_us_yield_curve": "/fixedincome/government/us_yield_curve",
"fixedincome_rate": "/fixedincome/rate",
"fixedincome_rate_ameribor": "/fixedincome/rate/ameribor",
"fixedincome_rate_dpcredit": "/fixedincome/rate/dpcredit",
"fixedincome_rate_ecb": "/fixedincome/rate/ecb",
"fixedincome_rate_effr": "/fixedincome/rate/effr",
"fixedincome_rate_effr_forecast": "/fixedincome/rate/effr_forecast",
"fixedincome_rate_estr": "/fixedincome/rate/estr",
"fixedincome_rate_iorb": "/fixedincome/rate/iorb",
"fixedincome_rate_sonia": "/fixedincome/rate/sonia",
"fixedincome_sofr": "/fixedincome/sofr",
"fixedincome_spreads": "/fixedincome/spreads",
"fixedincome_spreads_tcm": "/fixedincome/spreads/tcm",
"fixedincome_spreads_tcm_effr": "/fixedincome/spreads/tcm_effr",
"fixedincome_spreads_treasury_effr": "/fixedincome/spreads/treasury_effr",
"index": "/index",
"index_available": "/index/available",
"index_constituents": "/index/constituents",
"index_market": "/index/market",
"index_price": "/index/price",
"index_price_historical": "/index/price/historical",
"news": "/news",
"news_company": "/news/company",
"news_world": "/news/world",
"regulators": "/regulators",
"regulators_sec": "/regulators/sec",
"regulators_sec_cik_map": "/regulators/sec/cik_map",
"regulators_sec_institutions_search": "/regulators/sec/institutions_search",
"regulators_sec_rss_litigation": "/regulators/sec/rss_litigation",
"regulators_sec_schema_files": "/regulators/sec/schema_files",
"regulators_sec_sic_search": "/regulators/sec/sic_search",
"regulators_sec_symbol_map": "/regulators/sec/symbol_map"
}

41065
openbb_platform/openbb/assets/reference.json vendored Normal file
View File

File diff suppressed because it is too large Load Diff

783
website/generate_platform_v4_markdown.py
View File

@@ -1,20 +1,17 @@
"""Platform V4 Markdown Generator Script."""
# pylint: disable=too-many-lines
import inspect
import argparse
import json
import re
import shutil
import subprocess
from pathlib import Path
from typing import Any, Callable, Dict, List, Optional
from typing import Dict, List, Union
from openbb_core.app.model.example import Example
from openbb_core.app.provider_interface import ProviderInterface
from openbb_core.app.router import RouterLoader
from openbb_core.app.static.package_builder import DocstringGenerator, MethodDefinition
import toml
from openbb_core.app.static.utils.console import Console
from openbb_core.provider import standard_models
from pydantic_core import PydanticUndefined
from packaging import specifiers
# Number of spaces to substitute tabs for indentation
TAB_WIDTH = 4
@@ -22,6 +19,10 @@ TAB_WIDTH = 4
# Maximum number of commands to display on the cards
MAX_COMMANDS = 8
# Path to the Platform directory and the reference.json file
PLATFORM_PATH = Path(__file__).parent.parent / "openbb_platform"
REFERENCE_FILE_PATH = Path(PLATFORM_PATH / "openbb/assets/reference.json")
# Paths to use for generating and storing the markdown files
WEBSITE_PATH = Path(__file__).parent.absolute()
SEO_METADATA_PATH = Path(WEBSITE_PATH / "metadata/platform_v4_seo_metadata.json")
@@ -34,450 +35,157 @@ PLATFORM_REFERENCE_IMPORT = "import ReferenceCard from '@site/src/components/Gen
PLATFORM_REFERENCE_UL_ELEMENT = '<ul className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 xl:grid-cols-4 gap-4 -ml-6">' # noqa: E501
def get_field_data_type(field_type: Any) -> str:
"""Get the implicit data type from the field type.
# pylint: disable=redefined-outer-name
def check_installed_packages(
console: Console,
debug: bool,
) -> None:
"""Checks if the installed packages are the same as those on the platform pyproject.toml file.
String manipulation is used to extract the implicit
data type from the field type.
Compares the versions of the installed packages with the versions specified in the pyproject.toml file.
The source of truth for the package versions is the pyproject.toml file, and the installed packages are
checked against the specified versions. If the installed packages do not satisfy the version requirements,
an error is raised.
Args:
field_type (Any): typing object field type
Returns:
str: String representation of the implicit field tzxype
Parameters
----------
console (Console): Console object to display messages and save logs
debug (bool): Flag to enable debug mode
"""
try:
if "BeforeValidator" in str(field_type):
field_type = "int"
def convert_poetry_version_specifier(
poetry_version: Union[str, Dict[str, str]]
) -> str:
"""
Convert a Poetry version specifier to a format compatible with the packaging library.
Handles both simple string specifiers and dictionary specifiers, extracting only the version value if it's a dict.
if "Optional" in str(field_type):
field_type = str(field_type.__args__[0])
Parameters
----------
poetry_version (Union[str, Dict[str, str]]):
Poetry version specifier
if "Annotated[" in str(field_type):
field_type = str(field_type).rsplit("[", maxsplit=1)[-1].split(",")[0]
Returns
-------
str:
Version specifier compatible with the packaging library
"""
if isinstance(poetry_version, dict):
poetry_version = poetry_version.get("version", "")
if "models" in str(field_type):
field_type = str(field_type).rsplit(".", maxsplit=1)[-1]
if isinstance(poetry_version, str):
if poetry_version.startswith("^"):
base_version = poetry_version[1:]
# Use regex to split the version and convert to integers only if they are purely numeric
parts = re.split(r"\.|\-", base_version)
try:
major, minor = (int(x) for x in parts[:2])
except ValueError:
# If conversion fails, return the original version specifier
return poetry_version
next_major_version = major + 1
# Construct a version specifier that represents the range.
return f">={base_version},<{next_major_version}.0.0"
field_type = (
str(field_type)
.replace("<class '", "")
.replace("'>", "")
.replace("typing.", "")
.replace("pydantic.types.", "")
.replace("openbb_core.provider.abstract.data.", "")
.replace("datetime.datetime", "datetime")
.replace("datetime.date", "date")
.replace("NoneType", "None")
.replace(", None", "")
)
except TypeError:
field_type = str(field_type)
if poetry_version.startswith("~"):
base_version = poetry_version[1:]
parts = re.split(r"\.|\-", base_version)
try:
major, minor = (int(x) for x in parts[:2])
except ValueError:
# If conversion fails, return the original version specifier
return poetry_version
next_minor_version = minor + 1
# Construct a version specifier that represents the range.
return f">={base_version},<{major}.{next_minor_version}.0"
return field_type
# No need to modify other specifiers, as they are compatible with packaging library
return poetry_version
def check_dependency(
package_name: str, version_spec: str, installed_packages_dict: Dict[str, str]
) -> None:
"""
Check if the installed package version satisfies the required version specifier.
Raises DependencyCheckError if the package is not installed or does not satisfy the version requirements.
def get_endpoint_examples(
path: str,
func: Callable,
examples: Optional[List[Example]],
) -> str:
"""Get the examples for the given standard model or function.
Parameters
----------
package_name (str):
Name of the package to check
version_spec (str):
Version specifier to check against
installed_packages_dict (Dict[str, str]):
Dictionary of installed packages and their versions
"""
installed_version = installed_packages_dict.get(package_name.lower())
if not installed_version:
raise Exception(f"{package_name} is not installed.")
For a given standard model or function, the examples are fetched from the
list of Example objects and formatted into a string.
converted_version_spec = convert_poetry_version_specifier(version_spec)
specifier_set = specifiers.SpecifierSet(converted_version_spec)
Args:
path (str): Path of the router.
func (Callable): Router endpoint function.
examples (Optional[List[Example]]): List of Examples (APIEx or PythonEx type)
for the endpoint.
if not specifier_set.contains(installed_version, prereleases=True):
message = f"{package_name} version {installed_version} does not satisfy the specified version {converted_version_spec}." # noqa: E501, pylint: disable=line-too-long
raise Exception(message)
Returns:
str: Formatted string containing the examples for the endpoint.
"""
sig = inspect.signature(func)
parameter_map = dict(sig.parameters)
formatted_params = MethodDefinition.format_params(
path=path, parameter_map=parameter_map
console.log("\n[CRITICAL] Ensuring all the extensions are installed before the script runs...") # fmt: skip
# Execute the pip list command once and store the output
pip_list_output = subprocess.run(
"pip list | grep openbb", # noqa: S607
shell=True, # noqa: S602
capture_output=True,
text=True,
check=False,
)
explicit_params = dict(formatted_params)
explicit_params.pop("extra_params", None)
param_types = {k: v.annotation for k, v in explicit_params.items()}
return DocstringGenerator.build_examples(
path.replace("/", "."),
param_types,
examples,
"website",
)
def get_provider_parameter_info(endpoint: Callable) -> Dict[str, str]:
"""Get the name, type, description, default value and optionality
information for the provider parameter.
Function signature is insepcted to get the parameters of the router
endpoint function. The provider parameter is then extracted from the
function type annotations then the information is extracted from it.
Args:
endpoint (Callable): Router endpoint function
Returns:
Dict[str, str]: Dictionary of the provider parameter information
"""
params_dict = endpoint.__annotations__
model_type = params_dict["provider_choices"].__args__[0]
provider_params_field = model_type.__dataclass_fields__["provider"]
# Type is Union[Literal[<provider_name>], None]
default = provider_params_field.type.__args__[0]
description = (
"The provider to use for the query, by default None. "
"If None, the provider specified in defaults is selected "
f"or '{default}' if there is no default."
)
provider_parameter_info = {
"name": provider_params_field.name,
"type": str(provider_params_field.type).replace("typing.", ""),
"description": description,
"default": default,
"optional": True,
"standard": True,
installed_packages = pip_list_output.stdout.splitlines()
installed_packages_dict = {
line.split()[0].lower(): line.split()[1] for line in installed_packages
}
return provider_parameter_info
# Load the pyproject.toml file once
with open(PLATFORM_PATH / "pyproject.toml") as f:
toml_dict = toml.load(f)
# Extract the openbb dependencies, excluding the python dependency
dependencies = toml_dict["tool"]["poetry"]["dependencies"]
dependencies.pop("python", None)
def get_provider_field_params(
model_map: Dict[str, Any],
params_type: str,
provider: str = "openbb",
) -> List[Dict[str, Any]]:
"""Get the fields of the given parameter type for the given provider
of the standard_model.
# Compare versions and check dependencies
for package, version_spec in dependencies.items():
normalized_package_name = package.replace("_", "-").lower()
try:
# Convert the version specifier before checking
converted_version_spec = convert_poetry_version_specifier(version_spec)
check_dependency(
normalized_package_name, converted_version_spec, installed_packages_dict
)
Args:
provider_map (Dict[str, Any]): Model Map containing the QueryParams and Data parameters
params_type (str): Parameters to fetch data for (QueryParams or Data)
provider (str, optional): Provider name. Defaults to "openbb".
Returns:
List[Dict[str, str]]: List of dictionaries containing the field name,
type, description, default, optional flag and standard flag for each provider.
"""
provider_field_params = []
expanded_types = MethodDefinition.TYPE_EXPANSION
for field, field_info in model_map[provider][params_type]["fields"].items():
# Determine the field type, expanding it if necessary and if params_type is "Parameters"
field_type = get_field_data_type(field_info.annotation)
if params_type == "QueryParams" and field in expanded_types:
expanded_type = get_field_data_type(expanded_types[field])
field_type = f"Union[{expanded_type}, {field_type}]"
cleaned_description = (
str(field_info.description)
.strip().replace("\n", " ").replace(" ", " ").replace('"', "'")
) # fmt: skip
# Add information for the providers supporting multiple symbols
if params_type == "QueryParams" and (
field_extra := field_info.json_schema_extra
):
multiple_items_list = field_extra.get("multiple_items_allowed", None)
if multiple_items_list:
multiple_items = ", ".join(multiple_items_list)
cleaned_description += (
f" Multiple items allowed for provider(s): {multiple_items}."
# Ensure debug_mode output shows the processed version specifier
if debug:
installed_version = installed_packages_dict.get(normalized_package_name)
console.log(
f"{normalized_package_name}: Specified version {converted_version_spec}, Installed version {installed_version}" # noqa: E501, pylint: disable=line-too-long
)
# Manually setting to List[<field_type>] for multiple items
# Should be removed if TYPE_EXPANSION is updated to include this
field_type = f"Union[{field_type}, List[{field_type}]]"
default_value = "" if field_info.default is PydanticUndefined else str(field_info.default) # fmt: skip
provider_field_params.append(
{
"name": field,
"type": field_type,
"description": cleaned_description,
"default": default_value,
"optional": not field_info.is_required(),
"standard": provider == "openbb",
}
)
return provider_field_params
def get_function_params_default_value(endpoint: Callable) -> Dict:
"""Get the default for the endpoint function parameters.
Args:
endpoint (Callable): Router endpoint function
Returns:
Dict: Endpoint function parameters and their default values
"""
default_values = {}
signature = inspect.signature(endpoint)
parameters = signature.parameters
for name, param in parameters.items():
if param.default is not inspect.Parameter.empty:
default_values[name] = param.default
else:
default_values[name] = ""
return default_values
def get_post_method_parameters_info(endpoint: Callable) -> List[Dict[str, str]]:
"""Get the parameters for the POST method endpoints.
Args:
endpoint (Callable): Router endpoint function
Returns:
List[Dict[str, str]]: List of dictionaries containing the name,
type, description, default and optionality of each parameter.
"""
parameters_info = []
descriptions = {}
parameters_default_values = get_function_params_default_value(endpoint)
section = endpoint.__doc__.split("Parameters")[1].split("Returns")[0] # type: ignore
lines = section.split("\n")
current_param = None
for line in lines:
cleaned_line = line.strip()
if ":" in cleaned_line: # This line names a parameter
current_param = cleaned_line.split(":")[0]
current_param = current_param.strip()
elif current_param: # This line describes the parameter
description = cleaned_line.strip()
descriptions[current_param] = description
# Reset current_param to ensure each description is
# correctly associated with the parameter
current_param = None
for param, param_type in endpoint.__annotations__.items():
if param == "return":
continue
parameters_info.append(
{
"name": param,
"type": get_field_data_type(param_type),
"description": descriptions.get(param, ""),
"default": parameters_default_values.get(param, ""),
"optional": "Optional" in str(param_type),
}
)
return parameters_info
def get_post_method_returns_info(endpoint: Callable) -> List[Dict[str, str]]:
"""Get the returns information for the POST method endpoints.
Args:
endpoint (Callable): Router endpoint function
Returns:
Dict[str, str]: Dictionary containing the name, type, description of the return value
"""
section = endpoint.__doc__.split("Parameters")[1].split("Returns")[-1] # type: ignore
description_lines = section.strip().split("\n")
description = description_lines[-1].strip() if len(description_lines) > 1 else ""
return_type = endpoint.__annotations__["return"].model_fields["results"].annotation
# Only one item is returned hence its a list with a single dictionary.
# Future changes to the return type will require changes to this code snippet.
return_info = [
{
"name": "results",
"type": get_field_data_type(return_type),
"description": description,
}
]
return return_info
# mypy: disable-error-code="attr-defined,arg-type"
def generate_reference_file() -> None:
"""Generate reference.json file using the ProviderInterface map."""
# ProviderInterface Map contains the model and its
# corresponding QueryParams and Data fields
pi_map = ProviderInterface().map
reference: Dict[str, Dict] = {}
# Fields for the reference dictionary to be used in the JSON file
REFERENCE_FIELDS = [
"deprecated",
"description",
"examples",
"parameters",
"returns",
"data",
]
# Router object is used to get the endpoints and their
# corresponding APIRouter object
router = RouterLoader.from_extensions()
route_map = {route.path: route for route in router.api_router.routes}
for path, route in route_map.items():
# Initialize the reference fields as empty dictionaries
reference[path] = {field: {} for field in REFERENCE_FIELDS}
# Route method is used to distinguish between GET and POST methods
route_method = route.methods
# Route endpoint is the callable function
route_func = route.endpoint
# Standard model is used as the key for the ProviderInterface Map dictionary
standard_model = route.openapi_extra["model"] if route_method == {"GET"} else ""
# Model Map contains the QueryParams and Data fields for each provider for a standard model
model_map = pi_map[standard_model] if standard_model else ""
# Add endpoint model for GET methods
reference[path]["model"] = standard_model
# Add endpoint deprecation details
deprecated_value = getattr(route, "deprecated", None)
reference[path]["deprecated"] = {
"flag": bool(deprecated_value),
"message": route.summary if deprecated_value else None,
}
# Add endpoint description
if route_method == {"GET"}:
reference[path]["description"] = route.description
elif route_method == {"POST"}:
# POST method router `description` attribute is unreliable as it may or
# may not contain the "Parameters" and "Returns" sections. Hence, the
# endpoint function docstring is used instead.
description = route.endpoint.__doc__.split("Parameters")[0].strip()
# Remove extra spaces in between the string
reference[path]["description"] = re.sub(" +", " ", description)
# Add endpoint examples
examples = route.openapi_extra["examples"]
reference[path]["examples"] = get_endpoint_examples(path, route_func, examples)
# Add endpoint parameters fields for standard provider
if route_method == {"GET"}:
# openbb provider is always present hence its the standard field
reference[path]["parameters"]["standard"] = get_provider_field_params(
model_map, "QueryParams"
)
# Add `provider` parameter fields to the openbb provider
provider_parameter_fields = get_provider_parameter_info(route_func)
reference[path]["parameters"]["standard"].append(provider_parameter_fields)
# Add endpoint data fields for standard provider
reference[path]["data"]["standard"] = get_provider_field_params(
model_map, "Data"
)
for provider in model_map:
if provider == "openbb":
continue
# Adds standard parameters to the provider parameters since they are
# inherited by the model.
# A copy is used to prevent the standard parameters fields from being
# modified.
reference[path]["parameters"][provider] = reference[path]["parameters"][
"standard"
].copy()
provider_query_params = get_provider_field_params(
model_map, "QueryParams", provider
)
reference[path]["parameters"][provider].extend(provider_query_params)
# Adds standard data fields to the provider data fields since they are
# inherited by the model.
# A copy is used to prevent the standard data fields from being modified.
reference[path]["data"][provider] = reference[path]["data"][
"standard"
].copy()
provider_data = get_provider_field_params(model_map, "Data", provider)
reference[path]["data"][provider].extend(provider_data)
elif route_method == {"POST"}:
# Add endpoint parameters fields for POST methods
reference[path]["parameters"]["standard"] = get_post_method_parameters_info(
route_func
)
# Add endpoint returns data
# Currently only OBBject object is returned
if route_method == {"GET"}:
reference[path]["returns"]["OBBject"] = [
{
"name": "results",
"type": f"List[{standard_model}]",
"description": "Serializable results.",
},
{
"name": "provider",
"type": f"Optional[{provider_parameter_fields['type']}]",
"description": "Provider name.",
},
{
"name": "warnings",
"type": "Optional[List[Warning_]]",
"description": "List of warnings.",
},
{
"name": "chart",
"type": "Optional[Chart]",
"description": "Chart object.",
},
{
"name": "extra",
"type": "Dict[str, Any]",
"description": "Extra info.",
},
]
elif route_method == {"POST"}:
reference[path]["returns"]["OBBject"] = get_post_method_returns_info(
route_func
)
# Dumping the reference dictionary as a JSON file
with open(PLATFORM_CONTENT_PATH / "reference.json", "w", encoding="utf-8") as f:
json.dump(reference, f, indent=4)
except Exception as e:
raise e
def create_reference_markdown_seo(path: str, description: str) -> str:
"""Create the SEO section for the markdown file.
Args:
path (str): Command path relative to the obb class
description (str): Description of the command
Parameters
----------
path (str):
Command path relative to the obb class
description (str):
Description of the command
Returns:
str: SEO section for the markdown file
Returns
-------
str:
SEO section for the markdown file
"""
with open(SEO_METADATA_PATH) as f:
@@ -518,13 +226,19 @@ def create_reference_markdown_intro(
) -> str:
"""Create the introduction section for the markdown file.
Args:
path (str): Command path relative to the obb class
description (str): Description of the command
deprecated (Dict[str, str]): Deprecated flag and message
Parameters
----------
path (str):
Command path relative to the obb class
description (str):
Description of the command
deprecated (Dict[str, str]):
Deprecated flag and message
Returns:
str: Introduction section for the markdown file
Returns
-------
str:
Introduction section for the markdown file
"""
deprecation_message = (
@@ -552,15 +266,20 @@ def create_reference_markdown_tabular_section(
) -> str:
"""Create the tabular section for the markdown file.
Args:
parameters (Dict[str, List[Dict[str, str]]]): Dictionary of
providers and their corresponding parameters
heading (str): Section heading for the tabular section
Parameters
----------
parameters (Dict[str, List[Dict[str, str]]]):
Dictionary of providers and their corresponding parameters
heading (str):
Section heading for the tabular section
Returns:
str: Tabular section for the markdown file
Returns
-------
str:
Tabular section for the markdown file
"""
standard_params_list = []
tables_list = []
# params_list is a list of dictionaries containing the
@@ -572,14 +291,18 @@ def create_reference_markdown_tabular_section(
for params in params_list
]
# Do not add default and optional columns in the Data section
# because James and Andrew don't like it
# Exclude default and optional columns in the Data section
if heading == "Data":
filtered_params = [
{k: v for k, v in params.items() if k not in ["default", "optional"]}
for params in filtered_params
]
if provider == "standard":
standard_params_list = filtered_params
else:
filtered_params = standard_params_list + filtered_params
# Parameter information for every provider is extracted from the dictionary
# and joined to form a row of the table.
# A `|` is added at the start and end of the row to create the table cell.
@@ -613,48 +336,39 @@ def create_reference_markdown_tabular_section(
return markdown
def create_reference_markdown_returns_section(returns: List[Dict[str, str]]) -> str:
def create_reference_markdown_returns_section(returns_content: str) -> str:
"""Create the returns section for the markdown file.
Args:
returns (List[Dict[str, str]]): List of dictionaries containing
the name, type and description of the returns
Parameters
----------
returns_content (str):
Returns section formatted as a string
Returns:
str: Returns section for the markdown file
Returns
-------
str:
Returns section for the markdown file
"""
returns_data = ""
for params in returns:
returns_data += f"{TAB_WIDTH*' '}{params['name']} : {params['type']}\n"
returns_data += f"{TAB_WIDTH*' '}{TAB_WIDTH*' '}{params['description']}\n\n"
# Remove the last two newline characters to render Returns section properly
returns_data = returns_data.rstrip("\n\n")
markdown = (
"---\n\n"
"## Returns\n\n"
"```python wordwrap\n"
"OBBject\n"
f"{returns_data}\n"
"```\n\n"
)
return markdown
return f"---\n\n## Returns\n\n```python wordwrap\n{returns_content}\n```\n\n"
def create_data_model_markdown(title: str, description: str, model: str) -> str:
"""Create the basic markdown file content for the data model.
Args:
title (str): Title of the data model
description (str): Description of the data model
model (str): Model name
Parameters
----------
title (str):
Title of the data model
description (str):
Description of the data model
model (str):
Model name
Returns:
str: Basic markdown file content for the data model
Returns
-------
str:
Basic markdown file content for the data model
"""
# File name is used in the import statement
@@ -702,11 +416,15 @@ def create_data_model_markdown(title: str, description: str, model: str) -> str:
def find_data_model_implementation_file(data_model: str) -> str:
"""Find the file name containing the data model class.
Args:
data_model (str): Data model name
Parameters
----------
data_model (str):
Data model name
Returns:
str: File name containing the data model class
Returns
-------
str:
File name containing the data model class
"""
# Function to search for the data model class in the file
@@ -732,8 +450,10 @@ def generate_reference_index_files(reference_content: Dict[str, str]) -> None:
"""Generate index.mdx and _category_.json files for directories and sub-directories
in the reference directory.
Args:
reference_content (Dict[str, str]): Endpoints and their corresponding descriptions.
Parameters
----------
reference_content (Dict[str, str]):
Endpoints and their corresponding descriptions.
"""
def generate_index_and_category(
@@ -864,13 +584,19 @@ def generate_reference_top_level_index() -> None:
def create_data_models_index(title: str, description: str, model: str) -> str:
"""Create the index content for the data models.
Args:
title (str): Title of the data model
description (str): Description of the data model
model (str): Model name
Parameters
----------
title (str):
Title of the data model
description (str):
Description of the data model
model (str):
Model name
Returns:
str: Index content for the data models
Returns
-------
str:
Index content for the data models
"""
# Get the first sentence of the description
@@ -891,8 +617,10 @@ def create_data_models_index(title: str, description: str, model: str) -> str:
def generate_data_models_index_files(content: str) -> None:
"""Generate index.mdx and _category_.json files for the data_models directory.
Args:
content (str): Content for the data models index file
Parameters
----------
content (str):
Content for the data models index file
"""
index_content = (
@@ -918,13 +646,19 @@ def generate_data_models_index_files(content: str) -> None:
def generate_markdown_file(path: str, markdown_content: str, directory: str) -> None:
"""Generate markdown file using the content of the specified path and directory.
Args:
path (str): Path to the markdown file
markdown_content (str): Content for the markdown file
directory (str): Directory to save the markdown file
Parameters
----------
path (str):
Path to the markdown file
markdown_content (str):
Content for the markdown file
directory (str):
Directory to save the markdown file
Raises:
ValueError: If the content type is invalid
Raises
------
ValueError:
If the content type is invalid
"""
# For reference, split the path to separate the
@@ -949,30 +683,37 @@ def generate_markdown_file(path: str, markdown_content: str, directory: str) ->
md_file.write(markdown_content)
def generate_platform_markdown() -> None:
# pylint: disable=redefined-outer-name
def generate_platform_markdown(
console: Console,
) -> None:
"""Generate markdown files for OpenBB Docusaurus website."""
data_models_index_content = []
reference_index_content_dict = {}
print("[CRITICAL] Ensure all the extensions are installed before running this script!") # fmt: skip
# Generate and read the reference.json file
print("[INFO] Generating the reference.json file...")
generate_reference_file()
with open(PLATFORM_CONTENT_PATH / "reference.json") as f:
reference = json.load(f)
console.log(f"\n[INFO] Reading the {REFERENCE_FILE_PATH} file...")
# Load the reference.json file
try:
with open(REFERENCE_FILE_PATH) as f:
reference = json.load(f)
except FileNotFoundError as exc:
raise FileNotFoundError(
"File not found! Please ensure the file exists."
) from exc
# Clear the platform/reference folder
print("[INFO] Clearing the platform/reference folder...")
console.log(f"\n[INFO] Clearing the {PLATFORM_REFERENCE_PATH} folder...")
shutil.rmtree(PLATFORM_REFERENCE_PATH, ignore_errors=True)
# Clear the platform/data_models folder
print("[INFO] Clearing the platform/data_models folder...")
console.log(f"\n[INFO] Clearing the {PLATFORM_DATA_MODELS_PATH} folder...")
shutil.rmtree(PLATFORM_DATA_MODELS_PATH, ignore_errors=True)
print(f"[INFO] Generating the markdown files for the {PLATFORM_REFERENCE_PATH} sub-directories...") # fmt: skip
print(f"[INFO] Generating the markdown files for the {PLATFORM_DATA_MODELS_PATH} directory...") # fmt: skip
console.log(
f"\n[INFO] Generating the markdown files for the {PLATFORM_REFERENCE_PATH} sub-directories..."
) # noqa: E501
console.log(f"\n[INFO] Generating the markdown files for the {PLATFORM_DATA_MODELS_PATH} directory...") # fmt: skip
for path, path_data in reference.items():
reference_markdown_content = ""
@@ -990,6 +731,7 @@ def generate_platform_markdown() -> None:
reference_markdown_content += create_reference_markdown_intro(
path[1:], description, path_data["deprecated"]
)
# reference_markdown_content += create_reference_markdown_examples(path_data["examples"])
reference_markdown_content += path_data["examples"]
if path_parameters_fields := path_data["parameters"]:
@@ -1036,10 +778,10 @@ def generate_platform_markdown() -> None:
generate_markdown_file(model, data_markdown_content, "data_models")
# Generate the index.mdx and _category_.json files for the reference directory
print(f"[INFO] Generating the index files for the {PLATFORM_REFERENCE_PATH} sub-directories...") # fmt: skip
console.log(f"\n[INFO] Generating the index files for the {PLATFORM_REFERENCE_PATH} sub-directories...") # fmt: skip
generate_reference_index_files(reference_index_content_dict)
print(
f"[INFO] Generating the index files for the {PLATFORM_REFERENCE_PATH} directory..."
console.log(
f"\n[INFO] Generating the index files for the {PLATFORM_REFERENCE_PATH} directory..."
)
generate_reference_top_level_index()
@@ -1048,10 +790,33 @@ def generate_platform_markdown() -> None:
data_models_index_content_str = "".join(data_models_index_content)
# Generate the index.mdx and _category_.json files for the data_models directory
print(f"[INFO] Generating the index files for the {PLATFORM_DATA_MODELS_PATH} directory...") # fmt: skip
console.log(f"\n[INFO] Generating the index files for the {PLATFORM_DATA_MODELS_PATH} directory...") # fmt: skip
generate_data_models_index_files(data_models_index_content_str)
print("[INFO] Markdown files generated successfully!")
console.log("\n[INFO] Markdown files generated successfully!")
if __name__ == "__main__":
generate_platform_markdown()
parser = argparse.ArgumentParser(
prog="Platform Markdown Generator V2",
description="Generate markdown files for the Platform website docs.",
)
parser.add_argument(
"-v",
"--verbose",
action="store_true",
help="Enable verbose output for debugging.",
)
args = parser.parse_args()
console = Console(True)
verbose = False
if args.verbose:
verbose = True
check_installed_packages(
console=console,
debug=verbose,
)
generate_platform_markdown(console=console)