diff --git a/openbb_platform/core/openbb_core/app/static/package_builder.py b/openbb_platform/core/openbb_core/app/static/package_builder.py
index 9e75ce8fc31..abfad97eda6 100644
--- a/openbb_platform/core/openbb_core/app/static/package_builder.py
+++ b/openbb_platform/core/openbb_core/app/static/package_builder.py
@@ -843,7 +843,7 @@ class DocstringGenerator:
" List of warnings.\n"
" chart : Optional[Chart]\n"
" Chart object.\n"
- " extra: Dict[str, Any]\n"
+ " extra : Dict[str, Any]\n"
" Extra info.\n"
)
obbject_description = obbject_description.replace("NoneType", "None")
diff --git a/openbb_platform/extensions/econometrics/openbb_econometrics/econometrics_router.py b/openbb_platform/extensions/econometrics/openbb_econometrics/econometrics_router.py
index 0c7b52529a0..2a1a941e553 100644
--- a/openbb_platform/extensions/econometrics/openbb_econometrics/econometrics_router.py
+++ b/openbb_platform/extensions/econometrics/openbb_econometrics/econometrics_router.py
@@ -353,10 +353,10 @@ def causality(
x_column: str,
lag: PositiveInt = 3,
) -> OBBject[Data]:
- """Perform Granger causality test to determine if X "causes" y.
+ """Perform Granger causality test to determine if X 'causes' y.
The Granger causality test is a statistical hypothesis test to determine if one time series is useful in
- forecasting another. While "causality" in this context does not imply a cause-and-effect relationship in
+ forecasting another. While 'causality' in this context does not imply a cause-and-effect relationship in
the philosophical sense, it does test whether changes in one variable are systematically followed by changes
in another variable, suggesting a predictive relationship. By specifying a lag, you set the number of periods to
look back in the time series to assess this relationship. This test is particularly useful in economic and
diff --git a/openbb_platform/extensions/quantitative/openbb_quantitative/rolling/rolling_router.py b/openbb_platform/extensions/quantitative/openbb_quantitative/rolling/rolling_router.py
index 6e464083aea..c2cc5c7a1ae 100644
--- a/openbb_platform/extensions/quantitative/openbb_quantitative/rolling/rolling_router.py
+++ b/openbb_platform/extensions/quantitative/openbb_quantitative/rolling/rolling_router.py
@@ -230,7 +230,7 @@ def quantile(
Quantiles are points dividing the range of a probability distribution into intervals with equal probabilities,
or dividing the sample in the same way. This function is useful for understanding the distribution of data
- within a specified window, allowing for analysis of trends, identification of outliers, and assessment of risk..
+ within a specified window, allowing for analysis of trends, identification of outliers, and assessment of risk.
Parameters:
data: List[Data]
diff --git a/openbb_platform/providers/polygon/openbb_polygon/models/currency_pairs.py b/openbb_platform/providers/polygon/openbb_polygon/models/currency_pairs.py
index 66adef15c48..2ea951781d2 100644
--- a/openbb_platform/providers/polygon/openbb_polygon/models/currency_pairs.py
+++ b/openbb_platform/providers/polygon/openbb_polygon/models/currency_pairs.py
@@ -26,7 +26,7 @@ class PolygonCurrencyPairsQueryParams(CurrencyPairsQueryParams):
default=None, description="Symbol of the pair to search."
)
date: Optional[dateType] = Field(
- default=datetime.now().date(), description=QUERY_DESCRIPTIONS.get("date", "")
+ default=None, description=QUERY_DESCRIPTIONS.get("date", "")
)
search: Optional[str] = Field(
default="",
@@ -159,6 +159,7 @@ class PolygonCurrencyPairsFetcher(
return all_data
+ # pylint: disable=unused-argument
@staticmethod
def transform_data(
query: PolygonCurrencyPairsQueryParams,
diff --git a/openbb_platform/providers/polygon/openbb_polygon/models/equity_nbbo.py b/openbb_platform/providers/polygon/openbb_polygon/models/equity_nbbo.py
index ab92bc81e02..e332745c99e 100644
--- a/openbb_platform/providers/polygon/openbb_polygon/models/equity_nbbo.py
+++ b/openbb_platform/providers/polygon/openbb_polygon/models/equity_nbbo.py
@@ -92,7 +92,7 @@ class PolygonEquityNBBOData(EquityNBBOData):
conditions: Optional[Union[str, List[int], List[str]]] = Field(
default=None, description="A list of condition codes.", alias="conditions"
)
- indicators: Optional[List] = Field(
+ indicators: Optional[List[int]] = Field(
default=None, description="A list of indicator codes.", alias="indicators"
)
sequence_num: Optional[int] = Field(
@@ -208,6 +208,7 @@ class PolygonEquityNBBOFetcher(
return data
+ # pylint: disable=unused-argument
@staticmethod
def transform_data(
query: PolygonEquityNBBOQueryParams,
diff --git a/openbb_platform/providers/sec/openbb_sec/models/schema_files.py b/openbb_platform/providers/sec/openbb_sec/models/schema_files.py
index 93ef045baf7..682d03985b1 100644
--- a/openbb_platform/providers/sec/openbb_sec/models/schema_files.py
+++ b/openbb_platform/providers/sec/openbb_sec/models/schema_files.py
@@ -23,7 +23,7 @@ class SecSchemaFilesQueryParams(CotSearchQueryParams):
class SecSchemaFilesData(Data):
"""SEC Schema Files List Data."""
- files: List = Field(description="Dictionary of URLs to SEC Schema Files")
+ files: List[str] = Field(description="Dictionary of URLs to SEC Schema Files")
class SecSchemaFilesFetcher(Fetcher[SecSchemaFilesQueryParams, SecSchemaFilesData]):
diff --git a/website/content/platform/development/_category_.json b/website/content/platform/development/_category_.json
index 8219ce2ae74..7a45206fc82 100644
--- a/website/content/platform/development/_category_.json
+++ b/website/content/platform/development/_category_.json
@@ -1,4 +1,4 @@
{
"label": "Development",
- "position": 6
-}
+ "position": 7
+}
\ No newline at end of file
diff --git a/website/generate_platform_v4_markdown.py b/website/generate_platform_v4_markdown.py
index c377a68f70c..66842d771f4 100644
--- a/website/generate_platform_v4_markdown.py
+++ b/website/generate_platform_v4_markdown.py
@@ -1,602 +1,698 @@
+"""Platform V4 Markdown Generator Script."""
+
+# pylint: disable=too-many-lines
+
import inspect
import json
import re
import shutil
-from inspect import Parameter, _empty, signature
from pathlib import Path
-from textwrap import shorten
-from typing import Any, Callable, Dict, List, TextIO, Tuple, Union
+from typing import Any, Callable, Dict, List
-from docstring_parser import parse
from openbb_core.app.provider_interface import ProviderInterface
-from openbb_core.app.static.package_builder import (
- DocstringGenerator,
- MethodDefinition,
- PathHandler,
-)
+from openbb_core.app.router import RouterLoader
+from openbb_core.app.static.package_builder import MethodDefinition
from openbb_core.provider import standard_models
-from pydantic.fields import FieldInfo
+from pydantic_core import PydanticUndefined
-website_path = Path(__file__).parent.absolute()
-SEO_META: Dict[str, Dict[str, Union[str, List[str]]]] = json.loads(
- (website_path / "metadata/platform_v4_seo_metadata.json").read_text()
-)
+# Number of spaces to substitute tabs for indentation
+TAB_WIDTH = 4
-REFERENCE_IMPORT_UL = """import ReferenceCard from "@site/src/components/General/NewReferenceCard";
+# Maximum number of commands to display on the cards
+MAX_COMMANDS = 8
-
-"""
+# Paths to use for generating and storing the markdown files
+WEBSITE_PATH = Path(__file__).parent.absolute()
+SEO_METADATA_PATH = Path(WEBSITE_PATH / "metadata/platform_v4_seo_metadata.json")
+PLATFORM_CONTENT_PATH = Path(WEBSITE_PATH / "content/platform")
+PLATFORM_REFERENCE_PATH = Path(WEBSITE_PATH / "content/platform/reference")
+PLATFORM_DATA_MODELS_PATH = Path(WEBSITE_PATH / "content/platform/data_models")
-reference_import = (
- 'import ReferenceCard from "@site/src/components/General/NewReferenceCard";\n\n'
-)
-refrence_ul_element = """"""
+# Imports used in the generated markdown files
+PLATFORM_REFERENCE_IMPORT = "import ReferenceCard from '@site/src/components/General/NewReferenceCard';" # fmt: skip
+PLATFORM_REFERENCE_UL_ELEMENT = '' # noqa: E501
-def get_docstring_meta(
- func: Callable, full_command_path: str, formatted_params: Dict[str, Parameter]
-) -> Dict[str, Any]:
- """Extracts the meta information from the docstring of a function with no standardized results model."""
- meta_command = {}
- doc_parsed = parse(func.__doc__) # type: ignore
+def get_field_data_type(field_type: Any) -> str:
+ """Get the implicit data type from the field type.
- cmd_params = []
- for param in doc_parsed.params:
- arg_default = (
- formatted_params[param.arg_name].default
- if param.arg_name in formatted_params
- else None
- )
- cmd_params.append(
- {
- "name": param.arg_name,
- "type": get_annotation_type(param.type_name),
- "default": (
- str(arg_default)
- if arg_default is not inspect.Parameter.empty
- else None
- ),
- "cleaned_type": re.sub(
- r"Literal\[([^\"\]]*)\]",
- f"Literal[{type(arg_default).__name__}]",
- get_annotation_type(
- formatted_params[param.arg_name].annotation
- if param.arg_name in formatted_params
- else param.type_name
- ),
- ),
- "optional": bool(arg_default is not inspect.Parameter.empty)
- or param.is_optional,
- "doc": param.description,
- }
- )
+ String manipulation is used to extract the implicit
+ data type from the field type.
- if doc_parsed.returns:
- meta_command["returns"] = {
- "type": doc_parsed.returns.type_name,
- "doc": doc_parsed.returns.description,
- }
+ Args:
+ field_type (Any): typing object field type
- examples = []
- for example in doc_parsed.examples:
- examples.append(
- {
- "snippet": example.snippet,
- "description": example.description.strip(), # type: ignore
- }
- )
-
- def_params = [
- f"{d['name']}: {d['cleaned_type']}{' = ' + d['default'] if d['default'] else ''}"
- for d in cmd_params
- ]
- meta_command.update(
- {
- "description": doc_parsed.short_description
- + (
- "\n\n" + doc_parsed.long_description
- if doc_parsed.long_description
- else ""
- ),
- "params": cmd_params,
- "func_def": f"{full_command_path}({', '.join(def_params)})",
- "examples": examples,
- }
- )
-
- return meta_command
-
-
-def generate_markdown(meta_command: dict):
- markdown = meta_command["header"]
-
- markdown += "\n\n"
- markdown += (
- "import Tabs from '@theme/Tabs';\nimport TabItem from '@theme/TabItem';\n\n"
- )
-
- markdown += generate_markdown_section(meta_command)
- return markdown
-
-
-def generate_markdown_section(meta: Dict[str, Any]):
- # Process description to handle docstring examples
- lines = meta["description"].split("\n")
- description = []
- example_code = []
- in_example_block = False
-
- for line in lines:
- if line.strip().startswith(">>>"):
- in_example_block = True
- # Remove leading '>>>' and spaces
- example_line = line.strip()[4:]
- example_code.append(example_line)
- else:
- if in_example_block:
- # We've reached the end of an example block
- in_example_block = False
- # Append the gathered example code as a block
- description.append("```python\n" + "\n".join(example_code) + "\n```\n")
- example_code = [] # Reset for the next example block
- # Add the current line to the description
- description.append(line.strip())
-
- prev_snippet = " "
- for example in meta.get("examples", []):
- if isinstance(example["snippet"], str) and ">>>" in example["snippet"]:
- snippet = example["snippet"].replace(">>> ", "")
- example_code.append(snippet)
- if example["description"] and prev_snippet != "":
- example_code.append(example["description"])
- prev_snippet = snippet.strip()
- elif example["description"]:
- example_code.append(example["description"])
- else:
- if example["description"]:
- example_code.append(example["description"])
- prev_snippet = ""
-
- # Join the description parts and handle any remaining example code
- if example_code: # If there's an example block at the end of the docstring
- if meta.get("examples", []):
- description.append("\nExample:\n-------\n")
- description.append("\n\n```python\n" + "\n".join(example_code) + "\n```")
-
- markdown_description = "\n".join(description)
-
- markdown = markdown_description
- markdown += "\n\n" if not markdown_description.endswith("\n\n") else ""
-
- # Only add function definition if there was no example code
- if not example_code and not re.search(r"```python", markdown):
- markdown += "```python wordwrap\n" + meta["func_def"] + "\n```\n\n"
-
- markdown += "---\n\n## Parameters\n\n"
- if meta["params"]:
- markdown += generate_params_markdown_section(meta)
- else:
- markdown += "This function does not take standardized parameters.\n\n"
-
- markdown += "---\n\n## Returns\n\n"
- if meta["returns"]:
- return_desc = meta["returns"]["doc"] if meta["returns"]["doc"] else ""
- markdown += f"```python wordwrap\n{return_desc}\n```\n\n"
- else:
- markdown += "This function does not return a standardized model\n\n"
-
- markdown += "---\n\n"
-
- return markdown
-
-
-def create_nested_menus_card(folder: Path, url: str, data_models: bool = False) -> str:
- sub_categories = [
- sub.stem
- for sub in folder.glob("**/**/*.md*")
- if sub.is_file() and sub.stem != "index"
- ]
- path = "reference" if not data_models else "data_models"
- categories = shorten(", ".join(sub_categories), width=116, placeholder="...")
- url = f"/platform/{path}/{url}/{folder.name}".replace("//", "/")
-
- index_card = f"""\n"""
- return index_card
-
-
-def create_cmd_cards(cmd_text: List[Dict[str, str]], data_models: bool = False) -> str:
- path = "reference" if not data_models else "data_models"
- cmd_cards = ""
- for cmd in cmd_text:
- title = cmd["title"].replace("_", " ")
- url = f"/platform/{path}/{cmd['url']}".replace("//", "/")
- description = shorten(f"{cmd['description']}", width=116, placeholder="...")
-
- if not data_models:
- title = title.title()
- url = f"{url}/{cmd['title']}"
-
- cmd_cards += f"""\n"""
- return cmd_cards
-
-
-def write_reference_index(
- reference_cards: Dict[Path, List[Dict[str, str]]],
- fname: str,
- path: Path,
- rel_path: Path,
- f: TextIO,
- data_models: bool = False,
-) -> None:
- """Write to the corresponding index.mdx file for a given folder, with the
- appropriate nested menus and command cards.
-
- Parameters
- ----------
- reference_cards : Dict[Path, List[Dict[str, str]]]
- Dictionary of command cards to be written to the index.mdx file.
- fname : str
- Name of the index.mdx file.
- path : Path
- Path to the folder to be written.
- rel_path : Path
- Relative path to the folder to be written.
- f : TextIO
- File to write to.
- data_models : bool, optional
- Whether the folder is a data_models folder, by default False
+ Returns:
+ str: String representation of the implicit field tzxype
"""
- f.write(f"# {fname}\n\n{REFERENCE_IMPORT_UL if data_models else reference_import}")
- sub_folders = [sub for sub in path.glob("*") if sub.is_dir()]
- menus = []
- for folder in sub_folders:
- menus.append(
- create_nested_menus_card(folder, "/".join(rel_path.parts), data_models)
+ try:
+ if "BeforeValidator" in str(field_type):
+ field_type = "int"
+
+ if "Optional" in str(field_type):
+ field_type = str(field_type.__args__[0])
+
+ if "Annotated[" in str(field_type):
+ field_type = str(field_type).rsplit("[", maxsplit=1)[-1].split(",")[0]
+
+ if "models" in str(field_type):
+ field_type = str(field_type).rsplit(".", maxsplit=1)[-1]
+
+ field_type = (
+ str(field_type)
+ .replace("", "")
+ .replace("typing.", "")
+ .replace("pydantic.types.", "")
+ .replace("openbb_core.provider.abstract.data.", "")
+ .replace("datetime.datetime", "datetime")
+ .replace("datetime.date", "date")
+ .replace("NoneType", "None")
+ .replace(", None", "")
)
+ except TypeError:
+ field_type = str(field_type)
- if sub_folders and not data_models:
- f.write(f"### Menus\n{refrence_ul_element}\n{''.join(menus)}
\n")
-
- folder_cmd_cards: List[Dict[str, str]] = reference_cards.get(path, {}) # type: ignore
-
- if folder_cmd_cards:
- if not data_models:
- f.write(f"\n\n### Commands\n{refrence_ul_element}\n")
- f.write(create_cmd_cards(folder_cmd_cards, data_models) + "
\n")
+ return field_type
-provider_interface = ProviderInterface()
+def get_provider_parameter_info(endpoint: Callable) -> Dict[str, str]:
+ """Get the name, type, description, default value and optionality
+ information for the provider parameter.
+ Function signature is insepcted to get the parameters of the router
+ endpoint function. The provider parameter is then extracted from the
+ function type annotations then the information is extracted from it.
-def get_annotation_type(annotation: Any) -> str:
- optional_regex = re.compile(r"Optional\[(.*)\]")
- annotated_regex = re.compile(r"Annotated\[(?P.*)\,.*\]")
+ Args:
+ endpoint (Callable): Router endpoint function
- subbed = re.sub(
- optional_regex,
- r"\1",
- str(annotation)
- .replace("", "")
- .replace("typing.", "")
- .replace("pydantic.types.", "")
- .replace("datetime.datetime", "datetime")
- .replace("datetime.date", "date")
- .replace("NoneType", "None")
- .replace(", None", "")
- .replace("openbb_core.provider.abstract.data.", "")
- .replace("pandas.core.frame.", "pd.")
- .replace("pandas.core.series.", "pd."),
+ Returns:
+ Dict[str, str]: Dictionary of the provider parameter information
+ """
+
+ params_dict = endpoint.__annotations__
+ model_type = params_dict["provider_choices"].__args__[0]
+ provider_params_field = model_type.__dataclass_fields__["provider"]
+
+ # Type is Union[Literal[], None]
+ default = provider_params_field.type.__args__[0]
+ description = (
+ "The provider to use for the query, by default None. "
+ "If None, the provider specified in defaults is selected "
+ f"or '{default}' if there is no default."
)
- if match := annotated_regex.match(subbed):
- subbed = match.group("type")
- return subbed
-
-
-def get_command_meta(path: str, route_map: Dict[str, Any]) -> Dict[str, Any]:
- route = PathHandler.get_route(path=path, route_map=route_map)
- if not route:
- return {}
-
- func = route.endpoint # type: ignore
- func_name = func.__name__
- model_name = route.openapi_extra.get("model", None) if route.openapi_extra else None
-
- sig = signature(func)
- parameter_map = dict(sig.parameters)
- formatted_params = MethodDefinition.format_params(path, parameter_map=parameter_map)
-
- meta_command: Dict[str, Union[List[Dict[str, str]], Any]] = {
- "name": func_name,
- "description": func.__doc__.strip() if func.__doc__ else "",
- "source_code_url": "",
- "func_def": "",
- "params": [],
- "provider_params": {},
- "returns": {},
- "schema": {},
- "model": model_name,
- "deprecated": route.deprecated or False,
- "deprecation_message": route.summary if route.deprecated else "",
+ provider_parameter_info = {
+ "name": provider_params_field.name,
+ "type": str(provider_params_field.type).replace("typing.", ""),
+ "description": description,
+ "default": default,
+ "optional": True,
+ "standard": True,
}
- # Extract the full path from the 'path' variable, excluding the method name
- path_components = path.strip("/").split("/")
- # Construct the full command path, e.g., 'obb.equity.estimates.consensus'
- full_command_path = "obb." + ".".join(path_components[:-1])
+ return provider_parameter_info
- # Now add the actual function name
- func_name = route.endpoint.__name__
- full_command_path += f".{func_name}"
- if model_name:
- providers = provider_interface.map[model_name]
+def get_provider_field_params(
+ model_map: Dict[str, Any],
+ params_type: str,
+ provider: str = "openbb",
+) -> List[Dict[str, Any]]:
+ """Get the fields of the given parameter type for the given provider
+ of the standard_model.
- obb_query_fields: Dict[str, FieldInfo] = providers["openbb"]["QueryParams"][
- "fields"
- ]
+ Args:
+ provider_map (Dict[str, Any]): Model Map containing the QueryParams and Data parameters
+ params_type (str): Parameters to fetch data for (QueryParams or Data)
+ provider (str, optional): Provider name. Defaults to "openbb".
- meta_command["description"] += "\n\n" + DocstringGenerator.generate_example(
- model_name=model_name,
- standard_params=obb_query_fields,
+ Returns:
+ List[Dict[str, str]]: List of dictionaries containing the field name,
+ type, description, default, optional flag and standard flag for each provider.
+ """
+
+ provider_field_params = []
+ expanded_types = MethodDefinition.TYPE_EXPANSION
+
+ for field, field_info in model_map[provider][params_type]["fields"].items():
+ # Determine the field type, expanding it if necessary and if params_type is "Parameters"
+ field_type = get_field_data_type(field_info.annotation)
+
+ if params_type == "QueryParams" and field in expanded_types:
+ expanded_type = get_field_data_type(expanded_types[field])
+ field_type = f"Union[{expanded_type}, {field_type}]"
+
+ cleaned_description = (
+ str(field_info.description)
+ .strip().replace("\n", " ").replace(" ", " ").replace('"', "'")
+ ) # fmt: skip
+
+ # Add information for the providers supporting multiple symbols
+ if params_type == "QueryParams" and field_info.json_schema_extra:
+ multiple_items = ", ".join(
+ field_info.json_schema_extra["multiple_items_allowed"]
+ )
+ cleaned_description += (
+ f" Multiple items allowed for provider(s): {multiple_items}."
+ )
+ # Manually setting to List[] for multiple items
+ # Should be removed if TYPE_EXPANSION is updated to include this
+ field_type = f"Union[{field_type}, List[{field_type}]]"
+
+ default_value = "" if field_info.default is PydanticUndefined else str(field_info.default) # fmt: skip
+
+ provider_field_params.append(
+ {
+ "name": field,
+ "type": field_type,
+ "description": cleaned_description,
+ "default": default_value,
+ "optional": not field_info.is_required(),
+ "standard": provider == "openbb",
+ }
)
- available_fields = list(obb_query_fields.keys())
- available_fields.extend(["chart", "provider"])
+ return provider_field_params
- for param in formatted_params.values():
- if param.name not in available_fields:
- continue
- if param.name == "provider":
- # pylint: disable=W0212
- param_type = param._annotation # type: ignore
- default = param._annotation.__args__[0].__args__[0] # type: ignore
- description = (
- "The provider to use for the query, by default None. "
- f"If None, the provider specified in defaults is selected or '{default}' if there is no default."
- ""
- )
- optional = "True"
+def get_function_params_default_value(endpoint: Callable) -> Dict:
+ """Get the default for the endpoint function parameters.
- elif param.name == "chart":
- param_type = "bool"
- description = "Whether to create a chart or not, by default False."
- optional = "True"
- default = "False"
- else:
- description = obb_query_fields[param.name].description # type: ignore
+ Args:
+ endpoint (Callable): Router endpoint function
- param_type = param.annotation
+ Returns:
+ Dict: Endpoint function parameters and their default values
+ """
- if param.default is Parameter.empty:
- default = ""
- optional = "False"
- else:
- default = param.default
- optional = "True"
+ default_values = {}
- meta_command["params"].append(
- {
- "name": param.name,
- "type": get_annotation_type(param_type),
- "default": (
- str(default)
- if not isinstance(default, str) or not default
- else f'"{default}"'
- ),
- "cleaned_type": re.sub(
- r"Literal\[([^\"\]]*)\]",
- f"Literal[{type(default).__name__}]",
- get_annotation_type(param_type),
- ),
- "optional": optional,
- "doc": (description or "").replace("\n", "
").strip(),
- }
+ signature = inspect.signature(endpoint)
+ parameters = signature.parameters
+
+ for name, param in parameters.items():
+ if param.default is not inspect.Parameter.empty:
+ default_values[name] = param.default
+ else:
+ default_values[name] = ""
+
+ return default_values
+
+
+def get_post_method_parameters_info(endpoint: Callable) -> List[Dict[str, str]]:
+ """Get the parameters for the POST method endpoints.
+
+ Args:
+ endpoint (Callable): Router endpoint function
+
+ Returns:
+ List[Dict[str, str]]: List of dictionaries containing the name,
+ type, description, default and optionality of each parameter.
+ """
+ parameters_info = []
+ descriptions = {}
+
+ parameters_default_values = get_function_params_default_value(endpoint)
+ section = endpoint.__doc__.split("Parameters")[1].split("Returns")[0] # type: ignore
+
+ lines = section.split("\n")
+ current_param = None
+ for line in lines:
+ cleaned_line = line.strip()
+
+ if ":" in cleaned_line: # This line names a parameter
+ current_param = cleaned_line.split(":")[0]
+ current_param = current_param.strip()
+ elif current_param: # This line describes the parameter
+ description = cleaned_line.strip()
+ descriptions[current_param] = description
+ # Reset current_param to ensure each description is
+ # correctly associated with the parameter
+ current_param = None
+
+ for param, param_type in endpoint.__annotations__.items():
+ if param == "return":
+ continue
+
+ parameters_info.append(
+ {
+ "name": param,
+ "type": get_field_data_type(param_type),
+ "description": descriptions.get(param, ""),
+ "default": parameters_default_values.get(param, ""),
+ "optional": "Optional" in str(param_type),
+ }
+ )
+
+ return parameters_info
+
+
+def get_post_method_returns_info(endpoint: Callable) -> List[Dict[str, str]]:
+ """Get the returns information for the POST method endpoints.
+
+ Args:
+ endpoint (Callable): Router endpoint function
+
+ Returns:
+ Dict[str, str]: Dictionary containing the name, type, description of the return value
+ """
+ section = endpoint.__doc__.split("Parameters")[1].split("Returns")[-1] # type: ignore
+ description_lines = section.strip().split("\n")
+ description = description_lines[-1].strip() if len(description_lines) > 1 else ""
+ return_type = endpoint.__annotations__["return"].model_fields["results"].annotation
+
+ # Only one item is returned hence its a list with a single dictionary.
+ # Future changes to the return type will require changes to this code snippet.
+ return_info = [
+ {
+ "name": "results",
+ "type": get_field_data_type(return_type),
+ "description": description,
+ }
+ ]
+
+ return return_info
+
+
+# mypy: disable-error-code="attr-defined,arg-type"
+def generate_reference_file() -> None:
+ """Generate reference.json file using the ProviderInterface map."""
+
+ # ProviderInterface Map contains the model and its
+ # corresponding QueryParams and Data fields
+ pi_map = ProviderInterface().map
+ reference: Dict[str, Dict] = {}
+
+ # Fields for the reference dictionary to be used in the JSON file
+ REFERENCE_FIELDS = [
+ "deprecated",
+ "description",
+ "examples",
+ "parameters",
+ "returns",
+ "data",
+ ]
+
+ # Router object is used to get the endpoints and their
+ # corresponding APIRouter object
+ router = RouterLoader.from_extensions()
+ route_map = {route.path: route for route in router.api_router.routes}
+
+ for path, route in route_map.items():
+ # Initialize the reference fields as empty dictionaries
+ reference[path] = {field: {} for field in REFERENCE_FIELDS}
+
+ # Route method is used to distinguish between GET and POST methods
+ route_method = route.methods
+
+ # Route endpoint is the callable function
+ route_func = route.endpoint
+
+ # Standard model is used as the key for the ProviderInterface Map dictionary
+ standard_model = route.openapi_extra["model"] if route_method == {"GET"} else ""
+
+ # Model Map contains the QueryParams and Data fields for each provider for a standard model
+ model_map = pi_map[standard_model] if standard_model else ""
+
+ # Add endpoint model for GET methods
+ reference[path]["model"] = standard_model
+
+ # Add endpoint deprecation details
+ deprecated_value = getattr(route, "deprecated", None)
+ reference[path]["deprecated"] = {
+ "flag": bool(deprecated_value),
+ "message": route.summary if deprecated_value else None,
+ }
+
+ # Add endpoint description
+ if route_method == {"GET"}:
+ reference[path]["description"] = route.description
+ elif route_method == {"POST"}:
+ # POST method router `description` attribute is unreliable as it may or
+ # may not contain the "Parameters" and "Returns" sections. Hence, the
+ # endpoint function docstring is used instead.
+ description = route.endpoint.__doc__.split("Parameters")[0].strip()
+ # Remove extra spaces in between the string
+ reference[path]["description"] = re.sub(" +", " ", description)
+
+ # Add endpoint examples
+ reference[path]["examples"] = route.openapi_extra.get("examples", [])
+
+ # Add endpoint parameters fields for standard provider
+ if route_method == {"GET"}:
+ # openbb provider is always present hence its the standard field
+ reference[path]["parameters"]["standard"] = get_provider_field_params(
+ model_map, "QueryParams"
)
- # Update the func_def to include the full path
- def_params = [
- f"{d['name']}: {d['cleaned_type']}{' = ' + d['default'] if d['default'] else ''}"
- for d in meta_command["params"]
+ # Add `provider` parameter fields to the openbb provider
+ provider_parameter_fields = get_provider_parameter_info(route_func)
+ reference[path]["parameters"]["standard"].append(provider_parameter_fields)
+
+ # Add endpoint data fields for standard provider
+ reference[path]["data"]["standard"] = get_provider_field_params(
+ model_map, "Data"
+ )
+
+ for provider in model_map:
+ if provider == "openbb":
+ continue
+
+ # Adds standard parameters to the provider parameters since they are
+ # inherited by the model.
+ # A copy is used to prevent the standard parameters fields from being
+ # modified.
+ reference[path]["parameters"][provider] = reference[path]["parameters"][
+ "standard"
+ ].copy()
+ provider_query_params = get_provider_field_params(
+ model_map, "QueryParams", provider
+ )
+ reference[path]["parameters"][provider].extend(provider_query_params)
+
+ # Adds standard data fields to the provider data fields since they are
+ # inherited by the model.
+ # A copy is used to prevent the standard data fields from being modified.
+ reference[path]["data"][provider] = reference[path]["data"][
+ "standard"
+ ].copy()
+ provider_data = get_provider_field_params(model_map, "Data", provider)
+ reference[path]["data"][provider].extend(provider_data)
+
+ elif route_method == {"POST"}:
+ # Add endpoint parameters fields for POST methods
+ reference[path]["parameters"]["standard"] = get_post_method_parameters_info(
+ route_func
+ )
+
+ # Add endpoint returns data
+ # Currently only OBBject object is returned
+ if route_method == {"GET"}:
+ reference[path]["returns"]["OBBject"] = [
+ {
+ "name": "results",
+ "type": f"List[{standard_model}]",
+ "description": "Serializable results.",
+ },
+ {
+ "name": "provider",
+ "type": f"Optional[{provider_parameter_fields['type']}]",
+ "description": "Provider name.",
+ },
+ {
+ "name": "warnings",
+ "type": "Optional[List[Warning_]]",
+ "description": "List of warnings.",
+ },
+ {
+ "name": "chart",
+ "type": "Optional[Chart]",
+ "description": "Chart object.",
+ },
+ {
+ "name": "extra",
+ "type": "Dict[str, Any]",
+ "description": "Extra info.",
+ },
+ ]
+
+ elif route_method == {"POST"}:
+ reference[path]["returns"]["OBBject"] = get_post_method_returns_info(
+ route_func
+ )
+
+ # Dumping the reference dictionary as a JSON file
+ with open(PLATFORM_CONTENT_PATH / "reference.json", "w", encoding="utf-8") as f:
+ json.dump(reference, f, indent=4)
+
+
+def create_reference_markdown_seo(path: str, description: str) -> str:
+ """Create the SEO section for the markdown file.
+
+ Args:
+ path (str): Command path relative to the obb class
+ description (str): Description of the command
+
+ Returns:
+ str: SEO section for the markdown file
+ """
+
+ with open(SEO_METADATA_PATH) as f:
+ seo_metadata = json.load(f)
+
+ # Formatting path to match the key format in the SEO metadata
+ path = path.replace("/", ".")
+
+ if seo_metadata.get(path, None):
+ cleaned_title = seo_metadata[path]["title"]
+ cleaned_description = (
+ seo_metadata[path]["description"]
+ .strip()
+ .replace("\n", " ")
+ .replace(" ", " ")
+ .replace('"', "'")
+ )
+ keywords = "- " + "\n- ".join(seo_metadata[path]["keywords"])
+ else:
+ # Get the router name as the title
+ cleaned_title = path.split(".")[-1]
+ # Get the first sentence of the description
+ cleaned_description = description.split(".")[0].strip()
+ keywords = "- " + "\n- ".join(path.split("."))
+
+ markdown = (
+ "---\n"
+ f'title: "{cleaned_title}"\n'
+ f'description: "{cleaned_description}"\n'
+ f"keywords:\n{keywords}\n"
+ )
+
+ return markdown
+
+
+def create_reference_markdown_intro(
+ path: str, description: str, deprecated: Dict[str, str]
+) -> str:
+ """Create the introduction section for the markdown file.
+
+ Args:
+ path (str): Command path relative to the obb class
+ description (str): Description of the command
+ deprecated (Dict[str, str]): Deprecated flag and message
+
+ Returns:
+ str: Introduction section for the markdown file
+ """
+
+ deprecation_message = (
+ ":::caution Deprecated\n" f"{deprecated['message']}\n" ":::\n\n"
+ if deprecated["flag"]
+ else ""
+ )
+
+ markdown = (
+ "---\n\n"
+ "import HeadTitle from '@site/src/components/General/HeadTitle.tsx';\n\n"
+ f'\n\n'
+ f"{deprecation_message}"
+ "\n\n"
+ "import Tabs from '@theme/Tabs';\n"
+ "import TabItem from '@theme/TabItem';\n\n"
+ f"{description}\n\n"
+ )
+
+ return markdown
+
+
+def create_reference_markdown_examples(examples: List[str]) -> str:
+ """Create the example section for the markdown file.
+
+ Args:
+ examples (List[str]): List of examples
+
+ Returns:
+ str: Example section for the markdown file
+ """
+
+ examples_str = "from openbb import obb\n" + "\n".join(examples)
+ markdown = f"Example\n-------\n\n```python\n{examples_str}\n```\n\n"
+ return markdown
+
+
+def create_reference_markdown_tabular_section(
+ parameters: Dict[str, List[Dict[str, str]]], heading: str
+) -> str:
+ """Create the tabular section for the markdown file.
+
+ Args:
+ parameters (Dict[str, List[Dict[str, str]]]): Dictionary of
+ providers and their corresponding parameters
+ heading (str): Section heading for the tabular section
+
+ Returns:
+ str: Tabular section for the markdown file
+ """
+
+ tables_list = []
+
+ # params_list is a list of dictionaries containing the
+ # information for all the parameters of the provider.
+ for provider, params_list in parameters.items():
+ # Exclude the standard parameters from the table
+ filtered_params = [
+ {k: v for k, v in params.items() if k != "standard"}
+ for params in params_list
]
- meta_command["func_def"] = f"{full_command_path}({', '.join(def_params)})"
- available_providers = re.sub(
- r"Optional\[Literal\[([^\]]*)\]",
- r"\1",
- str(
- getattr(formatted_params.get("provider", ""), "_annotation", "")
- ).replace("typing.", ""),
- )
- if sig.return_annotation != _empty:
- return_type = sig.return_annotation
- if hasattr(return_type, "__origin__"):
- return_type = return_type.__origin__
+ # Do not add default and optional columns in the Data section
+ # because James and Andrew don't like it
+ if heading == "Data":
+ filtered_params = [
+ {k: v for k, v in params.items() if k not in ["default", "optional"]}
+ for params in filtered_params
+ ]
- description = f"""OBBject
- results : List[{model_name}]
- Serializable results.
+ # Parameter information for every provider is extracted from the dictionary
+ # and joined to form a row of the table.
+ # A `|` is added at the start and end of the row to create the table cell.
+ params_table_rows = [
+ f"| {' | '.join(map(str, params.values()))} |" for params in filtered_params
+ ]
+ # All rows are joined to form the table.
+ params_table_rows_str = "\n".join(params_table_rows)
- provider : Optional[Literal[{available_providers}]
- Provider name.
+ if heading == "Parameters":
+ tables_list.append(
+ f"\n\n\n"
+ "| Name | Type | Description | Default | Optional |\n"
+ "| ---- | ---- | ----------- | ------- | -------- |\n"
+ f"{params_table_rows_str}\n"
+ "\n"
+ )
+ elif heading == "Data":
+ tables_list.append(
+ f"\n\n\n"
+ "| Name | Type | Description |\n"
+ "| ---- | ---- | ----------- |\n"
+ f"{params_table_rows_str}\n"
+ "\n"
+ )
- warnings : Optional[List[Warning_]]
- List of warnings.
+ # For easy debugging of the created strings
+ tables = "".join(tables_list)
+ markdown = f"---\n\n## {heading}\n\n\n{tables}\n\n\n"
- chart : Optional[Chart]
- Chart object.
-
- extra: Dict[str, Any]
- Extra info."""
-
- meta_command["returns"] = {
- "type": return_type.__name__,
- "doc": description,
- }
-
- standard, provider_extras, provider_params = {}, {}, {} # type: ignore
-
- for provider_name, model_details in providers.items():
- data_fields: Dict[str, FieldInfo] = model_details["Data"]["fields"]
- query_fields: Dict[str, FieldInfo] = model_details["QueryParams"]["fields"]
-
- if provider_name == "openbb":
- for name, field in data_fields.items():
- standard[name] = {
- "type": get_annotation_type(field.annotation),
- "doc": (field.description or "").replace("\n", "
").strip(),
- }
- else:
- for name, field in query_fields.items():
- if name not in obb_query_fields:
- provider_params.setdefault(provider_name, {})[name] = {
- "type": get_annotation_type(field.annotation),
- "doc": (field.description or "")
- .replace("\n", "
")
- .strip(),
- "optional": "True" if not field.is_required() else "False",
- "default": str(field.default).replace(
- "PydanticUndefined", ""
- ),
- }
- for name, field in data_fields.items():
- if name not in providers["openbb"]["Data"]["fields"]:
- provider_extras.setdefault(provider_name, {})[name] = {
- "type": get_annotation_type(field.annotation),
- "doc": (field.description or "")
- .replace("\n", "
")
- .strip(),
- }
-
- meta_command["schema"] = {
- "standard": standard,
- "provider_extras": provider_extras,
- }
- meta_command["provider_params"] = provider_params
-
- if not meta_command.get("params", None):
- meta_command.update(
- get_docstring_meta(func, full_command_path, formatted_params)
- )
-
- ref_path = full_command_path.replace("obb.", "")
- cmd_keywords = "\n- ".join(ref_path.split("."))
- default_desc = (
- meta_command.get("description", "").split(".").pop(0).strip().replace(":", "")
- )
- header = (
- f"title: {func_name}\ndescription: {default_desc}\nkeywords:\n- {cmd_keywords}"
- )
-
- if seo_meta := SEO_META.get(
- ref_path, SEO_META.get((".".join(path.split("/")[-2:])), None)
- ):
- keywords = "\n- ".join(seo_meta["keywords"])
- header = f"title: {seo_meta['title']}\ndescription: {seo_meta['description']}\nkeywords:\n- {keywords}"
-
- title = ref_path.split(".")
- title[0] += " "
-
- meta_command[
- "header"
- ] = f"""---\n{header}\n---\n
-import HeadTitle from '@site/src/components/General/HeadTitle.tsx';
-
-\n\n"""
-
- if meta_command["deprecated"]:
- meta_command[
- "header"
- ] += f":::caution Deprecated\n{meta_command['deprecation_message']}\n:::\n\n"
-
- return meta_command
-
-
-def generate_data_markdown_section(meta: Dict[str, Any], command: bool = True):
- if not meta.get("schema", None):
- return ""
-
- markdown = "## Data\n\n" if command else ""
- markdown += """
-\n\n"""
- markdown += "| Name | Type | Description |\n"
- markdown += "| ---- | ---- | ----------- |\n"
- standard = ""
- for name, field in meta["schema"]["standard"].items():
- standard += f"| {name} | {field['type']} | {field['doc']} |\n"
- markdown += standard
- markdown += "\n\n"
-
- for provider, fields in meta["schema"]["provider_extras"].items():
- markdown += f"\n\n"
- markdown += "| Name | Type | Description |\n"
- markdown += "| ---- | ---- | ----------- |\n"
- markdown += standard
- for name, field in fields.items():
- markdown += f"| {name} | {field['type']} | {field['doc']} |\n"
- markdown += "\n\n"
-
- markdown += "\n\n"
return markdown
-def generate_params_markdown_section(meta: Dict[str, Any]):
- if not meta.get("params", None):
- return ""
+def create_reference_markdown_returns_section(returns: List[Dict[str, str]]) -> str:
+ """Create the returns section for the markdown file.
- markdown = """
-\n\n"""
- markdown += "| Name | Type | Description | Default | Optional |\n"
- markdown += "| ---- | ---- | ----------- | ------- | -------- |\n"
- standard = ""
- for param in meta["params"]:
- standard += f"| {param['name']} | {param['type']} | {param['doc']} | {param['default']} | {param['optional']} |\n"
+ Args:
+ returns (List[Dict[str, str]]): List of dictionaries containing
+ the name, type and description of the returns
- markdown += standard
- markdown += "\n\n"
+ Returns:
+ str: Returns section for the markdown file
+ """
- for provider, fields in meta["provider_params"].items():
- markdown += f"\n\n"
- markdown += "| Name | Type | Description | Default | Optional |\n"
- markdown += "| ---- | ---- | ----------- | ------- | -------- |\n"
- markdown += standard
- for name, field in fields.items():
- markdown += f"| {name} | {field['type']} | {field['doc']} | {field['default']} | {field['optional']} |\n"
- markdown += "\n\n"
+ returns_data = ""
+
+ for params in returns:
+ returns_data += f"{TAB_WIDTH*' '}{params['name']} : {params['type']}\n"
+ returns_data += f"{TAB_WIDTH*' '}{TAB_WIDTH*' '}{params['description']}\n\n"
+
+ # Remove the last two newline characters to render Returns section properly
+ returns_data = returns_data.rstrip("\n\n")
+
+ markdown = (
+ "---\n\n"
+ "## Returns\n\n"
+ "```python wordwrap\n"
+ "OBBject\n"
+ f"{returns_data}\n"
+ "```\n\n"
+ )
- markdown += "\n\n"
return markdown
-def generate_data_model_card_info(meta: Dict[str, Any]) -> Tuple[str, str]:
- description = meta["description"]
+def create_data_model_markdown(title: str, description: str, model: str) -> str:
+ """Create the basic markdown file content for the data model.
- split_description = list(filter(None, description.split("."))) # type: ignore
- title = split_description[0]
- description = ".".join(split_description[1:]) if len(split_description) > 1 else ""
+ Args:
+ title (str): Title of the data model
+ description (str): Description of the data model
+ model (str): Model name
- return title, description
+ Returns:
+ str: Basic markdown file content for the data model
+ """
+
+ # File name is used in the import statement
+ model_import_file = find_data_model_implementation_file(model)
+
+ # Get the first sentence of the description
+ cleaned_description = description.split(".")[0].strip()
+ # SEO section for the markdown file
+ seo_section = (
+ "---\n"
+ f'title: "{title}"\n'
+ f'description: "{cleaned_description}"\n'
+ "---\n\n"
+ )
+ # Introduction section for the markdown file
+ intro_section = (
+ "\n\n"
+ "import Tabs from '@theme/Tabs';\n"
+ "import TabItem from '@theme/TabItem';\n\n"
+ "---\n\n"
+ "## Implementation details\n\n"
+ )
+ # Tabular section for the markdown file
+ tables_section = (
+ "### Class names\n\n"
+ "| Model name | Parameters class | Data class |\n"
+ "| ---------- | ---------------- | ---------- |\n"
+ f"| `{model}` | `{model}QueryParams` | `{model}Data` |\n"
+ )
+ # Import statement for the markdown file
+ import_section = (
+ "\n### Import Statement\n\n"
+ "```python\n"
+ f"from openbb_core.provider.standard_models.{model_import_file} import (\n"
+ f"{model}Data,\n{model}QueryParams,\n"
+ ")\n"
+ "```\n\n"
+ )
+ # For easy debugging of the created strings
+ markdown = seo_section + intro_section + tables_section + import_section
+
+ return markdown
def find_data_model_implementation_file(data_model: str) -> str:
+ """Find the file name containing the data model class.
+
+ Args:
+ data_model (str): Data model name
+
+ Returns:
+ str: File name containing the data model class
+ """
+
+ # Function to search for the data model class in the file
def search_in_file(file_path: Path, search_string: str) -> bool:
with open(file_path, encoding="utf-8", errors="ignore") as file:
if search_string in file.read():
return True
return False
+ # Path is derived using the openbb-core package from the
+ # provider/standard_models folder
standard_models_path = Path(standard_models.__file__).parent
file_name = ""
@@ -607,195 +703,331 @@ def find_data_model_implementation_file(data_model: str) -> str:
return file_name
-def generate_implementation_details_markdown_section(data_model: str) -> str:
- markdown = "---\n\n## Implementation details\n\n"
- markdown += "### Class names\n\n"
- markdown += "| Model name | Parameters class | Data class |\n"
- markdown += "| ---------- | ---------------- | ---------- |\n"
- markdown += f"| `{data_model}` | `{data_model}QueryParams` | `{data_model}Data` |\n"
- markdown += "\n### Import Statement\n\n"
- markdown += "```python\n"
+def generate_reference_index_files(reference_content: Dict[str, str]) -> None:
+ """Generate index.mdx and _category_.json files for directories and sub-directories
+ in the reference directory.
- file_name = find_data_model_implementation_file(data_model)
+ Args:
+ reference_content (Dict[str, str]): Endpoints and their corresponding descriptions.
+ """
- markdown += f"from openbb_core.provider.standard_models.{file_name} import (\n"
- markdown += f"{data_model}Data,\n{data_model}QueryParams,\n"
- markdown += ")\n```"
+ def generate_index_and_category(
+ path: Path, parent_label: str = "Reference", position: int = 5
+ ):
+ # Check for sub-directories and markdown files in the current directory
+ sub_dirs = [d for d in path.iterdir() if d.is_dir()]
+ markdown_files = [
+ f for f in path.iterdir() if f.is_file() and f.suffix == ".md"
+ ]
- return markdown
+ # Generate _category_.json for the current directory
+ category_content = {"label": parent_label, "position": position}
+ with open(path / "_category_.json", "w", encoding="utf-8") as f:
+ json.dump(category_content, f, indent=2)
+
+ # Initialize index.mdx content with the parent label and import statement
+ index_content = f"# {parent_label}\n\n{PLATFORM_REFERENCE_IMPORT}\n\n"
+
+ # Menus section for sub-directories
+ if sub_dirs:
+ index_content += "### Menus\n"
+ index_content += PLATFORM_REFERENCE_UL_ELEMENT + "\n"
+ for sub_dir in sub_dirs:
+ # Initialize the sub-directory description
+ sub_dir_description = ""
+ # Capitalize the sub-directory name to use as a title for display
+ title = sub_dir.name.capitalize()
+ # Get the relative path of the sub-directory from the platform reference path
+ # and convert it to POSIX style for consistency across OS
+ sub_dir_path = sub_dir.relative_to(PLATFORM_REFERENCE_PATH).as_posix()
+ # List all markdown files in the sub-directory, excluding the index.mdx file,
+ # to include in the description
+ sub_dir_markdown_files = [
+ f.stem for f in sub_dir.glob("*.md") if f.name != "index.mdx"
+ ]
+ # If there are markdown files found, append their names to the sub-directory
+ # description, separated by commas
+ if sub_dir_markdown_files:
+ if len(sub_dir_markdown_files) <= MAX_COMMANDS:
+ sub_dir_description += ", ".join(sub_dir_markdown_files)
+ else:
+ sub_dir_description += (
+ f"{', '.join(sub_dir_markdown_files[:MAX_COMMANDS])},..."
+ )
+
+ url = f"/platform/reference/{sub_dir_path}"
+ index_content += f'\n'
+ index_content += "
\n\n"
+
+ # Commands section for markdown files
+ if markdown_files:
+ index_content += "### Commands\n"
+ index_content += PLATFORM_REFERENCE_UL_ELEMENT + "\n"
+ for file in markdown_files:
+ # Check if the current file is not the index file to avoid self-referencing
+ if file.name != "index.mdx":
+ # Extract the file name without extension to use as a title
+ title = file.stem.replace("_", " ")
+ # Generate a relative file path from the PLATFORM_REFERENCE_PATH,
+ # remove the file extension, and convert it to POSIX path format
+ # for consistency across OS
+ file_path = file.relative_to(PLATFORM_REFERENCE_PATH).with_suffix("").as_posix() # fmt: skip
+ # Attempt to fetch the file's description from reference_content
+ # using its path,split by the first period to get the first sentence,
+ # and default to an empty string if not found
+ file_description = reference_content.get(f"/{file_path}", "").split(".")[0] # fmt: skip
+ url = f"/platform/reference/{file_path}"
+ index_content += f'\n'
+ index_content += "\n\n"
+
+ # Save the index.mdx file
+ with open(path / "index.mdx", "w", encoding="utf-8") as f:
+ f.write(index_content)
+
+ # Recursively generate for sub-directories
+ for i, sub_dir in enumerate(sub_dirs, start=1):
+ generate_index_and_category(sub_dir, sub_dir.name.capitalize(), i)
+
+ # Start the recursive generation from the PLATFORM_REFERENCE_PATH
+ generate_index_and_category(PLATFORM_REFERENCE_PATH)
+
+
+def generate_reference_top_level_index() -> None:
+ """Generate the top-level index.mdx file for the reference directory."""
+
+ # Get the sub-directories in the reference directory
+ reference_dirs = [d for d in PLATFORM_REFERENCE_PATH.iterdir() if d.is_dir()]
+ reference_dirs.sort()
+ reference_cards_content = ""
+
+ for dir_path in reference_dirs:
+ # Sub-directory name is used as the title for the ReferenceCard component
+ title = dir_path.name
+ markdown_files = []
+
+ # Recursively find all markdown files in the directory and subdirectories
+ for file in dir_path.rglob("*.md"):
+ markdown_files.append(file.stem)
+
+ # Format description as a comma-separated string
+ if len(markdown_files) <= MAX_COMMANDS:
+ description_str = f"{', '.join(markdown_files)}"
+ else:
+ description_str = f"{', '.join(markdown_files[:MAX_COMMANDS])},..."
+
+ reference_cards_content += (
+ f"\n"
+ )
+
+ index_content = (
+ "# Reference\n\n"
+ f"{PLATFORM_REFERENCE_IMPORT}\n\n"
+ "\n"
+ f"{reference_cards_content}"
+ "
\n"
+ )
+
+ # Generate the top-level index.mdx file for the reference directory
+ with (PLATFORM_REFERENCE_PATH / "index.mdx").open("w", encoding="utf-8") as f:
+ f.write(index_content)
+
+
+def create_data_models_index(title: str, description: str, model: str) -> str:
+ """Create the index content for the data models.
+
+ Args:
+ title (str): Title of the data model
+ description (str): Description of the data model
+ model (str): Model name
+
+ Returns:
+ str: Index content for the data models
+ """
+
+ # Get the first sentence of the description
+ description = description.split(".")[0].strip()
+
+ # For easy debugging of the created strings
+ index_content = (
+ "\n"
+ )
+
+ return index_content
+
+
+def generate_data_models_index_files(content: str) -> None:
+ """Generate index.mdx and _category_.json files for the data_models directory.
+
+ Args:
+ content (str): Content for the data models index file
+ """
+
+ index_content = (
+ "# Data Models\n\n"
+ f"{PLATFORM_REFERENCE_IMPORT}\n\n"
+ "\n"
+ )
+
+ # Generate the index.mdx file for the data_models directory
+ with open(PLATFORM_DATA_MODELS_PATH / "index.mdx", "w", encoding="utf-8") as f:
+ f.write(index_content)
+
+ # Generate the _category_.json file for the data_models directory
+ category_content = {"label": "Data Models", "position": 6}
+ with open(
+ PLATFORM_DATA_MODELS_PATH / "_category_.json", "w", encoding="utf-8"
+ ) as f:
+ json.dump(category_content, f, indent=2)
+
+
+def generate_markdown_file(path: str, markdown_content: str, directory: str) -> None:
+ """Generate markdown file using the content of the specified path and directory.
+
+ Args:
+ path (str): Path to the markdown file
+ markdown_content (str): Content for the markdown file
+ directory (str): Directory to save the markdown file
+
+ Raises:
+ ValueError: If the content type is invalid
+ """
+
+ # For reference, split the path to separate the
+ # directory structure from the file name
+ if directory == "reference":
+ parts = path.strip("/").split("/")
+ file_name = f"{parts[-1]}.md"
+ directory_path = PLATFORM_REFERENCE_PATH / "/".join(parts[:-1])
+ # For data models, the file name is derived from the last
+ # part of the path, and there's no additional directory structure
+ elif directory == "data_models":
+ file_name = f"{path.split('/')[-1]}.md"
+ directory_path = PLATFORM_DATA_MODELS_PATH
+ else:
+ raise ValueError(f"Invalid directory: {directory}")
+
+ # Ensure the directory exists
+ directory_path.mkdir(parents=True, exist_ok=True)
+
+ # Generate the markdown file for the specified path and directory
+ with open(directory_path / file_name, "w", encoding="utf-8") as md_file:
+ md_file.write(markdown_content)
def generate_platform_markdown() -> None:
"""Generate markdown files for OpenBB Docusaurus website."""
- route_map = PathHandler.build_route_map()
- path_list = sorted(PathHandler.build_path_list(route_map=route_map))
- print("Generating markdown files...")
- kwargs = {"encoding": "utf-8", "newline": "\n"}
- content_path = website_path / "content/platform/reference"
- data_models_path = website_path / "content/platform/data_models"
- reference_cards: Dict[Path, List[Dict[str, str]]] = {}
- data_reference_cards: Dict[Path, List[Dict[str, str]]] = {}
+ data_models_index_content = []
+ reference_index_content_dict = {}
- for file in content_path.glob("*"):
- if file.is_file():
- file.unlink()
- else:
- shutil.rmtree(file)
- for file in data_models_path.glob("*"):
- if file.is_file():
- file.unlink()
- else:
- shutil.rmtree(file)
+ print("[CRITICAL] Ensure all the extensions are installed before running this script!") # fmt: skip
- for path in path_list:
- meta_command = get_command_meta(path, route_map)
- if not meta_command:
- continue
- func = PathHandler.get_route(path=path, route_map=route_map).endpoint
+ # Generate and read the reference.json file
+ print("[INFO] Generating the reference.json file...")
+ generate_reference_file()
+ with open(PLATFORM_CONTENT_PATH / "reference.json") as f:
+ reference = json.load(f)
- func_name = func.__name__
- if func_name == "index":
- func_name = "index_cmd"
+ # Clear the platform/reference folder
+ print("[INFO] Clearing the platform/reference folder...")
+ shutil.rmtree(PLATFORM_REFERENCE_PATH, ignore_errors=True)
- folder = "/".join(path.strip("/").split("/")[:-1])
- filepath = content_path / folder / f"{func_name}.md"
+ # Clear the platform/data_models folder
+ print("[INFO] Clearing the platform/data_models folder...")
+ shutil.rmtree(PLATFORM_DATA_MODELS_PATH, ignore_errors=True)
- markdown = generate_markdown(meta_command=meta_command)
+ print(f"[INFO] Generating the markdown files for the {PLATFORM_REFERENCE_PATH} sub-directories...") # fmt: skip
+ print(f"[INFO] Generating the markdown files for the {PLATFORM_DATA_MODELS_PATH} directory...") # fmt: skip
- if data_model := meta_command.get("model", None):
- ## title is the desc here - clean this later
- (
- data_model_card_title,
- data_model_card_description,
- ) = generate_data_model_card_info(meta_command)
+ for path, path_data in reference.items():
+ reference_markdown_content = ""
+ data_markdown_content = ""
- title = re.sub(
+ description = path_data["description"]
+ path_parameters_fields = path_data["parameters"]
+ path_data_fields = path_data["data"]
+
+ reference_index_content_dict[path] = description
+
+ reference_markdown_content = create_reference_markdown_seo(
+ path[1:], description
+ )
+ reference_markdown_content += create_reference_markdown_intro(
+ path[1:], description, path_data["deprecated"]
+ )
+ reference_markdown_content += create_reference_markdown_examples(
+ path_data["examples"]
+ )
+
+ if path_parameters_fields := path_data["parameters"]:
+ reference_markdown_content += create_reference_markdown_tabular_section(
+ path_parameters_fields, "Parameters"
+ )
+
+ reference_markdown_content += create_reference_markdown_returns_section(
+ path_data["returns"]["OBBject"]
+ )
+
+ if path_data_fields := path_data["data"]:
+ reference_markdown_content += create_reference_markdown_tabular_section(
+ path_data_fields, "Data"
+ )
+
+ generate_markdown_file(path, reference_markdown_content, "reference")
+
+ if model := path_data["model"]:
+ data_markdown_title = re.sub(
r"([A-Z]{1}[a-z]+)|([A-Z]{3}|[SP500]|[EU])([A-Z]{1}[a-z]+)|([A-Z]{5,})", # noqa: W605
lambda m: (
f"{m.group(1) or m.group(4)} ".title()
if not any([m.group(2), m.group(3)])
else f"{m.group(2)} {m.group(3)} "
),
- data_model,
+ path_data["model"],
).strip()
-
- data_markdown = (
- f"---\ntitle: {title}\n"
- f"description: {data_model_card_title}\n---\n\n"
- "\n\n"
- "import Tabs from '@theme/Tabs';\nimport TabItem from '@theme/TabItem';\n\n"
+ data_markdown_content = create_data_model_markdown(
+ data_markdown_title,
+ description,
+ model,
+ )
+ data_markdown_content += create_reference_markdown_tabular_section(
+ path_parameters_fields, "Parameters"
+ )
+ data_markdown_content += create_reference_markdown_tabular_section(
+ path_data_fields, "Data"
+ )
+ data_models_index_content.append(
+ create_data_models_index(data_markdown_title, description, model)
)
- data_markdown += generate_implementation_details_markdown_section(
- data_model
- )
+ generate_markdown_file(model, data_markdown_content, "data_models")
- data_model_markdown = generate_data_markdown_section(meta_command)
- data_markdown += "\n\n## Parameters\n\n"
- if meta_command["params"]:
- data_markdown += generate_params_markdown_section(meta_command)
-
- data_markdown += data_model_markdown
- markdown += data_model_markdown
-
- data_filepath = data_models_path / f"{data_model}.md"
-
- data_reference_cards.setdefault(data_filepath.parent, []).append(
- dict(
- title=title,
- description=data_model_card_title or "",
- url=data_models_path.relative_to(data_models_path) / data_model,
- )
- )
- data_filepath.parent.mkdir(parents=True, exist_ok=True)
- with open(data_filepath, "w", **kwargs) as f: # type: ignore
- f.write(data_markdown)
-
- reference_cards.setdefault(filepath.parent, []).append(
- dict(
- title=func_name,
- description=func.__doc__ or "",
- url="/".join((content_path / folder).relative_to(content_path).parts),
- )
- )
-
- filepath.parent.mkdir(parents=True, exist_ok=True)
- with open(filepath, "w", **kwargs) as f: # type: ignore
- f.write(markdown)
-
- reference_cards = dict(sorted(reference_cards.items(), key=lambda item: item[0]))
- data_reference_cards = {
- folder: sorted(cards, key=lambda item: item["title"])
- for folder, cards in data_reference_cards.items()
- }
- with open(content_path / "index.mdx", "w", **kwargs) as f: # type: ignore
- fname = "OpenBB Platform Reference"
- rel_path = content_path.relative_to(content_path)
- write_reference_index(reference_cards, fname, content_path, rel_path, f)
-
- with open(data_models_path / "index.mdx", "w", **kwargs) as f: # type: ignore
- fname = "Data Models"
- rel_path = data_models_path.relative_to(data_models_path)
- write_reference_index(
- data_reference_cards, fname, data_models_path, rel_path, f, True
- )
-
- with open(content_path / "_category_.json", "w", **kwargs) as f: # type: ignore
- # Generate category json
- f.write(json.dumps({"label": "Reference", "position": 5}, indent=2))
-
- def gen_category_json(fname: str, path: Path, position: int = 1):
- """Generate category json"""
- with open(path / "_category_.json", "w", **kwargs) as f: # type: ignore
- f.write(
- json.dumps({"label": fname.title(), "position": position}, indent=2)
- )
-
- with open(path / "index.mdx", "w", **kwargs) as f: # type: ignore
- rel_path = path.relative_to(content_path)
- write_reference_index(reference_cards, fname, path, rel_path, f)
-
- def gen_category_recursive(nested_path: Path):
- """Generate category json recursively"""
- position = 1
- for folder in nested_path.iterdir():
- if folder.is_dir():
- gen_category_json(folder.name, folder, position)
- gen_category_recursive(folder) # pylint: disable=cell-var-from-loop
- position += 1
-
- gen_category_recursive(content_path)
- print(f"Markdown files generated, check the {content_path} folder.")
-
-
-def save_metadata(path: Path) -> Dict[str, Dict[str, Union[str, List[str]]]]:
- """Save SEO metadata"""
- regex = re.compile(
- r"---\ntitle: (.*)\ndescription: (.*)\nkeywords:(.*)\n---\n\nimport HeadTitle",
- re.MULTILINE | re.DOTALL,
+ # Generate the index.mdx and _category_.json files for the reference directory
+ print(f"[INFO] Generating the index files for the {PLATFORM_REFERENCE_PATH} sub-directories...") # fmt: skip
+ generate_reference_index_files(reference_index_content_dict)
+ print(
+ f"[INFO] Generating the index files for the {PLATFORM_REFERENCE_PATH} directory..."
)
+ generate_reference_top_level_index()
- metadata = {}
- for file in path.rglob("*/**/*.md"):
- context = file.read_text(encoding="utf-8")
- match = regex.search(context)
- if match:
- title, description, keywords = match.groups()
- key = file.relative_to(path).as_posix().removesuffix(".md")
- metadata[key.replace("/", ".")] = {
- "title": title,
- "description": description,
- "keywords": [
- keyword.strip() for keyword in keywords.split("\n- ") if keyword
- ],
- }
+ # Sort the data models index content alphabetically to display in the same order
+ data_models_index_content.sort()
+ data_models_index_content_str = "".join(data_models_index_content)
- filepath = website_path / "metadata/platform_v4_seo_metadata2.json"
- with open(filepath, "w", encoding="utf-8", newline="\n") as f:
- json.dump(metadata, f, indent=2)
-
- return metadata
+ # Generate the index.mdx and _category_.json files for the data_models directory
+ print(f"[INFO] Generating the index files for the {PLATFORM_DATA_MODELS_PATH} directory...") # fmt: skip
+ generate_data_models_index_files(data_models_index_content_str)
+ print("[INFO] Markdown files generated successfully!")
if __name__ == "__main__":
diff --git a/website/tests/test_platform_docs.py b/website/tests/test_platform_docs.py
new file mode 100644
index 00000000000..e8565e80d0b
--- /dev/null
+++ b/website/tests/test_platform_docs.py
@@ -0,0 +1,308 @@
+import datetime
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+from typing import List, Literal, Optional, Union
+
+import pytest
+from pydantic import BaseModel, Field
+
+sys.path.append(str(Path(__file__).resolve().parent.parent))
+
+# pylint: disable=unused-argument,redefined-outer-name,wrong-import-position
+
+from generate_platform_v4_markdown import ( # noqa: E402
+ get_field_data_type,
+ get_function_params_default_value,
+ get_post_method_parameters_info,
+ get_post_method_returns_info,
+ get_provider_field_params,
+ get_provider_parameter_info,
+)
+
+
+# Mock Classes
+class MockField:
+ def __init__(self, annotation):
+ self.annotation = annotation
+
+
+class MockReturnType:
+ def __init__(self):
+ self.model_fields = {"results": MockField(annotation="List[str]")}
+
+
+@dataclass
+class MockProvider(BaseModel):
+ provider: Literal["provider1", "provider2"] = "provider1"
+
+
+class MockModel(BaseModel):
+ provider: Optional[MockProvider] = Field(default=None)
+
+
+# Mock Pydantic models to simulate your data structure
+class OpenBBQueryParams(BaseModel):
+ field1: Optional[int] = Field(default=None, description="A sample integer field")
+ date: datetime.date = Field(
+ default=None, description="A datetime.date type field (expansion type)"
+ )
+
+
+class OpenBBData(BaseModel):
+ field1: Literal["a", "b"] = Field(
+ default="a", description="A sample Union field with Literal['a', 'b']"
+ )
+ field2: Optional[int] = Field(
+ default=10,
+ description="A field with annotated type",
+ gt=0,
+ )
+
+
+class Provider1QueryParams(BaseModel):
+ field1: Optional[List[str]] = Field(
+ default=None, description="A sample List[str] field"
+ )
+
+
+class EmptyModel(BaseModel):
+ pass
+
+
+@pytest.fixture
+def mock_data():
+ return {
+ "openbb": {
+ "QueryParams": {"fields": OpenBBQueryParams().model_fields},
+ "Data": {"fields": OpenBBData().model_fields},
+ },
+ "provider1": {
+ "QueryParams": {"fields": Provider1QueryParams().model_fields},
+ "Data": {"fields": EmptyModel().model_fields},
+ },
+ }
+
+
+@pytest.fixture
+def mock_endpoint():
+ def endpoint(param1: int, param2: Optional[str] = "default"):
+ pass # Mock endpoint function
+
+ return endpoint
+
+
+class MockEndpointSetup:
+ def __init__(self):
+ self.mock_endpoint = self._create_mock_endpoint()
+
+ def _create_mock_endpoint(self):
+ def mock_endpoint(provider_choices: Optional[MockProvider] = None):
+ return ["result1", "result2"]
+
+ # Simulate annotations with a dataclass model
+ mock_endpoint.__annotations__ = {
+ "provider_choices": Optional[MockProvider],
+ "return": List[str],
+ }
+
+ return mock_endpoint
+
+
+@pytest.fixture
+def mock_endpoint_setup():
+ return MockEndpointSetup()
+
+
+# Test Classes
+class TestFieldDataType:
+ @pytest.mark.parametrize(
+ "input_type,expected_output",
+ [
+ (int, "int"),
+ (str, "str"),
+ (Optional[int], "int"),
+ (Union[int, None], "int"),
+ ],
+ )
+ def test_get_field_data_type(self, input_type, expected_output):
+ assert get_field_data_type(input_type) == expected_output
+
+
+class TestProviderFieldParams:
+ def test_get_provider_field_params(self, mock_data, monkeypatch):
+ monkeypatch.setattr(
+ "generate_platform_v4_markdown.get_field_data_type",
+ get_field_data_type,
+ )
+ # Define expected_result based on mock_data and logic in get_provider_field_params
+ expected_result_1 = [
+ {
+ "name": "field1",
+ "type": "int",
+ "description": "A sample integer field",
+ "default": "None",
+ "optional": True,
+ "standard": True,
+ },
+ {
+ "name": "date",
+ "type": "Union[str, date]",
+ "description": "A datetime.date type field (expansion type)",
+ "default": "None",
+ "optional": True,
+ "standard": True,
+ },
+ ]
+
+ expected_result_2 = [
+ {
+ "name": "field1",
+ "type": "List[str]",
+ "description": "A sample List[str] field",
+ "default": "None",
+ "optional": True,
+ "standard": False,
+ },
+ ]
+
+ expected_result_3 = [
+ {
+ "name": "field1",
+ "type": "Literal['a', 'b']",
+ "description": "A sample Union field with Literal['a', 'b']",
+ "default": "a",
+ "optional": True,
+ "standard": True,
+ },
+ {
+ "name": "field2",
+ "type": "int",
+ "description": "A field with annotated type",
+ "default": "10",
+ "optional": True,
+ "standard": True,
+ },
+ ]
+
+ result = get_provider_field_params(mock_data, "QueryParams")
+ assert result == expected_result_1
+
+ result = get_provider_field_params(mock_data, "QueryParams", "provider1")
+ assert result == expected_result_2
+
+ result = get_provider_field_params(mock_data, "Data")
+ assert result == expected_result_3
+
+
+class TestProviderParameterInfo:
+ def test_get_provider_parameter_info(self, mock_endpoint_setup):
+ # Access the mock_endpoint from the fixture
+ mock_endpoint = mock_endpoint_setup.mock_endpoint
+
+ expected = {
+ "name": "provider",
+ "type": "Literal['provider1', 'provider2']",
+ "description": "The provider to use for the query, by default None. If None, the provider specified in defaults is selected or 'provider1' if there is no default.", # noqa: E501
+ "default": "provider1",
+ "optional": True,
+ "standard": True,
+ }
+
+ result = get_provider_parameter_info(mock_endpoint)
+ assert result == expected
+
+
+class TestFunctionParamsDefaultValue:
+ def test_get_function_params_default_value(self, mock_endpoint):
+ expected_result = {
+ "param1": "",
+ "param2": "default",
+ }
+
+ result = get_function_params_default_value(mock_endpoint)
+ assert (
+ result == expected_result
+ ), "Failed to correctly identify default parameter values."
+
+
+class TestPostMethodParametersInfo:
+ def test_get_post_method_parameters_info(self, monkeypatch):
+ monkeypatch.setattr(
+ "generate_platform_v4_markdown.get_field_data_type",
+ get_field_data_type,
+ )
+
+ def mock_endpoint(param1: int, param2: Optional[str] = "default"):
+ """
+ A mock endpoint function.
+
+ Parameters
+ ----------
+ param1 : int
+ The first parameter.
+ param2 : Optional[str]
+ The second parameter, optional.
+ """
+ return
+
+ expected_result = [
+ {
+ "name": "param1",
+ "type": "int",
+ "description": "The first parameter.",
+ "default": "",
+ "optional": False,
+ },
+ {
+ "name": "param2",
+ "type": "str",
+ "description": "The second parameter, optional.",
+ "default": "default",
+ "optional": True,
+ },
+ ]
+
+ result = get_post_method_parameters_info(mock_endpoint)
+ assert (
+ result == expected_result
+ ), "Failed to correctly extract POST method parameters info."
+
+
+class TestPostMethodReturnsInfo:
+ def test_get_post_method_returns_info(self, monkeypatch):
+ monkeypatch.setattr(
+ "generate_platform_v4_markdown.get_field_data_type",
+ get_field_data_type,
+ )
+
+ class MockReturnModel(BaseModel):
+ results: List[str]
+
+ def mock_endpoint() -> MockReturnModel:
+ """
+ A mock endpoint function.
+
+ Parameters
+ ----------
+ None
+
+ Returns
+ -------
+ MockReturnModel
+ A list of strings as results.
+ """
+ return
+
+ expected_result = [
+ {
+ "name": "results",
+ "type": "List[str]",
+ "description": "A list of strings as results.",
+ }
+ ]
+
+ result = get_post_method_returns_info(mock_endpoint)
+ assert (
+ result == expected_result
+ ), "Failed to correctly extract POST method returns info."