jerry/OpenBB
1
0
Fork 0
mirror of https://github.com/OpenBB-finance/OpenBB.git synced 2026-10-27 04:31:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update SDK Docs Generation (#4262)

Browse Source 
* update generate_sdk_markdown.py to use sdk trailmap class

* Update trailmap.py

* Update trailmap.py
This commit is contained in:
teh_coderer
2023-02-15 16:49:07 -06:00
committed by GitHub
parent 69b4d117fa
commit 3747424950
8 changed files with 178 additions and 265 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
generate_sdk.py
View File

@@ -528,13 +528,12 @@ def generate_sdk(sort: bool = False) -> None:
if __name__ == "__main__":
sys_args = sys.argv
sort = False
if len(sys_args) > 1:
if sys_args[1] == "sort":
sort = True
sort_csv = False
if len(sys.argv) > 1:
if sys.argv[1] == "sort":
sort_csv = True
console.print("\n\n[bright_magenta]Sorting CSV...[/]\n")
else:
console.print("[red]Invalid argument.\n Accepted arguments: sort[/]")
generate_sdk(sort)
generate_sdk(sort_csv)

44
openbb_terminal/forecast/forecast_view.py
View File

@@ -249,6 +249,23 @@ def show_df(
export: str = "",
sheet_name: str = None,
):
"""Show a dataframe in a table
Parameters
----------
data: pd.DataFrame
The dataframe to show
limit: int
The number of rows to show
limit_col: int
The number of columns to show
name: str
The name of the dataframe
export: str
Format to export data
sheet_name: str
Optionally specify the name of the sheet the data is exported to.
"""
console.print(
f"[green]{name} dataset has shape (row, column): {data.shape}\n[/green]"
)
@@ -279,6 +296,19 @@ def show_df(
def describe_df(
data: pd.DataFrame, name: str = "", export: str = "", sheet_name: str = None
):
"""Show descriptive statistics for a dataframe
Parameters
----------
data: pd.DataFrame
The dataframe to show
name: str
The name of the dataframe
export: str
Format to export data
sheet_name: str
Optionally specify the name of the sheet the data is exported to.
"""
new_df = forecast_model.describe_df(data)
print_rich_table(
new_df,
@@ -299,6 +329,20 @@ def describe_df(
def export_df(
data: pd.DataFrame, export: str, name: str = "", sheet_name: str = None
) -> None:
"""Export a dataframe to a file
Parameters
----------
data: pd.DataFrame
The dataframe to export
export: str
The format to export the dataframe to
name: str
The name of the dataframe
sheet_name: str
Optionally specify the name of the sheet the data is exported to.
"""
export_data(
export,
os.path.dirname(os.path.abspath(__file__)),

3
openbb_terminal/sdk.py
View File

@@ -217,9 +217,11 @@ class OpenBBSDK:
`delete`: Delete a column from a dataframe\n
`delta`: Calculate the %change of a variable based on a specific column\n
`desc`: Returns statistics for a given df\n
`desc_chart`: Show descriptive statistics for a dataframe\n
`ema`: A moving average provides an indication of the trend of the price movement\n
`expo`: Performs Probabilistic Exponential Smoothing forecasting\n
`expo_chart`: Display Probabilistic Exponential Smoothing forecast\n
`export`: Export a dataframe to a file\n
`linregr`: Perform Linear Regression Forecasting\n
`linregr_chart`: Display Linear Regression Forecasting\n
`load`: Load custom file into dataframe.\n
@@ -244,6 +246,7 @@ class OpenBBSDK:
`season_chart`: Plot seasonality from a dataset\n
`seasonalnaive`: Performs Seasonal Naive forecasting\n
`seasonalnaive_chart`: Display SeasonalNaive Model\n
`show`: Show a dataframe in a table\n
`signal`: A price signal based on short/long term price.\n
`sto`: Stochastic Oscillator %K and %D : A stochastic oscillator is a momentum indicator comparing a particular closing\n
`tcn`: Perform TCN forecasting\n

4
openbb_terminal/sdk_core/models/forecast_sdk_model.py
View File

@@ -28,9 +28,11 @@ class ForecastRoot(Category):
`delete`: Delete a column from a dataframe\n
`delta`: Calculate the %change of a variable based on a specific column\n
`desc`: Returns statistics for a given df\n
`desc_chart`: Show descriptive statistics for a dataframe\n
`ema`: A moving average provides an indication of the trend of the price movement\n
`expo`: Performs Probabilistic Exponential Smoothing forecasting\n
`expo_chart`: Display Probabilistic Exponential Smoothing forecast\n
`export`: Export a dataframe to a file\n
`linregr`: Perform Linear Regression Forecasting\n
`linregr_chart`: Display Linear Regression Forecasting\n
`load`: Load custom file into dataframe.\n
@@ -55,6 +57,7 @@ class ForecastRoot(Category):
`season_chart`: Plot seasonality from a dataset\n
`seasonalnaive`: Performs Seasonal Naive forecasting\n
`seasonalnaive_chart`: Display SeasonalNaive Model\n
`show`: Show a dataframe in a table\n
`signal`: A price signal based on short/long term price.\n
`sto`: Stochastic Oscillator %K and %D : A stochastic oscillator is a momentum indicator comparing a particular closing\n
`tcn`: Perform TCN forecasting\n
@@ -136,7 +139,6 @@ class ForecastRoot(Category):
lib.forecast_seasonalnaive_view.display_seasonalnaive_forecast
)
self.show = lib.forecast_view.show_df
self.show_chart = lib.forecast_view.show_df
self.signal = lib.forecast_model.add_signal
self.sto = lib.forecast_model.add_sto
self.tcn = lib.forecast_tcn_model.get_tcn_data

2
openbb_terminal/sdk_core/trail_map_forecasting.csv vendored
View File

@@ -30,7 +30,7 @@ forecast.rsi,forecast_model.add_rsi,
forecast.rwd,forecast_rwd_model.get_rwd_data,forecast_rwd_view.display_rwd_forecast
forecast.season,,forecast_view.display_seasonality
forecast.seasonalnaive,forecast_seasonalnaive_model.get_seasonalnaive_data,forecast_seasonalnaive_view.display_seasonalnaive_forecast
forecast.show,forecast_view.show_df,forecast_view.show_df
forecast.show,forecast_view.show_df,
forecast.signal,forecast_model.add_signal,
forecast.sto,forecast_model.add_sto,
forecast.tcn,forecast_tcn_model.get_tcn_data,forecast_tcn_view.display_tcn_forecast
1 trail model view
30 forecast.rwd forecast_rwd_model.get_rwd_data forecast_rwd_view.display_rwd_forecast
31 forecast.season forecast_view.display_seasonality
32 forecast.seasonalnaive forecast_seasonalnaive_model.get_seasonalnaive_data forecast_seasonalnaive_view.display_seasonalnaive_forecast
33 forecast.show forecast_view.show_df forecast_view.show_df
34 forecast.signal forecast_model.add_signal
35 forecast.sto forecast_model.add_sto
36 forecast.tcn forecast_tcn_model.get_tcn_data forecast_tcn_view.display_tcn_forecast

133
openbb_terminal/sdk_core/trailmap.py
View File

@@ -1,22 +1,44 @@
import importlib
import inspect
from pathlib import Path
from types import FunctionType
from typing import Dict, List, Optional
from typing import Any, Callable, Dict, ForwardRef, List, Optional
import pandas as pd
import openbb_terminal.sdk_core.sdk_init as lib
from openbb_terminal.core.config.paths import PACKAGE_DIRECTORY
from openbb_terminal.rich_config import console
from openbb_terminal.sdk_core.sdk_helpers import clean_attr_desc
from openbb_terminal.sdk_core.sdk_init import (
FORECASTING_TOOLKIT_ENABLED,
OPTIMIZATION_TOOLKIT_ENABLED,
)
def get_signature_parameters(
function: Callable[..., Any], globalns: Dict[str, Any]
) -> Dict[str, inspect.Parameter]:
signature = inspect.signature(function)
params = {}
cache: dict[str, Any] = {}
for name, parameter in signature.parameters.items():
annotation = parameter.annotation
if annotation is parameter.empty:
params[name] = parameter
continue
if annotation is None:
params[name] = parameter.replace(annotation=type(None))
continue
if isinstance(annotation, ForwardRef):
annotation = annotation.__forward_arg__
if isinstance(annotation, str):
annotation = eval(annotation, globalns, cache) # pylint: disable=W0123
params[name] = parameter.replace(annotation=annotation)
return params
class FuncAttr:
def __init__(self, func: Optional[str] = None):
def __init__(self) -> None:
self.short_doc: Optional[str] = None
self.long_doc: Optional[str] = None
self.func_def: Optional[str] = None
@@ -25,15 +47,11 @@ class FuncAttr:
self.full_path: Optional[str] = None
self.func_unwrapped: Optional[FunctionType] = None
self.func_wrapped: Optional[FunctionType] = None
if func:
self.get_func_attrs(func)
self.params: Dict[str, inspect.Parameter] = {}
def get_func_attrs(self, func: str) -> None:
attr = getattr(
importlib.import_module("openbb_terminal.sdk_core.sdk_init"),
func.split(".")[0],
)
func_attr = getattr(attr, func.split(".")[1])
module_path, function_name = func.rsplit(".", 1)
func_attr = getattr(getattr(lib, module_path), function_name)
self.func_wrapped = func_attr
add_lineon = 0
@@ -45,50 +63,63 @@ class FuncAttr:
self.func_unwrapped = func_attr
self.lineon = inspect.getsourcelines(func_attr)[1] + add_lineon
self.func_def = self.get_definition()
self.long_doc = func_attr.__doc__
self.short_doc = clean_attr_desc(func_attr)
for k, p in get_signature_parameters(func_attr, func_attr.__globals__).items():
self.params[k] = p
self.path = inspect.getfile(func_attr)
full_path = (
inspect.getfile(func_attr).replace("\\", "/").split("openbb_terminal/")[1]
)
self.full_path = f"openbb_terminal/{full_path}"
def get_definition(self) -> str:
def get_definition(
self, location_path: list, class_attr: str, view: bool = False
) -> str:
"""Creates the function definition to be used in SDK docs."""
funcspec = inspect.getfullargspec(self.func_unwrapped)
funcspec = self.params
definition = ""
added_comma = False
for arg in funcspec.args:
for arg, param in funcspec.items():
annotation = (
funcspec.annotations[arg] if arg in funcspec.annotations else "Any"
(
str(param.annotation)
.replace("<class '", "")
.replace("'>", "")
.replace("typing.", "")
.replace("pandas.core.frame.", "pd.")
.replace("pandas.core.series.", "pd.")
.replace("openbb_terminal.portfolio.", "")
)
if param.annotation != inspect.Parameter.empty
else "Any"
)
annotation = (
str(annotation)
.replace("<class '", "")
.replace("'>", "")
.replace("typing.", "")
.replace("pandas.core.frame.", "pd.")
.replace("pandas.core.series.", "pd.")
.replace("openbb_terminal.portfolio.", "")
)
definition += f"{arg}: {annotation}, "
added_comma = True
if added_comma:
definition = definition[:-2]
default = ""
if param.default is not param.empty:
arg_default = (
param.default
if param.default is not inspect.Parameter.empty
else "None"
)
default = (
f" = {arg_default}"
if not isinstance(arg_default, str)
else f' = "{arg_default}"'
)
definition += f"{arg}: {annotation}{default}, "
return_def = (
funcspec.annotations["return"].__name__
if "return" in funcspec.annotations
and hasattr(funcspec.annotations["return"], "__name__")
and funcspec.annotations["return"] is not None
else "None"
)
definition = f"def {self.func_unwrapped.__name__}({definition }) -> {return_def}" # type: ignore
return definition
definition = definition.rstrip(", ")
if location_path[0] == "root":
location_path[0] = ""
trail = ".".join([t for t in location_path if t != ""])
sdk_name = class_attr if not view else f"{class_attr}_chart"
sdk_path = f"{f'openbb.{trail}' if trail else 'openbb'}.{sdk_name}"
return f"{sdk_path}({definition })"
# pylint: disable=R0903
@@ -106,8 +137,14 @@ class Trailmap:
self.model_func: Optional[str] = f"lib.{model}" if model else None
self.view_func: Optional[str] = f"lib.{view}" if view else None
self.func_attrs: Dict[str, FuncAttr] = {}
self.func_attrs["model"] = FuncAttr(self.model)
self.func_attrs["view"] = FuncAttr(self.view) # type: ignore
self.func_attrs["model"] = FuncAttr()
self.func_attrs["view"] = FuncAttr()
for k, cls in self.func_attrs.items():
if getattr(self, f"{k}_func"):
cls.get_func_attrs(getattr(self, f"{k}"))
cls.func_def = cls.get_definition(
tmap, self.class_attr, view=k == "view"
)
def get_trailmaps(sort: bool = False) -> List[Trailmap]:
@@ -119,7 +156,7 @@ def get_trailmaps(sort: bool = False) -> List[Trailmap]:
PACKAGE_DIRECTORY / "sdk_core" / "trail_map_optimization.csv"
)
def load_csv(path: Path = None) -> pd.DataFrame:
def load_csv(path: Optional[Path] = None) -> Dict[str, Dict[str, str]]:
path = path or MAP_PATH
df = pd.read_csv(path, keep_default_na=False)
df = df.set_index("trail")
@@ -139,13 +176,13 @@ def get_trailmaps(sort: bool = False) -> List[Trailmap]:
def load():
map_dict = load_csv(path=MAP_PATH)
if FORECASTING_TOOLKIT_ENABLED:
if lib.FORECASTING_TOOLKIT_ENABLED:
map_forecasting_dict = load_csv(path=MAP_FORECASTING_PATH)
map_dict.update(map_forecasting_dict)
else:
print_error("Forecasting")
if OPTIMIZATION_TOOLKIT_ENABLED:
if lib.OPTIMIZATION_TOOLKIT_ENABLED:
map_optimization_dict = load_csv(path=MAP_OPTIMIZATION_PATH)
map_dict.update(map_optimization_dict)
else:

36
tests/website/test_generate_docs.py
View File

@@ -3,6 +3,7 @@ from typing import Optional
import pytest
try:
from openbb_terminal.sdk_core.trailmap import get_signature_parameters
from website import (
generate_sdk_markdown as gen_sdk,
generate_terminal_markdown as gen_term,
@@ -42,25 +43,30 @@ def mock_func(arg1: Optional[str] = "Test", arg2: Optional[bool] = True) -> bool
# pylint:disable=too-few-public-methods
class MockFuncAttrs:
"""Mock function attributes"""
def __init__(self):
self.lineon = 69
self.full_path = "test/mock_func.py"
self.long_doc = mock_func.__doc__
self.func_unwrapped = mock_func
self.func_def = (
'openbb.mock(arg1: Optional[str] = "Test", arg2: Optional[bool] = True)'
)
self.params = {}
for k, p in get_signature_parameters(mock_func, mock_func.__globals__).items():
self.params[k] = p
class MockTrailMap:
"""Mock trail map"""
def __init__(self):
self.trailmap = ""
self.func_attr = {"model": mock_func}
self.class_attr = {"model": "mock"}
self.lineon = {"model": 69}
self.full_path = {"model": "test/mock_func.py"}
self.model = "mock_func"
self.long_doc = {"model": mock_func.__doc__}
self.func_def = {
"model": 'openbb.mock(arg1: Optional[str] = "Test", arg2: Optional[bool] = True)'
}
self.params = {"model": {}}
for k, p in gen_sdk.get_signature_parameters(
mock_func, mock_func.__globals__
).items():
self.params["model"][k] = p
self.func_attrs = {}
self.func_attrs["model"] = MockFuncAttrs()
self.model = mock_func
self.class_attr = "mock"
EXPECTED_OUTPUT = """Stuff here or stuff there, it doesn't matter, it's everywhere.

210
website/generate_sdk_markdown.py
View File

@@ -1,210 +1,27 @@
import csv # noqa: I001
import importlib
import inspect
import json
import os
import shutil
from pathlib import Path
from types import FunctionType
from typing import Any, Callable, Dict, ForwardRef, List, Literal, Optional, Union
from typing import Any, Dict, Literal, Union
import pandas as pd
from docstring_parser import parse
from openbb_terminal.core.library.trail_map import (
FORECASTING_TOOLKIT_ENABLED as FORECASTING,
MISCELLANEOUS_DIRECTORY,
OPTIMIZATION_TOOLKIT_ENABLED as OPTIMIZATION,
)
from openbb_terminal.rich_config import console
from openbb_terminal.sdk_core.trailmap import Trailmap, get_trailmaps
from website.controller_doc_classes import sub_names_full as subnames
MAP_PATH = MISCELLANEOUS_DIRECTORY / "library" / "trail_map.csv"
MAP_FORECASTING_PATH = MISCELLANEOUS_DIRECTORY / "library" / "trail_map_forecasting.csv"
MAP_OPTIMIZATION_PATH = (
MISCELLANEOUS_DIRECTORY / "library" / "trail_map_optimization.csv"
)
website_path = Path(__file__).parent.absolute()
def clean_attr_desc(attr: Optional[FunctionType] = None) -> Optional[str]:
"""Clean the attribute description."""
if attr.__doc__ is None:
return None
return (
attr.__doc__.splitlines()[1].lstrip()
if not attr.__doc__.splitlines()[0]
else attr.__doc__.splitlines()[0].lstrip()
if attr.__doc__
else ""
)
def get_signature_parameters(
function: Callable[..., Any], globalns: Dict[str, Any]
) -> Dict[str, inspect.Parameter]:
signature = inspect.signature(function)
params = {}
cache: dict[str, Any] = {}
for name, parameter in signature.parameters.items():
annotation = parameter.annotation
if annotation is parameter.empty:
params[name] = parameter
continue
if annotation is None:
params[name] = parameter.replace(annotation=type(None))
continue
if isinstance(annotation, ForwardRef):
annotation = annotation.__forward_arg__
if isinstance(annotation, str):
annotation = eval(annotation, globalns, cache) # pylint: disable=W0123
params[name] = parameter.replace(annotation=annotation)
return params
class Trailmap:
def __init__(self, trailmap: str, model: str, view: Optional[str] = None):
tmap = trailmap.split(".")
if len(tmap) == 1:
tmap = ["", tmap[0]]
self.class_attr: str = tmap.pop(-1)
self.location_path = tmap
self.model = model
self.view = view if view else None
self.short_doc: Dict[str, Optional[str]] = {}
self.long_doc: Dict[str, str] = {}
self.lineon: Dict[str, int] = {}
self.full_path: Dict[str, str] = {}
self.func_def: Dict[str, str] = {}
self.func_attr: Dict[str, FunctionType] = {}
self.params: Dict[str, Dict[str, inspect.Parameter]] = {}
self.get_docstrings()
def get_docstrings(self) -> None:
"""Gets the function docstrings. We get the short and long docstrings."""
for key, func in zip(["model", "view"], [self.model, self.view]):
if func:
module_path, function_name = func.rsplit(".", 1)
module = importlib.import_module(module_path)
func_attr = getattr(module, function_name)
add_juan = 0
if hasattr(func_attr, "__wrapped__"):
func_attr = func_attr.__wrapped__
if hasattr(func_attr, "__wrapped__"):
func_attr = func_attr.__wrapped__
add_juan = 1
self.func_attr[key] = func_attr
self.lineon[key] = inspect.getsourcelines(func_attr)[1] + add_juan
self.long_doc[key] = func_attr.__doc__
self.short_doc[key] = clean_attr_desc(func_attr)
self.params[key] = {}
for k, p in get_signature_parameters(
func_attr, func_attr.__globals__
).items():
self.params[key][k] = p
self.func_def[key] = self.get_definition(key)
full_path = (
inspect.getfile(self.func_attr[key])
.replace("\\", "/")
.split("openbb_terminal/")[1]
)
self.full_path[key] = f"openbb_terminal/{full_path}"
def get_definition(self, key: str) -> str:
"""Creates the function definition to be used in SDK docs."""
funcspec = self.params[key]
definition = ""
added_comma = False
for arg in funcspec:
annotation = (
(
str(funcspec[arg].annotation)
.replace("<class '", "")
.replace("'>", "")
.replace("typing.", "")
.replace("pandas.core.frame.", "pd.")
.replace("pandas.core.series.", "pd.")
.replace("openbb_terminal.portfolio.", "")
)
if funcspec[arg].annotation != inspect.Parameter.empty
else "Any"
)
default = ""
if funcspec[arg].default is not funcspec[arg].empty:
arg_default = (
funcspec[arg].default
if funcspec[arg].default is not inspect.Parameter.empty
else "None"
)
default = (
f" = {arg_default}"
if not isinstance(arg_default, str)
else f' = "{arg_default}"'
)
definition += f"{arg}: {annotation}{default}, "
added_comma = True
if added_comma:
definition = definition[:-2]
trail = ".".join([t for t in self.location_path if t != ""])
sdk_name = self.class_attr if key != "view" else f"{self.class_attr}_chart"
sdk_path = f"{f'openbb.{trail}' if trail else 'openbb'}.{sdk_name}"
definition = f"{sdk_path}({definition })"
return definition
def get_trailmaps() -> List[Trailmap]:
trailmaps = []
def sort_csv(path: Path) -> None:
columns = ["trail", "model", "view"]
df = pd.read_csv(path, usecols=columns, keep_default_na=False)
df.set_index("trail", inplace=True)
df.sort_index(inplace=True)
df.to_csv(path, index=True)
for tmap_csv in [MAP_PATH, MAP_FORECASTING_PATH, MAP_OPTIMIZATION_PATH]:
sort_csv(tmap_csv)
if tmap_csv == MAP_FORECASTING_PATH and not FORECASTING:
console.print(
"[bold red]Forecasting is disabled. Forecasting will not be included in the Generation of Docs[/bold red]"
)
break
if tmap_csv == MAP_OPTIMIZATION_PATH and not OPTIMIZATION:
console.print(
"[bold red]Optimization is disabled. Optimization will not be included in the Generation of Docs[/bold red]" # noqa: E501
)
break
with open(tmap_csv) as csvfile:
reader = csv.reader(csvfile, delimiter=",")
next(reader)
for row in reader:
trailmaps.append(Trailmap(*row))
return trailmaps
def get_function_meta(trailmap: Trailmap, trail_type: Literal["model", "view"]):
"""Gets the function meta data."""
if trailmap.func_attr[trail_type] is None:
func_attr = trailmap.func_attrs[trail_type]
if not func_attr.func_unwrapped:
return None
doc_parsed = parse(trailmap.long_doc[trail_type])
line = trailmap.lineon[trail_type]
path = trailmap.full_path[trail_type]
func_def = trailmap.func_def[trail_type]
doc_parsed = parse(func_attr.long_doc)
line = func_attr.lineon
path = func_attr.full_path
func_def = func_attr.func_def
source_code_url = (
"https://github.com/OpenBB-finance/OpenBBTerminal/tree/main/"
+ path
@@ -215,8 +32,8 @@ def get_function_meta(trailmap: Trailmap, trail_type: Literal["model", "view"]):
params = []
for param in doc_parsed.params:
arg_default = (
trailmap.params[trail_type][param.arg_name].default
if param.arg_name in trailmap.params[trail_type]
func_attr.params[param.arg_name].default
if param.arg_name in func_attr.params
else None
)
params.append(
@@ -403,6 +220,9 @@ def main() -> bool:
shutil.rmtree(file)
for trailmap in trailmaps:
try:
if trailmap.location_path[0] == "root":
trailmap.location_path[0] = ""
functions_dict = add_todict(
functions_dict, trailmap.location_path, trailmap
)
@@ -420,7 +240,9 @@ def main() -> bool:
with open(filepath, "w", **kwargs) as f: # type: ignore
f.write(markdown)
except Exception as e:
print(f"Error generating {trailmap.class_attr} - {e}")
print(
f"Error generating {trailmap.location_path} {trailmap.class_attr} - {e}"
)
return False
functions_dict = {