jerry/OpenBB
1
0
Fork 0
mirror of https://github.com/OpenBB-finance/OpenBB.git synced 2026-10-27 04:31:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Docs/fix doc generations (#5775)

Browse Source 
* few changes

* Update get_search_results.md

* updates

* undo build

* Update test_generate_docs.py

* Update controller_doc_classes.py

* Update params_controller.py

* Update params_controller.py
This commit is contained in:
teh_coderer
2023-11-21 16:10:45 -05:00
committed by GitHub
parent 40dece279d
commit 1e4feffd77
12 changed files with 25432 additions and 180 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
openbb_platform/extensions/econometrics/openbb_econometrics/econometrics_router.py
View File

@@ -39,7 +39,7 @@ def correlation_matrix(data: List[Data]) -> OBBject[List[Data]]:
Returns
-------
OBBject[List[Data]]
OBBject[List[Data]]:
Correlation matrix.
"""
df = basemodel_to_df(data)
@@ -72,12 +72,12 @@ def ols_regression(
Input dataset.
y_column: str
Target column.
x_columns: str
x_columns: List[str]
List of columns to use as exogenous variables.
Returns
-------
OBBject[Dict]
OBBject[Dict]:
OBBject with the results being model and results objects.
"""
X = sm.add_constant(get_target_columns(basemodel_to_df(data), x_columns))
@@ -101,12 +101,12 @@ def ols_regression_summary(
Input dataset.
y_column: str
Target column.
x_columns: str
x_columns: List[str]
List of columns to use as exogenous variables.
Returns
-------
OBBject[Dict]
OBBject[Data]:
OBBject with the results being summary object.
"""
X = sm.add_constant(get_target_columns(basemodel_to_df(data), x_columns))
@@ -166,12 +166,12 @@ def autocorrelation(
Input dataset.
y_column: str
Target column.
x_columns: str
x_columns: List[str]
List of columns to use as exogenous variables.
Returns
-------
OBBject[Data]
OBBject[Dict]:
OBBject with the results being the score from the test.
"""
X = sm.add_constant(get_target_columns(basemodel_to_df(data), x_columns))
@@ -195,14 +195,14 @@ def residual_autocorrelation(
Input dataset.
y_column: str
Target column.
x_columns: str
x_columns: List[str]
List of columns to use as exogenous variables.
lags: PositiveInt
Number of lags to use in the test.
Returns
-------
OBBject[Data]
OBBject[Data]:
OBBject with the results being the score from the test.
"""
X = sm.add_constant(get_target_columns(basemodel_to_df(data), x_columns))
@@ -239,7 +239,7 @@ def cointegration(
Returns
-------
OBBject[Data]
OBBject[Data]:
OBBject with the results being the score from the test.
"""
pairs = list(combinations(columns, 2))
@@ -287,7 +287,7 @@ def causality(
Returns
-------
OBBject[Data]
OBBject[Data]:
OBBject with the results being the score from the test.
"""
X = get_target_column(basemodel_to_df(data), x_column)
@@ -322,13 +322,13 @@ def unit_root(
Input dataset.
column: str
Data columns to check unit root
regression: str
regression: Literal["c", "ct", "ctt"]
Regression type to use in the test. Either "c" for constant only, "ct" for constant and trend, or "ctt" for
constant, trend, and trend-squared.
Returns
-------
OBBject[Data]
OBBject[Data]:
OBBject with the results being the score from the test.
"""
dataset = get_target_column(basemodel_to_df(data), column)
@@ -357,12 +357,12 @@ def panel_random_effects(
Input dataset.
y_column: str
Target column.
x_columns: str
x_columns: List[str]
List of columns to use as exogenous variables.
Returns
-------
OBBject[Dict]
OBBject[Dict]:
OBBject with the fit model returned
"""
X = get_target_columns(basemodel_to_df(data), x_columns)
@@ -386,12 +386,12 @@ def panel_between(
Input dataset.
y_column: str
Target column.
x_columns: str
x_columns: List[str]
List of columns to use as exogenous variables.
Returns
-------
OBBject[Dict]
OBBject[Dict]:
OBBject with the fit model returned
"""
X = get_target_columns(basemodel_to_df(data), x_columns)
@@ -415,12 +415,12 @@ def panel_pooled(
Input dataset.
y_column: str
Target column.
x_columns: str
x_columns: List[str]
List of columns to use as exogenous variables.
Returns
-------
OBBject[Dict]
OBBject[Dict]:
OBBject with the fit model returned
"""
X = get_target_columns(basemodel_to_df(data), x_columns)
@@ -444,12 +444,12 @@ def panel_fixed(
Input dataset.
y_column: str
Target column.
x_columns: str
x_columns: List[str]
List of columns to use as exogenous variables.
Returns
-------
OBBject[Dict]
OBBject[Dict]:
OBBject with the fit model returned
"""
X = get_target_columns(basemodel_to_df(data), x_columns)
@@ -473,12 +473,12 @@ def panel_first_difference(
Input dataset.
y_column: str
Target column.
x_columns: str
x_columns: List[str]
List of columns to use as exogenous variables.
Returns
-------
OBBject[Dict]
OBBject[Dict]:
OBBject with the fit model returned
"""
X = get_target_columns(basemodel_to_df(data), x_columns)
@@ -502,12 +502,12 @@ def panel_fmac(
Input dataset.
y_column: str
Target column.
x_columns: str
x_columns: List[str]
List of columns to use as exogenous variables.
Returns
-------
OBBject[Dict]
OBBject[Dict]:
OBBject with the fit model returned
"""
X = get_target_columns(basemodel_to_df(data), x_columns)

1
tests/website/test_generate_docs.py
View File

@@ -67,6 +67,7 @@ class MockTrailMap:
self.func_attrs["model"] = MockFuncAttrs()
self.model = mock_func
self.class_attr = "mock"
self.location_path = []
EXPECTED_OUTPUT = """Stuff here or stuff there, it doesn't matter, it's everywhere.

8
website/controller_doc_classes.py
View File

@@ -13,10 +13,12 @@ from unittest.mock import patch
import pandas as pd
from pandas.tseries.holiday import USFederalHolidayCalendar
from rich.console import Console, Theme
import openbb_terminal
import openbb_terminal.config_terminal as cfg
from openbb_terminal.core.plots.backend import plots_backend
from openbb_terminal.core.plots.plotly_helper import theme
from openbb_terminal.core.session.current_system import set_system_variable
from openbb_terminal.core.session.current_user import get_current_user # noqa: F401
from openbb_terminal.decorators import disable_check_api
@@ -27,13 +29,13 @@ from openbb_terminal.helper_funcs import (
set_command_location,
)
from openbb_terminal.parent_classes import BaseController
from openbb_terminal.rich_config import console
from openbb_terminal.sdk import openbb
from openbb_terminal.stocks.comparison_analysis import finviz_compare_model
set_system_variable("TEST_MODE", True)
set_system_variable("LOG_COLLECT", False)
disable_check_api()
console = Console(theme=Theme(theme.console_style), highlight=False, soft_wrap=True)
CRYPTO_DATA = openbb.crypto.load("BTC", to_symbol="usd", source="YahooFinance")
@@ -77,6 +79,7 @@ sub_folders_abbr = {
"comparison_analysis": "ca",
"dark_pool_shorts": "dps",
"portfolio_optimization": "po",
"companieshouse": "companieshouse",
"quantitative_analysis": "qa",
"technical_analysis": "ta",
"tradinghours": "th",
@@ -90,6 +93,7 @@ sub_names_full = {
"alt": "Alternative",
"ba": "Behavioural Analysis",
"ca": "Comparison Analysis",
"companieshouse": "Companies House",
"crypto": "Cryptocurrency",
"dd": "Due Diligence",
"defi": "DeFi",
@@ -275,11 +279,11 @@ class ControllerDoc:
self.image_exportable: Dict[str, bool] = {}
self.ignore = [
"call_help",
"call_new",
"call_exit",
"call_clear",
"call_cls",
"call_quit",
"call_about",
"call_reset",
"call_support",
"call_wiki",

238
website/generate_platform_v4_markdown.py
View File

@@ -1,27 +1,108 @@
import inspect
import json
import re
import shutil
from inspect import Parameter, _empty, signature
from pathlib import Path
from textwrap import shorten
from typing import Any, Dict, List, TextIO, Tuple, Union
from typing import Any, Callable, Dict, List, TextIO, Tuple, Union
from docstring_parser import parse
from openbb_core.app.provider_interface import ProviderInterface
from openbb_core.app.static.package_builder import MethodDefinition, PathHandler
from openbb_provider import standard_models
from pydantic.fields import FieldInfo
website_path = Path(__file__).parent.absolute()
SEO_META: Dict[str, Dict[str, Union[str, List[str]]]] = json.loads(
(website_path / "metadata/platform_v4_seo_metadata.json").read_text()
)
REFERENCE_IMPORT = """
import ReferenceCard from "@site/src/components/General/ReferenceCard";
REFERENCE_IMPORT_UL = """import ReferenceCard from "@site/src/components/General/NewReferenceCard";
<ul className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 xl:grid-cols-4 gap-4 -ml-6">
"""
reference_import = (
'import ReferenceCard from "@site/src/components/General/NewReferenceCard";\n\n'
)
refrence_ul_element = """<ul className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 xl:grid-cols-4 gap-4 -ml-6">"""
def get_docstring_meta(
func: Callable, full_command_path: str, formatted_params: Dict[str, Parameter]
) -> Dict[str, Any]:
"""Extracts the meta information from the docstring of a function with no standardized results model."""
meta_command = {}
doc_parsed = parse(func.__doc__) # type: ignore
cmd_params = []
for param in doc_parsed.params:
arg_default = (
formatted_params[param.arg_name].default
if param.arg_name in formatted_params
else None
)
cmd_params.append(
{
"name": param.arg_name,
"type": get_annotation_type(param.type_name),
"default": str(arg_default)
if arg_default is not inspect.Parameter.empty
else None,
"cleaned_type": re.sub(
r"Literal\[([^\"\]]*)\]",
f"Literal[{type(arg_default).__name__}]",
get_annotation_type(
formatted_params[param.arg_name].annotation
if param.arg_name in formatted_params
else param.type_name
),
),
"optional": bool(arg_default is not inspect.Parameter.empty)
or param.is_optional,
"doc": param.description,
}
)
if doc_parsed.returns:
meta_command["returns"] = {
"type": doc_parsed.returns.type_name,
"doc": doc_parsed.returns.description,
}
examples = []
for example in doc_parsed.examples:
examples.append(
{
"snippet": example.snippet,
"description": example.description.strip(), # type: ignore
}
)
def_params = [
f"{d['name']}: {d['cleaned_type']}{' = ' + d['default'] if d['default'] else ''}"
for d in cmd_params
]
meta_command.update(
{
"description": doc_parsed.short_description
+ (
"\n\n" + doc_parsed.long_description
if doc_parsed.long_description
else ""
),
"params": cmd_params,
"func_def": f"{full_command_path}({', '.join(def_params)})",
"examples": examples,
}
)
return meta_command
def generate_markdown(meta_command: dict):
markdown = f"---\ntitle: {meta_command['name']}\ndescription: OpenBB Platform Function\n---\n\n"
markdown = meta_command["header"]
markdown += "<!-- markdownlint-disable MD012 MD031 MD033 -->\n\n"
markdown += (
@@ -55,6 +136,21 @@ def generate_markdown_section(meta: Dict[str, Any]):
# Add the current line to the description
description.append(line.strip())
prev_snippet = " "
for example in meta.get("examples", []):
if isinstance(example["snippet"], str) and ">>>" in example["snippet"]:
snippet = example["snippet"].replace(">>> ", "")
example_code.append(snippet)
if example["description"] and prev_snippet != "":
example_code.append(example["description"])
prev_snippet = snippet.strip()
elif example["description"]:
example_code.append(example["description"])
else:
if example["description"]:
example_code.append(example["description"])
prev_snippet = ""
# Join the description parts and handle any remaining example code
if example_code: # If there's an example block at the end of the docstring
description.append("```python\n" + "\n".join(example_code) + "\n```\n")
@@ -94,10 +190,10 @@ def create_nested_menus_card(folder: Path, url: str, data_models: bool = False)
url = f"/platform/{path}/{url}/{folder.name}".replace("//", "/")
index_card = f"""<ReferenceCard
title="{folder.name}"
description="{categories}"
url="{url}"
/>\n"""
title="{folder.name.replace("_", " ").title()}"
description="{categories}"
url="{url}"
/>\n"""
return index_card
@@ -110,10 +206,10 @@ def create_cmd_cards(cmd_text: List[Dict[str, str]], data_models: bool = False)
url = f"{url}/{cmd['title']}"
description = shorten(f"{cmd['description']}", width=116, placeholder="...")
cmd_cards += f"""<ReferenceCard
title="{cmd["title"]}"
description="{description.replace('"', "'")}"
title="{cmd["title"].replace("_", " ").title()}"
description="{description.split(".").pop(0).strip().replace(":", "").replace('"', "'")}"
url="{url}"
command="true"
command
/>\n"""
return cmd_cards
@@ -144,18 +240,24 @@ def write_reference_index(
data_models : bool, optional
Whether the folder is a data_models folder, by default False
"""
f.write(f"# {fname}\n{REFERENCE_IMPORT}")
for folder in path.glob("*"):
if folder.is_dir():
f.write(
create_nested_menus_card(folder, "/".join(rel_path.parts), data_models)
)
f.write(f"# {fname}\n\n{REFERENCE_IMPORT_UL if data_models else reference_import}")
sub_folders = [sub for sub in path.glob("*") if sub.is_dir()]
menus = []
for folder in sub_folders:
menus.append(
create_nested_menus_card(folder, "/".join(rel_path.parts), data_models)
)
if sub_folders and not data_models:
f.write(f"### Menus\n{refrence_ul_element}\n{''.join(menus)}</ul>\n")
folder_cmd_cards: List[Dict[str, str]] = reference_cards.get(path, {}) # type: ignore
if folder_cmd_cards:
f.write(create_cmd_cards(folder_cmd_cards, data_models))
f.write("</ul>\n")
if folder_cmd_cards:
if not data_models:
f.write(f"\n\n### Commands\n{refrence_ul_element}\n")
f.write(create_cmd_cards(folder_cmd_cards, data_models) + "</ul>\n")
provider_interface = ProviderInterface()
@@ -176,7 +278,10 @@ def get_annotation_type(annotation: Any) -> str:
.replace("datetime.datetime", "datetime")
.replace("datetime.date", "date")
.replace("NoneType", "None")
.replace(", None", ""),
.replace(", None", "")
.replace("openbb_provider.abstract.data.", "")
.replace("pandas.core.frame.", "pd.")
.replace("pandas.core.series.", "pd."),
)
if match := annotated_regex.match(subbed):
subbed = match.group("type")
@@ -209,6 +314,15 @@ def get_command_meta(path: str, route_map: Dict[str, Any]) -> Dict[str, Any]:
"model": model_name,
}
# Extract the full path from the 'path' variable, excluding the method name
path_components = path.strip("/").split("/")
# Construct the full command path, e.g., 'obb.equity.estimates.consensus'
full_command_path = "obb." + ".".join(path_components[:-1])
# Now add the actual function name
func_name = route.endpoint.__name__
full_command_path += f".{func_name}"
if model_name:
providers = provider_interface.map[model_name]
@@ -266,15 +380,6 @@ def get_command_meta(path: str, route_map: Dict[str, Any]) -> Dict[str, Any]:
}
)
# Extract the full path from the 'path' variable, excluding the method name
path_components = path.strip("/").split("/")
# Construct the full command path, e.g., 'obb.equity.estimates.consensus'
full_command_path = "obb." + ".".join(path_components[:-1])
# Now add the actual function name
func_name = route.endpoint.__name__
full_command_path += f".{func_name}"
# Update the func_def to include the full path
def_params = [
f"{d['name']}: {d['cleaned_type']}{' = ' + d['default'] if d['default'] else ''}"
@@ -351,6 +456,36 @@ def get_command_meta(path: str, route_map: Dict[str, Any]) -> Dict[str, Any]:
}
meta_command["provider_params"] = provider_params
if not meta_command.get("params", None):
meta_command.update(
get_docstring_meta(func, full_command_path, formatted_params)
)
ref_path = full_command_path.replace("obb.", "")
cmd_keywords = "\n- ".join(ref_path.split("."))
default_desc = (
meta_command.get("description", "").split(".").pop(0).strip().replace(":", "")
)
header = (
f"title: {func_name}\ndescription: {default_desc}\nkeywords:\n- {cmd_keywords}"
)
if seo_meta := SEO_META.get(
ref_path, SEO_META.get((".".join(path.split("/")[-2:])), None)
):
keywords = "\n- ".join(seo_meta["keywords"])
header = f"title: {seo_meta['title']}\ndescription: {seo_meta['description']}\nkeywords:\n- {keywords}"
title = ref_path.split(".")
title[0] += " "
meta_command[
"header"
] = f"""---\n{header}\n---\n
import HeadTitle from '@site/src/components/General/HeadTitle.tsx';
<HeadTitle title="{'/'.join(title)} - Reference | OpenBB Platform Docs" />\n\n"""
return meta_command
@@ -483,10 +618,12 @@ def generate_platform_markdown() -> None:
if not meta_command:
continue
func = PathHandler.get_route(path=path, route_map=route_map).endpoint
folder, func_name = path.split("/")[-2:]
func_name = func.__name__
if func_name == "index":
func_name = "index_cmd"
folder = "/".join(path.strip("/").split("/")[:-1])
filepath = content_path / folder / f"{func_name}.md"
markdown = generate_markdown(meta_command=meta_command)
@@ -562,8 +699,12 @@ def generate_platform_markdown() -> None:
# Generate category json
f.write(json.dumps({"label": "Reference", "position": 5}, indent=2))
def gen_category_json(fname: str, path: Path):
def gen_category_json(fname: str, path: Path, position: int = 1):
"""Generate category json"""
with open(path / "_category_.json", "w", **kwargs) as f: # type: ignore
f.write(
json.dumps({"label": fname.title(), "position": position}, indent=2)
)
with open(path / "index.mdx", "w", **kwargs) as f: # type: ignore
rel_path = path.relative_to(content_path)
@@ -571,14 +712,45 @@ def generate_platform_markdown() -> None:
def gen_category_recursive(nested_path: Path):
"""Generate category json recursively"""
position = 1
for folder in nested_path.iterdir():
if folder.is_dir():
gen_category_json(folder.name, folder)
gen_category_json(folder.name, folder, position)
gen_category_recursive(folder) # pylint: disable=cell-var-from-loop
position += 1
gen_category_recursive(content_path)
print(f"Markdown files generated, check the {content_path} folder.")
def save_metadata(path: Path) -> Dict[str, Dict[str, Union[str, List[str]]]]:
"""Save SEO metadata"""
regex = re.compile(
r"---\ntitle: (.*)\ndescription: (.*)\nkeywords:(.*)\n---\n\nimport HeadTitle",
re.MULTILINE | re.DOTALL,
)
metadata = {}
for file in path.rglob("*/**/*.md"):
context = file.read_text(encoding="utf-8")
match = regex.search(context)
if match:
title, description, keywords = match.groups()
key = file.relative_to(path).as_posix().removesuffix(".md")
metadata[key.replace("/", ".")] = {
"title": title,
"description": description,
"keywords": [
keyword.strip() for keyword in keywords.split("\n- ") if keyword
],
}
filepath = website_path / "metadata/platform_v4_seo_metadata2.json"
with open(filepath, "w", encoding="utf-8", newline="\n") as f:
json.dump(metadata, f, indent=2)
return metadata
if __name__ == "__main__":
generate_platform_markdown()

155
website/generate_sdk_v3_markdown.py
View File

@@ -1,9 +1,10 @@
import inspect
import json
import re
import shutil
from pathlib import Path
from textwrap import shorten
from typing import Any, Dict, List, Literal, TextIO
from typing import Any, Dict, List, Literal, TextIO, Union
from docstring_parser import parse
@@ -13,12 +14,16 @@ from openbb_terminal.core.session.current_system import set_system_variable
set_system_variable("TEST_MODE", True)
set_system_variable("LOG_COLLECT", False)
website_path = Path(__file__).parent.absolute()
CONTENT_PATH = website_path / "content/sdk/reference"
SEO_META: Dict[str, Dict[str, Union[str, List[str]]]] = json.loads(
(website_path / "metadata/sdk_v3_seo_metadata.json").read_text()
)
reference_import = """
import ReferenceCard from "@site/src/components/General/ReferenceCard";
reference_import = (
'import ReferenceCard from "@site/src/components/General/NewReferenceCard";\n\n'
)
<ul className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 xl:grid-cols-4 gap-4 -ml-6">
"""
refrence_ul_element = """<ul className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 xl:grid-cols-4 gap-4 -ml-6">"""
def get_function_meta(trailmap: Trailmap, trail_type: Literal["model", "view"]):
@@ -44,7 +49,7 @@ def get_function_meta(trailmap: Trailmap, trail_type: Literal["model", "view"]):
params.append(
{
"name": param.arg_name,
"doc": param.description,
"doc": param.description if param.description else "",
"type": param.type_name,
"default": arg_default
if arg_default is not inspect.Parameter.empty
@@ -70,19 +75,43 @@ def get_function_meta(trailmap: Trailmap, trail_type: Literal["model", "view"]):
"description": example.description.strip(), # type: ignore
}
)
return {
desc = doc_parsed.short_description or doc_parsed.long_description
cmd_meta = {
"name": trailmap.class_attr,
"path": path,
"function_name": function_name,
"func_def": func_def,
"source_code_url": source_code_url,
"description": doc_parsed.short_description,
"description": desc if desc else "",
"params": params,
"returns": returns,
"examples": examples,
}
default_desc = (doc_parsed.short_description or " .").split(".").pop(
0
).strip().replace(":", "") or trailmap.class_attr
keywords = "\n- ".join(
k for k in trailmap.location_path + [trailmap.class_attr] if k
)
header = f"title: {trailmap.class_attr}\ndescription: {default_desc}\nkeywords:\n- {keywords}"
key = f"{'.'.join(trailmap.location_path)}.{trailmap.class_attr}"
if seo_meta := SEO_META.get(key, None):
keywords = "\n- ".join(seo_meta["keywords"])
header = f"title: {seo_meta['title']}\ndescription: {seo_meta['description']}\nkeywords:\n- {keywords}"
cmd_meta[
"header"
] = f"""---\n{header}\n---\n
import HeadTitle from '@site/src/components/General/HeadTitle.tsx';
<HeadTitle title="{key} - Reference | OpenBB SDK Docs" />\n\n"""
return cmd_meta
def generate_markdown(meta_model: dict, meta_view: dict):
main_model = meta_model
@@ -90,16 +119,11 @@ def generate_markdown(meta_model: dict, meta_view: dict):
if not meta_view:
raise ValueError("No model or view")
main_model = meta_view
markdown = f"""---
title: {main_model["name"]}
description: OpenBB SDK Function
---\n\n"""
markdown = main_model["header"]
if meta_view and meta_model:
markdown += """import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';\n\n"""
markdown += f"# {main_model['name']}\n\n"
if meta_view and meta_model:
markdown += f"""<Tabs>
<TabItem value="model" label="Model" default>\n
@@ -173,14 +197,16 @@ def create_nested_menus_card(folder: Path, url: str) -> str:
for sub in folder.glob("**/**/*.md*")
if sub.is_file() and sub.stem != "index"
]
categories = shorten(", ".join(sub_categories), width=116, placeholder="...")
categories = shorten(
", ".join(sub_categories or [""]), width=116, placeholder="..."
)
url = f"/sdk/reference/{url}/{folder.name}".replace("//", "/")
index_card = f"""<ReferenceCard
title="{folder.name}"
description="{categories}"
url="{url}"
/>\n"""
title="{folder.name.replace("_", " ").title()}"
description="{categories}"
url="{url}"
/>\n"""
return index_card
@@ -190,10 +216,10 @@ def create_cmd_cards(cmd_text: List[Dict[str, str]]) -> str:
url = f"/sdk/reference/{cmd['url']}/{cmd['title']}".replace("//", "/")
description = shorten(f"{cmd['description']}", width=116, placeholder="...")
cmd_cards += f"""<ReferenceCard
title="{cmd["title"]}"
description="{description}"
title="{cmd["title"].replace("_", " ").title()}"
description="{description.replace('None', '')}"
url="{url}"
command="true"
command
/>\n"""
return cmd_cards
@@ -221,16 +247,26 @@ def write_reference_index(
f : TextIO
File to write to.
"""
f.write(f"# {fname}\n{reference_import}")
for folder in path.glob("*"):
if folder.is_dir():
f.write(create_nested_menus_card(folder, "/".join(rel_path.parts)))
f.write(f"# {fname}\n\n{reference_import}")
sub_folders = [sub for sub in path.glob("*") if sub.is_dir()]
menus = []
for folder in sub_folders:
menus.append(create_nested_menus_card(folder, "/".join(rel_path.parts)))
if menus:
f.write(f"### Menus\n{refrence_ul_element}\n{''.join(menus)}</ul>\n")
folder_cmd_cards: List[Dict[str, str]] = reference_cards.get(path, {}) # type: ignore
if folder_cmd_cards:
f.write(create_cmd_cards(folder_cmd_cards))
f.write("</ul>\n")
if folder_cmd_cards:
f.writelines(
[
f"\n\n### Commands\n{refrence_ul_element}\n",
create_cmd_cards(folder_cmd_cards),
"</ul>\n",
]
)
def main() -> bool:
@@ -240,10 +276,9 @@ def main() -> bool:
kwargs = {"encoding": "utf-8", "newline": "\n"}
print("Generating markdown files...")
content_path = website_path / "content/sdk/reference"
reference_cards: Dict[Path, List[Dict[str, str]]] = {}
for file in content_path.glob("*"):
for file in CONTENT_PATH.glob("*"):
if file.is_file():
file.unlink()
else:
@@ -263,7 +298,7 @@ def main() -> bool:
trailmap.class_attr = "index_cmd"
filepath = (
content_path
CONTENT_PATH
/ "/".join(trailmap.location_path)
/ f"{trailmap.class_attr}.md"
)
@@ -292,34 +327,70 @@ def main() -> bool:
# Sort reference_cards
reference_cards = dict(sorted(reference_cards.items(), key=lambda item: item[0]))
with open(content_path / "index.mdx", "w", **kwargs) as f: # type: ignore
with open(CONTENT_PATH / "index.mdx", "w", **kwargs) as f: # type: ignore
fname = "OpenBB SDK Reference"
rel_path = content_path.relative_to(content_path)
write_reference_index(reference_cards, fname, content_path, rel_path, f)
rel_path = CONTENT_PATH.relative_to(CONTENT_PATH)
write_reference_index(reference_cards, fname, CONTENT_PATH, rel_path, f)
with open(content_path / "_category_.json", "w", **kwargs) as f: # type: ignore
with open(CONTENT_PATH / "_category_.json", "w", **kwargs) as f: # type: ignore
"""Generate category json"""
f.write(json.dumps({"label": "SDK Reference", "position": 4}, indent=2))
f.write(json.dumps({"label": "Reference", "position": 4}, indent=2))
def gen_category_json(fname: str, path: Path):
def gen_category_json(fname: str, path: Path, position: int = 1):
"""Generate category json"""
with open(path / "_category_.json", "w", **kwargs) as f: # type: ignore
f.write(
json.dumps({"label": fname.title(), "position": position}, indent=2)
)
with open(path / "index.mdx", "w", **kwargs) as f: # type: ignore
rel_path = path.relative_to(content_path)
rel_path = path.relative_to(CONTENT_PATH)
write_reference_index(reference_cards, fname, path, rel_path, f)
def gen_category_recursive(nested_path: Path):
"""Generate category json recursively"""
position = 1
for folder in nested_path.iterdir():
if folder.is_dir():
gen_category_json(folder.name, folder)
gen_category_json(folder.name, folder, position)
gen_category_recursive(folder) # pylint: disable=cell-var-from-loop
position += 1
gen_category_recursive(content_path)
gen_category_recursive(CONTENT_PATH)
print("Markdown files generated, check the functions folder")
return True
def save_metadata() -> Dict[str, Dict[str, Union[str, List[str]]]]:
"""Save SEO metadata to json file."""
regex = re.compile(
r"---\ntitle: (.*)\ndescription: (.*)\nkeywords:(.*)\n---\n\nimport HeadTitle",
re.MULTILINE | re.DOTALL,
)
metadata = {}
for file in CONTENT_PATH.rglob("*/**/*.md"):
context = file.read_text(encoding="utf-8")
match = regex.search(context)
if match:
title, description, keywords = match.groups()
key = file.relative_to(CONTENT_PATH).as_posix().removesuffix(".md")
metadata[key.replace("/", ".")] = {
"title": title,
"description": description,
"keywords": [
keyword.strip() for keyword in keywords.split("\n- ") if keyword
],
}
filepath = website_path / "metadata/sdk_v3_seo_metadata.json"
with open(filepath, "w", encoding="utf-8", newline="\n") as f:
json.dump(metadata, f, indent=2)
return metadata
if __name__ == "__main__":
main()

183
website/generate_terminal_v3_markdown.py
View File

@@ -1,5 +1,4 @@
import json
import os
import re
import shutil
import traceback
@@ -10,22 +9,30 @@ from typing import Dict, List, TextIO, Union
from openbb_terminal.core.session.current_system import set_system_variable
from openbb_terminal.core.session.current_user import get_current_user
from openbb_terminal.rich_config import console
from website.controller_doc_classes import (
ControllerDoc,
LoadControllersDoc,
console,
sub_names_full as subnames,
)
set_system_variable("TEST_MODE", True)
set_system_variable("LOG_COLLECT", False)
website_path = Path(__file__).parent.absolute()
CONTENT_PATH: Path = website_path / "content/terminal/reference"
EXAMPLES_META: Dict[str, Dict[str, Union[str, List[str]]]] = json.loads(
(website_path / "metadata/terminal_v3_examples.json").read_text()
)
SEO_META: Dict[str, Dict[str, Union[str, List[str]]]] = json.loads(
(website_path / "metadata/terminal_v3_seo_metadata.json").read_text()
)
reference_import = """
import ReferenceCard from "@site/src/components/General/NewReferenceCard";
reference_import = (
'import ReferenceCard from "@site/src/components/General/NewReferenceCard";\n\n'
)
refrence_ul_element = """<ul className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 xl:grid-cols-4 gap-4 -ml-6">"""
<ul className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 xl:grid-cols-4 gap-4 -ml-6">
"""
USER_PATH = (
f"{get_current_user().preferences.USER_DATA_DIRECTORY}",
@@ -55,32 +62,15 @@ def existing_markdown_file_examples(
if sub in ["ba", "ta", "qa"]:
trail.remove(trailmap.split(".")[0])
examples_path = (
f"old_content/terminal/{'/'.join(trail)}/{cmd_dict['cmd_name']}/_index.md"
)
examples_dict: Dict[str, Union[str, List[str]]] = {}
trail.append(cmd_dict["cmd_name"])
if os.path.exists(website_path / examples_path):
with open(website_path / examples_path, encoding="utf-8") as f:
content = f.read()
examples: str = ""
if "Example:" in content:
example_split = content.split("Example:")[1].split("```")
if example_split and len(example_split) > 1:
examples = f"{example_split[1].strip()}"
examples_dict["example"] = examples
images = [
x for x in content.split("\n") if x.startswith("!") and "TODO" not in x
]
examples_dict["images"] = images
return examples_dict
return EXAMPLES_META.get(".".join(trail), {})
# pylint: disable=isinstance-second-argument-not-valid-type
def process_cmd_parsers(ctrl: ControllerDoc) -> List[Dict[str, str]]:
def process_cmd_parsers(
ctrl: ControllerDoc, ctrl_trailmap: str
) -> List[Dict[str, str]]:
"""Process command parsers from ControllerDoc"""
commands = []
@@ -145,12 +135,32 @@ def process_cmd_parsers(ctrl: ControllerDoc) -> List[Dict[str, str]]:
# We do this to fix multiline docstrings for the markdown
desc = " ".join(desc.split()).replace(*USER_PATH)
cmd_name = cmd.replace("call_", "")
param = {
"cmd_name": cmd.replace("call_", ""),
"cmd_name": cmd_name,
"actions": actions,
"usage": " ".join(parser.format_usage().split()).replace("usage: ", ""),
"description": desc if desc else "",
}
default_desc = (desc or " .").split(".").pop(0).strip().replace(":", "")
header = f"title: {cmd_name}\ndescription: {default_desc}\nkeywords:\n- {ctrl_trailmap}\n- {cmd_name}"
key = f"{ctrl_trailmap}.{cmd_name}"
if cmd_meta := SEO_META.get(key, None):
keywords = "\n- ".join(cmd_meta["keywords"])
header = f"title: {cmd_meta['title']}\ndescription: {cmd_meta['description']}\nkeywords:\n- {keywords}"
title = key.split(".")
title[0] += " "
param[
"header"
] = f"""---\n{header}\n---\n
import HeadTitle from '@site/src/components/General/HeadTitle.tsx';
<HeadTitle title="{'/'.join(title)} - Reference | OpenBB Terminal Docs" />\n\n"""
commands.append(param)
return commands # type: ignore
@@ -176,22 +186,14 @@ def generate_markdown(
if not cmd_meta:
raise ValueError("No command metadata found")
markdown = f"""---
########### THIS FILE IS AUTO GENERATED - ANY CHANGES WILL BE VOID ###########
title: {cmd_meta["cmd_name"]}
description: OpenBB Terminal Function
---\n\n"""
# head meta https://docusaurus.io/docs/markdown-features/head-metadata
markdown += f"# {cmd_meta['cmd_name']}\n\n{cmd_meta['description']}\n\n"
markdown = f"{cmd_meta['description']}\n\n"
markdown += f"### Usage\n\n```python wordwrap\n{cmd_meta['usage']}\n```\n\n"
markdown += "---\n\n## Parameters\n\n"
if cmd_meta["actions"]:
markdown += (
"| Name | Parameter | Description | Default | Optional | Choices |\n"
)
markdown += (
"| ---- | --------- | ----------- | ------- | -------- | ------- |\n"
)
@@ -221,16 +223,16 @@ description: OpenBB Terminal Function
def create_nested_menus_card(folder: Path, url: str) -> str:
sub_categories = [
sub.stem
for sub in folder.glob("**/**/*.md*")
for sub in folder.rglob("**/**/*.md*")
if sub.is_file() and sub.stem != "index"
]
categories = shorten(", ".join(sub_categories), width=116, placeholder="...")
url = f"/terminal/reference/{url}/{folder.name}".replace("//", "/")
index_card = f"""<ReferenceCard
title="{folder.name}"
description="{categories}"
url="{url}"
title="{folder.name.title()}"
description="{categories}"
url="{url}"
/>\n"""
return index_card
@@ -242,10 +244,10 @@ def create_cmd_cards(cmd_text: List[Dict[str, str]]) -> str:
description = shorten(cmd["description"], width=116, placeholder="...")
cmd_cards += f"""<ReferenceCard
title="{cmd["title"]}"
title="{cmd["title"].title()}"
description="{description.replace('"', "'")}"
url="{url}"
command="true"
command
/>\n"""
return cmd_cards
@@ -273,16 +275,26 @@ def write_reference_index(
f : TextIO
File to write to.
"""
f.write(f"# {fname}\n{reference_import}")
for folder in path.glob("*"):
if folder.is_dir():
f.write(create_nested_menus_card(folder, "/".join(rel_path.parts)))
f.write(f"# {fname}\n\n{reference_import}")
sub_folders = [sub for sub in path.glob("*") if sub.is_dir()]
menus = []
for folder in sub_folders:
menus.append(create_nested_menus_card(folder, "/".join(rel_path.parts)))
if menus:
f.write(f"### Menus\n{refrence_ul_element}\n{''.join(menus)}</ul>\n")
folder_cmd_cards: List[Dict[str, str]] = reference_cards.get(path, {}) # type: ignore
if folder_cmd_cards:
f.write(create_cmd_cards(folder_cmd_cards))
f.write("</ul>\n")
if folder_cmd_cards:
f.writelines(
[
f"\n\n### Commands\n{refrence_ul_element}\n",
create_cmd_cards(folder_cmd_cards),
"</ul>\n",
]
)
def main() -> bool:
@@ -298,10 +310,10 @@ def main() -> bool:
console.print(
"[bright_yellow]Generating markdown files... Don't ignore any errors now[/bright_yellow]"
)
content_path: Path = website_path / "content/terminal/reference"
reference_cards: Dict[Path, List[Dict[str, str]]] = {}
global_cmds = {"askobb": False, "hold": False, "about": False}
for file in content_path.glob("*"):
for file in CONTENT_PATH.glob("*"):
if file.is_file():
file.unlink()
else:
@@ -310,18 +322,25 @@ def main() -> bool:
for ctrl_trailmap in ctrls:
try:
ctrl = load_ctrls.get_controller_doc(ctrl_trailmap)
ctrl_cmds = process_cmd_parsers(ctrl)
ctrl_cmds = process_cmd_parsers(ctrl, ctrl_trailmap)
for cmd in ctrl_cmds:
trail = ctrl_trailmap.split(".")
if (created := global_cmds.get(cmd["cmd_name"], None)) is not None:
if created:
continue
global_cmds[cmd["cmd_name"]] = True
trail = []
examples = existing_markdown_file_examples(ctrl_trailmap, cmd)
markdown = generate_markdown(cmd, examples)
markdown = cmd["header"] + generate_markdown(cmd, examples)
if cmd["cmd_name"] == "index":
cmd["cmd_name"] = "index_cmd"
trail = ctrl_trailmap.split(".")
filepath = CONTENT_PATH / f"{'/'.join(trail + [cmd['cmd_name']])}.md"
filepath = content_path / f"{'/'.join(trail)}/{cmd['cmd_name']}.md"
reference_cards.setdefault(filepath.parent, []).append(
dict(
title=cmd["cmd_name"],
@@ -344,34 +363,36 @@ def main() -> bool:
reference_cards = dict(sorted(reference_cards.items(), key=lambda item: item[0]))
# Generate root "_category_.json" file
with open(content_path / "_category_.json", "w", **wopen_kwargs) as f: # type: ignore
with open(CONTENT_PATH / "_category_.json", "w", **wopen_kwargs) as f: # type: ignore
f.write(json.dumps({"label": "Reference", "position": 4}, indent=2))
# Generate root "index.md" file
with open(content_path / "index.mdx", "w", **wopen_kwargs) as f: # type: ignore
with open(CONTENT_PATH / "index.mdx", "w", **wopen_kwargs) as f: # type: ignore
fname = "OpenBB Terminal Features"
rel_path = content_path.relative_to(content_path)
write_reference_index(reference_cards, fname, content_path, rel_path, f)
rel_path = CONTENT_PATH.relative_to(CONTENT_PATH)
write_reference_index(reference_cards, fname, CONTENT_PATH, rel_path, f)
def gen_category_json(fname: str, path: Path):
def gen_category_json(fname: str, path: Path, position: int = 1):
"""Generate category json"""
fname = subnames[fname.lower()] if fname.lower() in subnames else fname.title()
with open(path / "_category_.json", "w", **wopen_kwargs) as f: # type: ignore
f.write(json.dumps({"label": fname}, indent=2))
f.write(json.dumps({"label": fname, "position": position}, indent=2))
with open(path / "index.mdx", "w", **wopen_kwargs) as f: # type: ignore
rel_path = path.relative_to(content_path)
rel_path = path.relative_to(CONTENT_PATH)
write_reference_index(reference_cards, fname, path, rel_path, f)
def gen_category_recursive(nested_path: Path):
"""Generate category json recursively"""
position = 1
for folder in nested_path.iterdir():
if folder.is_dir():
gen_category_json(folder.name, folder)
gen_category_json(folder.name, folder, position)
gen_category_recursive(folder) # pylint: disable=cell-var-from-loop
position += 1
# Generate modules/sub category json and index files
gen_category_recursive(content_path)
gen_category_recursive(CONTENT_PATH)
console.print(
"[green]Markdown files generated, check the website/content/terminal/reference/ folder[/green]"
@@ -380,5 +401,35 @@ def main() -> bool:
return True
def save_metadata() -> Dict[str, Dict[str, Union[str, List[str]]]]:
"""Save SEO metadata to json file."""
regex = re.compile(
r"---\ntitle: (.*)\ndescription: (.*)\nkeywords:(.*)\n---\n\nimport HeadTitle",
re.MULTILINE | re.DOTALL,
)
metadata = {}
for file in CONTENT_PATH.rglob("*/**/*.md"):
context = file.read_text(encoding="utf-8")
match = regex.search(context)
if match:
title, description, keywords = match.groups()
key = file.relative_to(CONTENT_PATH).as_posix().removesuffix(".md")
metadata[key.replace("/", ".")] = {
"title": title,
"description": description,
"keywords": [
keyword.strip() for keyword in keywords.split("\n- ") if keyword
],
}
filepath = website_path / "metadata/sdk_v3_seo_metadata.json"
with open(filepath, "w", encoding="utf-8", newline="\n") as f:
json.dump(metadata, f, indent=2)
return metadata
if __name__ == "__main__":
main()

13
website/generate_widgets_library.py
View File

@@ -1,4 +1,5 @@
from pathlib import Path
from typing import Dict
import requests
@@ -7,10 +8,10 @@ def main() -> bool:
widgets_json_url = "https://raw.githubusercontent.com/OpenBB-finance/widgets-library/main/widgets.json"
response = requests.get(widgets_json_url, timeout=10)
widgets_data = response.json()
widgets_data: Dict[str, Dict[str, dict]] = response.json()
# Process the data into the desired format
all_widgets = {}
all_widgets: Dict[str, list] = {}
for widget_id, widget_info in widgets_data.items():
if widget_info.get("disabled", False):
continue
@@ -49,7 +50,7 @@ import NewReferenceCard from "@site/src/components/General/NewReferenceCard";
base_path = website_path / "content" / "pro" / "widgets-library"
index_path = base_path / "index.mdx"
with index_path.open("w") as index_file:
with index_path.open("w", encoding="utf-8", newline="\n") as index_file:
index_file.write(f"{text}\n")
for category, widgets in all_widgets.items():
@@ -69,12 +70,12 @@ import NewReferenceCard from "@site/src/components/General/NewReferenceCard";
title="{widget['name']}"
description="{widget['description']}"
url="/pro/widgets-library/{category}/{widget['widgetId']}"
chevron={{false}}
command
/>"""
text += "\n</ul>"
index_path = category_path / "index.mdx"
with index_path.open("w") as index_file:
with index_path.open("w", encoding="utf-8", newline="\n") as index_file:
index_file.write(f"{text}\n")
# Create each individual widget file
@@ -106,7 +107,7 @@ import HeadTitle from '@site/src/components/General/HeadTitle.tsx';
{'**Source:** ' + ','.join(widget['source']) if widget['source'] else ''}
"""
with widget_path.open("w") as widget_file:
with widget_path.open("w", encoding="utf-8", newline="\n") as widget_file:
widget_file.write(f"{text}\n")
return True

4012
website/metadata/platform_v4_seo_metadata.json vendored Normal file
View File

File diff suppressed because it is too large Load Diff

7993
website/metadata/sdk_v3_seo_metadata.json vendored Normal file
View File

File diff suppressed because it is too large Load Diff

2289
website/metadata/terminal_v3_examples.json vendored Normal file
View File

File diff suppressed because one or more lines are too long

10649
website/metadata/terminal_v3_seo_metadata.json vendored Normal file
View File

File diff suppressed because it is too large Load Diff

21
website/src/components/General/NewReferenceCard.tsx vendored
View File

@@ -10,26 +10,33 @@ export default function NewReferenceCard({
url,
description,
command,
chevron = true,
}: {
title: string;
url: string;
description: string;
command: string;
chevron?: boolean;
}) {
// TODO: Waiting for mockup of how to display the command vs menu item
const something = command ? "" : "";
const { pathname } = useLocation();
const cleanedPath = pathname.replace(/\/v\d+/, "");
const chevron =
!cleanedPath.includes("/reference") &&
!cleanedPath.includes("/widgets-library/");
return (
<Link
className={clsx(
"rounded border-2 hover:!text-black dark:hover:!text-white !no-underline p-6 cursor-pointer relative overflow-hidden hover:-translate-y-2 transform transition-all duration-300 ease-in-out font-bold shadow-md",
{
"hover:bg-[#669DCB] border-[#669DCB] dark:hover:bg-[#004A87] dark:border-[#004A87]": cleanedPath.startsWith("/terminal") || cleanedPath.startsWith("/pro"),
"hover:bg-[#b186bb] border-[#b186bb] dark:hover:bg-[#3a204f] dark:border-[#3a204f]": cleanedPath.startsWith("/bot"),
"hover:bg-[#F5B166] border-[#F5B166] dark:hover:bg-[#511d11] dark:border-[#511d11]": cleanedPath.startsWith("/sdk") || cleanedPath.startsWith("/platform"),
"hover:bg-[#669DCB] border-[#669DCB] dark:hover:bg-[#004A87] dark:border-[#004A87]":
cleanedPath.startsWith("/terminal") ||
cleanedPath.startsWith("/pro"),
"hover:bg-[#b186bb] border-[#b186bb] dark:hover:bg-[#3a204f] dark:border-[#3a204f]":
cleanedPath.startsWith("/bot"),
"hover:bg-[#F5B166] border-[#F5B166] dark:hover:bg-[#511d11] dark:border-[#511d11]":
cleanedPath.startsWith("/sdk") ||
cleanedPath.startsWith("/platform"),
header_docs:
!cleanedPath.startsWith("/terminal") &&
!cleanedPath.startsWith("/pro") &&
@@ -49,7 +56,9 @@ export default function NewReferenceCard({
{chevron && <ChevronRightIcon className="ml-auto mr-4" />}
</div>
{description ? (
<p className="text-grey-900 dark:text-grey-200 text-xs font-medium mt-2 mb-0">{description}</p>
<p className="text-grey-900 dark:text-grey-200 text-xs font-medium mt-2 mb-0">
{description}
</p>
) : null}
</Link>
);