jerry/OpenBB
1
0
Fork 0
mirror of https://github.com/OpenBB-finance/OpenBB.git synced 2026-10-30 05:52:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update SDK generation scripts

Browse Source
This commit is contained in:
Theodore Aptekarev
2023-02-23 16:36:48 +01:00
parent d6114efea3
commit f2b1a26c81
5 changed files with 92 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
CONTRIBUTING.md vendored
View File

@@ -49,6 +49,7 @@ Use your best judgment, and feel free to propose changes to this document in a p
- [Pre Commit Hooks](#pre-commit-hooks)
- [Coding](#coding)
- [Git Process](#git-process)
- [Branch Naming Conventions](#branch-naming-conventions)
- [Add a Test](#add-a-test)
- [Installers](#installers)
@@ -309,7 +310,7 @@ In addition, note the `self.load_class` which allows to not create a new DarkPoo
In order to add a command to the SDK, follow these steps:
1. If you've created a new model or view file, add the import with an alias to `openbb_terminal/sdk_core/sdk_init.py` following this structure:
1. If you've created a new model or view file, add the import with an alias to `openbb_terminal/core/sdk/sdk_init.py` following this structure:
```python
# Stocks - Fundamental Analysis
@@ -320,7 +321,7 @@ In order to add a command to the SDK, follow these steps:
)
```
2. Go to the `trail_map.csv` file located in `openbb_terminal/sdk_core`, which should look like this:
2. Go to the `trail_map.csv` file located in `openbb_terminal/core/sdk`, which should look like this:
```csv
trail,model,view
@@ -339,7 +340,7 @@ The added line of the file should look like this:
stocks.dps.shorted,stocks_dps_yahoofinance_model.get_most_shorted,stocks_dps_yahoofinance_view.display_most_shorted
```
4. Generate the SDK files by running `python generate_sdk.py` from the root of the project. This will automatically generate the SDK `openbb_terminal/sdk.py`, corresponding `openbb_terminal/sdk_core/controllers/` and `openbb_terminal/sdk_core/models/` class files.
4. Generate the SDK files by running `python generate_sdk.py` from the root of the project. This will automatically generate the SDK `openbb_terminal/sdk.py`, corresponding `openbb_terminal/core/sdk/controllers/` and `openbb_terminal/core/sdk/models/` class files.
To sort the `trail_map.csv` file and generate the SDK files, run the following command

152
generate_sdk.py
View File

@@ -1,13 +1,14 @@
"""Generates the sdk files from the trailmaps."""
import os
import re
import subprocess
import subprocess # nosec: B404
import sys
from pathlib import Path
from typing import Any, Dict, List, Optional, TextIO
from openbb_terminal.core.sdk.sdk_helpers import get_sdk_imports_text
from openbb_terminal.core.sdk.trailmap import Trailmap, get_trailmaps
from openbb_terminal.rich_config import console
from openbb_terminal.sdk_core.sdk_helpers import get_sdk_imports_text
from openbb_terminal.sdk_core.trailmap import Trailmap, get_trailmaps
REPO_ROOT = Path(__file__).parent.joinpath("openbb_terminal").resolve()
@@ -62,17 +63,22 @@ class SDKLogger:
openbb = OpenBBSDK()\r\r
"""
disable_lines = "# flake8: noqa\r# pylint: disable=C0301,R0902,R0903\r"
disable_lines = (
"# ######### THIS FILE IS AUTO GENERATED - ANY CHANGES WILL BE VOID ######### #\r"
"# flake8: noqa\r# pylint: disable=C0301,R0902,R0903\r"
)
class BuildCategoryModelClasses:
"""Builds the command tree for the SDK."""
def __init__(self, trailmaps: List[Trailmap]) -> None:
self.trailmaps = trailmaps
self.categories: Dict[str, Any] = {}
self.import_modules: Dict[str, Any] = {}
self.root_modules: Dict[str, Any] = {}
self.import_cat_class = (
"from openbb_terminal.sdk_core.sdk_helpers import Category\r"
"from openbb_terminal.core.sdk.sdk_helpers import Category\r"
)
for tmap in self.trailmaps:
@@ -80,22 +86,25 @@ class BuildCategoryModelClasses:
self.categories = self.add_todict(self.categories, local, tmap)
for folder in ["models", "controllers"]:
if not (REPO_ROOT / f"sdk_core/{folder}").exists():
(REPO_ROOT / f"sdk_core/{folder}").mkdir()
if not (REPO_ROOT / f"sdk_core/{folder}/__init__.py").exists():
with open(REPO_ROOT / f"sdk_core/{folder}/__init__.py", "w") as f:
if not (REPO_ROOT / f"core/sdk/{folder}").exists():
(REPO_ROOT / f"core/sdk/{folder}").mkdir()
if not (REPO_ROOT / f"core/sdk/{folder}/__init__.py").exists():
with open(REPO_ROOT / f"core/sdk/{folder}/__init__.py", "w") as f:
f.write(
"# ######### THIS FILE IS AUTO GENERATED - ANY CHANGES WILL BE VOID ######### #\r"
"# flake8: noqa\r# pylint: disable=unused-import,wrong-import-order\r\r"
)
if not (REPO_ROOT / "sdk_core/__init__.py").exists():
with open(REPO_ROOT / "sdk_core/__init__.py", "w") as f:
if not (REPO_ROOT / "core/sdk/__init__.py").exists():
with open(REPO_ROOT / "core/sdk/__init__.py", "w") as f:
f.write("")
def add_todict(self, d: dict, location_path: list, tmap: Trailmap) -> dict:
"""Adds the trailmap to the dictionary. A trailmap is a path to a function
in the sdk. This function creates the dictionary paths to the function."""
"""Add the trailmap to the dictionary.
A trailmap is a path to a function in the sdk.
This function creates the dictionary paths to the function.
"""
if location_path[0] not in d:
d[location_path[0]] = {}
@@ -107,36 +116,36 @@ class BuildCategoryModelClasses:
return d
def get_nested_dict(self, d: dict) -> dict:
"""Gets the nested dictionary of the category."""
"""Get the nested dictionary of the category."""
nested_dict = {}
for k, v in d.items():
if isinstance(v, dict):
nested_dict[k] = self.get_nested_dict(v)
return nested_dict
def get_subcat_fullname(self, cat: str) -> str:
"""Gets the full category name from the shortened category name."""
def get_subcategory_fullname(self, cat: str) -> str:
"""Get the full category name from the shortened category name."""
if cat in sub_names:
return sub_names[cat]
return cat.title().replace(" ", "")
def check_submodules(self, category: str) -> bool:
"""Checks if a category has submodules."""
"""Check if a category has submodules."""
return any(isinstance(d, dict) for d in self.categories[category].values())
def write_init_imports(self, importstr: str, filestr: str) -> None:
"""Checks if a category has submodules and adds the imports to the init file."""
regex = re.compile(importstr)
with open(REPO_ROOT / filestr) as init_file:
def write_init_imports(self, import_string: str, file_string: str) -> None:
"""Check if a category has submodules and adds the imports to the init file."""
regex = re.compile(import_string)
with open(REPO_ROOT / file_string) as init_file:
check_init = bool(regex.search(init_file.read()))
if not check_init:
with open(REPO_ROOT / filestr, "a") as init_file:
init_file.write(f"{importstr}\r")
if not check_init:
with open(REPO_ROOT / file_string, "a") as init_file:
init_file.write(f"{import_string}\r")
def write_sdk_class(
self, category: str, f: TextIO, cls_type: Optional[str] = ""
) -> None:
"""Writes the sdk class to the file.
"""Write the sdk class to the file.
Parameters
----------
@@ -152,13 +161,13 @@ class BuildCategoryModelClasses:
else:
class_name = f"{category.title()}{cls_type}"
f.write(
f'class {class_name}:\r """{self.get_subcat_fullname(category)} Module.\r'
f'class {class_name}:\r """{self.get_subcategory_fullname(category)} Module.\r'
)
def write_class_property(
self, category: str, f: TextIO, subcat: Optional[str] = ""
self, category: str, f: TextIO, subcategory: Optional[str] = ""
) -> None:
"""Writes the class property to the file.
"""Write the class property to the file.
Parameters
----------
@@ -166,18 +175,18 @@ class BuildCategoryModelClasses:
The category name.
f : TextIO
The file to write to.
subcat : Optional[str], optional
subcategory : Optional[str], optional
The subcategory name, by default None
"""
def_name = subcat if subcat else category
if subcat:
subcat = f" {self.get_subcat_fullname(subcat)}"
def_name = subcategory if subcategory else category
if subcategory:
subcategory = f" {self.get_subcategory_fullname(subcategory)}"
category = sub_names.get(category, category.title())
f.write(
f" @property\r def {def_name}(self):\r "
f'"""{category}{subcat} Submodule\r'
f'"""{category}{subcategory} Submodule\r'
)
def write_class_attr_docs(
@@ -186,9 +195,9 @@ class BuildCategoryModelClasses:
f: TextIO,
module: bool = True,
sdk_root: bool = False,
trail: list = None,
trail: Optional[list] = None,
) -> None:
"""Writes the class attribute docs to the category file.
"""Write the class attribute docs to the category file.
Parameters
----------
@@ -231,7 +240,7 @@ class BuildCategoryModelClasses:
def write_class_attributes(
self, d: dict, f: TextIO, cat: Optional[str] = None
) -> None:
"""Writes the class attributes to the category file.
"""Write the class attributes to the category file.
Parameters
----------
@@ -271,7 +280,7 @@ class BuildCategoryModelClasses:
f.write("\r\r")
def write_category(self, category: str, d: dict, f: TextIO) -> None:
"""Writes the category class to the file
"""Write the category class to the file.
Parameters
----------
@@ -282,22 +291,22 @@ class BuildCategoryModelClasses:
f : TextIO
The file to write to
"""
subname = self.get_subcat_fullname(category)
subcategory_name = self.get_subcategory_fullname(category)
# If this catergory has no attributes, then we don't write it to the file.
# If this category has no attributes, then we don't write it to the file.
if not any(isinstance(v, Trailmap) for v in d.values()):
return
self.root_modules[category] = f"{category.title().replace(' ', '')}Root"
f.write(f"class {self.root_modules[category]}(Category):\r")
f.write(f' """{subname.title()} Module\r')
f.write(f' """{subcategory_name.title()} Module\r')
self.write_class_attr_docs(d, f, trail=[category])
self.write_class_attributes(d, f, category)
def write_nested_category(self, category: str, d: dict, f: TextIO) -> None:
"""Writes the nested category classes.
"""Write the nested category classes.
Parameters
----------
@@ -311,10 +320,12 @@ class BuildCategoryModelClasses:
for nested_category, nested_dict in d.items():
if isinstance(nested_dict, Trailmap):
continue
subname = self.get_subcat_fullname(nested_category)
subcategory_name = self.get_subcategory_fullname(nested_category)
class_name = f"{category.title()}{subname.replace(' ', '')}(Category)"
f.write(f'class {class_name}:\r """{subname} Module.\r')
class_name = (
f"{category.title()}{subcategory_name.replace(' ', '')}(Category)"
)
f.write(f'class {class_name}:\r """{subcategory_name} Module.\r')
self.write_nested_submodule_docs(nested_dict, f)
@@ -327,7 +338,7 @@ class BuildCategoryModelClasses:
def write_submodule_doc(
self, k: str, f: TextIO, added: bool = False, indent: bool = False
) -> None:
"""Writes the submodules to the class docstring.
"""Write the submodules to the class docstring.
Parameters
----------
@@ -343,13 +354,13 @@ class BuildCategoryModelClasses:
add_indent = " " if indent else ""
if not added:
f.write("\r Submodules:\r")
subcat_name = self.get_subcat_fullname(k)
f.write(f"{add_indent} `{k}`: {subcat_name} Module\r")
subcategory_name = self.get_subcategory_fullname(k)
f.write(f"{add_indent} `{k}`: {subcategory_name} Module\r")
def write_nested_submodule_docs(
self, nested_dict: dict, f: TextIO, indent: bool = False
) -> None:
"""Writes the nested submodule docs to the class docstring.
"""Write the nested submodule docs to the class docstring.
Parameters
----------
@@ -361,9 +372,9 @@ class BuildCategoryModelClasses:
Whether or not to add an indent to the docstring, by default False
"""
added = False
nested_subcat = self.get_nested_dict(nested_dict)
if nested_subcat:
for k in nested_subcat:
nested_subcategory = self.get_nested_dict(nested_dict)
if nested_subcategory:
for k in nested_subcategory:
if isinstance(nested_dict[k], Trailmap):
continue
self.write_submodule_doc(k, f, added, indent)
@@ -371,8 +382,9 @@ class BuildCategoryModelClasses:
added = True
def write_category_file(self, category: str, d: dict) -> None:
"""Writes the category file. This is the file that contains the categories
and subcategories of the sdk.
"""Write the category file.
This is the file that contains the categories and subcategories of the sdk.
Parameters
----------
@@ -381,21 +393,21 @@ class BuildCategoryModelClasses:
d : dict
The category dictionary
"""
with open(REPO_ROOT / f"sdk_core/models/{category}_sdk_model.py", "w") as f:
with open(REPO_ROOT / f"core/sdk/models/{category}_sdk_model.py", "w") as f:
import_cat_class = self.import_cat_class
if category in self.root_modules:
import_cat_class = ""
category = self.root_modules[category]
f.write(
f"{disable_lines}{import_cat_class}import openbb_terminal.sdk_core.sdk_init as lib\r\r\r"
f"{disable_lines}{import_cat_class}import openbb_terminal.core.sdk.sdk_init as lib\r\r\r"
)
if category not in self.root_modules and any(
isinstance(v, Trailmap) for v in d.values()
):
self.write_init_imports(
f"from .{category}_sdk_model import {category.title()}Root",
"sdk_core/models/__init__.py",
"core/sdk/models/__init__.py",
)
self.write_category(category, d, f)
@@ -405,8 +417,9 @@ class BuildCategoryModelClasses:
f.truncate()
def write_sdk_controller_file(self, category: str, d: dict) -> None:
"""Writes the sdk controller file. This is the file that contains the
controller classes for the sdk.
"""Write the sdk controller file.
This is the file that contains the controller classes for the sdk.
Parameters
----------
@@ -417,10 +430,10 @@ class BuildCategoryModelClasses:
"""
added_init_imports = []
with open(
REPO_ROOT / f"sdk_core/controllers/{category}_sdk_controller.py", "w"
REPO_ROOT / f"core/sdk/controllers/{category}_sdk_controller.py", "w"
) as f:
f.write(
f"{disable_lines}from openbb_terminal.sdk_core.models "
f"{disable_lines}from openbb_terminal.core.sdk.models "
f"import {category}_sdk_model as model\r\r\r"
)
@@ -429,7 +442,7 @@ class BuildCategoryModelClasses:
):
self.write_init_imports(
f"from .{category}_sdk_controller import {category.title()}Controller",
"sdk_core/controllers/__init__.py",
"core/sdk/controllers/__init__.py",
)
added_init_imports.append(category)
@@ -446,15 +459,17 @@ class BuildCategoryModelClasses:
self.write_class_attr_docs(d[subcategory], f, False)
f.write(
f" return model.{category.title()}"
f"{self.get_subcat_fullname(subcategory).replace(' ', '')}()\r\r"
f"{self.get_subcategory_fullname(subcategory).replace(' ', '')}()\r\r"
)
f.seek(f.tell() - 1, os.SEEK_SET)
f.truncate()
def write_sdk_file(self) -> None:
"""Writes the main sdk file. This is the file that we initialize the SDK with openbb."""
"""Write the main sdk file.
This is the file that we initialize the SDK with openbb.
"""
with open(REPO_ROOT / "sdk.py", "w") as f:
f.write(
f'{get_sdk_imports_text()}class OpenBBSDK:\r """OpenBB SDK Class.\r'
@@ -477,12 +492,13 @@ class BuildCategoryModelClasses:
f.write("\r".join(sdk_openbb_var.splitlines()))
def build(self) -> None:
"""Builds the SDK."""
for path in ["sdk_core/models", "sdk_core/controllers"]:
"""Build the SDK."""
for path in ["core/sdk/models", "core/sdk/controllers"]:
for file in (REPO_ROOT / path).glob("*.py"):
if file.name == "__init__.py":
with open(file, "w") as f:
f.write(
"# ######### THIS FILE IS AUTO GENERATED - ANY CHANGES WILL BE VOID ######### #\r"
"# flake8: noqa\r"
"# pylint: disable=unused-import,wrong-import-order\r\r"
)
@@ -498,7 +514,7 @@ class BuildCategoryModelClasses:
self.write_sdk_controller_file(category, d)
self.write_sdk_file()
for path in ["sdk.py", "sdk_core", "sdk_core/models", "sdk_core/controllers"]:
for path in ["sdk.py", "core/sdk", "core/sdk/models", "core/sdk/controllers"]:
for file in (REPO_ROOT / path).glob("*.py"):
with open(file, "rb") as f:
content = f.read()
@@ -506,11 +522,11 @@ class BuildCategoryModelClasses:
f.write(content.replace(b"\r", b"\n"))
# We run black to make sure the code is formatted correctly
subprocess.check_call(["black", "openbb_terminal"])
subprocess.check_call(["black", "openbb_terminal"]) # nosec: B603, B607
def generate_sdk(sort: bool = False) -> None:
"""Main function to generate the SDK.
"""Generate the SDK.
Parameters
----------

13
openbb_terminal/sdk.py
View File

@@ -15,14 +15,13 @@ from openbb_terminal.helper_classes import TerminalStyle # noqa: F401
from openbb_terminal.reports import widget_helpers as widgets # noqa: F401
from openbb_terminal.reports.reports_controller import ReportController
import openbb_terminal.sdk_core.sdk_init as lib
from openbb_terminal.sdk_core import (
import openbb_terminal.core.sdk.sdk_init as lib
from openbb_terminal.core.sdk import (
controllers as ctrl,
models as model,
)
from openbb_terminal import feature_flags as obbff
from openbb_terminal.session.user import User
from openbb_terminal.terminal_helper import is_auth_enabled
if User.is_guest():
load_dotenv_and_reload_configs()
@@ -49,14 +48,6 @@ class OpenBBSDK:
self.logout = lib.sdk_session.logout
self.news = lib.common_feedparser_model.get_news
self.whoami = lib.sdk_session.whoami
self._try_to_login()
def _try_to_login(self):
if User.is_guest() and is_auth_enabled():
try:
self.login()
except Exception:
pass
@property
def alt(self):

2
tests/website/test_generate_docs.py
View File

@@ -3,7 +3,7 @@ from typing import Optional
import pytest
try:
from openbb_terminal.sdk_core.trailmap import get_signature_parameters
from openbb_terminal.core.sdk.trailmap import get_signature_parameters
from website import (
generate_sdk_markdown as gen_sdk,
generate_terminal_markdown as gen_term,

2
website/generate_sdk_markdown.py
View File

@@ -7,7 +7,7 @@ from typing import Any, Dict, Literal, Optional, Union
from docstring_parser import parse
from openbb_terminal.sdk_core.trailmap import Trailmap, get_trailmaps
from openbb_terminal.core.sdk.trailmap import Trailmap, get_trailmaps
from website.controller_doc_classes import sub_names_full as subnames
website_path = Path(__file__).parent.absolute()