From f2b1a26c81b8e5dee78e5cce23cd0adb73fe1992 Mon Sep 17 00:00:00 2001 From: Theodore Aptekarev Date: Thu, 23 Feb 2023 16:36:48 +0100 Subject: [PATCH] Update SDK generation scripts --- CONTRIBUTING.md | 7 +- generate_sdk.py | 152 +++++++++++++++------------- openbb_terminal/sdk.py | 13 +-- tests/website/test_generate_docs.py | 2 +- website/generate_sdk_markdown.py | 2 +- 5 files changed, 92 insertions(+), 84 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 292c6fd30df..7b7e8741a48 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -49,6 +49,7 @@ Use your best judgment, and feel free to propose changes to this document in a p - [Pre Commit Hooks](#pre-commit-hooks) - [Coding](#coding) - [Git Process](#git-process) + - [Branch Naming Conventions](#branch-naming-conventions) - [Add a Test](#add-a-test) - [Installers](#installers) @@ -309,7 +310,7 @@ In addition, note the `self.load_class` which allows to not create a new DarkPoo In order to add a command to the SDK, follow these steps: -1. If you've created a new model or view file, add the import with an alias to `openbb_terminal/sdk_core/sdk_init.py` following this structure: +1. If you've created a new model or view file, add the import with an alias to `openbb_terminal/core/sdk/sdk_init.py` following this structure: ```python # Stocks - Fundamental Analysis @@ -320,7 +321,7 @@ In order to add a command to the SDK, follow these steps: ) ``` -2. Go to the `trail_map.csv` file located in `openbb_terminal/sdk_core`, which should look like this: +2. Go to the `trail_map.csv` file located in `openbb_terminal/core/sdk`, which should look like this: ```csv trail,model,view @@ -339,7 +340,7 @@ The added line of the file should look like this: stocks.dps.shorted,stocks_dps_yahoofinance_model.get_most_shorted,stocks_dps_yahoofinance_view.display_most_shorted ``` -4. Generate the SDK files by running `python generate_sdk.py` from the root of the project. This will automatically generate the SDK `openbb_terminal/sdk.py`, corresponding `openbb_terminal/sdk_core/controllers/` and `openbb_terminal/sdk_core/models/` class files. +4. Generate the SDK files by running `python generate_sdk.py` from the root of the project. This will automatically generate the SDK `openbb_terminal/sdk.py`, corresponding `openbb_terminal/core/sdk/controllers/` and `openbb_terminal/core/sdk/models/` class files. To sort the `trail_map.csv` file and generate the SDK files, run the following command diff --git a/generate_sdk.py b/generate_sdk.py index 116c4baa76d..3f065849de9 100644 --- a/generate_sdk.py +++ b/generate_sdk.py @@ -1,13 +1,14 @@ +"""Generates the sdk files from the trailmaps.""" import os import re -import subprocess +import subprocess # nosec: B404 import sys from pathlib import Path from typing import Any, Dict, List, Optional, TextIO +from openbb_terminal.core.sdk.sdk_helpers import get_sdk_imports_text +from openbb_terminal.core.sdk.trailmap import Trailmap, get_trailmaps from openbb_terminal.rich_config import console -from openbb_terminal.sdk_core.sdk_helpers import get_sdk_imports_text -from openbb_terminal.sdk_core.trailmap import Trailmap, get_trailmaps REPO_ROOT = Path(__file__).parent.joinpath("openbb_terminal").resolve() @@ -62,17 +63,22 @@ class SDKLogger: openbb = OpenBBSDK()\r\r """ -disable_lines = "# flake8: noqa\r# pylint: disable=C0301,R0902,R0903\r" +disable_lines = ( + "# ######### THIS FILE IS AUTO GENERATED - ANY CHANGES WILL BE VOID ######### #\r" + "# flake8: noqa\r# pylint: disable=C0301,R0902,R0903\r" +) class BuildCategoryModelClasses: + """Builds the command tree for the SDK.""" + def __init__(self, trailmaps: List[Trailmap]) -> None: self.trailmaps = trailmaps self.categories: Dict[str, Any] = {} self.import_modules: Dict[str, Any] = {} self.root_modules: Dict[str, Any] = {} self.import_cat_class = ( - "from openbb_terminal.sdk_core.sdk_helpers import Category\r" + "from openbb_terminal.core.sdk.sdk_helpers import Category\r" ) for tmap in self.trailmaps: @@ -80,22 +86,25 @@ class BuildCategoryModelClasses: self.categories = self.add_todict(self.categories, local, tmap) for folder in ["models", "controllers"]: - if not (REPO_ROOT / f"sdk_core/{folder}").exists(): - (REPO_ROOT / f"sdk_core/{folder}").mkdir() - if not (REPO_ROOT / f"sdk_core/{folder}/__init__.py").exists(): - with open(REPO_ROOT / f"sdk_core/{folder}/__init__.py", "w") as f: + if not (REPO_ROOT / f"core/sdk/{folder}").exists(): + (REPO_ROOT / f"core/sdk/{folder}").mkdir() + if not (REPO_ROOT / f"core/sdk/{folder}/__init__.py").exists(): + with open(REPO_ROOT / f"core/sdk/{folder}/__init__.py", "w") as f: f.write( + "# ######### THIS FILE IS AUTO GENERATED - ANY CHANGES WILL BE VOID ######### #\r" "# flake8: noqa\r# pylint: disable=unused-import,wrong-import-order\r\r" ) - if not (REPO_ROOT / "sdk_core/__init__.py").exists(): - with open(REPO_ROOT / "sdk_core/__init__.py", "w") as f: + if not (REPO_ROOT / "core/sdk/__init__.py").exists(): + with open(REPO_ROOT / "core/sdk/__init__.py", "w") as f: f.write("") def add_todict(self, d: dict, location_path: list, tmap: Trailmap) -> dict: - """Adds the trailmap to the dictionary. A trailmap is a path to a function - in the sdk. This function creates the dictionary paths to the function.""" + """Add the trailmap to the dictionary. + A trailmap is a path to a function in the sdk. + This function creates the dictionary paths to the function. + """ if location_path[0] not in d: d[location_path[0]] = {} @@ -107,36 +116,36 @@ class BuildCategoryModelClasses: return d def get_nested_dict(self, d: dict) -> dict: - """Gets the nested dictionary of the category.""" + """Get the nested dictionary of the category.""" nested_dict = {} for k, v in d.items(): if isinstance(v, dict): nested_dict[k] = self.get_nested_dict(v) return nested_dict - def get_subcat_fullname(self, cat: str) -> str: - """Gets the full category name from the shortened category name.""" + def get_subcategory_fullname(self, cat: str) -> str: + """Get the full category name from the shortened category name.""" if cat in sub_names: return sub_names[cat] return cat.title().replace(" ", "") def check_submodules(self, category: str) -> bool: - """Checks if a category has submodules.""" + """Check if a category has submodules.""" return any(isinstance(d, dict) for d in self.categories[category].values()) - def write_init_imports(self, importstr: str, filestr: str) -> None: - """Checks if a category has submodules and adds the imports to the init file.""" - regex = re.compile(importstr) - with open(REPO_ROOT / filestr) as init_file: + def write_init_imports(self, import_string: str, file_string: str) -> None: + """Check if a category has submodules and adds the imports to the init file.""" + regex = re.compile(import_string) + with open(REPO_ROOT / file_string) as init_file: check_init = bool(regex.search(init_file.read())) - if not check_init: - with open(REPO_ROOT / filestr, "a") as init_file: - init_file.write(f"{importstr}\r") + if not check_init: + with open(REPO_ROOT / file_string, "a") as init_file: + init_file.write(f"{import_string}\r") def write_sdk_class( self, category: str, f: TextIO, cls_type: Optional[str] = "" ) -> None: - """Writes the sdk class to the file. + """Write the sdk class to the file. Parameters ---------- @@ -152,13 +161,13 @@ class BuildCategoryModelClasses: else: class_name = f"{category.title()}{cls_type}" f.write( - f'class {class_name}:\r """{self.get_subcat_fullname(category)} Module.\r' + f'class {class_name}:\r """{self.get_subcategory_fullname(category)} Module.\r' ) def write_class_property( - self, category: str, f: TextIO, subcat: Optional[str] = "" + self, category: str, f: TextIO, subcategory: Optional[str] = "" ) -> None: - """Writes the class property to the file. + """Write the class property to the file. Parameters ---------- @@ -166,18 +175,18 @@ class BuildCategoryModelClasses: The category name. f : TextIO The file to write to. - subcat : Optional[str], optional + subcategory : Optional[str], optional The subcategory name, by default None """ - def_name = subcat if subcat else category - if subcat: - subcat = f" {self.get_subcat_fullname(subcat)}" + def_name = subcategory if subcategory else category + if subcategory: + subcategory = f" {self.get_subcategory_fullname(subcategory)}" category = sub_names.get(category, category.title()) f.write( f" @property\r def {def_name}(self):\r " - f'"""{category}{subcat} Submodule\r' + f'"""{category}{subcategory} Submodule\r' ) def write_class_attr_docs( @@ -186,9 +195,9 @@ class BuildCategoryModelClasses: f: TextIO, module: bool = True, sdk_root: bool = False, - trail: list = None, + trail: Optional[list] = None, ) -> None: - """Writes the class attribute docs to the category file. + """Write the class attribute docs to the category file. Parameters ---------- @@ -231,7 +240,7 @@ class BuildCategoryModelClasses: def write_class_attributes( self, d: dict, f: TextIO, cat: Optional[str] = None ) -> None: - """Writes the class attributes to the category file. + """Write the class attributes to the category file. Parameters ---------- @@ -271,7 +280,7 @@ class BuildCategoryModelClasses: f.write("\r\r") def write_category(self, category: str, d: dict, f: TextIO) -> None: - """Writes the category class to the file + """Write the category class to the file. Parameters ---------- @@ -282,22 +291,22 @@ class BuildCategoryModelClasses: f : TextIO The file to write to """ - subname = self.get_subcat_fullname(category) + subcategory_name = self.get_subcategory_fullname(category) - # If this catergory has no attributes, then we don't write it to the file. + # If this category has no attributes, then we don't write it to the file. if not any(isinstance(v, Trailmap) for v in d.values()): return self.root_modules[category] = f"{category.title().replace(' ', '')}Root" f.write(f"class {self.root_modules[category]}(Category):\r") - f.write(f' """{subname.title()} Module\r') + f.write(f' """{subcategory_name.title()} Module\r') self.write_class_attr_docs(d, f, trail=[category]) self.write_class_attributes(d, f, category) def write_nested_category(self, category: str, d: dict, f: TextIO) -> None: - """Writes the nested category classes. + """Write the nested category classes. Parameters ---------- @@ -311,10 +320,12 @@ class BuildCategoryModelClasses: for nested_category, nested_dict in d.items(): if isinstance(nested_dict, Trailmap): continue - subname = self.get_subcat_fullname(nested_category) + subcategory_name = self.get_subcategory_fullname(nested_category) - class_name = f"{category.title()}{subname.replace(' ', '')}(Category)" - f.write(f'class {class_name}:\r """{subname} Module.\r') + class_name = ( + f"{category.title()}{subcategory_name.replace(' ', '')}(Category)" + ) + f.write(f'class {class_name}:\r """{subcategory_name} Module.\r') self.write_nested_submodule_docs(nested_dict, f) @@ -327,7 +338,7 @@ class BuildCategoryModelClasses: def write_submodule_doc( self, k: str, f: TextIO, added: bool = False, indent: bool = False ) -> None: - """Writes the submodules to the class docstring. + """Write the submodules to the class docstring. Parameters ---------- @@ -343,13 +354,13 @@ class BuildCategoryModelClasses: add_indent = " " if indent else "" if not added: f.write("\r Submodules:\r") - subcat_name = self.get_subcat_fullname(k) - f.write(f"{add_indent} `{k}`: {subcat_name} Module\r") + subcategory_name = self.get_subcategory_fullname(k) + f.write(f"{add_indent} `{k}`: {subcategory_name} Module\r") def write_nested_submodule_docs( self, nested_dict: dict, f: TextIO, indent: bool = False ) -> None: - """Writes the nested submodule docs to the class docstring. + """Write the nested submodule docs to the class docstring. Parameters ---------- @@ -361,9 +372,9 @@ class BuildCategoryModelClasses: Whether or not to add an indent to the docstring, by default False """ added = False - nested_subcat = self.get_nested_dict(nested_dict) - if nested_subcat: - for k in nested_subcat: + nested_subcategory = self.get_nested_dict(nested_dict) + if nested_subcategory: + for k in nested_subcategory: if isinstance(nested_dict[k], Trailmap): continue self.write_submodule_doc(k, f, added, indent) @@ -371,8 +382,9 @@ class BuildCategoryModelClasses: added = True def write_category_file(self, category: str, d: dict) -> None: - """Writes the category file. This is the file that contains the categories - and subcategories of the sdk. + """Write the category file. + + This is the file that contains the categories and subcategories of the sdk. Parameters ---------- @@ -381,21 +393,21 @@ class BuildCategoryModelClasses: d : dict The category dictionary """ - with open(REPO_ROOT / f"sdk_core/models/{category}_sdk_model.py", "w") as f: + with open(REPO_ROOT / f"core/sdk/models/{category}_sdk_model.py", "w") as f: import_cat_class = self.import_cat_class if category in self.root_modules: import_cat_class = "" category = self.root_modules[category] f.write( - f"{disable_lines}{import_cat_class}import openbb_terminal.sdk_core.sdk_init as lib\r\r\r" + f"{disable_lines}{import_cat_class}import openbb_terminal.core.sdk.sdk_init as lib\r\r\r" ) if category not in self.root_modules and any( isinstance(v, Trailmap) for v in d.values() ): self.write_init_imports( f"from .{category}_sdk_model import {category.title()}Root", - "sdk_core/models/__init__.py", + "core/sdk/models/__init__.py", ) self.write_category(category, d, f) @@ -405,8 +417,9 @@ class BuildCategoryModelClasses: f.truncate() def write_sdk_controller_file(self, category: str, d: dict) -> None: - """Writes the sdk controller file. This is the file that contains the - controller classes for the sdk. + """Write the sdk controller file. + + This is the file that contains the controller classes for the sdk. Parameters ---------- @@ -417,10 +430,10 @@ class BuildCategoryModelClasses: """ added_init_imports = [] with open( - REPO_ROOT / f"sdk_core/controllers/{category}_sdk_controller.py", "w" + REPO_ROOT / f"core/sdk/controllers/{category}_sdk_controller.py", "w" ) as f: f.write( - f"{disable_lines}from openbb_terminal.sdk_core.models " + f"{disable_lines}from openbb_terminal.core.sdk.models " f"import {category}_sdk_model as model\r\r\r" ) @@ -429,7 +442,7 @@ class BuildCategoryModelClasses: ): self.write_init_imports( f"from .{category}_sdk_controller import {category.title()}Controller", - "sdk_core/controllers/__init__.py", + "core/sdk/controllers/__init__.py", ) added_init_imports.append(category) @@ -446,15 +459,17 @@ class BuildCategoryModelClasses: self.write_class_attr_docs(d[subcategory], f, False) f.write( f" return model.{category.title()}" - f"{self.get_subcat_fullname(subcategory).replace(' ', '')}()\r\r" + f"{self.get_subcategory_fullname(subcategory).replace(' ', '')}()\r\r" ) f.seek(f.tell() - 1, os.SEEK_SET) f.truncate() def write_sdk_file(self) -> None: - """Writes the main sdk file. This is the file that we initialize the SDK with openbb.""" + """Write the main sdk file. + This is the file that we initialize the SDK with openbb. + """ with open(REPO_ROOT / "sdk.py", "w") as f: f.write( f'{get_sdk_imports_text()}class OpenBBSDK:\r """OpenBB SDK Class.\r' @@ -477,12 +492,13 @@ class BuildCategoryModelClasses: f.write("\r".join(sdk_openbb_var.splitlines())) def build(self) -> None: - """Builds the SDK.""" - for path in ["sdk_core/models", "sdk_core/controllers"]: + """Build the SDK.""" + for path in ["core/sdk/models", "core/sdk/controllers"]: for file in (REPO_ROOT / path).glob("*.py"): if file.name == "__init__.py": with open(file, "w") as f: f.write( + "# ######### THIS FILE IS AUTO GENERATED - ANY CHANGES WILL BE VOID ######### #\r" "# flake8: noqa\r" "# pylint: disable=unused-import,wrong-import-order\r\r" ) @@ -498,7 +514,7 @@ class BuildCategoryModelClasses: self.write_sdk_controller_file(category, d) self.write_sdk_file() - for path in ["sdk.py", "sdk_core", "sdk_core/models", "sdk_core/controllers"]: + for path in ["sdk.py", "core/sdk", "core/sdk/models", "core/sdk/controllers"]: for file in (REPO_ROOT / path).glob("*.py"): with open(file, "rb") as f: content = f.read() @@ -506,11 +522,11 @@ class BuildCategoryModelClasses: f.write(content.replace(b"\r", b"\n")) # We run black to make sure the code is formatted correctly - subprocess.check_call(["black", "openbb_terminal"]) + subprocess.check_call(["black", "openbb_terminal"]) # nosec: B603, B607 def generate_sdk(sort: bool = False) -> None: - """Main function to generate the SDK. + """Generate the SDK. Parameters ---------- diff --git a/openbb_terminal/sdk.py b/openbb_terminal/sdk.py index f9a9ed7d82a..55c5f1d9319 100644 --- a/openbb_terminal/sdk.py +++ b/openbb_terminal/sdk.py @@ -15,14 +15,13 @@ from openbb_terminal.helper_classes import TerminalStyle # noqa: F401 from openbb_terminal.reports import widget_helpers as widgets # noqa: F401 from openbb_terminal.reports.reports_controller import ReportController -import openbb_terminal.sdk_core.sdk_init as lib -from openbb_terminal.sdk_core import ( +import openbb_terminal.core.sdk.sdk_init as lib +from openbb_terminal.core.sdk import ( controllers as ctrl, models as model, ) from openbb_terminal import feature_flags as obbff from openbb_terminal.session.user import User -from openbb_terminal.terminal_helper import is_auth_enabled if User.is_guest(): load_dotenv_and_reload_configs() @@ -49,14 +48,6 @@ class OpenBBSDK: self.logout = lib.sdk_session.logout self.news = lib.common_feedparser_model.get_news self.whoami = lib.sdk_session.whoami - self._try_to_login() - - def _try_to_login(self): - if User.is_guest() and is_auth_enabled(): - try: - self.login() - except Exception: - pass @property def alt(self): diff --git a/tests/website/test_generate_docs.py b/tests/website/test_generate_docs.py index fa99b6ce728..9d4f1c096ca 100644 --- a/tests/website/test_generate_docs.py +++ b/tests/website/test_generate_docs.py @@ -3,7 +3,7 @@ from typing import Optional import pytest try: - from openbb_terminal.sdk_core.trailmap import get_signature_parameters + from openbb_terminal.core.sdk.trailmap import get_signature_parameters from website import ( generate_sdk_markdown as gen_sdk, generate_terminal_markdown as gen_term, diff --git a/website/generate_sdk_markdown.py b/website/generate_sdk_markdown.py index fbd374c3343..988bbf3a85e 100644 --- a/website/generate_sdk_markdown.py +++ b/website/generate_sdk_markdown.py @@ -7,7 +7,7 @@ from typing import Any, Dict, Literal, Optional, Union from docstring_parser import parse -from openbb_terminal.sdk_core.trailmap import Trailmap, get_trailmaps +from openbb_terminal.core.sdk.trailmap import Trailmap, get_trailmaps from website.controller_doc_classes import sub_names_full as subnames website_path = Path(__file__).parent.absolute()