[SDK v4] General improvements related with the charting extension (#5283)

* introducing fig on the chart model

* charting manager to expect fig and content. also, adding a 1st example of its usage for stocks/load route

* leveraging fig.show on command output

* introducing the render_chart function

* using render_chart() on the command output

* drop to_plotly_json function

* a way of removing fields from the api only

* remove references to to_plotly_json

* fix charting_router to use new convention on returning fig and content

* adjustments on the builder script

* ruff and black

* typo

* stocks_news chart implementation

* repr on chart model

* adding table style config

* removing to_plotly_json from the charting manager

* typo and a little change on the way to_table behaves so we can leverage inheritance

* introducing OpenBBFigureTable as a workarround to have tables on the sdk

* renaming to to_chart

* multiples implementation

* black

openbb_sdk/extensions/charting/openbb_charting/__init__.py

@@ -3,4 +3,4 @@ from openbb_charting.core.backend import (
     create_backend,
     get_backend,
 )
-from openbb_charting.core.to_plotly_json import to_plotly_json
+from openbb_charting.core.to_chart import to_chart

openbb_sdk/extensions/charting/openbb_charting/builder.py

@@ -1,9 +1,12 @@
 from inspect import getmembers, isfunction
 from typing import List
 
+from openbb_charting import charting_router
 from openbb_core.app.router import RouterLoader
 
-CHARTING_ROUTER = "./charting_router.py"
+# refers to the root of the repo, not the root of the extension
+# this script should be run from the root of the repo
+CHARTING_ROUTER = "openbb_sdk/extensions/charting/openbb_charting/charting_router.py"
 
 
 def get_routes() -> List[str]:
@@ -18,15 +21,13 @@ def create_charting_functions_w_routes(routes_to_add: List[str]):
             f.write(
                 f"""
 
-def {route}(**kwargs):
+def {route}(**kwargs) -> Tuple[OpenBBFigure, Dict[str, Any]]:
     raise NotImplementedError("Command `{original_route}` does not have a charting function")
 """
             )
 
 
 def get_charting_functions() -> List[str]:
-    from . import charting_router  # pylint: disable=import-outside-toplevel
-
     functions = getmembers(charting_router, isfunction)
     if functions:
         return [function[0] for function in functions]

openbb_sdk/extensions/charting/openbb_charting/core/chart_style.py

@@ -137,7 +137,7 @@ class ChartStyle:
 
         for attr, ext in zip(
             ["plt_styles_available", "console_styles_available"],
-            [".pltstyle", ".richstyle.json"],
+            [".pltstyle.json", ".richstyle.json"],
         ):
             for file in folder.rglob(f"*{ext}"):
                 getattr(self, attr)[file.name.replace(ext, "")] = file

openbb_sdk/extensions/charting/openbb_charting/core/openbb_figure.py

@@ -338,9 +338,7 @@ class OpenBBFigure(go.Figure):
         for i, (x_i, name_i, color_i) in enumerate(zip(valid_x, name, colors)):
             if not color_i:
                 color_i = (  # noqa: PLW2901
-                    self._self._theme.up_color
-                    if i % 2 == 0
-                    else self._self._theme.down_color
+                    self._theme.up_color if i % 2 == 0 else self._theme.down_color
                 )
 
             res_mean, res_std = np.mean(x_i), np.std(x_i)
@@ -386,7 +384,7 @@ class OpenBBFigure(go.Figure):
                     name=name_i if len(name) < 2 else name[1],
                     mode="markers",
                     marker=dict(
-                        color=self._self._theme.down_color,
+                        color=self._theme.down_color,
                         symbol="line-ns-open",
                         size=8,
                     ),
@@ -1381,7 +1379,7 @@ class OpenBBFigure(go.Figure):
         height = kwargs.pop("height", len(data.index) * 28 + 25)
         width = kwargs.pop("width", sum(columnwidth) * 8.7)
 
-        fig: OpenBBFigure = cls()
+        fig = OpenBBFigure(**kwargs)
         fig.add_table(
             header=dict(values=header_values),
             cells=dict(

openbb_sdk/extensions/charting/openbb_charting/core/openbb_figure_table.py

@@ -0,0 +1,95 @@
+from typing import List, Optional, Union
+
+import pandas as pd
+
+from openbb_sdk.extensions.charting.openbb_charting.core.openbb_figure import (
+    OpenBBFigure,
+)
+
+
+class OpenBBFigureTable(OpenBBFigure):
+    def __init__(
+        self,
+        tabular_data: pd.DataFrame,
+        columnwidth: Optional[List[Union[int, float]]] = None,
+    ):
+        super().__init__()
+        self._tabular_data = tabular_data
+        self._columnwidth = columnwidth
+
+    def show(
+        self,
+        show_index: Optional[bool] = True,
+        index_name: Optional[str] = "",
+        headers: Optional[Union[List[str], pd.Index]] = None,
+        title: Optional[str] = "",
+        source: Optional[str] = "",
+        external: Optional[bool] = False,
+    ) -> Optional["OpenBBFigureTable"]:
+        if external:
+            return self
+
+        if not self._backend.isatty:
+            tmp_fig = self.to_table()
+            tmp_fig.show()
+        else:
+            # Make a copy of the dataframe to avoid SettingWithCopyWarning
+            df = self._tabular_data.copy()
+
+            show_index = not isinstance(df.index, pd.RangeIndex) and show_index
+            #  convert non-str that are not timestamp or int into str
+            # eg) praw.models.reddit.subreddit.Subreddit
+            for col in df.columns:
+                try:
+                    if not any(
+                        isinstance(df[col].iloc[x], pd.Timestamp)
+                        for x in range(min(10, len(df)))
+                    ):
+                        df[col] = pd.to_numeric(df[col], errors="ignore")
+                except (ValueError, TypeError):
+                    df[col] = df[col].astype(str)
+
+            def _get_headers(_headers: Union[List[str], pd.Index]) -> List[str]:
+                """Check if headers are valid and return them."""
+                output = _headers
+                if isinstance(_headers, pd.Index):
+                    output = list(_headers)
+                if len(output) != len(df.columns):
+                    raise (
+                        ValueError(
+                            "Length of headers does not match length of DataFrame"
+                        )
+                    )
+                return output
+
+            df_outgoing = df.copy()
+            # If headers are provided, use them
+            if headers is not None:
+                # We check if headers are valid
+                df_outgoing.columns = _get_headers(headers)
+
+            if show_index and index_name not in df_outgoing.columns:
+                # If index name is provided, we use it
+                df_outgoing.index.name = index_name or "Index"
+                df_outgoing = df_outgoing.reset_index()
+
+            for col in df_outgoing.columns:
+                if col == "":
+                    df_outgoing = df_outgoing.rename(columns={col: "  "})
+
+            self._backend.send_table(
+                df_table=df_outgoing,
+                title=title,
+                source=source,  # type: ignore
+                theme=self._charting_settings.table_style,
+            )
+
+    def to_table(
+        self,
+        print_index: bool = True,
+        **kwargs,
+    ) -> "OpenBBFigure":
+        data = self._tabular_data
+        columnwidth = self._columnwidth
+
+        return super().to_table(data, columnwidth, print_index, **kwargs)

openbb_sdk/extensions/charting/openbb_charting/core/to_chart.py

@@ -1,12 +1,13 @@
-from typing import Any, Dict, Optional, Union
+from typing import Any, Dict, Optional, Tuple, Union
 
 import pandas as pd
+from openbb_charting.core.openbb_figure import OpenBBFigure
 from openbb_charting.core.plotly_ta.data_classes import ChartIndicators
 from openbb_charting.core.plotly_ta.ta_class import PlotlyTA
 from openbb_core.app.model.charts.charting_settings import ChartingSettings
 
 
-def to_plotly_json(
+def to_chart(
     charting_settings: ChartingSettings,
     data: Union[pd.DataFrame, pd.Series],
     indicators: Optional[Union[ChartIndicators, Dict[str, Dict[str, Any]]]] = None,
@@ -15,7 +16,7 @@ def to_plotly_json(
     volume: bool = True,
     prepost: bool = False,
     volume_ticks_x: int = 7,
-) -> str:
+) -> Tuple[OpenBBFigure, Dict[str, Any]]:
     """
     Returns the plotly json representation of the chart.
     This function is used so it can be called at the module level and used out of the box,
@@ -57,5 +58,6 @@ def to_plotly_json(
         prepost=prepost,
         volume_ticks_x=volume_ticks_x,
     )
+    content = fig.show(external=True).to_plotly_json()
 
-    return fig.show(external=True).to_plotly_json()
+    return fig, content

openbb_sdk/extensions/charting/openbb_charting/styles/default/tables.pltstyle.json

@@ -0,0 +1,102 @@
+{
+    "data": {
+        "candlestick": [
+            {
+                "decreasing": {
+                    "fillcolor": "#e4003a",
+                    "line": {
+                        "color": "#e4003a"
+                    }
+                },
+                "increasing": {
+                    "fillcolor": "#00ACFF",
+                    "line": {
+                        "color": "#00ACFF"
+                    }
+                },
+                "type": "candlestick"
+            }
+        ]
+    },
+    "layout": {
+        "annotationdefaults": {
+            "showarrow": false
+        },
+        "autotypenumbers": "strict",
+        "colorway": [
+            "#ffed00",
+            "#ef7d00",
+            "#e4003a",
+            "#c13246",
+            "#822661",
+            "#48277c",
+            "#005ca9",
+            "#00aaff",
+            "#9b30d9",
+            "#af005f",
+            "#5f00af",
+            "#af87ff"
+        ],
+        "dragmode": "pan",
+        "font": {
+            "family": "Fira Code",
+            "size": 18
+        },
+        "hoverlabel": {
+            "align": "left"
+        },
+        "mapbox": {
+            "style": "light"
+        },
+        "hovermode": "x",
+        "legend": {
+            "bgcolor": "rgba(0, 0, 0, 0)",
+            "x": 0.01,
+            "xanchor": "left",
+            "y": 0.99,
+            "yanchor": "top",
+            "font": {
+                "size": 15
+            }
+        },
+        "paper_bgcolor": "white",
+        "plot_bgcolor": "white",
+        "xaxis": {
+            "automargin": true,
+            "autorange": true,
+            "rangeslider": {
+                "visible": false
+            },
+            "showgrid": true,
+            "showline": true,
+            "tickfont": {
+                "size": 14
+            },
+            "zeroline": false,
+            "tick0": 1,
+            "title": {
+                "standoff": 20
+            },
+            "linecolor": "#F5EFF3",
+            "mirror": true,
+            "ticks": "outside"
+        },
+        "yaxis": {
+            "anchor": "x",
+            "automargin": true,
+            "fixedrange": false,
+            "zeroline": false,
+            "showgrid": true,
+            "showline": true,
+            "side": "right",
+            "tick0": 0.5,
+            "title": {
+                "standoff": 20
+            },
+            "gridcolor": "#283442",
+            "linecolor": "#F5EFF3",
+            "mirror": true,
+            "ticks": "outside"
+        }
+    }
+}

openbb_sdk/sdk/core/openbb_core/api/router/commands.py

@@ -1,3 +1,4 @@
+import inspect
 from functools import partial, wraps
 from inspect import Parameter, Signature, signature
 from typing import Any, Callable, Dict, Tuple, TypeVar
@@ -7,9 +8,11 @@ from fastapi.routing import APIRoute
 from openbb_core.api.dependency.user import get_user
 from openbb_core.app.command_runner import CommandRunner
 from openbb_core.app.model.command_context import CommandContext
+from openbb_core.app.model.command_output import CommandOutput
 from openbb_core.app.model.user_settings import UserSettings
 from openbb_core.app.router import RouterLoader
 from openbb_core.app.service.system_service import SystemService
+from pydantic import BaseModel
 from typing_extensions import Annotated, ParamSpec
 
 T = TypeVar("T")
@@ -73,6 +76,52 @@ def build_new_signature(func):
     )
 
 
+def validate_output(c_out: CommandOutput) -> CommandOutput:
+    """
+    Validate CommandOutput object.
+    Checks against the CommandOutput schema and removes fields that contain the
+    `exclude_from_api` extra `pydantic.Field` kwarg.
+    Note that the modification to the `CommandOutput` object is done in-place.
+
+    Parameters
+    ----------
+    c_out : CommandOutput
+        CommandOutput object to validate.
+
+    Returns
+    -------
+    CommandOutput
+        Validated CommandOutput object.
+    """
+
+    def is_model(type_):
+        return inspect.isclass(type_) and issubclass(type_, BaseModel)
+
+    def exclude_fields_from_api(key: str, value: Any):
+        type_ = type(value)
+
+        # case where 1st layer field needs to be excluded
+        if key in c_out.__fields__ and c_out.__fields__[key].field_info.extra.get(
+            "exclude_from_api", None
+        ):
+            delattr(c_out, key)
+
+        # if it's a model with nested fields
+        elif is_model(type_):
+            for field in type_.__fields__.values():
+                if field.field_info.extra.get("exclude_from_api", None):
+                    delattr(value, field.name)
+
+                # if it's a yet a nested model we need to go deeper in the recursion
+                elif is_model(field.type_):
+                    exclude_fields_from_api(field.name, getattr(value, field.name))
+
+    for k, v in c_out:
+        exclude_fields_from_api(k, v)
+
+    return c_out
+
+
 def build_api_wrapper(
     command_runner: CommandRunner,
     route: APIRoute,
@@ -97,7 +146,7 @@ def build_api_wrapper(
         execute = partial(command_runner.run_once, user_settings, path)
         journal_entry = execute(*args, **kwargs)
 
-        return journal_entry.output
+        return validate_output(journal_entry.output)
 
     return wrapper
 

openbb_sdk/sdk/core/openbb_core/app/charting_manager.py

@@ -144,10 +144,28 @@ class ChartingManager(metaclass=SingletonMeta):
         create_backend_func(charting_settings=charting_settings)
         get_backend_func().start(debug=charting_settings.debug_mode)
 
-    def to_plotly_json(self, **kwargs) -> str:
+    def to_chart(self, **kwargs) -> Chart:
         """
-        Returns the plotly json representation of the chart.
+        Returns the chart object.
+
+        Parameters
+        ----------
+        **kwargs
+            Keyword arguments to be passed to the charting extension.
+
+        Returns
+        -------
+        Chart
+            Chart object.
+
+        Raises
+        ------
+        ChartingManagerError
+            If charting extension is not installed.
+        Exception
+            If the charting extension module does not contain the `to_chart` function.
         """
+
         if not self._charting_extension_installed:
             raise ChartingManagerError(
                 f"Charting extension `{self._charting_extension}` is not installed"
@@ -156,12 +174,19 @@ class ChartingManager(metaclass=SingletonMeta):
 
         # Dynamically import the charting module
         backend_module = import_module(self._charting_extension)
-        # Get the plotly json function from the charting module
-        to_plotly_json_func = getattr(backend_module, "to_plotly_json")
+        # Get the `to_chart` function from the charting module
+        to_chart_func = getattr(backend_module, "to_chart")
+
         # Add the charting settings to the kwargs
         kwargs["charting_settings"] = self._charting_settings
 
-        return to_plotly_json_func(**kwargs)
+        fig, content = to_chart_func(**kwargs)
+
+        return Chart(
+            content=content,
+            format=self.get_chart_format(self._charting_extension),
+            fig=fig,
+        )
 
     def chart(
         self,
@@ -211,7 +236,11 @@ class ChartingManager(metaclass=SingletonMeta):
         kwargs["command_output_item"] = command_output_item
         kwargs["charting_settings"] = self._charting_settings
 
+        charting_function = self.get_chart_function(self._charting_extension, route)
+        fig, content = charting_function(**kwargs)
+
         return Chart(
-            content=self.get_chart_function(self._charting_extension, route)(**kwargs),
+            content=content,
             format=self.get_chart_format(self._charting_extension),
+            fig=fig,
         )

openbb_sdk/sdk/core/openbb_core/app/model/charts/chart.py

@@ -2,7 +2,7 @@ from enum import Enum
 from typing import Any, Dict, Optional
 
 from openbb_core.app.model.abstract.error import Error
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
 
 
 class ChartFormat(str, Enum):
@@ -10,28 +10,30 @@ class ChartFormat(str, Enum):
 
 
 class Chart(BaseModel):
-    content: Optional[Dict[str, Any]] = None
-    format: Optional[ChartFormat] = ChartFormat.plotly
-    error: Optional[Error] = None
+    content: Optional[Dict[str, Any]] = Field(
+        default=None,
+        description="Raw textual representation of the chart.",
+    )
+    format: Optional[ChartFormat] = Field(
+        default=ChartFormat.plotly,
+        description="Complementary attribute to the `content` attribute. It specifies the format of the chart.",
+    )
+    error: Optional[Error] = Field(
+        default=None,
+        description="Exception caught during the computation of the `Chart`.",
+    )
+    fig: Optional[Any] = Field(
+        default=None,
+        description="The figure object.",
+        exclude_from_api=True,
+    )
 
     class Config:
         validate_assignment = True
 
-    def show(self):
-        """Shows the chart in PyWry, browser or notebook."""
-
-        # TODO : this method should be handled by the ChartingManager so that
-        # this model class doesn't have specific knowledge on the charting extension
-
-        # pylint: disable=import-outside-toplevel
-        if self.format == ChartFormat.plotly:
-            from openbb_charting.core.openbb_figure import OpenBBFigure
-
-            chart = OpenBBFigure()
-
-            chart.update(self.content)
-
-            chart.show()
-
-        else:
-            raise ValueError(f"Chart format {self.format} not supported.")
+    def __repr__(self) -> str:
+        return (
+            self.__class__.__name__
+            + "\n\n"
+            + "\n".join([f"{k}: {v}" for k, v in self.dict().items()])
+        )

openbb_sdk/sdk/core/openbb_core/app/model/charts/charting_settings.py

@@ -43,9 +43,8 @@ class ChartingSettings:
         self.user_uuid: Optional[str] = (
             user_settings.profile.hub_session.user_uuid if has_hub else None
         )
-        self.user_exports_directory: str = user_settings.preferences.export_directory
-        self.user_styles_directory: str = (
-            user_settings.preferences.user_styles_directory
-        )
+        self.user_exports_directory = user_settings.preferences.export_directory
+        self.user_styles_directory = user_settings.preferences.user_styles_directory
         # Theme
         self.chart_style: str = user_settings.preferences.chart_style
+        self.table_style = user_settings.preferences.table_style

openbb_sdk/sdk/core/openbb_core/app/model/command_output.py

@@ -1,4 +1,4 @@
-from typing import Any, Dict, Generic, List, Optional, TypeVar, Union
+from typing import Dict, Generic, List, Optional, TypeVar, Union
 
 import pandas as pd
 from pydantic import Field
@@ -8,7 +8,7 @@ from openbb_core.app.charting_manager import ChartingManager
 from openbb_core.app.model.abstract.error import Error
 from openbb_core.app.model.abstract.tagged import Tagged
 from openbb_core.app.model.abstract.warning import Warning_
-from openbb_core.app.model.charts.chart import Chart, ChartFormat
+from openbb_core.app.model.charts.chart import Chart
 from openbb_core.app.provider_interface import get_provider_interface
 from openbb_core.app.utils import basemodel_to_df
 
@@ -105,41 +105,31 @@ class CommandOutput(GenericModel, Generic[T], Tagged):
 
         return results
 
-    def to_plotly_json(
-        self, create_chart: bool = True, **kwargs
-    ) -> Optional[Dict[str, Any]]:
+    def to_chart(self, **kwargs):
         """
-        Returns the plotly json representation of the chart.
-        If the chart was already computed it return the plotly json representation of the chart.
-        Otherwise it computes the chart based on the available data and provided kwargs.
+        Create or update the `Chart`.
+        Note that the `chart` attribute is composed by: `content`, `format` and `fig`.
 
         Parameters
         ----------
-        create_chart : bool, optional
-            If True, it creates the chart object and populates the respective field on the object, by default True.
         **kwargs
             Keyword arguments to be passed to the charting extension.
-            This implies that the user has to know the charting extension API; this is the case
-            because the charting extension may vary on user settings.
+            This implies that the user has some knowledge on the charting extension API.
+            This is the case because the charting extension may vary on user preferences.
+
+        Returns
+        -------
+        chart.fig
+            The chart figure.
         """
+        cm = ChartingManager()
+        kwargs["data"] = self.to_dataframe()
 
-        if self.chart and not kwargs:
-            plotly_json = self.chart.content
-        else:
-            cm = ChartingManager()
-            kwargs["data"] = self.to_dataframe()
-            plotly_json = cm.to_plotly_json(**kwargs)
-
-            if create_chart:
-                try:
-                    self.chart = Chart(content=plotly_json, format=ChartFormat.plotly)
-                except Exception as e:
-                    self.chart = Chart(error=Error(message=str(e)))
-
-        return plotly_json
+        self.chart = cm.to_chart(**kwargs)
+        return self.chart.fig
 
     def show(self):
         """Displays chart."""
-        if not self.chart:
+        if not self.chart and not self.chart.fig:
             raise OpenBBError("Chart not found.")
-        self.chart.show()
+        self.chart.fig.show()

openbb_sdk/sdk/core/openbb_core/app/model/preferences.py

@@ -16,6 +16,7 @@ class Preferences(BaseModel):
     plot_open_export: bool = (
         False  # Whether to open plot image exports after they are created
     )
+    table_style: Literal["dark", "light"] = "dark"
     request_timeout: PositiveInt = 15
 
     class Config:

openbb_sdk/sdk/provider/README.md

@@ -34,7 +34,7 @@ Close for closing price column name in data from one provider might be coded as
 - **Provider** : Entry point class for each provider extension. Contains information about the provider, it's required credentials and available fetchers.
 - **RegistryLoader** : Loads the registry with the provider extensions installed.
 - **Registry** : Maintains a registry of provider extensions installed.
-- **RegistryMap** : Stores the complete characterization of each provider. It centralizes information like required credentials, standardised and extra query parameteres/data by provider.
+- **RegistryMap** : Stores the complete characterization of each provider. It centralizes information like required credentials, standardised and extra query parameters/data by provider.
 - **QueryExecutor** : Executes a given query, routing it to the respective provider and fetcher.
 
 ## Installation