Merge branch 'develop' of https://github.com/OpenBB-finance/OpenBBTerminal into develop

.github/workflows/intel_macos_build.yml

@@ -5,6 +5,7 @@ env:
   OPENBB_LOG_COLLECT: false
   OPENBB_USE_ION: false
   OPENBB_USE_PROMPT_TOOLKIT: false
+  OPENBB_FILE_OVERWRITE: true
   PIP_DEFAULT_TIMEOUT: 100
   PYTHONNOUSERSITE: 1
 

.github/workflows/linting.yml

@@ -5,6 +5,7 @@ env:
   OPENBB_LOG_COLLECT: false
   OPENBB_USE_ION: false
   OPENBB_USE_PROMPT_TOOLKIT: false
+  OPENBB_FILE_OVERWRITE: true
   PIP_DEFAULT_TIMEOUT: 100
 
 on:
@@ -48,7 +49,6 @@ jobs:
       - run: ruff .
       - run: mypy --ignore-missing-imports openbb_terminal
       - run: shopt -s globstar && pyupgrade --py38-plus **/*.py
-      - run: safety check
       - run: |
           pylint terminal.py openbb_terminal tests
 

.github/workflows/m1_macos_build.yml

@@ -5,6 +5,7 @@ env:
   OPENBB_LOG_COLLECT: false
   OPENBB_USE_ION: false
   OPENBB_USE_PROMPT_TOOLKIT: false
+  OPENBB_FILE_OVERWRITE: true
   PIP_DEFAULT_TIMEOUT: 100
   PYTHONNOUSERSITE: 1
 
@@ -58,10 +59,10 @@ jobs:
 
       - name: Install Pyinstaller
         run: poetry install -E installer
-  
+
       - name: Poetry Install Portfolio Optimization and Forecasting Toolkits
         run: poetry install -E all
-  
+
       - name: Install Specific Papermill
         run: |
           pip uninstall papermill -y

.github/workflows/mac.yml

@@ -5,6 +5,7 @@ env:
   OPENBB_USE_ION: false
   OPENBB_USE_PROMPT_TOOLKIT: false
   PIP_DEFAULT_TIMEOUT: 100
+  OPENBB_FILE_OVERWRITE: true
   PYTHONNOUSERSITE: 1
 on:
   push:

.github/workflows/nightly-build.yml

@@ -6,6 +6,7 @@ env:
   OPENBB_USE_ION: false
   OPENBB_USE_PROMPT_TOOLKIT: false
   PIP_DEFAULT_TIMEOUT: 100
+  OPENBB_FILE_OVERWRITE: true
   PYTHONNOUSERSITE: 1
 
 on:
@@ -24,7 +25,7 @@ jobs:
         uses: benc-uk/workflow-dispatch@v1
         with:
           workflow: windows.yml
-  
+
   trigger-macos-build:
     runs-on: ubuntu-latest
     steps:
@@ -32,7 +33,7 @@ jobs:
         uses: benc-uk/workflow-dispatch@v1
         with:
           workflow: m1_macos_build.yml
-  
+
   trigger-intel-build:
     runs-on: ubuntu-latest
     steps:

.github/workflows/test.yml

@@ -4,6 +4,7 @@ env:
   OPENBB_LOG_COLLECT: false
   OPENBB_USE_ION: false
   OPENBB_USE_PROMPT_TOOLKIT: false
+  OPENBB_FILE_OVERWRITE: true
   PIP_DEFAULT_TIMEOUT: 100
 
 on: [workflow_dispatch]

.github/workflows/unit-test.yml

@@ -5,6 +5,7 @@ env:
   OPENBB_LOG_COLLECT: false
   OPENBB_USE_ION: false
   OPENBB_USE_PROMPT_TOOLKIT: false
+  OPENBB_FILE_OVERWRITE: true
   PIP_DEFAULT_TIMEOUT: 100
 
 on:

.github/workflows/windows.yml

@@ -4,6 +4,7 @@ env:
   OPENBB_LOG_COLLECT: false
   OPENBB_USE_ION: false
   OPENBB_USE_PROMPT_TOOLKIT: false
+  OPENBB_FILE_OVERWRITE: true
   PIP_DEFAULT_TIMEOUT: 100
   PYTHONNOUSERSITE: 1
 on:

openbb_terminal/featflags_controller.py

@@ -42,6 +42,7 @@ class FeatureFlagsController(BaseController):
         "rich",
         "richpanel",
         "tbhint",
+        "overwrite",
     ]
     PATH = "/featflags/"
 
@@ -73,9 +74,15 @@ class FeatureFlagsController(BaseController):
         mt.add_setting("watermark", obbff.USE_WATERMARK)
         mt.add_setting("cmdloc", obbff.USE_CMD_LOCATION_FIGURE)
         mt.add_setting("tbhint", obbff.TOOLBAR_HINT)
+        mt.add_setting("overwrite", obbff.FILE_OVERWITE)
 
         console.print(text=mt.menu_text, menu="Feature Flags")
 
+    def call_overwrite(self, _):
+        """Process overwrite command"""
+        obbff.FILE_OVERWITE = not obbff.FILE_OVERWITE
+        set_key(USER_ENV_FILE, "OPENBB_FILE_OVERWITE", str(obbff.FILE_OVERWITE))
+
     def call_retryload(self, _):
         """Process retryload command"""
         obbff.RETRY_WITH_LOAD = not obbff.RETRY_WITH_LOAD

openbb_terminal/feature_flags.py

@@ -31,6 +31,9 @@ else:
     WITH_GIT = True
 
 
+# Always overwrite exported files
+FILE_OVERWITE = load_env_vars("OPENBB_FILE_OVERWRITE", strtobool, False, "featflags")
+
 # Retry unknown commands with `load`
 RETRY_WITH_LOAD = load_env_vars("OPENBB_RETRY_WITH_LOAD", strtobool, False, "featflags")
 

openbb_terminal/forecast/anom_model.py

@@ -0,0 +1,62 @@
+# pylint: disable=too-many-arguments
+"""Quantile Anomaly Detection Model"""
+__docformat__ = "numpy"
+
+import logging
+from typing import Tuple, Union, List
+
+import pandas as pd
+
+from darts import TimeSeries
+from darts.ad import KMeansScorer
+from darts.ad import QuantileDetector
+from openbb_terminal.decorators import log_start_end
+
+from openbb_terminal.forecast import helpers
+
+logger = logging.getLogger(__name__)
+
+
+@log_start_end(log=logger)
+def get_anomaly_detection_data(
+    data: Union[pd.Series, pd.DataFrame],
+    target_column: str = "close",
+    train_split: float = 0.6,
+) -> Tuple[List[TimeSeries], List[TimeSeries], List[TimeSeries]]:
+    """Get Quantile Anomaly Detection Data
+
+    Parameters
+    ----------
+    data: Union[pd.Series, pd.DataFrame]
+
+    Input Data
+    -------
+    target_column: str
+        Target column to forecast. Defaults to "close".
+    train_split: (float, optional)
+        Train/val split. Defaults to 0.85.
+
+    Returns
+    -------
+    Tuple[
+        List[TimeSeries],
+        List[TimeSeries],
+        List[TimeSeries],
+        ]
+    """
+    use_scalers = False
+
+    _, ticker_series = helpers.get_series(data, target_column, is_scaler=use_scalers)
+
+    # split dataset into train and val
+    train, val = ticker_series.split_before(train_split)
+
+    scorer = KMeansScorer(k=2, window=5)
+    scorer.fit(train)
+    anom_score = scorer.score(val)
+
+    detector = QuantileDetector(high_quantile=0.99)
+    detector.fit(scorer.score(train))
+    binary_anom = detector.detect(anom_score)
+
+    return ticker_series, anom_score, binary_anom

openbb_terminal/forecast/anom_view.py

@@ -0,0 +1,96 @@
+"""Quantile Anomaly Detector View"""
+__docformat__ = "numpy"
+
+import logging
+from typing import Union, List, Optional
+from datetime import datetime
+
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+
+from openbb_terminal.forecast import anom_model
+from openbb_terminal.decorators import log_start_end
+from openbb_terminal.forecast import helpers
+from openbb_terminal.helper_funcs import (
+    plot_autoscale,
+)
+from openbb_terminal.config_terminal import theme
+from openbb_terminal.config_plot import PLOT_DPI
+from openbb_terminal.rich_config import console
+
+logger = logging.getLogger(__name__)
+# pylint: disable=too-many-arguments
+
+
+@log_start_end(log=logger)
+def display_anomaly_detection(
+    data: Union[pd.Series, pd.DataFrame],
+    dataset_name="",
+    target_column: str = "close",
+    train_split: float = 0.6,
+    start_date: Optional[datetime] = None,
+    end_date: Optional[datetime] = None,
+    external_axes: Optional[List[plt.axes]] = None,
+):
+    """Display Quantile Anomaly Detection
+
+    Parameters
+    ----------
+    data: Union[pd.Series, pd.DataFrame]
+
+    Input Data
+    ----------
+    target_column: str
+        Target column to forecast. Defaults to "close".
+    train_split: (float, optional)
+        Train/val split. Defaults to 0.85.
+    forecast_horizon: (int, optional)
+        Forecast horizon. Defaults to 5.
+    export: (str, optional)
+        Export data to csv. Defaults to "".
+    start_date: (Optional[datetime], optional)
+        Start date. Defaults to None.
+    end_date: (Optional[datetime], optional)
+        End date. Defaults to None.
+    external_axes: (Optional[List[plt.axes]], optional)
+        External axes. Defaults to None.
+
+    Returns
+    -------
+    None
+    """
+    data = helpers.clean_data(data, start_date, end_date, target_column)
+    if not helpers.check_data(data, target_column):
+        return
+
+    (ticker_series, _, binary_anom) = anom_model.get_anomaly_detection_data(
+        data=data,
+        target_column=target_column,
+        train_split=train_split,
+    )
+
+    if not external_axes:
+        _, ax = plt.subplots(figsize=plot_autoscale(), dpi=PLOT_DPI)
+    else:
+        ax = external_axes[0]
+
+    ticker_series.plot(label=target_column, ax=ax)
+
+    ticker_max = np.max(ticker_series.values())
+    (binary_anom * int(ticker_max)).plot(
+        label="detected binary anomaly", c="red", ax=ax
+    )
+    # set min and max for y axis
+    ax.set_ylim([-1, ticker_max * 1.1])
+
+    ax.set_title(f"Quantile Anomaly Detection for {dataset_name}")
+    ax.set_xlabel("Date")
+
+    console.print(
+        f" [green]Quantile Anomaly Detection for {dataset_name} calculated.[/green]"
+    )
+
+    theme.style_primary_axis(ax)
+    if external_axes is None:
+        theme.visualize_output()

openbb_terminal/forecast/forecast_controller.py

@@ -71,6 +71,7 @@ from openbb_terminal.forecast import (
     helpers,
     trans_view,
     nhits_view,
+    anom_view,
 )
 
 from openbb_terminal.common import common_model
@@ -133,6 +134,7 @@ class ForecastController(BaseController):
         "season",
         "which",
         "nhits",
+        "anom",
     ]
     pandas_plot_choices = [
         "line",
@@ -303,6 +305,9 @@ class ForecastController(BaseController):
         mt.add_cmd("tcn", self.files)
         mt.add_cmd("trans", self.files)
         mt.add_cmd("tft", self.files)
+        mt.add_raw("\n")
+        mt.add_info("_anomaly_")
+        mt.add_cmd("anom", self.files)
 
         console.print(text=mt.menu_text, menu="Forecast")
 
@@ -3127,3 +3132,48 @@ class ForecastController(BaseController):
                 naive=ns_parser.naive,
                 export_pred_raw=ns_parser.export_pred_raw,
             )
+
+    @log_start_end(log=logger)
+    def call_anom(self, other_args: List[str]):
+        """Process ANOM command"""
+        parser = argparse.ArgumentParser(
+            formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+            add_help=False,
+            prog="anom",
+            description="""
+                Perform a Quantile Anomaly detection on a given dataset:
+                https://unit8co.github.io/darts/generated_api/darts.ad.detectors.quantile_detector.html
+            """,
+        )
+
+        # if user does not put in --dataset
+        if other_args and "-" not in other_args[0][0]:
+            other_args.insert(0, "--dataset")
+
+        parser = self.add_standard_args(
+            parser,
+            train_split=True,
+            target_column=True,
+            target_dataset=True,
+            forecast_only=True,
+            start=True,
+            end=True,
+        )
+        ns_parser = self.parse_known_args_and_warn(
+            parser,
+            other_args,
+            export_allowed=EXPORT_ONLY_FIGURES_ALLOWED,
+        )
+
+        if ns_parser:
+            if not helpers.check_parser_input(ns_parser, self.datasets):
+                return
+
+            anom_view.display_anomaly_detection(
+                data=self.datasets[ns_parser.target_dataset],
+                dataset_name=ns_parser.target_dataset,
+                target_column=ns_parser.target_column,
+                train_split=ns_parser.train_split,
+                start_date=ns_parser.s_start_date,
+                end_date=ns_parser.s_end_date,
+            )

openbb_terminal/helper_funcs.py

@@ -5,7 +5,7 @@ import argparse
 import io
 import logging
 from pathlib import Path
-from typing import List, Union, Optional, Dict
+from typing import List, Union, Optional, Dict, Tuple
 from functools import lru_cache
 from datetime import datetime, timedelta
 from datetime import date as d
@@ -1322,8 +1322,34 @@ def compose_export_path(func_name: str, dir_path: str) -> Path:
     return full_path
 
 
+def ask_file_overwrite(file_path: str) -> Tuple[bool, bool]:
+    """Helper to provide a prompt for overwriting existing files.
+
+    Returns two values, the first is a boolean indicating if the file exists and the
+    second is a boolean indicating if the user wants to overwrite the file.
+    """
+    # Jeroen asked for a flag to overwrite no matter what
+    if obbff.FILE_OVERWITE:
+        return False, True
+    if os.path.exists(file_path):
+        overwrite = input("\nFile already exists. Overwrite? [y/n]: ").lower()
+        if overwrite == "y":
+            # File exists and user wants to overwrite
+            return True, True
+        # File exists and user does not want to overwrite
+        return True, False
+    # File does not exist
+    return False, True
+
+
+# This is a false positive on pylint and being tracked in pylint #3060
+# pylint: disable=abstract-class-instantiated
 def export_data(
-    export_type: str, dir_path: str, func_name: str, df: pd.DataFrame = pd.DataFrame()
+    export_type: str,
+    dir_path: str,
+    func_name: str,
+    df: pd.DataFrame = pd.DataFrame(),
+    sheet_name: str = None,
 ) -> None:
     """Export data to a file.
 
@@ -1337,6 +1363,8 @@ def export_data(
         Name of the command that invokes this function
     df : pd.Dataframe
         Dataframe of data to save
+    sheet_name : str
+        If provided.  The name of the sheet to save in excel file
     """
     if export_type:
         export_path = compose_export_path(func_name, dir_path)
@@ -1372,19 +1400,62 @@ def export_data(
             )
 
             if exp_type.endswith("csv"):
+                exists, overwrite = ask_file_overwrite(saved_path)
+                if exists and not overwrite:
+                    return
                 df.to_csv(saved_path)
             elif exp_type.endswith("json"):
+                exists, overwrite = ask_file_overwrite(saved_path)
+                if exists and not overwrite:
+                    return
                 df.reset_index(drop=True, inplace=True)
                 df.to_json(saved_path)
             elif exp_type.endswith("xlsx"):
-                df.to_excel(saved_path, index=True, header=True)
+
+                if sheet_name is None:
+                    exists, overwrite = ask_file_overwrite(saved_path)
+                    if exists and not overwrite:
+                        return
+                    df.to_excel(saved_path, index=True, header=True)
+
+                else:
+                    if os.path.exists(saved_path):
+                        with pd.ExcelWriter(
+                            saved_path,
+                            mode="a",
+                            if_sheet_exists="new",
+                            engine="openpyxl",
+                        ) as writer:
+                            df.to_excel(
+                                writer, sheet_name=sheet_name, index=True, header=True
+                            )
+                    else:
+                        with pd.ExcelWriter(
+                            saved_path,
+                            engine="openpyxl",
+                        ) as writer:
+                            df.to_excel(
+                                writer, sheet_name=sheet_name, index=True, header=True
+                            )
             elif exp_type.endswith("png"):
+                exists, overwrite = ask_file_overwrite(saved_path)
+                if exists and not overwrite:
+                    return
                 plt.savefig(saved_path)
             elif exp_type.endswith("jpg"):
+                exists, overwrite = ask_file_overwrite(saved_path)
+                if exists and not overwrite:
+                    return
                 plt.savefig(saved_path)
             elif exp_type.endswith("pdf"):
+                exists, overwrite = ask_file_overwrite(saved_path)
+                if exists and not overwrite:
+                    return
                 plt.savefig(saved_path)
             elif exp_type.endswith("svg"):
+                exists, overwrite = ask_file_overwrite(saved_path)
+                if exists and not overwrite:
+                    return
                 plt.savefig(saved_path)
             else:
                 console.print("\nWrong export file specified.")

openbb_terminal/miscellaneous/i18n/en.yml

@@ -1064,4 +1064,6 @@ en:
   forecast/trans: Transformer Network (feat. Past covariates)
   forecast/tft: Temporal Fusion Transformer Network(feat. Past covariates)
   forecast/nhits: Neural Hierarchical Interpolation (feat. Past covariates)
+  forecast/_anomaly_: Anomaly Detection
+  forecast/anom: Quantile Anomaly Detector
   forecast/_comingsoon_: Coming Soon

openbb_terminal/miscellaneous/library/trail_map_forecasting.csv

@@ -36,3 +36,4 @@ forecast.tcn,openbb_terminal.forecast.tcn_model.get_tcn_data,openbb_terminal.for
 forecast.tft,openbb_terminal.forecast.tft_model.get_tft_data,openbb_terminal.forecast.tft_view.display_tft_forecast
 forecast.theta,openbb_terminal.forecast.theta_model.get_theta_data,openbb_terminal.forecast.theta_view.display_theta_forecast
 forecast.trans,openbb_terminal.forecast.trans_model.get_trans_data,openbb_terminal.forecast.trans_view.display_trans_forecast
+forecast.anom,openbb_terminal.forecast.anom_model.get_anomaly_detection_data,openbb_terminal.forecast.anom_view.display_anomaly_detection

openbb_terminal/parent_classes.py

@@ -753,6 +753,16 @@ class BaseController(metaclass=ABCMeta):
                 help=help_export,
             )
 
+            # If excel is an option, add the sheet name
+            if export_allowed == EXPORT_ONLY_RAW_DATA_ALLOWED:
+                parser.add_argument(
+                    "--sheet-name",
+                    dest="sheet_name",
+                    default=None,
+                    nargs="+",
+                    help="Name of excel sheet to save data to. Only valid for .xlsx files.",
+                )
+
         if raw:
             parser.add_argument(
                 "--raw",
@@ -1147,6 +1157,9 @@ class StockBaseController(BaseController, metaclass=ABCMeta):
                     os.path.dirname(os.path.abspath(__file__)),
                     f"load_{self.ticker}",
                     self.stock.copy(),
+                    sheet_name=" ".join(ns_parser.sheet_name)
+                    if ns_parser.sheet_name
+                    else None,
                 )
 
 

tests/openbb_terminal/forecast/test_anom_model.py

@@ -0,0 +1,11 @@
+import pytest
+from tests.openbb_terminal.forecast import conftest
+
+try:
+    from openbb_terminal.forecast import anom_model
+except ImportError:
+    pytest.skip(allow_module_level=True)
+
+
+def test_get_expo_model(tsla_csv):
+    conftest.test_model(anom_model.get_anomaly_detection_data, tsla_csv)

tests/openbb_terminal/forecast/test_anom_view.py

@@ -0,0 +1,14 @@
+import pytest
+
+try:
+    from openbb_terminal.forecast import anom_view
+except ImportError:
+    pytest.skip(allow_module_level=True)
+
+
+def test_display_anom_forecast(tsla_csv):
+    anom_view.display_anomaly_detection(
+        tsla_csv,
+        target_column="close",
+        start_window=0.5,
+    )

tests/openbb_terminal/portfolio/csv/test_portfolio_model/test_tracking_error.csv

@@ -5,9 +5,7 @@ Total       NaN
 dtype: float64"
 ytd,-
 3m,-
-6m,"Adj Close   NaN
-Total       NaN
-dtype: float64"
+6m,-
 1y,"Adj Close   NaN
 Total       NaN
 dtype: float64"