diff --git a/.github/workflows/intel_macos_build.yml b/.github/workflows/intel_macos_build.yml index bdb01afcd4e..5e274e00dbe 100644 --- a/.github/workflows/intel_macos_build.yml +++ b/.github/workflows/intel_macos_build.yml @@ -5,6 +5,7 @@ env: OPENBB_LOG_COLLECT: false OPENBB_USE_ION: false OPENBB_USE_PROMPT_TOOLKIT: false + OPENBB_FILE_OVERWRITE: true PIP_DEFAULT_TIMEOUT: 100 PYTHONNOUSERSITE: 1 diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index ba3ccdabc20..d6edf443d61 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -5,6 +5,7 @@ env: OPENBB_LOG_COLLECT: false OPENBB_USE_ION: false OPENBB_USE_PROMPT_TOOLKIT: false + OPENBB_FILE_OVERWRITE: true PIP_DEFAULT_TIMEOUT: 100 on: @@ -48,7 +49,6 @@ jobs: - run: ruff . - run: mypy --ignore-missing-imports openbb_terminal - run: shopt -s globstar && pyupgrade --py38-plus **/*.py - - run: safety check - run: | pylint terminal.py openbb_terminal tests diff --git a/.github/workflows/m1_macos_build.yml b/.github/workflows/m1_macos_build.yml index 19131d86899..24fa3beb712 100644 --- a/.github/workflows/m1_macos_build.yml +++ b/.github/workflows/m1_macos_build.yml @@ -5,6 +5,7 @@ env: OPENBB_LOG_COLLECT: false OPENBB_USE_ION: false OPENBB_USE_PROMPT_TOOLKIT: false + OPENBB_FILE_OVERWRITE: true PIP_DEFAULT_TIMEOUT: 100 PYTHONNOUSERSITE: 1 @@ -58,10 +59,10 @@ jobs: - name: Install Pyinstaller run: poetry install -E installer - + - name: Poetry Install Portfolio Optimization and Forecasting Toolkits run: poetry install -E all - + - name: Install Specific Papermill run: | pip uninstall papermill -y diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 440df2f0971..2bca5fa0910 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -5,6 +5,7 @@ env: OPENBB_USE_ION: false OPENBB_USE_PROMPT_TOOLKIT: false PIP_DEFAULT_TIMEOUT: 100 + OPENBB_FILE_OVERWRITE: true PYTHONNOUSERSITE: 1 on: push: diff --git a/.github/workflows/nightly-build.yml b/.github/workflows/nightly-build.yml index 42e734dab2e..ce8e87e7d76 100644 --- a/.github/workflows/nightly-build.yml +++ b/.github/workflows/nightly-build.yml @@ -6,6 +6,7 @@ env: OPENBB_USE_ION: false OPENBB_USE_PROMPT_TOOLKIT: false PIP_DEFAULT_TIMEOUT: 100 + OPENBB_FILE_OVERWRITE: true PYTHONNOUSERSITE: 1 on: @@ -24,7 +25,7 @@ jobs: uses: benc-uk/workflow-dispatch@v1 with: workflow: windows.yml - + trigger-macos-build: runs-on: ubuntu-latest steps: @@ -32,7 +33,7 @@ jobs: uses: benc-uk/workflow-dispatch@v1 with: workflow: m1_macos_build.yml - + trigger-intel-build: runs-on: ubuntu-latest steps: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8491355f1c6..c6cd57ebaf5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,6 +4,7 @@ env: OPENBB_LOG_COLLECT: false OPENBB_USE_ION: false OPENBB_USE_PROMPT_TOOLKIT: false + OPENBB_FILE_OVERWRITE: true PIP_DEFAULT_TIMEOUT: 100 on: [workflow_dispatch] diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index fd0ae41914a..c0aaa4e260f 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -5,6 +5,7 @@ env: OPENBB_LOG_COLLECT: false OPENBB_USE_ION: false OPENBB_USE_PROMPT_TOOLKIT: false + OPENBB_FILE_OVERWRITE: true PIP_DEFAULT_TIMEOUT: 100 on: diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 5b470bdd0fc..6c5763f28b4 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -4,6 +4,7 @@ env: OPENBB_LOG_COLLECT: false OPENBB_USE_ION: false OPENBB_USE_PROMPT_TOOLKIT: false + OPENBB_FILE_OVERWRITE: true PIP_DEFAULT_TIMEOUT: 100 PYTHONNOUSERSITE: 1 on: diff --git a/openbb_terminal/featflags_controller.py b/openbb_terminal/featflags_controller.py index 8470d349766..b99a7040d83 100644 --- a/openbb_terminal/featflags_controller.py +++ b/openbb_terminal/featflags_controller.py @@ -42,6 +42,7 @@ class FeatureFlagsController(BaseController): "rich", "richpanel", "tbhint", + "overwrite", ] PATH = "/featflags/" @@ -73,9 +74,15 @@ class FeatureFlagsController(BaseController): mt.add_setting("watermark", obbff.USE_WATERMARK) mt.add_setting("cmdloc", obbff.USE_CMD_LOCATION_FIGURE) mt.add_setting("tbhint", obbff.TOOLBAR_HINT) + mt.add_setting("overwrite", obbff.FILE_OVERWITE) console.print(text=mt.menu_text, menu="Feature Flags") + def call_overwrite(self, _): + """Process overwrite command""" + obbff.FILE_OVERWITE = not obbff.FILE_OVERWITE + set_key(USER_ENV_FILE, "OPENBB_FILE_OVERWITE", str(obbff.FILE_OVERWITE)) + def call_retryload(self, _): """Process retryload command""" obbff.RETRY_WITH_LOAD = not obbff.RETRY_WITH_LOAD diff --git a/openbb_terminal/feature_flags.py b/openbb_terminal/feature_flags.py index 4673c17652f..86da3e1787f 100644 --- a/openbb_terminal/feature_flags.py +++ b/openbb_terminal/feature_flags.py @@ -31,6 +31,9 @@ else: WITH_GIT = True +# Always overwrite exported files +FILE_OVERWITE = load_env_vars("OPENBB_FILE_OVERWRITE", strtobool, False, "featflags") + # Retry unknown commands with `load` RETRY_WITH_LOAD = load_env_vars("OPENBB_RETRY_WITH_LOAD", strtobool, False, "featflags") diff --git a/openbb_terminal/forecast/anom_model.py b/openbb_terminal/forecast/anom_model.py new file mode 100644 index 00000000000..6017c80a86a --- /dev/null +++ b/openbb_terminal/forecast/anom_model.py @@ -0,0 +1,62 @@ +# pylint: disable=too-many-arguments +"""Quantile Anomaly Detection Model""" +__docformat__ = "numpy" + +import logging +from typing import Tuple, Union, List + +import pandas as pd + +from darts import TimeSeries +from darts.ad import KMeansScorer +from darts.ad import QuantileDetector +from openbb_terminal.decorators import log_start_end + +from openbb_terminal.forecast import helpers + +logger = logging.getLogger(__name__) + + +@log_start_end(log=logger) +def get_anomaly_detection_data( + data: Union[pd.Series, pd.DataFrame], + target_column: str = "close", + train_split: float = 0.6, +) -> Tuple[List[TimeSeries], List[TimeSeries], List[TimeSeries]]: + """Get Quantile Anomaly Detection Data + + Parameters + ---------- + data: Union[pd.Series, pd.DataFrame] + + Input Data + ------- + target_column: str + Target column to forecast. Defaults to "close". + train_split: (float, optional) + Train/val split. Defaults to 0.85. + + Returns + ------- + Tuple[ + List[TimeSeries], + List[TimeSeries], + List[TimeSeries], + ] + """ + use_scalers = False + + _, ticker_series = helpers.get_series(data, target_column, is_scaler=use_scalers) + + # split dataset into train and val + train, val = ticker_series.split_before(train_split) + + scorer = KMeansScorer(k=2, window=5) + scorer.fit(train) + anom_score = scorer.score(val) + + detector = QuantileDetector(high_quantile=0.99) + detector.fit(scorer.score(train)) + binary_anom = detector.detect(anom_score) + + return ticker_series, anom_score, binary_anom diff --git a/openbb_terminal/forecast/anom_view.py b/openbb_terminal/forecast/anom_view.py new file mode 100644 index 00000000000..04d339593ff --- /dev/null +++ b/openbb_terminal/forecast/anom_view.py @@ -0,0 +1,96 @@ +"""Quantile Anomaly Detector View""" +__docformat__ = "numpy" + +import logging +from typing import Union, List, Optional +from datetime import datetime + +import pandas as pd +import numpy as np +import matplotlib.pyplot as plt + +from openbb_terminal.forecast import anom_model +from openbb_terminal.decorators import log_start_end +from openbb_terminal.forecast import helpers +from openbb_terminal.helper_funcs import ( + plot_autoscale, +) +from openbb_terminal.config_terminal import theme +from openbb_terminal.config_plot import PLOT_DPI +from openbb_terminal.rich_config import console + +logger = logging.getLogger(__name__) +# pylint: disable=too-many-arguments + + +@log_start_end(log=logger) +def display_anomaly_detection( + data: Union[pd.Series, pd.DataFrame], + dataset_name="", + target_column: str = "close", + train_split: float = 0.6, + start_date: Optional[datetime] = None, + end_date: Optional[datetime] = None, + external_axes: Optional[List[plt.axes]] = None, +): + """Display Quantile Anomaly Detection + + Parameters + ---------- + data: Union[pd.Series, pd.DataFrame] + + Input Data + ---------- + target_column: str + Target column to forecast. Defaults to "close". + train_split: (float, optional) + Train/val split. Defaults to 0.85. + forecast_horizon: (int, optional) + Forecast horizon. Defaults to 5. + export: (str, optional) + Export data to csv. Defaults to "". + start_date: (Optional[datetime], optional) + Start date. Defaults to None. + end_date: (Optional[datetime], optional) + End date. Defaults to None. + external_axes: (Optional[List[plt.axes]], optional) + External axes. Defaults to None. + + Returns + ------- + None + """ + data = helpers.clean_data(data, start_date, end_date, target_column) + if not helpers.check_data(data, target_column): + return + + (ticker_series, _, binary_anom) = anom_model.get_anomaly_detection_data( + data=data, + target_column=target_column, + train_split=train_split, + ) + + if not external_axes: + _, ax = plt.subplots(figsize=plot_autoscale(), dpi=PLOT_DPI) + else: + ax = external_axes[0] + + ticker_series.plot(label=target_column, ax=ax) + + ticker_max = np.max(ticker_series.values()) + (binary_anom * int(ticker_max)).plot( + label="detected binary anomaly", c="red", ax=ax + ) + # set min and max for y axis + ax.set_ylim([-1, ticker_max * 1.1]) + + ax.set_title(f"Quantile Anomaly Detection for {dataset_name}") + ax.set_xlabel("Date") + + console.print( + f" [green]Quantile Anomaly Detection for {dataset_name} calculated.[/green]" + ) + + theme.style_primary_axis(ax) + if external_axes is None: + theme.visualize_output() diff --git a/openbb_terminal/forecast/forecast_controller.py b/openbb_terminal/forecast/forecast_controller.py index fca6ed08c11..0768b3f0fc3 100644 --- a/openbb_terminal/forecast/forecast_controller.py +++ b/openbb_terminal/forecast/forecast_controller.py @@ -71,6 +71,7 @@ from openbb_terminal.forecast import ( helpers, trans_view, nhits_view, + anom_view, ) from openbb_terminal.common import common_model @@ -133,6 +134,7 @@ class ForecastController(BaseController): "season", "which", "nhits", + "anom", ] pandas_plot_choices = [ "line", @@ -303,6 +305,9 @@ class ForecastController(BaseController): mt.add_cmd("tcn", self.files) mt.add_cmd("trans", self.files) mt.add_cmd("tft", self.files) + mt.add_raw("\n") + mt.add_info("_anomaly_") + mt.add_cmd("anom", self.files) console.print(text=mt.menu_text, menu="Forecast") @@ -3127,3 +3132,48 @@ class ForecastController(BaseController): naive=ns_parser.naive, export_pred_raw=ns_parser.export_pred_raw, ) + + @log_start_end(log=logger) + def call_anom(self, other_args: List[str]): + """Process ANOM command""" + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + add_help=False, + prog="anom", + description=""" + Perform a Quantile Anomaly detection on a given dataset: + https://unit8co.github.io/darts/generated_api/darts.ad.detectors.quantile_detector.html + """, + ) + + # if user does not put in --dataset + if other_args and "-" not in other_args[0][0]: + other_args.insert(0, "--dataset") + + parser = self.add_standard_args( + parser, + train_split=True, + target_column=True, + target_dataset=True, + forecast_only=True, + start=True, + end=True, + ) + ns_parser = self.parse_known_args_and_warn( + parser, + other_args, + export_allowed=EXPORT_ONLY_FIGURES_ALLOWED, + ) + + if ns_parser: + if not helpers.check_parser_input(ns_parser, self.datasets): + return + + anom_view.display_anomaly_detection( + data=self.datasets[ns_parser.target_dataset], + dataset_name=ns_parser.target_dataset, + target_column=ns_parser.target_column, + train_split=ns_parser.train_split, + start_date=ns_parser.s_start_date, + end_date=ns_parser.s_end_date, + ) diff --git a/openbb_terminal/helper_funcs.py b/openbb_terminal/helper_funcs.py index 031eaa74442..87f92001c66 100644 --- a/openbb_terminal/helper_funcs.py +++ b/openbb_terminal/helper_funcs.py @@ -5,7 +5,7 @@ import argparse import io import logging from pathlib import Path -from typing import List, Union, Optional, Dict +from typing import List, Union, Optional, Dict, Tuple from functools import lru_cache from datetime import datetime, timedelta from datetime import date as d @@ -1322,8 +1322,34 @@ def compose_export_path(func_name: str, dir_path: str) -> Path: return full_path +def ask_file_overwrite(file_path: str) -> Tuple[bool, bool]: + """Helper to provide a prompt for overwriting existing files. + + Returns two values, the first is a boolean indicating if the file exists and the + second is a boolean indicating if the user wants to overwrite the file. + """ + # Jeroen asked for a flag to overwrite no matter what + if obbff.FILE_OVERWITE: + return False, True + if os.path.exists(file_path): + overwrite = input("\nFile already exists. Overwrite? [y/n]: ").lower() + if overwrite == "y": + # File exists and user wants to overwrite + return True, True + # File exists and user does not want to overwrite + return True, False + # File does not exist + return False, True + + +# This is a false positive on pylint and being tracked in pylint #3060 +# pylint: disable=abstract-class-instantiated def export_data( - export_type: str, dir_path: str, func_name: str, df: pd.DataFrame = pd.DataFrame() + export_type: str, + dir_path: str, + func_name: str, + df: pd.DataFrame = pd.DataFrame(), + sheet_name: str = None, ) -> None: """Export data to a file. @@ -1337,6 +1363,8 @@ def export_data( Name of the command that invokes this function df : pd.Dataframe Dataframe of data to save + sheet_name : str + If provided. The name of the sheet to save in excel file """ if export_type: export_path = compose_export_path(func_name, dir_path) @@ -1372,19 +1400,62 @@ def export_data( ) if exp_type.endswith("csv"): + exists, overwrite = ask_file_overwrite(saved_path) + if exists and not overwrite: + return df.to_csv(saved_path) elif exp_type.endswith("json"): + exists, overwrite = ask_file_overwrite(saved_path) + if exists and not overwrite: + return df.reset_index(drop=True, inplace=True) df.to_json(saved_path) elif exp_type.endswith("xlsx"): - df.to_excel(saved_path, index=True, header=True) + + if sheet_name is None: + exists, overwrite = ask_file_overwrite(saved_path) + if exists and not overwrite: + return + df.to_excel(saved_path, index=True, header=True) + + else: + if os.path.exists(saved_path): + with pd.ExcelWriter( + saved_path, + mode="a", + if_sheet_exists="new", + engine="openpyxl", + ) as writer: + df.to_excel( + writer, sheet_name=sheet_name, index=True, header=True + ) + else: + with pd.ExcelWriter( + saved_path, + engine="openpyxl", + ) as writer: + df.to_excel( + writer, sheet_name=sheet_name, index=True, header=True + ) elif exp_type.endswith("png"): + exists, overwrite = ask_file_overwrite(saved_path) + if exists and not overwrite: + return plt.savefig(saved_path) elif exp_type.endswith("jpg"): + exists, overwrite = ask_file_overwrite(saved_path) + if exists and not overwrite: + return plt.savefig(saved_path) elif exp_type.endswith("pdf"): + exists, overwrite = ask_file_overwrite(saved_path) + if exists and not overwrite: + return plt.savefig(saved_path) elif exp_type.endswith("svg"): + exists, overwrite = ask_file_overwrite(saved_path) + if exists and not overwrite: + return plt.savefig(saved_path) else: console.print("\nWrong export file specified.") diff --git a/openbb_terminal/miscellaneous/i18n/en.yml b/openbb_terminal/miscellaneous/i18n/en.yml index d6c7947fd3b..ee231f93bc1 100644 --- a/openbb_terminal/miscellaneous/i18n/en.yml +++ b/openbb_terminal/miscellaneous/i18n/en.yml @@ -1064,4 +1064,6 @@ en: forecast/trans: Transformer Network (feat. Past covariates) forecast/tft: Temporal Fusion Transformer Network(feat. Past covariates) forecast/nhits: Neural Hierarchical Interpolation (feat. Past covariates) + forecast/_anomaly_: Anomaly Detection + forecast/anom: Quantile Anomaly Detector forecast/_comingsoon_: Coming Soon diff --git a/openbb_terminal/miscellaneous/library/trail_map_forecasting.csv b/openbb_terminal/miscellaneous/library/trail_map_forecasting.csv index e6bd66fca92..00216c42e95 100644 --- a/openbb_terminal/miscellaneous/library/trail_map_forecasting.csv +++ b/openbb_terminal/miscellaneous/library/trail_map_forecasting.csv @@ -36,3 +36,4 @@ forecast.tcn,openbb_terminal.forecast.tcn_model.get_tcn_data,openbb_terminal.for forecast.tft,openbb_terminal.forecast.tft_model.get_tft_data,openbb_terminal.forecast.tft_view.display_tft_forecast forecast.theta,openbb_terminal.forecast.theta_model.get_theta_data,openbb_terminal.forecast.theta_view.display_theta_forecast forecast.trans,openbb_terminal.forecast.trans_model.get_trans_data,openbb_terminal.forecast.trans_view.display_trans_forecast +forecast.anom,openbb_terminal.forecast.anom_model.get_anomaly_detection_data,openbb_terminal.forecast.anom_view.display_anomaly_detection diff --git a/openbb_terminal/parent_classes.py b/openbb_terminal/parent_classes.py index c719edccd7f..5f2e06f746c 100644 --- a/openbb_terminal/parent_classes.py +++ b/openbb_terminal/parent_classes.py @@ -753,6 +753,16 @@ class BaseController(metaclass=ABCMeta): help=help_export, ) + # If excel is an option, add the sheet name + if export_allowed == EXPORT_ONLY_RAW_DATA_ALLOWED: + parser.add_argument( + "--sheet-name", + dest="sheet_name", + default=None, + nargs="+", + help="Name of excel sheet to save data to. Only valid for .xlsx files.", + ) + if raw: parser.add_argument( "--raw", @@ -1147,6 +1157,9 @@ class StockBaseController(BaseController, metaclass=ABCMeta): os.path.dirname(os.path.abspath(__file__)), f"load_{self.ticker}", self.stock.copy(), + sheet_name=" ".join(ns_parser.sheet_name) + if ns_parser.sheet_name + else None, ) diff --git a/tests/openbb_terminal/forecast/test_anom_model.py b/tests/openbb_terminal/forecast/test_anom_model.py new file mode 100644 index 00000000000..736d0f7b511 --- /dev/null +++ b/tests/openbb_terminal/forecast/test_anom_model.py @@ -0,0 +1,11 @@ +import pytest +from tests.openbb_terminal.forecast import conftest + +try: + from openbb_terminal.forecast import anom_model +except ImportError: + pytest.skip(allow_module_level=True) + + +def test_get_expo_model(tsla_csv): + conftest.test_model(anom_model.get_anomaly_detection_data, tsla_csv) diff --git a/tests/openbb_terminal/forecast/test_anom_view.py b/tests/openbb_terminal/forecast/test_anom_view.py new file mode 100644 index 00000000000..a35eec8e3a7 --- /dev/null +++ b/tests/openbb_terminal/forecast/test_anom_view.py @@ -0,0 +1,14 @@ +import pytest + +try: + from openbb_terminal.forecast import anom_view +except ImportError: + pytest.skip(allow_module_level=True) + + +def test_display_anom_forecast(tsla_csv): + anom_view.display_anomaly_detection( + tsla_csv, + target_column="close", + start_window=0.5, + ) diff --git a/tests/openbb_terminal/portfolio/csv/test_portfolio_model/test_tracking_error.csv b/tests/openbb_terminal/portfolio/csv/test_portfolio_model/test_tracking_error.csv index 8dec8d7ca01..c2b7ce9b12e 100644 --- a/tests/openbb_terminal/portfolio/csv/test_portfolio_model/test_tracking_error.csv +++ b/tests/openbb_terminal/portfolio/csv/test_portfolio_model/test_tracking_error.csv @@ -5,9 +5,7 @@ Total NaN dtype: float64" ytd,- 3m,- -6m,"Adj Close NaN -Total NaN -dtype: float64" +6m,- 1y,"Adj Close NaN Total NaN dtype: float64"