diff --git a/.gitignore b/.gitignore index f34aa31f0eb..502886f06cc 100644 --- a/.gitignore +++ b/.gitignore @@ -61,3 +61,6 @@ darts_logs/ # User data custom_imports/*.csv custom_imports/*/*.csv + +# lightning logs +lightning_logs/ diff --git a/build/conda/conda-3-9-env-full.yaml b/build/conda/conda-3-9-env-full.yaml index 33392f4c800..634a3c6692e 100644 --- a/build/conda/conda-3-9-env-full.yaml +++ b/build/conda/conda-3-9-env-full.yaml @@ -5,4 +5,4 @@ dependencies: - python=3.9.6 - pip - poetry=1.1.13 - - lightgbm=3.3.3 + - lightgbm=3.3.5 diff --git a/openbb_terminal/forecast/brnn_model.py b/openbb_terminal/forecast/brnn_model.py index 4a1341f0a65..2cef8210b7c 100644 --- a/openbb_terminal/forecast/brnn_model.py +++ b/openbb_terminal/forecast/brnn_model.py @@ -38,6 +38,7 @@ def get_brnn_data( model_save_name: str = "brnn_model", force_reset: bool = True, save_checkpoints: bool = True, + metric: str = "mape", ) -> Tuple[ Optional[List[TimeSeries]], Optional[List[TimeSeries]], @@ -84,6 +85,8 @@ def get_brnn_data( discarded). Defaults to True. save_checkpoints: bool Whether or not to automatically save the untrained model and checkpoints from training. Defaults to True. + metric: str + Metric to use for model selection. Defaults to "mape". Returns ------- @@ -174,4 +177,5 @@ def get_brnn_data( train_split, forecast_horizon, n_predict, + metric, ) diff --git a/openbb_terminal/forecast/brnn_view.py b/openbb_terminal/forecast/brnn_view.py index 34b4225001d..1bfdde3f3c2 100644 --- a/openbb_terminal/forecast/brnn_view.py +++ b/openbb_terminal/forecast/brnn_view.py @@ -43,6 +43,7 @@ def display_brnn_forecast( end_date: Optional[datetime] = None, naive: bool = False, export_pred_raw: bool = False, + metric: str = "mape", external_axes: Optional[List[plt.axes]] = None, ): """Display BRNN forecast @@ -102,6 +103,10 @@ def display_brnn_forecast( naive: bool Whether to show the naive baseline. This just assumes the closing price will be the same as the previous day's closing price. Defaults to False. + export_pred_raw: bool + Whether to export the raw predicted values. Defaults to False. + metric: str + The metric to use for the model. Defaults to "mape". external_axes: Optional[List[plt.axes]] External axes to plot on """ @@ -137,6 +142,7 @@ def display_brnn_forecast( model_save_name=model_save_name, force_reset=force_reset, save_checkpoints=save_checkpoints, + metric=metric, ) if ticker_series == []: return @@ -160,6 +166,7 @@ def display_brnn_forecast( forecast_only=forecast_only, naive=naive, export_pred_raw=export_pred_raw, + metric=metric, external_axes=external_axes, ) if residuals: diff --git a/openbb_terminal/forecast/expo_model.py b/openbb_terminal/forecast/expo_model.py index 4c117a90f78..74ace57dd59 100644 --- a/openbb_terminal/forecast/expo_model.py +++ b/openbb_terminal/forecast/expo_model.py @@ -8,7 +8,7 @@ from typing import List, Optional, Tuple, Union import pandas as pd from darts import TimeSeries -from darts.metrics import mape +from darts.metrics import mape, mse, rmse, smape from darts.models import ExponentialSmoothing from darts.utils.utils import ModelMode, SeasonalityMode from numpy import ndarray @@ -40,6 +40,7 @@ def get_expo_data( n_predict: int = 5, start_window: float = 0.85, forecast_horizon: int = 5, + metric: str = "mape", ) -> Tuple[ List[TimeSeries], List[TimeSeries], @@ -76,6 +77,8 @@ def get_expo_data( Size of sliding window from start of timeseries and onwards forecast_horizon: int Number of days to forecast when backtesting and retraining historical + metric: str + Metric to use for backtesting. Defaults to MAPE. Returns ------- @@ -149,8 +152,19 @@ def get_expo_data( # we have the historical fcast, now lets train on entire set and predict. best_model.fit(ticker_series) probabilistic_forecast = best_model.predict(int(n_predict), num_samples=500) - precision = mape(actual_series=ticker_series, pred_series=historical_fcast_es) - console.print(f"Exponential smoothing obtains MAPE: {precision:.2f}% \n") + + if metric == "rmse": + precision = rmse(actual_series=ticker_series, pred_series=historical_fcast_es) + elif metric == "mse": + precision = mse(actual_series=ticker_series, pred_series=historical_fcast_es) + elif metric == "mape": + precision = mape(actual_series=ticker_series, pred_series=historical_fcast_es) + elif metric == "smape": + precision = smape(actual_series=ticker_series, pred_series=historical_fcast_es) + + console.print( + f"Exponential smoothing obtains {metric.upper()}: {precision:.2f}% \n" + ) return ( ticker_series, diff --git a/openbb_terminal/forecast/expo_view.py b/openbb_terminal/forecast/expo_view.py index de41d84f49c..79771b537c7 100644 --- a/openbb_terminal/forecast/expo_view.py +++ b/openbb_terminal/forecast/expo_view.py @@ -36,6 +36,7 @@ def display_expo_forecast( end_date: Optional[datetime] = None, naive: bool = False, export_pred_raw: bool = False, + metric: str = "mape", external_axes: Optional[List[plt.axes]] = None, ): """Display Probabilistic Exponential Smoothing forecast @@ -80,6 +81,10 @@ def display_expo_forecast( naive: bool Whether to show the naive baseline. This just assumes the closing price will be the same as the previous day's closing price. Defaults to False. + export_pred_raw: bool + Whether to export the raw predicted values. Defaults to False. + metric: str + The metric to use when backtesting. Defaults to "mape". external_axes: Optional[List[plt.axes]] External axes to plot on """ @@ -103,6 +108,7 @@ def display_expo_forecast( n_predict=n_predict, start_window=start_window, forecast_horizon=forecast_horizon, + metric=metric, ) if ticker_series == []: @@ -127,6 +133,7 @@ def display_expo_forecast( forecast_only=forecast_only, naive=naive, export_pred_raw=export_pred_raw, + metric=metric, external_axes=external_axes, ) if residuals: diff --git a/openbb_terminal/forecast/forecast_controller.py b/openbb_terminal/forecast/forecast_controller.py index 1252f27731c..2da91ffbfa5 100644 --- a/openbb_terminal/forecast/forecast_controller.py +++ b/openbb_terminal/forecast/forecast_controller.py @@ -356,6 +356,7 @@ class ForecastController(BaseController): naive: bool = False, explainability_raw: bool = False, export_pred_raw: bool = False, + metric: bool = False, ): if hidden_size: parser.add_argument( @@ -531,7 +532,7 @@ class ForecastController(BaseController): action="store", dest="model_type", default="LSTM", - help='Either a string specifying the RNN module type ("RNN", "LSTM" or "GRU")', + help='Enter a string specifying the RNN module type ("RNN", "LSTM" or "GRU")', ) if dropout is not None: parser.add_argument( @@ -631,6 +632,17 @@ class ForecastController(BaseController): help="Export predictions to a csv file.", ) + if metric: + parser.add_argument( + "--metric", + type=str, + action="store", + dest="metric", + default="mape", + choices=["rmse", "mse", "mape", "smape"], + help="Calculate precision based on a specific metric (rmse, mse, mape)", + ) + # if user does not put in --dataset return parser @@ -2114,6 +2126,7 @@ class ForecastController(BaseController): end=True, naive=True, export_pred_raw=True, + metric=True, ) ns_parser = self.parse_known_args_and_warn( parser, @@ -2143,6 +2156,7 @@ class ForecastController(BaseController): end_date=ns_parser.s_end_date, naive=ns_parser.naive, export_pred_raw=ns_parser.export_pred_raw, + metric=ns_parser.metric, ) @log_start_end(log=logger) @@ -2175,6 +2189,7 @@ class ForecastController(BaseController): end=True, naive=True, export_pred_raw=True, + metric=True, ) ns_parser = self.parse_known_args_and_warn( parser, @@ -2202,6 +2217,7 @@ class ForecastController(BaseController): end_date=ns_parser.s_end_date, naive=ns_parser.naive, export_pred_raw=ns_parser.export_pred_raw, + metric=ns_parser.metric, ) @log_start_end(log=logger) @@ -2261,6 +2277,7 @@ class ForecastController(BaseController): end=True, naive=True, export_pred_raw=True, + metric=True, ) ns_parser = self.parse_known_args_and_warn( parser, @@ -2297,6 +2314,7 @@ class ForecastController(BaseController): end_date=ns_parser.s_end_date, naive=ns_parser.naive, export_pred_raw=ns_parser.export_pred_raw, + metric=ns_parser.metric, ) @log_start_end(log=logger) @@ -2373,6 +2391,7 @@ class ForecastController(BaseController): end=True, naive=True, export_pred_raw=True, + metric=True, ) ns_parser = self.parse_known_args_and_warn( parser, @@ -2415,6 +2434,7 @@ class ForecastController(BaseController): end_date=ns_parser.s_end_date, naive=ns_parser.naive, export_pred_raw=ns_parser.export_pred_raw, + metric=ns_parser.metric, ) @log_start_end(log=logger) @@ -2482,6 +2502,7 @@ class ForecastController(BaseController): end=True, naive=True, export_pred_raw=True, + metric=True, ) ns_parser = self.parse_known_args_and_warn( parser, @@ -2525,6 +2546,7 @@ class ForecastController(BaseController): end_date=ns_parser.s_end_date, naive=ns_parser.naive, export_pred_raw=ns_parser.export_pred_raw, + metric=ns_parser.metric, ) @log_start_end(log=logger) @@ -2561,6 +2583,7 @@ class ForecastController(BaseController): naive=True, explainability_raw=True, export_pred_raw=True, + metric=True, ) ns_parser = self.parse_known_args_and_warn( parser, @@ -2595,6 +2618,7 @@ class ForecastController(BaseController): naive=ns_parser.naive, explainability_raw=ns_parser.explainability_raw, export_pred_raw=ns_parser.export_pred_raw, + metric=ns_parser.metric, ) @log_start_end(log=logger) @@ -2630,6 +2654,7 @@ class ForecastController(BaseController): naive=True, explainability_raw=True, export_pred_raw=True, + metric=True, ) ns_parser = self.parse_known_args_and_warn( parser, @@ -2663,6 +2688,7 @@ class ForecastController(BaseController): naive=ns_parser.naive, explainability_raw=ns_parser.explainability_raw, export_pred_raw=ns_parser.export_pred_raw, + metric=ns_parser.metric, ) @log_start_end(log=logger) @@ -2714,6 +2740,7 @@ class ForecastController(BaseController): end=True, naive=True, export_pred_raw=True, + metric=True, ) ns_parser = self.parse_known_args_and_warn( parser, @@ -2755,6 +2782,7 @@ class ForecastController(BaseController): end_date=ns_parser.s_end_date, naive=ns_parser.naive, export_pred_raw=ns_parser.export_pred_raw, + metric=ns_parser.metric, ) @log_start_end(log=logger) @@ -2842,6 +2870,7 @@ class ForecastController(BaseController): end=True, naive=True, export_pred_raw=True, + metric=True, ) ns_parser = self.parse_known_args_and_warn( parser, @@ -2885,6 +2914,7 @@ class ForecastController(BaseController): end_date=ns_parser.s_end_date, naive=ns_parser.naive, export_pred_raw=ns_parser.export_pred_raw, + metric=ns_parser.metric, ) @log_start_end(log=logger) @@ -2958,6 +2988,7 @@ class ForecastController(BaseController): end=True, naive=True, export_pred_raw=True, + metric=True, ) ns_parser = self.parse_known_args_and_warn( parser, @@ -3001,6 +3032,7 @@ class ForecastController(BaseController): end_date=ns_parser.s_end_date, naive=ns_parser.naive, export_pred_raw=ns_parser.export_pred_raw, + metric=ns_parser.metric, ) @log_start_end(log=logger) @@ -3092,6 +3124,7 @@ class ForecastController(BaseController): end=True, naive=True, export_pred_raw=True, + metric=True, ) ns_parser = self.parse_known_args_and_warn( parser, @@ -3136,6 +3169,7 @@ class ForecastController(BaseController): end_date=ns_parser.s_end_date, naive=ns_parser.naive, export_pred_raw=ns_parser.export_pred_raw, + metric=ns_parser.metric, ) @log_start_end(log=logger) diff --git a/openbb_terminal/forecast/helpers.py b/openbb_terminal/forecast/helpers.py index 19df1151904..6e6750bcd23 100644 --- a/openbb_terminal/forecast/helpers.py +++ b/openbb_terminal/forecast/helpers.py @@ -11,7 +11,7 @@ import pandas as pd from darts import TimeSeries from darts.dataprocessing.transformers import MissingValuesFiller, Scaler from darts.explainability.shap_explainer import ShapExplainer -from darts.metrics import mape +from darts.metrics import mape, mse, rmse, smape from darts.models.forecasting.torch_forecasting_model import GlobalForecastingModel from darts.utils.statistics import plot_residuals_analysis from pytorch_lightning.callbacks.early_stopping import EarlyStopping @@ -477,6 +477,7 @@ def plot_forecast( forecast_only: bool = False, naive: bool = False, export_pred_raw: bool = False, + metric: str = "mape", external_axes: Optional[List[plt.axes]] = None, ): quant_kwargs = {} @@ -503,7 +504,16 @@ def plot_forecast( if naive: # show naive forecast shift timeseries by 1 naive_fcast = naive_fcast.drop_before(historical_fcast.start_time()) - naive_precision = mape(ticker_series, naive_fcast) + + # calculate precision based on metric + if metric == "rsme": + naive_precision = rmse(ticker_series, naive_fcast) + elif metric == "mse": + naive_precision = mse(ticker_series, naive_fcast) + elif metric == "mape": + naive_precision = mape(ticker_series, naive_fcast) + elif metric == "smape": + naive_precision = smape(ticker_series, naive_fcast) naive_fcast.plot( label=f"Naive+1: {naive_precision:.2f}%", @@ -516,7 +526,7 @@ def plot_forecast( pred_label += " w/ past covs" predicted_values.plot(label=pred_label, **quant_kwargs, color="#00AAFF") ax.set_title( - f"{name} for <{ticker_name}> for next [{n_predict}] days (MAPE={precision:.2f}%)" + f"{name} for <{ticker_name}> for next [{n_predict}] days ({metric.upper()}={precision:.2f}%)" ) ax.set_ylabel(target_col) ax.set_xlabel("Date") @@ -719,6 +729,7 @@ def get_prediction( train_split: float, forecast_horizon: int, n_predict: int, + metric: str, ): _, val = ticker_series.split_before(train_split) @@ -771,10 +782,17 @@ def get_prediction( else: prediction = best_model.predict(series=ticker_series, n=n_predict) - precision = mape( - actual_series=val, pred_series=historical_fcast - ) # mape = mean average percentage error - console.print(f"{model_name} model obtains MAPE: {precision:.2f}% \n") + # calculate precision based on metric (rmse, mse, mape) + if metric == "rmse": + precision = rmse(actual_series=val, pred_series=historical_fcast) + elif metric == "mse": + precision = mse(actual_series=val, pred_series=historical_fcast) + elif metric == "mape": + precision = mape(actual_series=val, pred_series=historical_fcast) + elif metric == "smape": + precision = smape(actual_series=val, pred_series=historical_fcast) + + console.print(f"{model_name} model obtains {metric.upper()}: {precision:.2f}% \n") # scale back if use_scalers and isinstance(scaler, Scaler): diff --git a/openbb_terminal/forecast/linregr_model.py b/openbb_terminal/forecast/linregr_model.py index 5e4e4d91fc4..70a60654cd0 100644 --- a/openbb_terminal/forecast/linregr_model.py +++ b/openbb_terminal/forecast/linregr_model.py @@ -27,6 +27,7 @@ def get_linear_regression_data( output_chunk_length: int = 5, lags: Union[int, List[int]] = 14, random_state: Optional[int] = None, + metric: str = "mape", ) -> Tuple[ List[TimeSeries], List[TimeSeries], @@ -56,6 +57,8 @@ def get_linear_regression_data( lagged target values to predict the next time step random_state: Optional[int] The state for the model + metric: str + The metric to use for the model. Defaults to "mape". Returns ------- @@ -112,4 +115,5 @@ def get_linear_regression_data( train_split, forecast_horizon, n_predict, + metric, ) diff --git a/openbb_terminal/forecast/linregr_view.py b/openbb_terminal/forecast/linregr_view.py index 6bc09879edc..2b3815338cd 100644 --- a/openbb_terminal/forecast/linregr_view.py +++ b/openbb_terminal/forecast/linregr_view.py @@ -35,6 +35,7 @@ def display_linear_regression( naive: bool = False, explainability_raw: bool = False, export_pred_raw: bool = False, + metric: str = "mape", external_axes: Optional[List[plt.axes]] = None, ): """Display Linear Regression Forecasting @@ -74,6 +75,12 @@ def display_linear_regression( naive: bool Whether to show the naive baseline. This just assumes the closing price will be the same as the previous day's closing price. Defaults to False. + explainability_raw: bool + Whether to show the raw explainability data. Defaults to False. + export_pred_raw: bool + Whether to export the raw prediction data. Defaults to False. + metric: str + The metric to use for the model. Defaults to "mape". external_axes: Optional[List[plt.axes]] External axes to plot on """ @@ -100,6 +107,7 @@ def display_linear_regression( forecast_horizon=forecast_horizon, output_chunk_length=output_chunk_length, lags=lags, + metric=metric, ) probabilistic = True @@ -121,6 +129,7 @@ def display_linear_regression( forecast_only=forecast_only, naive=naive, export_pred_raw=export_pred_raw, + metric=metric, external_axes=external_axes, ) if residuals: diff --git a/openbb_terminal/forecast/nbeats_model.py b/openbb_terminal/forecast/nbeats_model.py index acb8480bfbe..57bca236fce 100644 --- a/openbb_terminal/forecast/nbeats_model.py +++ b/openbb_terminal/forecast/nbeats_model.py @@ -37,6 +37,7 @@ def get_NBEATS_data( model_save_name: str = "nbeats_model", force_reset: bool = True, save_checkpoints: bool = True, + metric: str = "mape", ) -> Tuple[ Optional[List[TimeSeries]], Optional[List[TimeSeries]], @@ -88,6 +89,8 @@ def get_NBEATS_data( save_checkpoints: bool Whether or not to automatically save the untrained model and checkpoints from training. Defaults to True. + metric: str + Metric to use for model selection. Defaults to "mape". Returns ------- @@ -168,4 +171,5 @@ def get_NBEATS_data( train_split, forecast_horizon, n_predict, + metric, ) diff --git a/openbb_terminal/forecast/nbeats_view.py b/openbb_terminal/forecast/nbeats_view.py index 283f6dac602..e790ce09aff 100644 --- a/openbb_terminal/forecast/nbeats_view.py +++ b/openbb_terminal/forecast/nbeats_view.py @@ -44,6 +44,7 @@ def display_nbeats_forecast( end_date: Optional[datetime] = None, naive: bool = False, export_pred_raw: bool = False, + metric: str = "mape", external_axes: Optional[List[plt.axes]] = None, ): """Display NBEATS forecast @@ -108,6 +109,10 @@ def display_nbeats_forecast( naive: bool Whether to show the naive baseline. This just assumes the closing price will be the same as the previous day's closing price. Defaults to False. + export_pred_raw: bool + Whether to export the raw predicted values. Defaults to False. + metric: str + Metric to use for evaluation. Defaults to "mape". external_axes: Optional[List[plt.axes]] External axes to plot on """ @@ -145,6 +150,7 @@ def display_nbeats_forecast( model_save_name=model_save_name, force_reset=force_reset, save_checkpoints=save_checkpoints, + metric=metric, ) if ticker_series == []: return @@ -168,6 +174,7 @@ def display_nbeats_forecast( forecast_only=forecast_only, naive=naive, export_pred_raw=export_pred_raw, + metric=metric, external_axes=external_axes, ) if residuals: diff --git a/openbb_terminal/forecast/nhits_model.py b/openbb_terminal/forecast/nhits_model.py index 43a3d1ed0f4..312b1d6e0ad 100644 --- a/openbb_terminal/forecast/nhits_model.py +++ b/openbb_terminal/forecast/nhits_model.py @@ -43,6 +43,7 @@ def get_nhits_data( model_save_name: str = "nhits_model", force_reset: bool = True, save_checkpoints: bool = True, + metric: str = "mape", ) -> Tuple[ Optional[List[TimeSeries]], Optional[List[TimeSeries]], @@ -109,6 +110,8 @@ def get_nhits_data( discarded). Defaults to True. save_checkpoints: bool Whether or not to automatically save the untrained model and checkpoints from training. Defaults to True. + metric: str + Metric to use for model selection. Defaults to "mape". Returns ------- @@ -197,4 +200,5 @@ def get_nhits_data( train_split, forecast_horizon, n_predict, + metric, ) diff --git a/openbb_terminal/forecast/nhits_view.py b/openbb_terminal/forecast/nhits_view.py index d4ec81ecdee..42fc8d06d79 100644 --- a/openbb_terminal/forecast/nhits_view.py +++ b/openbb_terminal/forecast/nhits_view.py @@ -49,6 +49,7 @@ def display_nhits_forecast( end_date: Optional[datetime] = None, naive: bool = False, export_pred_raw: bool = False, + metric: str = "mape", external_axes: Optional[List[plt.axes]] = None, ): """Display Nhits forecast @@ -128,6 +129,10 @@ def display_nhits_forecast( naive: bool Whether to show the naive baseline. This just assumes the closing price will be the same as the previous day's closing price. Defaults to False. + export_pred_raw: bool + Whether to export the raw predicted values. Defaults to False. + metric: str + Metric to use for evaluation. Defaults to "mape". external_axes: Optional[List[plt.axes]] External axes to plot on """ @@ -169,6 +174,7 @@ def display_nhits_forecast( model_save_name=model_save_name, force_reset=force_reset, save_checkpoints=save_checkpoints, + metric=metric, ) if ticker_series == []: return @@ -192,6 +198,7 @@ def display_nhits_forecast( forecast_only=forecast_only, naive=naive, export_pred_raw=export_pred_raw, + metric=metric, external_axes=external_axes, ) if residuals: diff --git a/openbb_terminal/forecast/regr_model.py b/openbb_terminal/forecast/regr_model.py index 19871d10f24..12057e4bea3 100644 --- a/openbb_terminal/forecast/regr_model.py +++ b/openbb_terminal/forecast/regr_model.py @@ -26,6 +26,7 @@ def get_regression_data( forecast_horizon: int = 5, output_chunk_length: int = 5, lags: Union[int, List[int]] = 14, + metric: str = "mape", ) -> Tuple[ List[TimeSeries], List[TimeSeries], List[TimeSeries], float, type[RegressionModel] ]: @@ -49,6 +50,8 @@ def get_regression_data( The length of the forecast of the model. Defaults to 1. lags: Union[int, List[int]] lagged target values to predict the next time step + metric: str + Metric to use for evaluation. Defaults to "mape". Returns ------- @@ -99,4 +102,5 @@ def get_regression_data( train_split, forecast_horizon, n_predict, + metric, ) diff --git a/openbb_terminal/forecast/regr_view.py b/openbb_terminal/forecast/regr_view.py index 71e34bf4ea5..f680dedb590 100644 --- a/openbb_terminal/forecast/regr_view.py +++ b/openbb_terminal/forecast/regr_view.py @@ -35,6 +35,7 @@ def display_regression( naive: bool = False, explainability_raw: bool = False, export_pred_raw: bool = False, + metric: str = "mape", external_axes: Optional[List[plt.axes]] = None, ): """Display Regression Forecasting @@ -74,6 +75,8 @@ def display_regression( naive: bool Whether to show the naive baseline. This just assumes the closing price will be the same as the previous day's closing price. Defaults to False. + metric: str + The metric to use for the forecast. Defaults to "mape". external_axes: Optional[List[plt.axes]] External axes to plot on """ @@ -100,6 +103,7 @@ def display_regression( forecast_horizon=forecast_horizon, output_chunk_length=output_chunk_length, lags=lags, + metric=metric, ) probabilistic = False helpers.plot_forecast( @@ -120,6 +124,7 @@ def display_regression( forecast_only=forecast_only, naive=naive, export_pred_raw=export_pred_raw, + metric=metric, external_axes=external_axes, ) if residuals: diff --git a/openbb_terminal/forecast/rnn_model.py b/openbb_terminal/forecast/rnn_model.py index 5f860bb43e0..8744ef98338 100644 --- a/openbb_terminal/forecast/rnn_model.py +++ b/openbb_terminal/forecast/rnn_model.py @@ -36,6 +36,7 @@ def get_rnn_data( input_chunk_size: int = 14, force_reset: bool = True, save_checkpoints: bool = True, + metric: str = "mape", ) -> Tuple[ Optional[List[type[TimeSeries]]], Optional[List[type[TimeSeries]]], @@ -77,6 +78,8 @@ def get_rnn_data( save_checkpoints: bool Whether or not to automatically save the untrained model and checkpoints from training. Defaults to True. + metric: str + Metric to use for model selection. Defaults to "mape". Returns ------- @@ -145,4 +148,5 @@ def get_rnn_data( train_split, forecast_horizon, n_predict, + metric, ) diff --git a/openbb_terminal/forecast/rnn_view.py b/openbb_terminal/forecast/rnn_view.py index 31f25916575..cbf1360f8cd 100644 --- a/openbb_terminal/forecast/rnn_view.py +++ b/openbb_terminal/forecast/rnn_view.py @@ -42,6 +42,7 @@ def display_rnn_forecast( end_date: Optional[datetime] = None, naive: bool = False, export_pred_raw: bool = False, + metric: str = "mape", external_axes: Optional[List[plt.axes]] = None, ): """Display RNN forecast @@ -94,6 +95,10 @@ def display_rnn_forecast( naive: bool Whether to show the naive baseline. This just assumes the closing price will be the same as the previous day's closing price. Defaults to False. + export_pred_raw: bool + Whether to export the raw predictions. Defaults to False. + metric: str + The metric to use for the forecast. Defaults to "mape". external_axes: Optional[List[plt.axes]] External axes to plot on """ @@ -124,6 +129,7 @@ def display_rnn_forecast( input_chunk_size=input_chunk_size, force_reset=force_reset, save_checkpoints=save_checkpoints, + metric=metric, ) if ticker_series == []: return @@ -148,6 +154,7 @@ def display_rnn_forecast( forecast_only=forecast_only, naive=naive, export_pred_raw=export_pred_raw, + metric=metric, external_axes=external_axes, ) if residuals: diff --git a/openbb_terminal/forecast/tcn_model.py b/openbb_terminal/forecast/tcn_model.py index 1d0c7467d39..fe4604b1404 100644 --- a/openbb_terminal/forecast/tcn_model.py +++ b/openbb_terminal/forecast/tcn_model.py @@ -37,6 +37,7 @@ def get_tcn_data( model_save_name: str = "tcn_model", force_reset: bool = True, save_checkpoints: bool = True, + metric: str = "mape", ) -> Tuple[ List[Optional[TimeSeries]], List[Optional[TimeSeries]], @@ -86,6 +87,8 @@ def get_tcn_data( save_checkpoints: bool Whether or not to automatically save the untrained model and checkpoints from training. Defaults to True. + metric: str + Metric to use for model selection. Defaults to "mape". Returns ------- @@ -166,4 +169,5 @@ def get_tcn_data( train_split, forecast_horizon, n_predict, + metric, ) diff --git a/openbb_terminal/forecast/tcn_view.py b/openbb_terminal/forecast/tcn_view.py index 5e7cf5aec94..13ff4003761 100644 --- a/openbb_terminal/forecast/tcn_view.py +++ b/openbb_terminal/forecast/tcn_view.py @@ -44,6 +44,7 @@ def display_tcn_forecast( end_date: Optional[datetime] = None, naive: bool = False, export_pred_raw: bool = False, + metric: str = "mape", external_axes: Optional[List[plt.axes]] = None, ): """Display TCN forecast @@ -104,6 +105,10 @@ def display_tcn_forecast( naive: bool Whether to show the naive baseline. This just assumes the closing price will be the same as the previous day's closing price. Defaults to False. + export_pred_raw: bool + Whether to export the raw predicted values. Defaults to False. + metric: str + The metric to use for the model. Defaults to "mape". external_axes: Optional[List[plt.axes]] External axes to plot on """ @@ -141,6 +146,7 @@ def display_tcn_forecast( model_save_name=model_save_name, force_reset=force_reset, save_checkpoints=save_checkpoints, + metric=metric, ) if ticker_series == []: return @@ -165,6 +171,7 @@ def display_tcn_forecast( naive=naive, export_pred_raw=export_pred_raw, external_axes=external_axes, + metric=metric, ) if residuals: helpers.plot_residuals( diff --git a/openbb_terminal/forecast/tft_model.py b/openbb_terminal/forecast/tft_model.py index 136ab21f074..41420ee2af3 100644 --- a/openbb_terminal/forecast/tft_model.py +++ b/openbb_terminal/forecast/tft_model.py @@ -43,6 +43,7 @@ def get_tft_data( model_save_name: str = "tft_model", force_reset: bool = True, save_checkpoints: bool = True, + metric: str = "mape", ) -> Tuple[ Optional[List[TimeSeries]], Optional[List[TimeSeries]], @@ -101,6 +102,8 @@ def get_tft_data( save_checkpoints: (bool, optional) Whether or not to automatically save the untrained model and checkpoints from training. Defaults to True. + metric: (str, optional) + Metric to use for model selection. Defaults to "mape". Returns ------- @@ -204,4 +207,5 @@ def get_tft_data( train_split, forecast_horizon, n_predict, + metric, ) diff --git a/openbb_terminal/forecast/tft_view.py b/openbb_terminal/forecast/tft_view.py index 3064027500c..8264b921d2a 100644 --- a/openbb_terminal/forecast/tft_view.py +++ b/openbb_terminal/forecast/tft_view.py @@ -45,6 +45,7 @@ def display_tft_forecast( end_date: Optional[datetime] = None, naive: bool = False, export_pred_raw: bool = False, + metric: str = "mape", external_axes: Optional[List[plt.axes]] = None, ): """Display Temporal Fusion Transformer forecast @@ -105,6 +106,8 @@ def display_tft_forecast( naive: bool Whether to show the naive baseline. This just assumes the closing price will be the same as the previous day's closing price. Defaults to False. + metric: str + The metric to use for the model. Defaults to "mape". external_axes:Optional[List[plt.axes]] External axes to plot on """ @@ -143,6 +146,7 @@ def display_tft_forecast( model_save_name=model_save_name, force_reset=force_reset, save_checkpoints=save_checkpoints, + metric=metric, ) if ticker_series == []: return @@ -166,6 +170,7 @@ def display_tft_forecast( forecast_only=forecast_only, naive=naive, export_pred_raw=export_pred_raw, + metric=metric, external_axes=external_axes, ) if residuals: diff --git a/openbb_terminal/forecast/theta_model.py b/openbb_terminal/forecast/theta_model.py index 74ef6523933..0f3f1c1ebe3 100644 --- a/openbb_terminal/forecast/theta_model.py +++ b/openbb_terminal/forecast/theta_model.py @@ -8,7 +8,7 @@ from typing import List, Optional, Tuple, Union import numpy as np import pandas as pd from darts import TimeSeries -from darts.metrics import mape +from darts.metrics import mape, mse, rmse, smape from darts.models import Theta from darts.utils.utils import SeasonalityMode from statsmodels.tools.sm_exceptions import ConvergenceWarning @@ -34,6 +34,7 @@ def get_theta_data( n_predict: int = 5, start_window: float = 0.85, forecast_horizon: int = 5, + metric: str = "mape", ) -> Tuple[ Optional[List[TimeSeries]], Optional[List[TimeSeries]], @@ -64,7 +65,9 @@ def get_theta_data( start_window: float Size of sliding window from start of timeseries and onwards forecast_horizon: int - Number of days to forecast when backtesting and retraining historical + Number of days to forecast when backtesting and retraining historical data + metric: str + Metric to use when backtesting and retraining historical data. Defaults to "mape". Returns ------- @@ -135,10 +138,23 @@ def get_theta_data( # fit model on entire series for final prediction best_theta_model_final.fit(ticker_series) prediction = best_theta_model_final.predict(int(n_predict)) - precision = mape( - actual_series=ticker_series, pred_series=historical_fcast_theta - ) # mape = mean average precision error - console.print(f"Theta Model obtains MAPE: {precision:.2f}% \n") + + if metric == "rmse": + precision = rmse( + actual_series=ticker_series, pred_series=historical_fcast_theta + ) + elif metric == "mse": + precision = mse(actual_series=ticker_series, pred_series=historical_fcast_theta) + elif metric == "mape": + precision = mape( + actual_series=ticker_series, pred_series=historical_fcast_theta + ) + elif metric == "smape": + precision = smape( + actual_series=ticker_series, pred_series=historical_fcast_theta + ) + + console.print(f"Theta Model obtains {metric.upper()}: {precision:.2f}% \n") return ( ticker_series, diff --git a/openbb_terminal/forecast/theta_view.py b/openbb_terminal/forecast/theta_view.py index 428e95c1663..041b865fbf7 100644 --- a/openbb_terminal/forecast/theta_view.py +++ b/openbb_terminal/forecast/theta_view.py @@ -34,6 +34,7 @@ def display_theta_forecast( end_date: Optional[datetime] = None, naive: bool = False, export_pred_raw: bool = False, + metric: str = "mape", external_axes: Optional[List[plt.axes]] = None, ): """Display Theta forecast @@ -73,6 +74,10 @@ def display_theta_forecast( naive: bool Whether to show the naive baseline. This just assumes the closing price will be the same as the previous day's closing price. Defaults to False. + export_pred_raw: bool + Whether to export the raw predicted values. Defaults to False. + metric: str + The metric to use for backtesting. Defaults to "mape". external_axes:Optional[List[plt.axes]] External axes to plot on """ @@ -95,6 +100,7 @@ def display_theta_forecast( target_column=target_column, start_window=start_window, forecast_horizon=forecast_horizon, + metric=metric, ) if ticker_series == []: return @@ -119,6 +125,7 @@ def display_theta_forecast( naive=naive, export_pred_raw=export_pred_raw, external_axes=external_axes, + metric=metric, ) if residuals: diff --git a/openbb_terminal/forecast/trans_model.py b/openbb_terminal/forecast/trans_model.py index ac89b255820..21cac569567 100644 --- a/openbb_terminal/forecast/trans_model.py +++ b/openbb_terminal/forecast/trans_model.py @@ -40,6 +40,7 @@ def get_trans_data( model_save_name: str = "trans_model", force_reset: bool = True, save_checkpoints: bool = True, + metric: str = "mape", ) -> Tuple[ Optional[List[TimeSeries]], Optional[List[TimeSeries]], @@ -94,6 +95,8 @@ def get_trans_data( discarded). Defaults to True. save_checkpoints: bool Whether or not to automatically save the untrained model and checkpoints from training. Defaults to True. + metric: str + Metric to use for model selection. Defaults to "mape". Returns ------- @@ -176,4 +179,5 @@ def get_trans_data( train_split, forecast_horizon, n_predict, + metric, ) diff --git a/openbb_terminal/forecast/trans_view.py b/openbb_terminal/forecast/trans_view.py index fda46a687a6..4ca64c1b5b4 100644 --- a/openbb_terminal/forecast/trans_view.py +++ b/openbb_terminal/forecast/trans_view.py @@ -47,6 +47,7 @@ def display_trans_forecast( end_date: Optional[datetime] = None, naive: bool = False, export_pred_raw: bool = False, + metric: str = "mape", external_axes: Optional[List[plt.axes]] = None, ): """Display Transformer forecast @@ -112,6 +113,10 @@ def display_trans_forecast( naive: bool Whether to show the naive baseline. This just assumes the closing price will be the same as the previous day's closing price. Defaults to False. + export_pred_raw: bool + Whether to export the raw predicted values. Defaults to False. + metric: str + The metric to use for the model. Defaults to "mape". external_axes: Optional[List[plt.axes]] External axes to plot on """ @@ -152,6 +157,7 @@ def display_trans_forecast( model_save_name=model_save_name, force_reset=force_reset, save_checkpoints=save_checkpoints, + metric=metric, ) if ticker_series == []: return @@ -176,6 +182,7 @@ def display_trans_forecast( naive=naive, export_pred_raw=export_pred_raw, external_axes=external_axes, + metric=metric, ) if residuals: helpers.plot_residuals(