From bbd802bf02fd7c106c709108022b01c7ba77f895 Mon Sep 17 00:00:00 2001 From: martinb-bb <105685594+martinb-bb@users.noreply.github.com> Date: Wed, 8 Feb 2023 13:51:22 -0500 Subject: [PATCH] adding rmse, mse to regr --- .../forecast/forecast_controller.py | 16 +++++++++- openbb_terminal/forecast/helpers.py | 29 ++++++++++++++----- openbb_terminal/forecast/regr_model.py | 4 +++ openbb_terminal/forecast/regr_view.py | 5 ++++ 4 files changed, 46 insertions(+), 8 deletions(-) diff --git a/openbb_terminal/forecast/forecast_controller.py b/openbb_terminal/forecast/forecast_controller.py index 1252f27731c..6d244bbb0bb 100644 --- a/openbb_terminal/forecast/forecast_controller.py +++ b/openbb_terminal/forecast/forecast_controller.py @@ -356,6 +356,7 @@ class ForecastController(BaseController): naive: bool = False, explainability_raw: bool = False, export_pred_raw: bool = False, + metric: bool = False, ): if hidden_size: parser.add_argument( @@ -531,7 +532,7 @@ class ForecastController(BaseController): action="store", dest="model_type", default="LSTM", - help='Either a string specifying the RNN module type ("RNN", "LSTM" or "GRU")', + help='Enter a string specifying the RNN module type ("RNN", "LSTM" or "GRU")', ) if dropout is not None: parser.add_argument( @@ -631,6 +632,17 @@ class ForecastController(BaseController): help="Export predictions to a csv file.", ) + if metric: + parser.add_argument( + "--metric", + type=str, + action="store", + dest="metric", + default="mape", + choices=["rmse", "mse", "mape"], + help="Calculate precision based on a specific metric (rmse, mse, mape)", + ) + # if user does not put in --dataset return parser @@ -2561,6 +2573,7 @@ class ForecastController(BaseController): naive=True, explainability_raw=True, export_pred_raw=True, + metric=True, ) ns_parser = self.parse_known_args_and_warn( parser, @@ -2595,6 +2608,7 @@ class ForecastController(BaseController): naive=ns_parser.naive, explainability_raw=ns_parser.explainability_raw, export_pred_raw=ns_parser.export_pred_raw, + metric=ns_parser.metric, ) @log_start_end(log=logger) diff --git a/openbb_terminal/forecast/helpers.py b/openbb_terminal/forecast/helpers.py index 19df1151904..76b25664400 100644 --- a/openbb_terminal/forecast/helpers.py +++ b/openbb_terminal/forecast/helpers.py @@ -11,7 +11,7 @@ import pandas as pd from darts import TimeSeries from darts.dataprocessing.transformers import MissingValuesFiller, Scaler from darts.explainability.shap_explainer import ShapExplainer -from darts.metrics import mape +from darts.metrics import mape, rmse, mse from darts.models.forecasting.torch_forecasting_model import GlobalForecastingModel from darts.utils.statistics import plot_residuals_analysis from pytorch_lightning.callbacks.early_stopping import EarlyStopping @@ -477,6 +477,7 @@ def plot_forecast( forecast_only: bool = False, naive: bool = False, export_pred_raw: bool = False, + metric: str = "mape", external_axes: Optional[List[plt.axes]] = None, ): quant_kwargs = {} @@ -503,7 +504,14 @@ def plot_forecast( if naive: # show naive forecast shift timeseries by 1 naive_fcast = naive_fcast.drop_before(historical_fcast.start_time()) - naive_precision = mape(ticker_series, naive_fcast) + + # calculate precision based on metric + if metric == "rsme": + naive_precision = rmse(ticker_series, naive_fcast) + elif metric == "mse": + naive_precision = mse(ticker_series, naive_fcast) + elif metric == "mape": + naive_precision = mape(ticker_series, naive_fcast) naive_fcast.plot( label=f"Naive+1: {naive_precision:.2f}%", @@ -516,7 +524,7 @@ def plot_forecast( pred_label += " w/ past covs" predicted_values.plot(label=pred_label, **quant_kwargs, color="#00AAFF") ax.set_title( - f"{name} for <{ticker_name}> for next [{n_predict}] days (MAPE={precision:.2f}%)" + f"{name} for <{ticker_name}> for next [{n_predict}] days ({metric.upper()}={precision:.2f}%)" ) ax.set_ylabel(target_col) ax.set_xlabel("Date") @@ -719,6 +727,7 @@ def get_prediction( train_split: float, forecast_horizon: int, n_predict: int, + metric: str, ): _, val = ticker_series.split_before(train_split) @@ -771,10 +780,16 @@ def get_prediction( else: prediction = best_model.predict(series=ticker_series, n=n_predict) - precision = mape( - actual_series=val, pred_series=historical_fcast - ) # mape = mean average percentage error - console.print(f"{model_name} model obtains MAPE: {precision:.2f}% \n") + # calculate precision based on metric (rmse, mse, mape) + if metric == "rmse": + precision = rmse(actual_series=val, pred_series=historical_fcast) + elif metric == "mse": + precision = mse(actual_series=val, pred_series=historical_fcast) + elif metric == "mape": + precision = mape(actual_series=val, pred_series=historical_fcast) + + # capalize metic name + console.print(f"{model_name} model obtains {metric.upper()}: {precision:.2f}% \n") # scale back if use_scalers and isinstance(scaler, Scaler): diff --git a/openbb_terminal/forecast/regr_model.py b/openbb_terminal/forecast/regr_model.py index 19871d10f24..12057e4bea3 100644 --- a/openbb_terminal/forecast/regr_model.py +++ b/openbb_terminal/forecast/regr_model.py @@ -26,6 +26,7 @@ def get_regression_data( forecast_horizon: int = 5, output_chunk_length: int = 5, lags: Union[int, List[int]] = 14, + metric: str = "mape", ) -> Tuple[ List[TimeSeries], List[TimeSeries], List[TimeSeries], float, type[RegressionModel] ]: @@ -49,6 +50,8 @@ def get_regression_data( The length of the forecast of the model. Defaults to 1. lags: Union[int, List[int]] lagged target values to predict the next time step + metric: str + Metric to use for evaluation. Defaults to "mape". Returns ------- @@ -99,4 +102,5 @@ def get_regression_data( train_split, forecast_horizon, n_predict, + metric, ) diff --git a/openbb_terminal/forecast/regr_view.py b/openbb_terminal/forecast/regr_view.py index 71e34bf4ea5..f680dedb590 100644 --- a/openbb_terminal/forecast/regr_view.py +++ b/openbb_terminal/forecast/regr_view.py @@ -35,6 +35,7 @@ def display_regression( naive: bool = False, explainability_raw: bool = False, export_pred_raw: bool = False, + metric: str = "mape", external_axes: Optional[List[plt.axes]] = None, ): """Display Regression Forecasting @@ -74,6 +75,8 @@ def display_regression( naive: bool Whether to show the naive baseline. This just assumes the closing price will be the same as the previous day's closing price. Defaults to False. + metric: str + The metric to use for the forecast. Defaults to "mape". external_axes: Optional[List[plt.axes]] External axes to plot on """ @@ -100,6 +103,7 @@ def display_regression( forecast_horizon=forecast_horizon, output_chunk_length=output_chunk_length, lags=lags, + metric=metric, ) probabilistic = False helpers.plot_forecast( @@ -120,6 +124,7 @@ def display_regression( forecast_only=forecast_only, naive=naive, export_pred_raw=export_pred_raw, + metric=metric, external_axes=external_axes, ) if residuals: