From a8223abb5a997d4c8e836fdec526b4fe2ef74303 Mon Sep 17 00:00:00 2001 From: Jamie Date: Sun, 9 May 2021 20:00:10 -0400 Subject: [PATCH] update helper functions and reintroduce loops --- .../neural_networks_view.py | 99 +++++++++---------- .../prediction_techniques/pred_helper.py | 97 +++++++++++++++++- 2 files changed, 136 insertions(+), 60 deletions(-) diff --git a/gamestonk_terminal/prediction_techniques/neural_networks_view.py b/gamestonk_terminal/prediction_techniques/neural_networks_view.py index 2d149a5b7d2..9a606154f4a 100644 --- a/gamestonk_terminal/prediction_techniques/neural_networks_view.py +++ b/gamestonk_terminal/prediction_techniques/neural_networks_view.py @@ -30,6 +30,8 @@ from gamestonk_terminal.prediction_techniques.pred_helper import ( price_prediction_backtesting_color, print_prediction_kpis, prepare_scale_train_valid_test, + forecast, + plot_data_predictions, ) from gamestonk_terminal import feature_flags as gtff @@ -1032,69 +1034,56 @@ def lstm(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): X_dates_valid, y_dates_train, y_dates_valid, - forecast_data, - dates_valid, + forecast_data_input, + dates_forecast_input, scaler, ) = prepare_scale_train_valid_test(df_stock["5. adjusted close"], ns_parser) - # Build Neural Network model - model = build_neural_network_model( - cfg_nn_models.Long_Short_Term_Memory, ns_parser.n_inputs, ns_parser.n_days + print( + f"Training on {X_train.shape[0]} samples. Using {X_valid.shape[0]} samples for validation. {ns_parser.n_loops} loops" ) - model.compile(optimizer=ns_parser.s_optimizer, loss=ns_parser.s_loss) - print(X_train.shape, X_valid.shape) - model.fit( - X_train.reshape(X_train.shape[0], X_train.shape[1], 1), - y_train, - epochs=ns_parser.n_epochs, - verbose=True, - validation_data=(X_valid, y_valid), + future_dates = get_next_stock_market_days( + dates_forecast_input[-1], n_next_days=ns_parser.n_days ) - preds = model.predict(X_valid.reshape(X_valid.shape[0], X_valid.shape[1], 1)) + preds = np.zeros((ns_parser.n_loops, X_valid.shape[0], ns_parser.n_days)) + forecast_data = np.zeros((ns_parser.n_loops, ns_parser.n_days)) + for i in range(ns_parser.n_loops): + # Build Neural Network model + model = build_neural_network_model( + cfg_nn_models.Long_Short_Term_Memory, + ns_parser.n_inputs, + ns_parser.n_days, + ) + model.compile(optimizer=ns_parser.s_optimizer, loss=ns_parser.s_loss) - plt.scatter(df_stock.index, df_stock["5. adjusted close"].values, s= 3) - for i in range(len(y_valid)): - plt.plot(y_dates_valid[i], scaler.inverse_transform(preds[i].reshape(-1,1)), "r", lw=3) - plt.fill_between(y_dates_valid[i], scaler.inverse_transform(preds[i].reshape(-1,1)), - scaler.inverse_transform(y_valid[i].reshape(-1,1))) - plt.show() - # for idx_loop in range(ns_parser.n_loops): - # Train our model - # model.fit( - # stock_x, - # stock_y, - # epochs=ns_parser.n_epochs, - # batch_size=ns_parser.n_batch_size, - # verbose=1, - # ) - # print("") + model.fit( + X_train.reshape(X_train.shape[0], X_train.shape[1], 1), + y_train, + epochs=ns_parser.n_epochs, + verbose=True, + batch_size=ns_parser.n_batch_size, + validation_data=(X_valid, y_valid), + ) - # print(model.summary()) - # print("") + preds[i] = model.predict( + X_valid.reshape(X_valid.shape[0], X_valid.shape[1], 1) + ).reshape(X_valid.shape[0], ns_parser.n_days) + forecast_data[i] = forecast( + forecast_data_input, future_dates, model, scaler + ).values.flat - # # Prediction - # yhat = model.predict( - # stock_train_data[-ns_parser.n_inputs :].reshape( - # 1, ns_parser.n_inputs, 1 - # ), - # verbose=0, - # ) - - # if idx_loop == 0: - # Re-scale the data back, plot, and print the results - # df_pred = _rescale_data( - # df_stock, ns_parser, scaler, yhat, idx_loop - # ).to_frame() - # else: - # df_pred = df_pred.join( - # _rescale_data( - # df_stock, ns_parser, scaler, yhat, idx_loop - # ).to_frame() - # ) - - # _plot_and_print_results( - # df_stock, ns_parser, df_future, df_pred, "LSTM", s_ticker - # ) + forecast_data_df = pd.DataFrame(forecast_data.T, index=future_dates) + print(forecast_data_df) + plot_data_predictions( + df_stock, + preds.median(axis=0), + y_valid, + y_dates_valid, + scaler, + f"LSTM Model on {s_ticker}", + forecast_data_df, + ) + print("") except Exception as e: print(e) diff --git a/gamestonk_terminal/prediction_techniques/pred_helper.py b/gamestonk_terminal/prediction_techniques/pred_helper.py index 4258863570e..e7f42abc582 100644 --- a/gamestonk_terminal/prediction_techniques/pred_helper.py +++ b/gamestonk_terminal/prediction_techniques/pred_helper.py @@ -66,7 +66,6 @@ def prepare_scale_train_valid_test( n_input_days = ns_parser.n_inputs n_predict_days = ns_parser.n_days - parsed_end_date = (ns_parser.s_end_date or df_stock.index[-1]) - timedelta(days= n_input_days) test_size = ns_parser.valid_split # Pre-process data @@ -79,9 +78,17 @@ def prepare_scale_train_valid_test( elif ns_parser.s_preprocessing == "normalization": scaler = Normalizer() - forecast_input = 1 - test_data = df_stock[df_stock.index > parsed_end_date] - train_data = df_stock[df_stock.index < parsed_end_date] + # Test data is used for forcasting. Takes the last n_input_days data points. + # These points are not fed into training + + if ns_parser.s_end_date: + df_stock = df_stock[df_stock.index <= ns_parser.s_end_date] + if n_input_days + n_predict_days < df_stock.shape[0]: + print("Cannot train enough input days to predict with loaded dataframe") + return + + test_data = df_stock.iloc[-n_input_days:] + train_data = df_stock.iloc[:-n_input_days] dates = train_data.index dates_test = test_data.index scaler = scaler @@ -140,6 +147,86 @@ def prepare_scale_train_valid_test( ) +def forecast(input_values, future_dates, model, scaler): + + future_values = scaler.inverse_transform( + model.predict(input_values.reshape(1, -1, 1)).reshape(-1, 1) + ) + df_future = pd.DataFrame( + future_values, index=future_dates, columns=["Predicted Price"] + ) + return df_future + + +def plot_data_predictions( + df_stock, preds, y_valid, y_dates_valid, scaler, title, forecast_data +): + + plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) + plt.plot( + df_stock.index, + df_stock["5. adjusted close"].values, + "-ob", + lw=1, + ms=2, + label="Prices", + ) + for i in range(len(y_valid) - 1): + plt.plot( + y_dates_valid[i], + scaler.inverse_transform(preds[i].reshape(-1, 1)), + "r", + lw=3, + ) + plt.fill_between( + y_dates_valid[i], + scaler.inverse_transform(preds[i].reshape(-1, 1)).ravel(), + scaler.inverse_transform(y_valid[i].reshape(-1, 1)).ravel(), + color="k", + alpha=0.2, + ) + plt.plot( + y_dates_valid[-1], + scaler.inverse_transform(preds[-1].reshape(-1, 1)), + "r", + lw=3, + label="Predicions", + ) + plt.fill_between( + y_dates_valid[-1], + scaler.inverse_transform(preds[-1].reshape(-1, 1)).ravel(), + scaler.inverse_transform(y_valid[-1].reshape(-1, 1)).ravel(), + color="k", + alpha=0.2, + ) + plt.axvspan( + df_stock.index[-1], forecast_data.index[-1], facecolor="tab:orange", alpha=0.2 + ) + _, _, ymin, ymax = plt.axis() + plt.vlines( + df_stock.index[-1], + ymin, + ymax, + colors="k", + linewidth=3, + linestyle="--", + color="k", + ) + plt.plot(forecast_data.index, forecast_data.values, "-ok", ms=5) + plt.legend(loc=0) + plt.xlim(df_stock.index[0], forecast_data.index[-1] + timedelta(days=1)) + plt.xlabel("Time") + plt.ylabel("Share Price ($)") + plt.grid(b=True, which="major", color="#666666", linestyle="-") + plt.minorticks_on() + plt.grid(b=True, which="minor", color="#999999", linestyle="-", alpha=0.2) + plt.title(title) + if gtff.USE_ION: + plt.ion() + plt.show() + print("") + + def get_backtesting_data( df_stock: pd.DataFrame, parsed_end_date: datetime, pred_days: int ) -> Tuple[pd.DataFrame, pd.DataFrame, bool]: @@ -150,7 +237,7 @@ def get_backtesting_data( df_stock: pd.DataFrame Dataframe of stock prices parsed_end_date: datetime - Date that seperates test from prediction + Date that separates test from prediction pred_days: int Number of days to predict