From 0a5e763d740f9cf302636a6ac2e7c6513293b923 Mon Sep 17 00:00:00 2001 From: Jamie Date: Tue, 4 May 2021 12:35:39 -0400 Subject: [PATCH 01/29] ARIMA edit (change -o from pdq to p,d,q) --- gamestonk_terminal/feature_flags.py | 4 +-- .../prediction_techniques/README.md | 2 +- .../{arima.py => arima_view.py} | 30 +++++++++++++++---- .../prediction_techniques/pred_controller.py | 7 +++-- 4 files changed, 32 insertions(+), 11 deletions(-) rename gamestonk_terminal/prediction_techniques/{arima.py => arima_view.py} (95%) diff --git a/gamestonk_terminal/feature_flags.py b/gamestonk_terminal/feature_flags.py index 5b32fb003fa..061b6688f82 100644 --- a/gamestonk_terminal/feature_flags.py +++ b/gamestonk_terminal/feature_flags.py @@ -14,10 +14,10 @@ USE_FLAIR = os.getenv("GTFF_USE_FLAIR") or "stars" USE_ION = strtobool(os.getenv("GTFF_USE_ION", "True")) # Enable Prompt Toolkit -USE_PROMPT_TOOLKIT = strtobool(os.getenv("GTFF_USE_PROMPT_TOOLKIT", "False")) +USE_PROMPT_TOOLKIT = strtobool(os.getenv("GTFF_USE_PROMPT_TOOLKIT", "True")) # Enable Prediction features -ENABLE_PREDICT = strtobool(os.getenv("GTFF_ENABLE_PREDICT", "False")) +ENABLE_PREDICT = strtobool(os.getenv("GTFF_ENABLE_PREDICT", "True")) # Enable FB Prophet ENABLE_FBPROPHET = strtobool(os.getenv("GTFF_ENABLE_FBPROPHET", "False")) diff --git a/gamestonk_terminal/prediction_techniques/README.md b/gamestonk_terminal/prediction_techniques/README.md index efd0a0f2fcd..0f137a5b06a 100644 --- a/gamestonk_terminal/prediction_techniques/README.md +++ b/gamestonk_terminal/prediction_techniques/README.md @@ -133,7 +133,7 @@ Auto-Regressive Integrated Moving Average: * -i : information criteria - used if auto_arima library is invoked. Default aic. * -s : weekly seasonality flag. Default False. * -r : results about ARIMA summary flag. Default False. - * -o : arima model order. If the model order is defined, auto_arima is not invoked, deeming information criteria useless.
Example: -o 514, where: + * -o : arima model order. If the model order is defined, auto_arima is not invoked, deeming information criteria useless.
Example: `-o 5,1,4` where: * p = 5 : order (number of time lags) of the autoregressive model. * d = 1 : degree of differencing (the number of times the data have had past values subtracted). * q = 4 : order of the moving-average model. diff --git a/gamestonk_terminal/prediction_techniques/arima.py b/gamestonk_terminal/prediction_techniques/arima_view.py similarity index 95% rename from gamestonk_terminal/prediction_techniques/arima.py rename to gamestonk_terminal/prediction_techniques/arima_view.py index e68ac8aad8b..7ad42ab824f 100644 --- a/gamestonk_terminal/prediction_techniques/arima.py +++ b/gamestonk_terminal/prediction_techniques/arima_view.py @@ -1,4 +1,8 @@ +""" Due Diligence Controller """ +__docformat__ = "numpy" + import argparse +from typing import List import datetime import matplotlib.pyplot as plt import pandas as pd @@ -25,7 +29,22 @@ from gamestonk_terminal import feature_flags as gtff register_matplotlib_converters() -def arima(l_args, s_ticker, df_stock): +def arima(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): + """ + ARIMA prediction + Parameters + ---------- + other_args: List[str] + Argparse arguments + s_ticker: str + ticker + df_stock: pd.DataFrame + Dataframe of prices + + Returns + ------- + + """ parser = argparse.ArgumentParser( add_help=False, prog="arima", @@ -74,7 +93,7 @@ def arima(l_args, s_ticker, df_stock): action="store", dest="s_order", type=str, - help="arima model order (p,d,q) in format: pdq.", + help="arima model order (p,d,q) in format: p,d,q.", ) parser.add_argument( "-r", @@ -95,7 +114,7 @@ def arima(l_args, s_ticker, df_stock): ) try: - ns_parser = parse_known_args_and_warn(parser, l_args) + ns_parser = parse_known_args_and_warn(parser, other_args) if not ns_parser: return @@ -134,7 +153,7 @@ def arima(l_args, s_ticker, df_stock): # Machine Learning model if ns_parser.s_order: - t_order = tuple([int(ord) for ord in list(ns_parser.s_order)]) + t_order = tuple([int(ord) for ord in ns_parser.s_order.split(",")]) model = ARIMA(df_stock["5. adjusted close"].values, order=t_order).fit() l_predictions = model.predict( start=len(df_stock["5. adjusted close"]) + 1, @@ -362,5 +381,4 @@ def arima(l_args, s_ticker, df_stock): print("") except Exception as e: - print(e) - print("") + print(e, "\n") diff --git a/gamestonk_terminal/prediction_techniques/pred_controller.py b/gamestonk_terminal/prediction_techniques/pred_controller.py index 200a93dd1b2..738c08dc56b 100644 --- a/gamestonk_terminal/prediction_techniques/pred_controller.py +++ b/gamestonk_terminal/prediction_techniques/pred_controller.py @@ -1,3 +1,6 @@ +""" Due Diligence Controller """ +__docformat__ = "numpy" + import argparse from typing import List from datetime import datetime @@ -9,7 +12,7 @@ from gamestonk_terminal import feature_flags as gtff from gamestonk_terminal.helper_funcs import get_flair from gamestonk_terminal.menu import session from gamestonk_terminal.prediction_techniques import ( - arima, + arima_view, ets, knn, neural_networks, @@ -152,7 +155,7 @@ class PredictionTechniquesController: def call_arima(self, other_args: List[str]): """Process arima command""" - arima.arima(other_args, self.ticker, self.stock) + arima_view.arima(other_args, self.ticker, self.stock) def call_mlp(self, other_args: List[str]): """Process mlp command""" From 3c5184e02a4618b781aaa3f783daa73dddb7eca4 Mon Sep 17 00:00:00 2001 From: Jamie Date: Tue, 4 May 2021 12:42:45 -0400 Subject: [PATCH 02/29] undo feature flag push --- gamestonk_terminal/feature_flags.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gamestonk_terminal/feature_flags.py b/gamestonk_terminal/feature_flags.py index 061b6688f82..5b32fb003fa 100644 --- a/gamestonk_terminal/feature_flags.py +++ b/gamestonk_terminal/feature_flags.py @@ -14,10 +14,10 @@ USE_FLAIR = os.getenv("GTFF_USE_FLAIR") or "stars" USE_ION = strtobool(os.getenv("GTFF_USE_ION", "True")) # Enable Prompt Toolkit -USE_PROMPT_TOOLKIT = strtobool(os.getenv("GTFF_USE_PROMPT_TOOLKIT", "True")) +USE_PROMPT_TOOLKIT = strtobool(os.getenv("GTFF_USE_PROMPT_TOOLKIT", "False")) # Enable Prediction features -ENABLE_PREDICT = strtobool(os.getenv("GTFF_ENABLE_PREDICT", "True")) +ENABLE_PREDICT = strtobool(os.getenv("GTFF_ENABLE_PREDICT", "False")) # Enable FB Prophet ENABLE_FBPROPHET = strtobool(os.getenv("GTFF_ENABLE_FBPROPHET", "False")) From 380848c38025b5b459c0b377897bfffbf29fd95c Mon Sep 17 00:00:00 2001 From: Jamie Date: Tue, 4 May 2021 14:39:29 -0400 Subject: [PATCH 03/29] ets to ets view --- .../prediction_techniques/arima_view.py | 2 +- .../{ets.py => ets_view.py} | 47 +++++++++++++++++-- .../prediction_techniques/pred_controller.py | 4 +- 3 files changed, 46 insertions(+), 7 deletions(-) rename gamestonk_terminal/prediction_techniques/{ets.py => ets_view.py} (94%) diff --git a/gamestonk_terminal/prediction_techniques/arima_view.py b/gamestonk_terminal/prediction_techniques/arima_view.py index 7ad42ab824f..930003ef9d7 100644 --- a/gamestonk_terminal/prediction_techniques/arima_view.py +++ b/gamestonk_terminal/prediction_techniques/arima_view.py @@ -1,4 +1,4 @@ -""" Due Diligence Controller """ +""" ARIMA Model View """ __docformat__ = "numpy" import argparse diff --git a/gamestonk_terminal/prediction_techniques/ets.py b/gamestonk_terminal/prediction_techniques/ets_view.py similarity index 94% rename from gamestonk_terminal/prediction_techniques/ets.py rename to gamestonk_terminal/prediction_techniques/ets_view.py index c26e6f28a5a..3e51ecef9bd 100644 --- a/gamestonk_terminal/prediction_techniques/ets.py +++ b/gamestonk_terminal/prediction_techniques/ets_view.py @@ -1,4 +1,8 @@ +""" ets model view """ +__docformat__ = "numpy" + import argparse +from typing import List import datetime import matplotlib.pyplot as plt import numpy as np @@ -45,7 +49,22 @@ def check_valid_seasonal(seasonal: str) -> str: ) -def exponential_smoothing(l_args, s_ticker, df_stock): +def exponential_smoothing(other_args:List[str], s_ticker:str, df_stock:pd.DataFrame): + """ + Perform exponential smoothing forecasting + Parameters + ---------- + other_args: List[str] + Argparse arguments + s_ticker: str + Loaded ticker + df_stock: pd.DataFrame + Loaded stock dataframe + + Returns + ------- + + """ parser = argparse.ArgumentParser( add_help=False, prog="ets", @@ -61,7 +80,6 @@ def exponential_smoothing(l_args, s_ticker, df_stock): Trend='Ad', Seasonal='N': Additive damped trend method Trend='Ad', Seasonal='A': Exponential Smoothing Trend='Ad', Seasonal='M': Holt-Winters’ damped method - Trend component: N: None, A: Additive, Ad: Additive Damped Seasonality component: N: None, A: Additive, M: Multiplicative """, @@ -114,7 +132,7 @@ def exponential_smoothing(l_args, s_ticker, df_stock): ) try: - ns_parser = parse_known_args_and_warn(parser, l_args) + ns_parser = parse_known_args_and_warn(parser, other_args) if not ns_parser: return @@ -391,7 +409,28 @@ def exponential_smoothing(l_args, s_ticker, df_stock): print("") -def get_exponential_smoothing_model(data, trend, seasonal, seasonal_periods): +def get_exponential_smoothing_model(data:pd.Series, trend, seasonal, seasonal_periods): + """ + Perform exponential smoothing + Parameters + ---------- + data: pd.Series + Series of closing values + trend: str + Trend component. One of [N, A, Ad] + seasonal: str + Seasonal component. One of [N, A, M] + seasonal_periods: int + Number of seaosnal periods in a year + + + Returns + ------- + model: + Exponential smoothing model + title: str + String describing selected model + """ if trend == "N": # None if seasonal == "N": # None title = "Trend='N', Seasonal='N': Simple Exponential Smoothing" diff --git a/gamestonk_terminal/prediction_techniques/pred_controller.py b/gamestonk_terminal/prediction_techniques/pred_controller.py index 738c08dc56b..64600e41d53 100644 --- a/gamestonk_terminal/prediction_techniques/pred_controller.py +++ b/gamestonk_terminal/prediction_techniques/pred_controller.py @@ -13,7 +13,7 @@ from gamestonk_terminal.helper_funcs import get_flair from gamestonk_terminal.menu import session from gamestonk_terminal.prediction_techniques import ( arima_view, - ets, + ets_view, knn, neural_networks, regression, @@ -129,7 +129,7 @@ class PredictionTechniquesController: def call_ets(self, other_args: List[str]): """Process ets command""" - ets.exponential_smoothing(other_args, self.ticker, self.stock) + ets_view.exponential_smoothing(other_args, self.ticker, self.stock) def call_knn(self, other_args: List[str]): """Process knn command""" From 08398163e1012b6966602132ca4fab6ff466aeca Mon Sep 17 00:00:00 2001 From: Jamie Date: Wed, 5 May 2021 11:04:28 -0400 Subject: [PATCH 04/29] update neural networks typing and add new preprocessing (add actual normalization and correctly rename nrormalization to mimax) --- .../config_neural_network_models.py | 6 +- .../prediction_techniques/README.md | 10 +- .../prediction_techniques/ets_view.py | 10 +- ...al_networks.py => neural_networks_view.py} | 106 ++++++++++++++---- .../prediction_techniques/pred_controller.py | 8 +- .../prediction_techniques/pred_helper.py | 12 +- tests/test_pred_neural_networks.py | 2 +- 7 files changed, 114 insertions(+), 40 deletions(-) rename gamestonk_terminal/prediction_techniques/{neural_networks.py => neural_networks_view.py} (92%) diff --git a/gamestonk_terminal/config_neural_network_models.py b/gamestonk_terminal/config_neural_network_models.py index 24358030438..9d788f9aa59 100644 --- a/gamestonk_terminal/config_neural_network_models.py +++ b/gamestonk_terminal/config_neural_network_models.py @@ -23,7 +23,7 @@ MultiLayer_Perceptron = [ {"Dense": {"units": 100, "activation": "relu"}}, {"Dense": {"units": 80, "activation": "relu"}}, {"Dense": {"units": 30, "activation": "relu"}}, - {"Dense": {"activation": "linear"}}, + {"Dense": {"activation": "relu"}}, ] Recurrent_Neural_Network = [ @@ -31,11 +31,11 @@ Recurrent_Neural_Network = [ {"SimpleRNN": {"units": 50, "activation": "linear", "return_sequences": True}}, {"Dropout": {"rate": 0.2}}, {"SimpleRNN": {"units": 21, "activation": "linear", "return_sequences": False}}, - {"Dense": {"activation": "linear"}}, + {"Dense": {"activation": "relu"}}, ] Long_Short_Term_Memory = [ {"LSTM": {"units": 25, "activation": "tanh", "return_sequences": True}}, {"LSTM": {"units": 15, "activation": "tanh", "return_sequences": False}}, - {"Dense": {"activation": "linear"}}, + {"Dense": {"activation": "relu"}}, ] diff --git a/gamestonk_terminal/prediction_techniques/README.md b/gamestonk_terminal/prediction_techniques/README.md index 0f137a5b06a..4646140a780 100644 --- a/gamestonk_terminal/prediction_techniques/README.md +++ b/gamestonk_terminal/prediction_techniques/README.md @@ -153,7 +153,7 @@ Facebook's Prophet: ## mlp ``` -usage: mlp [-d N_DAYS] [-i N_INPUTS] [-j N_JUMPS] [--epochs N_EPOCHS] [-p {normalization,standardization,none}] +usage: mlp [-d N_DAYS] [-i N_INPUTS] [-j N_JUMPS] [--epochs N_EPOCHS] [-p {minmax, normalization,standardization,none}] [-o {adam,adagrad,adadelta,adamax,ftrl,nadam,optimizer,rmsprop,sgd}] [-l {mae,mape,mse,msle}] [-e S_END_DATE] [--loops N_LOOPS] ``` MulitLayer Perceptron: @@ -161,7 +161,7 @@ MulitLayer Perceptron: * -i : number of days to use for prediction. Default 40. * -j : number of jumps in training data. Default 1. * --epochs : number of training epochs. Default 200. - * -p : pre-processing data. Default normalization. + * -p : pre-processing data. Default minmax. * -o : optimization technique. Default adam. * -l : loss function. Default mae. * -e : end date (format YYYY-MM-DD) of the stock - Backtesting. Default None. @@ -183,7 +183,7 @@ MultiLayer_Perceptron \ {'Dense': {'units':30, 'activation':'relu'} }, {'Dense': - {'activation':'linear'} }] + {'activation':'relu'} }] ``` ![mlp](https://user-images.githubusercontent.com/25267873/108604944-d2025400-73a8-11eb-9ab6-52972160cd2a.png) @@ -220,7 +220,7 @@ Recurrent_Neural_Network \ {'SimpleRNN': {'units':21, 'activation':'linear', 'return_sequences':False} }, {'Dense': - {'activation':'linear'} }] + {'activation':'relu'} }] ``` ![rnn](https://user-images.githubusercontent.com/25267873/108604940-d0d12700-73a8-11eb-837e-a5aa128942d9.png) @@ -260,7 +260,7 @@ Long_Short_Term_Memory \ {'LSTM': {'units':15, 'activation':'tanh', 'return_sequences':False} }, {'Dense': - {'activation':'linear'} }] + {'activation':'relu'} }] ``` ![lstm](https://user-images.githubusercontent.com/25267873/108604943-d2025400-73a8-11eb-83c5-edb4a2121cba.png) diff --git a/gamestonk_terminal/prediction_techniques/ets_view.py b/gamestonk_terminal/prediction_techniques/ets_view.py index 3e51ecef9bd..96238826ba9 100644 --- a/gamestonk_terminal/prediction_techniques/ets_view.py +++ b/gamestonk_terminal/prediction_techniques/ets_view.py @@ -2,7 +2,7 @@ __docformat__ = "numpy" import argparse -from typing import List +from typing import List, Union import datetime import matplotlib.pyplot as plt import numpy as np @@ -49,7 +49,7 @@ def check_valid_seasonal(seasonal: str) -> str: ) -def exponential_smoothing(other_args:List[str], s_ticker:str, df_stock:pd.DataFrame): +def exponential_smoothing(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): """ Perform exponential smoothing forecasting Parameters @@ -409,12 +409,14 @@ def exponential_smoothing(other_args:List[str], s_ticker:str, df_stock:pd.DataFr print("") -def get_exponential_smoothing_model(data:pd.Series, trend, seasonal, seasonal_periods): +def get_exponential_smoothing_model( + data: Union[pd.Series, np.ndarray], trend, seasonal, seasonal_periods +): """ Perform exponential smoothing Parameters ---------- - data: pd.Series + data: Union[pd.Series, np.ndarray] Series of closing values trend: str Trend component. One of [N, A, Ad] diff --git a/gamestonk_terminal/prediction_techniques/neural_networks.py b/gamestonk_terminal/prediction_techniques/neural_networks_view.py similarity index 92% rename from gamestonk_terminal/prediction_techniques/neural_networks.py rename to gamestonk_terminal/prediction_techniques/neural_networks_view.py index e3cabd309f9..498606321d5 100644 --- a/gamestonk_terminal/prediction_techniques/neural_networks.py +++ b/gamestonk_terminal/prediction_techniques/neural_networks_view.py @@ -1,4 +1,8 @@ +""" Neural Networks View""" +__docformat__ = "numpy" + import argparse +from typing import List, Any import datetime import os import traceback @@ -8,7 +12,7 @@ import matplotlib.pyplot as plt import pandas as pd from pandas.plotting import register_matplotlib_converters from TimeSeriesCrossValidation import splitTrain -from sklearn.preprocessing import MinMaxScaler, StandardScaler +from sklearn.preprocessing import MinMaxScaler, StandardScaler, Normalizer from tensorflow.keras.models import Sequential from tensorflow.keras.layers import LSTM, SimpleRNN, Dense, Dropout @@ -44,7 +48,9 @@ ORIGINAL_TF_XLA_FLAGS = os.environ.get("TF_XLA_FLAGS") ORIGINAL_TF_FORCE_GPU_ALLOW_GROWTH = os.environ.get("TF_FORCE_GPU_ALLOW_GROWTH") -def build_neural_network_model(Recurrent_Neural_Network, n_inputs, n_days): +def build_neural_network_model( + Recurrent_Neural_Network: List[Any], n_inputs: int, n_days: int +): model = Sequential() for idx_layer, d_layer in enumerate(Recurrent_Neural_Network): @@ -91,9 +97,24 @@ def build_neural_network_model(Recurrent_Neural_Network, n_inputs, n_days): return model -def _parse_args(prog, description, l_args): - """Create an argparser and parse l_args. Will print help if user requests it. - :return: ns_parser""" +def _parse_args(prog: str, description: str, other_args: List[str]): + """ + Create an argparser and parse other_args. Will print help if user requests it. + Parameters + ---------- + prog: str + Program for argparser + description: str + Description for argparser + other_args + Argparse arguments to pass + Returns + ------- + ns_parser: argparse.Namespace + + Parsed argument parser + """ + parser = argparse.ArgumentParser( prog=prog, description=description, @@ -140,8 +161,8 @@ def _parse_args(prog, description, l_args): "--pp", action="store", dest="s_preprocessing", - default="normalization", - choices=["normalization", "standardization", "none"], + default="minmax", + choices=["normalization", "standardization", "minmax", "none"], help="pre-processing data.", ) parser.add_argument( @@ -233,7 +254,7 @@ def _parse_args(prog, description, l_args): ) try: - ns_parser = parse_known_args_and_warn(parser, l_args) + ns_parser = parse_known_args_and_warn(parser, other_args) if not ns_parser: return None @@ -276,10 +297,27 @@ def _restore_env(): restore("TF_FORCE_GPU_ALLOW_GROWTH", ORIGINAL_TF_FORCE_GPU_ALLOW_GROWTH) -def _setup_backtesting(df_stock, ns_parser): - """Set up backtesting if enabled - :return: (df_stock, df_future), where df_future is None if s_end_date is not set. - :raises Exception: if configuration is invalid""" +def _setup_backtesting(df_stock: pd.DataFrame, ns_parser: argparse.Namespace): + """ + Setup backtesting when end data is specified + Parameters + ---------- + df_stock: pd.DataFrame + Stock data + ns_parser: argparse.Namespace + Parsed arguments + Returns + ------- + df_stock: pd.DataFrame + Data before end date + df_future: pd.DataFrame + Data after end date + Raises + ------- + Exception + If configuration is invalid + """ + df_future = None if ns_parser.s_end_date: if ns_parser.s_end_date < df_stock.index[0]: @@ -319,11 +357,16 @@ def _preprocess_split(df_stock, ns_parser): stock_train_data = scaler.fit_transform( np.array(df_stock["5. adjusted close"].values.reshape(-1, 1)) ) - elif ns_parser.s_preprocessing == "normalization": + elif ns_parser.s_preprocessing == "minmax": scaler = MinMaxScaler() stock_train_data = scaler.fit_transform( np.array(df_stock["5. adjusted close"].values.reshape(-1, 1)) ) + elif ns_parser.s_preprocessing == "normalization": + scaler = Normalizer() + stock_train_data = scaler.fit_transform( + np.array(df_stock["5. adjusted close"].values.reshape(-1, 1)) + ) else: # No pre-processing stock_train_data = np.array(df_stock["5. adjusted close"].values.reshape(-1, 1)) @@ -343,8 +386,10 @@ def _preprocess_split(df_stock, ns_parser): def _rescale_data(df_stock, ns_parser, scaler, yhat, idx_loop): """Re-scale the data back and return the prediction dataframe.""" - if (ns_parser.s_preprocessing == "standardization") or ( - ns_parser.s_preprocessing == "normalization" + if ( + (ns_parser.s_preprocessing == "standardization") + or (ns_parser.s_preprocessing == "normalization") + or (ns_parser.s_preprocessing == "minmax") ): y_pred_test_t = scaler.inverse_transform(yhat.tolist()) else: @@ -699,10 +744,25 @@ def _plot_and_print_results( print("") -def mlp(l_args, s_ticker, df_stock): +def mlp(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): + """ + Train a multi-layer perceptron model + Parameters + ---------- + other_args: List[str] + Argparse Arguments + s_ticker: str + Ticker + df_stock: pd.DataFrame + Loaded stock dataframe + + Returns + ------- + + """ try: ns_parser = _parse_args( - prog="mlp", description="""Multilayer Perceptron. """, l_args=l_args + prog="mlp", description="""Multilayer Perceptron. """, other_args=other_args ) if not ns_parser: return @@ -768,10 +828,12 @@ def mlp(l_args, s_ticker, df_stock): _restore_env() -def rnn(l_args, s_ticker, df_stock): +def rnn(other_args: List[str], s_ticker, df_stock): try: ns_parser = _parse_args( - prog="rnn", description="""Recurrent Neural Network. """, l_args=l_args + prog="rnn", + description="""Recurrent Neural Network. """, + other_args=other_args, ) if not ns_parser: return @@ -839,10 +901,12 @@ def rnn(l_args, s_ticker, df_stock): _restore_env() -def lstm(l_args, s_ticker, df_stock): +def lstm(other_args: List[str], s_ticker, df_stock): try: ns_parser = _parse_args( - prog="lstm", description="""Long-Short Term Memory. """, l_args=l_args + prog="lstm", + description="""Long-Short Term Memory. """, + other_args=other_args, ) if not ns_parser: return diff --git a/gamestonk_terminal/prediction_techniques/pred_controller.py b/gamestonk_terminal/prediction_techniques/pred_controller.py index 64600e41d53..3b92c7f058f 100644 --- a/gamestonk_terminal/prediction_techniques/pred_controller.py +++ b/gamestonk_terminal/prediction_techniques/pred_controller.py @@ -15,7 +15,7 @@ from gamestonk_terminal.prediction_techniques import ( arima_view, ets_view, knn, - neural_networks, + neural_networks_view, regression, sma, ) @@ -159,15 +159,15 @@ class PredictionTechniquesController: def call_mlp(self, other_args: List[str]): """Process mlp command""" - neural_networks.mlp(other_args, self.ticker, self.stock) + neural_networks_view.mlp(other_args, self.ticker, self.stock) def call_rnn(self, other_args: List[str]): """Process rnn command""" - neural_networks.rnn(other_args, self.ticker, self.stock) + neural_networks_view.rnn(other_args, self.ticker, self.stock) def call_lstm(self, other_args: List[str]): """Process lstm command""" - neural_networks.lstm(other_args, self.ticker, self.stock) + neural_networks_view.lstm(other_args, self.ticker, self.stock) if gtff.ENABLE_FBPROPHET: diff --git a/gamestonk_terminal/prediction_techniques/pred_helper.py b/gamestonk_terminal/prediction_techniques/pred_helper.py index c69ac5f0616..fd80ded4824 100644 --- a/gamestonk_terminal/prediction_techniques/pred_helper.py +++ b/gamestonk_terminal/prediction_techniques/pred_helper.py @@ -1,3 +1,6 @@ +""" Prediction helper functions """ +__docformat__ = "numpy" + import numpy as np import pandas as pd from colorama import Fore, Style @@ -11,6 +14,7 @@ from gamestonk_terminal import feature_flags as gtff def price_prediction_color(val: float, last_val: float) -> str: + """Set prediction to be a colored string""" if float(val) > last_val: color = Fore.GREEN else: @@ -19,6 +23,7 @@ def price_prediction_color(val: float, last_val: float) -> str: def print_pretty_prediction(df_pred: pd.DataFrame, last_price: float): + """Print predictions""" if gtff.USE_COLOR: print(f"Actual price: {Fore.YELLOW}{last_price:.2f} ${Style.RESET_ALL}\n") print("Prediction:") @@ -44,12 +49,14 @@ def print_pretty_prediction_nn(df_pred: pd.DataFrame, last_price: float): print(df_pred.to_string()) -def mean_absolute_percentage_error(y_true, y_pred): +def mean_absolute_percentage_error(y_true: np.ndarray, y_pred: np.ndarray) -> np.number: + """Calculate mean absolute percent error""" y_true, y_pred = np.array(y_true), np.array(y_pred) return np.mean(np.abs((y_true - y_pred) / y_true)) * 100 -def print_prediction_kpis(real, pred): +def print_prediction_kpis(real: np.ndarray, pred: np.ndarray): + """Print prediction statistics""" print("KPIs") print(f"MAPE: {mean_absolute_percentage_error(real, pred):.3f} %") print(f"R2: {r2_score(real, pred):.3f}") @@ -59,6 +66,7 @@ def print_prediction_kpis(real, pred): def price_prediction_backtesting_color(val: list) -> str: + """Add color to backtest data""" err_pct = 100 * (val[0] - val[1]) / val[1] if val[0] > val[1]: s_err_pct = f" {Fore.GREEN} +{err_pct:.2f} %" diff --git a/tests/test_pred_neural_networks.py b/tests/test_pred_neural_networks.py index cc369861296..303038f2a02 100644 --- a/tests/test_pred_neural_networks.py +++ b/tests/test_pred_neural_networks.py @@ -2,7 +2,7 @@ import unittest try: - from gamestonk_terminal.prediction_techniques.neural_networks import mlp + from gamestonk_terminal.prediction_techniques.neural_networks_view import mlp except ModuleNotFoundError as e: print("One of the optional packages seems to be missing") print(e) From 48b561f14ef05ab6fd52af5855f4c77f38aec440 Mon Sep 17 00:00:00 2001 From: Jamie Date: Wed, 5 May 2021 11:33:45 -0400 Subject: [PATCH 05/29] Docstring-ed and Typing-ed all functions in nn_view --- gamestonk_terminal/feature_flags.py | 4 +- .../neural_networks_helper.py | 0 .../neural_networks_view.py | 129 ++++++++++++++++-- 3 files changed, 118 insertions(+), 15 deletions(-) create mode 100644 gamestonk_terminal/prediction_techniques/neural_networks_helper.py diff --git a/gamestonk_terminal/feature_flags.py b/gamestonk_terminal/feature_flags.py index 5b32fb003fa..061b6688f82 100644 --- a/gamestonk_terminal/feature_flags.py +++ b/gamestonk_terminal/feature_flags.py @@ -14,10 +14,10 @@ USE_FLAIR = os.getenv("GTFF_USE_FLAIR") or "stars" USE_ION = strtobool(os.getenv("GTFF_USE_ION", "True")) # Enable Prompt Toolkit -USE_PROMPT_TOOLKIT = strtobool(os.getenv("GTFF_USE_PROMPT_TOOLKIT", "False")) +USE_PROMPT_TOOLKIT = strtobool(os.getenv("GTFF_USE_PROMPT_TOOLKIT", "True")) # Enable Prediction features -ENABLE_PREDICT = strtobool(os.getenv("GTFF_ENABLE_PREDICT", "False")) +ENABLE_PREDICT = strtobool(os.getenv("GTFF_ENABLE_PREDICT", "True")) # Enable FB Prophet ENABLE_FBPROPHET = strtobool(os.getenv("GTFF_ENABLE_FBPROPHET", "False")) diff --git a/gamestonk_terminal/prediction_techniques/neural_networks_helper.py b/gamestonk_terminal/prediction_techniques/neural_networks_helper.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/gamestonk_terminal/prediction_techniques/neural_networks_view.py b/gamestonk_terminal/prediction_techniques/neural_networks_view.py index 498606321d5..9c7bacbc5eb 100644 --- a/gamestonk_terminal/prediction_techniques/neural_networks_view.py +++ b/gamestonk_terminal/prediction_techniques/neural_networks_view.py @@ -2,7 +2,7 @@ __docformat__ = "numpy" import argparse -from typing import List, Any +from typing import List, Any, Union import datetime import os import traceback @@ -111,7 +111,6 @@ def _parse_args(prog: str, description: str, other_args: List[str]): Returns ------- ns_parser: argparse.Namespace - Parsed argument parser """ @@ -347,10 +346,32 @@ def _setup_backtesting(df_stock: pd.DataFrame, ns_parser: argparse.Namespace): return df_stock, df_future -def _preprocess_split(df_stock, ns_parser): - """Preprocess and split training data. - :return: (scaler, stock_train_data, stock_x, stock_y) - :raises Exception: if more training data is needed.""" +def _preprocess_split(df_stock: pd.DataFrame, ns_parser: argparse.Namespace): + """ + Preprocess and split dstock data + Parameters + ---------- + df_stock: pd.DataFrame + Dataframe of stock prices + ns_parser: argparse.Namespace + Parsed argparse arguments + + Returns + ------- + scaler: Union[MinMaxScaler, StandardScaler, Normalizer] + sklearn scaler used + stock_train_data: np.ndarray + All training data + stock_x: np.ndarray + Inputs for model + stock_y: np.ndarray + Outputs for model + Raises + ------- + Exception + Not enough training data is provided + """ + # Pre-process data if ns_parser.s_preprocessing == "standardization": scaler = StandardScaler() @@ -379,13 +400,40 @@ def _preprocess_split(df_stock, ns_parser): ) if not stock_x: raise Exception("Given the model parameters more training data is needed.") + stock_x = np.array(stock_x) stock_y = np.array(stock_y) + return scaler, stock_train_data, stock_x, stock_y -def _rescale_data(df_stock, ns_parser, scaler, yhat, idx_loop): - """Re-scale the data back and return the prediction dataframe.""" +def _rescale_data( + df_stock: pd.DataFrame, + ns_parser: argparse.Namespace, + scaler: Union[MinMaxScaler, StandardScaler, Normalizer], + yhat: np.ndarray, + idx_loop: int, +): + """ + Re-scale the data back and return the prediction dataframe. + Parameters + ---------- + df_stock: pd.DataFrame + Dataframe of stock prices + ns_parser: argparse.Namespace + Parsed argument parser + scaler: + One of the predefined sklearn scalers + yhat: np.ndarry + Predicted values + idx_loop: int + Loop index + + Returns + ------- + df_pred: pd.Series + Series of resclaled predictions + """ if ( (ns_parser.s_preprocessing == "standardization") or (ns_parser.s_preprocessing == "normalization") @@ -403,11 +451,36 @@ def _rescale_data(df_stock, ns_parser, scaler, yhat, idx_loop): df_pred = pd.Series(y_pred_test_t[0].tolist(), index=l_pred_days, name=column_name) return df_pred - def _plot_and_print_results( - df_stock, ns_parser, df_future, df_pred, model_name, s_ticker + df_stock: pd.DataFrame, + ns_parser: argparse.Namespace, + df_future: pd.DataFrame, + df_pred: pd.DataFrame, + model_name: str, + s_ticker: str, ): - """Plot and print the results.""" + """ + Plot and print results + Parameters + ---------- + df_stock: pd.DataFrame: + Dataframe of stock prices + ns_parser: argparse.Namespace + Parsed argument parser + df_future: pd.DataFrame + Dataframe of future prices + df_pred: pd.DataFrame + Dataframe of predicted prices + model_name: str + Model Name + s_ticker: str + Ticker + + Returns + ------- + + """ + # Plotting plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) plt.plot(df_stock.index, df_stock["5. adjusted close"], lw=3) @@ -828,7 +901,22 @@ def mlp(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): _restore_env() -def rnn(other_args: List[str], s_ticker, df_stock): +def rnn(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): + """ + Train a Recurrent Neural Network (rnn) + Parameters + ---------- + other_args:List[str] + Argparse arguments + s_ticker: str + Stock ticker + df_stock: pd.DataFrame + Dataframe of stock prices + + Returns + ------- + + """ try: ns_parser = _parse_args( prog="rnn", @@ -901,7 +989,22 @@ def rnn(other_args: List[str], s_ticker, df_stock): _restore_env() -def lstm(other_args: List[str], s_ticker, df_stock): +def lstm(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): + """ + Train a Long-Short-Term-Memory Neural Net (lstm) + Parameters + ---------- + other_args:List[str] + Argparse arguments + s_ticker: str + Stock ticker + df_stock: pd.DataFrame + Dataframe of stock prices + + Returns + ------- + + """ try: ns_parser = _parse_args( prog="lstm", From 2ca11aad296952effd1ed6c7fc36ac36ec3b38ae Mon Sep 17 00:00:00 2001 From: Jamie Date: Wed, 5 May 2021 11:34:32 -0400 Subject: [PATCH 06/29] Uncommit feature flags --- gamestonk_terminal/feature_flags.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gamestonk_terminal/feature_flags.py b/gamestonk_terminal/feature_flags.py index 061b6688f82..5b32fb003fa 100644 --- a/gamestonk_terminal/feature_flags.py +++ b/gamestonk_terminal/feature_flags.py @@ -14,10 +14,10 @@ USE_FLAIR = os.getenv("GTFF_USE_FLAIR") or "stars" USE_ION = strtobool(os.getenv("GTFF_USE_ION", "True")) # Enable Prompt Toolkit -USE_PROMPT_TOOLKIT = strtobool(os.getenv("GTFF_USE_PROMPT_TOOLKIT", "True")) +USE_PROMPT_TOOLKIT = strtobool(os.getenv("GTFF_USE_PROMPT_TOOLKIT", "False")) # Enable Prediction features -ENABLE_PREDICT = strtobool(os.getenv("GTFF_ENABLE_PREDICT", "True")) +ENABLE_PREDICT = strtobool(os.getenv("GTFF_ENABLE_PREDICT", "False")) # Enable FB Prophet ENABLE_FBPROPHET = strtobool(os.getenv("GTFF_ENABLE_FBPROPHET", "False")) From c5f4f7832c4a9305ebee84d6791a6e0a48ae25de Mon Sep 17 00:00:00 2001 From: Jamie Date: Wed, 5 May 2021 12:29:03 -0400 Subject: [PATCH 07/29] knn/fbprophet/regression/sma renamed + typed/docstring-ed --- .../{fbprophet.py => fbprophet_view.py} | 23 ++++++++++++-- .../{knn.py => knn_view.py} | 23 ++++++++++++-- .../neural_networks_view.py | 1 + .../prediction_techniques/pred_controller.py | 30 +++++++++++-------- .../{regression.py => regression_view.py} | 27 +++++++++++++++-- .../{sma.py => sma_view.py} | 25 ++++++++++++++-- 6 files changed, 108 insertions(+), 21 deletions(-) rename gamestonk_terminal/prediction_techniques/{fbprophet.py => fbprophet_view.py} (95%) rename gamestonk_terminal/prediction_techniques/{knn.py => knn_view.py} (95%) rename gamestonk_terminal/prediction_techniques/{regression.py => regression_view.py} (95%) rename gamestonk_terminal/prediction_techniques/{sma.py => sma_view.py} (95%) diff --git a/gamestonk_terminal/prediction_techniques/fbprophet.py b/gamestonk_terminal/prediction_techniques/fbprophet_view.py similarity index 95% rename from gamestonk_terminal/prediction_techniques/fbprophet.py rename to gamestonk_terminal/prediction_techniques/fbprophet_view.py index e2294791b6c..fe00655803e 100644 --- a/gamestonk_terminal/prediction_techniques/fbprophet.py +++ b/gamestonk_terminal/prediction_techniques/fbprophet_view.py @@ -1,4 +1,8 @@ +""" FB Prophet View""" +__docformat__ = "numpy" + import argparse +from typing import List import datetime import warnings import matplotlib.pyplot as plt @@ -26,7 +30,22 @@ register_matplotlib_converters() warnings.simplefilter("ignore") -def fbprophet(l_args, s_ticker, df_stock): +def fbprophet(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): + """ + Run FB Prophet model + Parameters + ---------- + other_args: List[str]: + List of argparse argumenst + s_ticker: str + Stock ticker + df_stock: pd.DataFrame + Dataframe of prices + + Returns + ------- + + """ parser = argparse.ArgumentParser( add_help=False, prog="fbprophet", @@ -58,7 +77,7 @@ def fbprophet(l_args, s_ticker, df_stock): ) try: - ns_parser = parse_known_args_and_warn(parser, l_args) + ns_parser = parse_known_args_and_warn(parser, other_args) if not ns_parser: return diff --git a/gamestonk_terminal/prediction_techniques/knn.py b/gamestonk_terminal/prediction_techniques/knn_view.py similarity index 95% rename from gamestonk_terminal/prediction_techniques/knn.py rename to gamestonk_terminal/prediction_techniques/knn_view.py index 01de4c62b84..2b70c555a29 100644 --- a/gamestonk_terminal/prediction_techniques/knn.py +++ b/gamestonk_terminal/prediction_techniques/knn_view.py @@ -1,4 +1,8 @@ +""" K Nearest Neighbors View""" +__docformat__ = "numpy" + import argparse +from typing import List import datetime import matplotlib.pyplot as plt import pandas as pd @@ -25,7 +29,22 @@ from gamestonk_terminal import feature_flags as gtff register_matplotlib_converters() -def k_nearest_neighbors(l_args, s_ticker, df_stock): +def k_nearest_neighbors(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): + """ + Train KNN model + Parameters + ---------- + other_args: List[str] + List of argparse arguments + s_ticker: str + Ticker + df_stock: pd.DataFrame + Dataframe of stock prices + + Returns + ------- + + """ parser = argparse.ArgumentParser( add_help=False, prog="knn", @@ -83,7 +102,7 @@ def k_nearest_neighbors(l_args, s_ticker, df_stock): ) try: - ns_parser = parse_known_args_and_warn(parser, l_args) + ns_parser = parse_known_args_and_warn(parser, other_args) if not ns_parser: return diff --git a/gamestonk_terminal/prediction_techniques/neural_networks_view.py b/gamestonk_terminal/prediction_techniques/neural_networks_view.py index 9c7bacbc5eb..41130ea932c 100644 --- a/gamestonk_terminal/prediction_techniques/neural_networks_view.py +++ b/gamestonk_terminal/prediction_techniques/neural_networks_view.py @@ -451,6 +451,7 @@ def _rescale_data( df_pred = pd.Series(y_pred_test_t[0].tolist(), index=l_pred_days, name=column_name) return df_pred + def _plot_and_print_results( df_stock: pd.DataFrame, ns_parser: argparse.Namespace, diff --git a/gamestonk_terminal/prediction_techniques/pred_controller.py b/gamestonk_terminal/prediction_techniques/pred_controller.py index 3b92c7f058f..b0df8b0cf02 100644 --- a/gamestonk_terminal/prediction_techniques/pred_controller.py +++ b/gamestonk_terminal/prediction_techniques/pred_controller.py @@ -14,10 +14,10 @@ from gamestonk_terminal.menu import session from gamestonk_terminal.prediction_techniques import ( arima_view, ets_view, - knn, + knn_view, neural_networks_view, - regression, - sma, + regression_view, + sma_view, ) @@ -125,7 +125,7 @@ class PredictionTechniquesController: def call_sma(self, other_args: List[str]): """Process sma command""" - sma.simple_moving_average(other_args, self.ticker, self.stock) + sma_view.simple_moving_average(other_args, self.ticker, self.stock) def call_ets(self, other_args: List[str]): """Process ets command""" @@ -133,24 +133,30 @@ class PredictionTechniquesController: def call_knn(self, other_args: List[str]): """Process knn command""" - knn.k_nearest_neighbors(other_args, self.ticker, self.stock) + knn_view.k_nearest_neighbors(other_args, self.ticker, self.stock) def call_linear(self, other_args: List[str]): """Process linear command""" - regression.regression(other_args, self.ticker, self.stock, regression.LINEAR) + regression_view.regression( + other_args, self.ticker, self.stock, regression_view.LINEAR + ) def call_quadratic(self, other_args: List[str]): """Process quadratic command""" - regression.regression(other_args, self.ticker, self.stock, regression.QUADRATIC) + regression_view.regression( + other_args, self.ticker, self.stock, regression_view.QUADRATIC + ) def call_cubic(self, other_args: List[str]): """Process cubic command""" - regression.regression(other_args, self.ticker, self.stock, regression.CUBIC) + regression_view.regression( + other_args, self.ticker, self.stock, regression_view.CUBIC + ) def call_regression(self, other_args: List[str]): """Process regression command""" - regression.regression( - other_args, self.ticker, self.stock, regression.USER_INPUT + regression_view.regression( + other_args, self.ticker, self.stock, regression_view.USER_INPUT ) def call_arima(self, other_args: List[str]): @@ -174,9 +180,9 @@ class PredictionTechniquesController: def call_prophet(self, other_args: List[str]): """Process prophet command""" # pylint: disable=import-outside-toplevel - from gamestonk_terminal.prediction_techniques import fbprophet + from gamestonk_terminal.prediction_techniques import fbprophet_view - fbprophet.fbprophet(other_args, self.ticker, self.stock) + fbprophet_view.fbprophet(other_args, self.ticker, self.stock) def menu(stock: pd.DataFrame, ticker: str, start: datetime, interval: str): diff --git a/gamestonk_terminal/prediction_techniques/regression.py b/gamestonk_terminal/prediction_techniques/regression_view.py similarity index 95% rename from gamestonk_terminal/prediction_techniques/regression.py rename to gamestonk_terminal/prediction_techniques/regression_view.py index 8e02f37b44b..6f81805f5a0 100644 --- a/gamestonk_terminal/prediction_techniques/regression.py +++ b/gamestonk_terminal/prediction_techniques/regression_view.py @@ -1,4 +1,8 @@ +""" Regression View""" +__docformat__ = "numpy" + import argparse +from typing import List import datetime import matplotlib.pyplot as plt import pandas as pd @@ -34,7 +38,26 @@ QUADRATIC = 2 CUBIC = 3 -def regression(l_args, s_ticker, df_stock, polynomial): +def regression( + other_args: List[str], s_ticker: str, df_stock: pd.DataFrame, polynomial: int +): + """ + Train a regression model + Parameters + ---------- + other_args: List[str] + Argparse arguments + s_ticker: str + Stock ticker + df_stock: pd.DataFrame + Dataframe of stock prices + polynomial: int + Order of polynomial + + Returns + ------- + + """ parser = argparse.ArgumentParser( add_help=False, prog="regression", @@ -95,7 +118,7 @@ def regression(l_args, s_ticker, df_stock, polynomial): ) try: - ns_parser = parse_known_args_and_warn(parser, l_args) + ns_parser = parse_known_args_and_warn(parser, other_args) if not ns_parser: return diff --git a/gamestonk_terminal/prediction_techniques/sma.py b/gamestonk_terminal/prediction_techniques/sma_view.py similarity index 95% rename from gamestonk_terminal/prediction_techniques/sma.py rename to gamestonk_terminal/prediction_techniques/sma_view.py index b371553343c..23fd35fb486 100644 --- a/gamestonk_terminal/prediction_techniques/sma.py +++ b/gamestonk_terminal/prediction_techniques/sma_view.py @@ -1,4 +1,8 @@ +""" SMA View""" +__docformat__ = "numpy" + import argparse +from typing import List import datetime import numpy as np import matplotlib.pyplot as plt @@ -27,7 +31,22 @@ register_matplotlib_converters() # pylint: disable=unused-argument -def simple_moving_average(l_args, s_ticker, df_stock): +def simple_moving_average(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): + """ + Run an SMA model + Parameters + ---------- + other_args: List[str] + Argparse arguments + s_ticker: str + Stock ticker + df_stock: pd.DataFrame + Dataframe of stock prices + + Returns + ------- + + """ parser = argparse.ArgumentParser( add_help=False, prog="sma", @@ -71,7 +90,7 @@ def simple_moving_average(l_args, s_ticker, df_stock): ) try: - ns_parser = parse_known_args_and_warn(parser, l_args) + ns_parser = parse_known_args_and_warn(parser, other_args) if not ns_parser: return @@ -109,7 +128,7 @@ def simple_moving_average(l_args, s_ticker, df_stock): df_stock = df_stock[: ns_parser.s_end_date] # Prediction data - l_predictions = list() + l_predictions = list() # type: List for pred_day in range(ns_parser.n_days): if pred_day < ns_parser.n_length: l_ma_stock = df_stock["5. adjusted close"].values[ From 7523fd7901fb1b4bb7771eea394cabcd0fa5f676 Mon Sep 17 00:00:00 2001 From: Jamie Date: Wed, 5 May 2021 15:20:14 -0400 Subject: [PATCH 08/29] make general "backtest" function / change default stock load date --- gamestonk_terminal/main_helper.py | 2 +- .../neural_networks_helper.py | 12 ++ .../prediction_techniques/pred_helper.py | 202 +++++++++++++++++- .../prediction_techniques/sma_view.py | 33 +-- 4 files changed, 218 insertions(+), 31 deletions(-) diff --git a/gamestonk_terminal/main_helper.py b/gamestonk_terminal/main_helper.py index 30ef54f0af0..8cd282145f6 100644 --- a/gamestonk_terminal/main_helper.py +++ b/gamestonk_terminal/main_helper.py @@ -145,7 +145,7 @@ def load(l_args, s_ticker, s_start, s_interval, df_stock): "-s", "--start", type=valid_date, - default="2015-01-01", + default="2019-01-01", dest="s_start_date", help="The starting date (format YYYY-MM-DD) of the stock", ) diff --git a/gamestonk_terminal/prediction_techniques/neural_networks_helper.py b/gamestonk_terminal/prediction_techniques/neural_networks_helper.py index e69de29bb2d..125105080ea 100644 --- a/gamestonk_terminal/prediction_techniques/neural_networks_helper.py +++ b/gamestonk_terminal/prediction_techniques/neural_networks_helper.py @@ -0,0 +1,12 @@ +"""Helper functions""" +__docformat__ = "numpy" +from typing import List, Tuple +from datetime import datetime +import pandas as pd +import numpy as np +from matplotlib import pyplot as plt +from gamestonk_terminal.helper_funcs import ( + get_next_stock_market_days, + patch_pandas_text_adjustment, + plot_autoscale, +) diff --git a/gamestonk_terminal/prediction_techniques/pred_helper.py b/gamestonk_terminal/prediction_techniques/pred_helper.py index fd80ded4824..6cd4624863e 100644 --- a/gamestonk_terminal/prediction_techniques/pred_helper.py +++ b/gamestonk_terminal/prediction_techniques/pred_helper.py @@ -1,16 +1,216 @@ """ Prediction helper functions """ __docformat__ = "numpy" +from typing import Tuple +from datetime import datetime import numpy as np import pandas as pd +from pandas.plotting import register_matplotlib_converters +import matplotlib.pyplot as plt from colorama import Fore, Style from sklearn.metrics import ( mean_absolute_error, r2_score, mean_squared_error, ) - +from gamestonk_terminal.helper_funcs import ( + get_next_stock_market_days, + patch_pandas_text_adjustment, + plot_autoscale, +) from gamestonk_terminal import feature_flags as gtff +from gamestonk_terminal.config_plot import PLOT_DPI + +register_matplotlib_converters() + + +def get_backtesting_data( + df_stock: pd.DataFrame, parsed_end_date: datetime, pred_days: int +) -> Tuple[pd.DataFrame, pd.DataFrame, bool]: + """ + + Parameters + ---------- + df_stock: pd.DataFrame + Dataframe of stock prices + parsed_end_date: datetime + Date that seperates test from prediction + pred_days: int + Number of days to predict + + Returns + ------- + df_future: pd.DataFrame + Dataframe of future prices + df_stock: pd.DataFrame + Dataframe of past prices + Flag: bool + Returns True if backtesting meets requirements, False if there are errors + + """ + if parsed_end_date < df_stock.index[0]: + print( + "Backtesting not allowed, since End Date is older than Start Date of historical data\n" + ) + return pd.DataFrame(), pd.DataFrame(), False + + if ( + parsed_end_date + < get_next_stock_market_days( + last_stock_day=df_stock.index[0], n_next_days=5 + pred_days + )[-1] + ): + print( + "Backtesting not allowed, since End Date is too close to Start Date to train model\n" + ) + return pd.DataFrame(), pd.DataFrame(), False + + future_index = get_next_stock_market_days( + last_stock_day=parsed_end_date, n_next_days=pred_days + ) + + if future_index[-1] > datetime.now(): + print( + "Backtesting not allowed, since End Date + Prediction days is in the future\n" + ) + return pd.DataFrame(), pd.DataFrame(), False + + df_future = df_stock[future_index[0] : future_index[-1]] + df_stock = df_stock[:parsed_end_date] + return df_future, df_stock, True + + +def plot_pred( + df_stock: pd.DataFrame, + df_future: pd.DataFrame, + df_pred: pd.DataFrame, + bt_flag: bool, +): + plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) + plt.plot(df_stock.index, df_stock["5. adjusted close"], lw=2) + # BACKTESTING + if ns_parser.s_end_date: + plt.title( + f"BACKTESTING: {ns_parser.n_length} Moving Average on {s_ticker} - {ns_parser.n_days} days prediction" + ) + else: + plt.title( + f"{ns_parser.n_length} Moving Average on {s_ticker} - {ns_parser.n_days} days prediction" + ) + plt.xlim(df_stock.index[0], get_next_stock_market_days(df_pred.index[-1], 1)[-1]) + plt.xlabel("Time") + plt.ylabel("Share Price ($)") + plt.grid(b=True, which="major", color="#666666", linestyle="-") + plt.minorticks_on() + plt.grid(b=True, which="minor", color="#999999", linestyle="-", alpha=0.2) + df_ma = df_stock["5. adjusted close"].rolling(window=ns_parser.n_length).mean() + plt.plot(df_ma.index, df_ma, lw=2, linestyle="--", c="tab:orange") + plt.plot( + [df_stock.index[-1], df_pred.index[0]], + [df_stock["5. adjusted close"].values[-1], df_pred.values[0]], + lw=1, + c="tab:green", + linestyle="--", + ) + plt.plot(df_pred.index, df_pred, lw=2, c="tab:green") + plt.axvspan( + df_stock.index[-1], df_pred.index[-1], facecolor="tab:orange", alpha=0.2 + ) + _, _, ymin, ymax = plt.axis() + plt.vlines(df_stock.index[-1], ymin, ymax, linewidth=1, linestyle="--", color="k") + + if bt_flag: + plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) + plt.subplot(211) + plt.plot( + df_future.index, + df_future["5. adjusted close"], + lw=2, + c="tab:blue", + ls="--", + ) + plt.plot(df_pred.index, df_pred, lw=2, c="green") + plt.scatter(df_future.index, df_future["5. adjusted close"], c="tab:blue", lw=3) + plt.plot( + [df_stock.index[-1], df_future.index[0]], + [ + df_stock["5. adjusted close"].values[-1], + df_future["5. adjusted close"].values[0], + ], + lw=2, + c="tab:blue", + ls="--", + ) + plt.scatter(df_pred.index, df_pred, c="green", lw=3) + plt.plot( + [df_stock.index[-1], df_pred.index[0]], + [df_stock["5. adjusted close"].values[-1], df_pred.values[0]], + lw=2, + c="green", + ls="--", + ) + plt.title("BACKTESTING: Real data price versus Prediction") + plt.xlim(df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)) + plt.xticks( + [df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)], + visible=True, + ) + plt.ylabel("Share Price ($)") + plt.grid(b=True, which="major", color="#666666", linestyle="-") + plt.minorticks_on() + plt.grid(b=True, which="minor", color="#999999", linestyle="-", alpha=0.2) + plt.legend(["Real data", "Prediction data"]) + plt.xticks([]) + + plt.subplot(212) + plt.axhline(y=0, color="k", linestyle="--", linewidth=2) + plt.plot( + df_future.index, + 100 + * (df_pred.values - df_future["5. adjusted close"].values) + / df_future["5. adjusted close"].values, + lw=2, + c="red", + ) + plt.scatter( + df_future.index, + 100 + * (df_pred.values - df_future["5. adjusted close"].values) + / df_future["5. adjusted close"].values, + c="red", + lw=5, + ) + plt.title("BACKTESTING: Error between Real data and Prediction [%]") + plt.plot( + [df_stock.index[-1], df_future.index[0]], + [ + 0, + 100 + * (df_pred.values[0] - df_future["5. adjusted close"].values[0]) + / df_future["5. adjusted close"].values[0], + ], + lw=2, + ls="--", + c="red", + ) + plt.xlim(df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)) + plt.xticks( + [df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)], + visible=True, + ) + plt.xlabel("Time") + plt.ylabel("Prediction Error (%)") + plt.grid(b=True, which="major", color="#666666", linestyle="-") + plt.minorticks_on() + plt.grid(b=True, which="minor", color="#999999", linestyle="-", alpha=0.2) + plt.legend(["Real data", "Prediction data"]) + + if gtff.USE_ION: + plt.ion() + + plt.show() + + pass def price_prediction_color(val: float, last_val: float) -> str: diff --git a/gamestonk_terminal/prediction_techniques/sma_view.py b/gamestonk_terminal/prediction_techniques/sma_view.py index 23fd35fb486..902939018a9 100644 --- a/gamestonk_terminal/prediction_techniques/sma_view.py +++ b/gamestonk_terminal/prediction_techniques/sma_view.py @@ -22,6 +22,7 @@ from gamestonk_terminal.prediction_techniques.pred_helper import ( print_pretty_prediction, price_prediction_backtesting_color, print_prediction_kpis, + get_backtesting_data, ) from gamestonk_terminal.config_plot import PLOT_DPI @@ -30,7 +31,6 @@ from gamestonk_terminal import feature_flags as gtff register_matplotlib_converters() -# pylint: disable=unused-argument def simple_moving_average(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): """ Run an SMA model @@ -96,37 +96,12 @@ def simple_moving_average(other_args: List[str], s_ticker: str, df_stock: pd.Dat # BACKTESTING if ns_parser.s_end_date: - - if ns_parser.s_end_date < df_stock.index[0]: - print( - "Backtesting not allowed, since End Date is older than Start Date of historical data\n" - ) - return - - if ( - ns_parser.s_end_date - < get_next_stock_market_days( - last_stock_day=df_stock.index[0], n_next_days=5 + ns_parser.n_days - )[-1] - ): - print( - "Backtesting not allowed, since End Date is too close to Start Date to train model\n" - ) - return - - future_index = get_next_stock_market_days( - last_stock_day=ns_parser.s_end_date, n_next_days=ns_parser.n_days + df_future, df_stock, bt_flag = get_backtesting_data( + df_stock, ns_parser.s_end_date, ns_parser.n_days ) - - if future_index[-1] > datetime.datetime.now(): - print( - "Backtesting not allowed, since End Date + Prediction days is in the future\n" - ) + if not bt_flag: return - df_future = df_stock[future_index[0] : future_index[-1]] - df_stock = df_stock[: ns_parser.s_end_date] - # Prediction data l_predictions = list() # type: List for pred_day in range(ns_parser.n_days): From 978438c8d314803ba7f240e65498c67a89389a24 Mon Sep 17 00:00:00 2001 From: Jamie Date: Thu, 6 May 2021 09:54:49 -0400 Subject: [PATCH 09/29] starting to refactor data preprosess --- .../prediction_techniques/knn_view.py | 29 ++-------- .../prediction_techniques/pred_helper.py | 58 ++++++++++++++++--- .../prediction_techniques/sma_view.py | 4 +- 3 files changed, 55 insertions(+), 36 deletions(-) diff --git a/gamestonk_terminal/prediction_techniques/knn_view.py b/gamestonk_terminal/prediction_techniques/knn_view.py index 2b70c555a29..950c57b46d2 100644 --- a/gamestonk_terminal/prediction_techniques/knn_view.py +++ b/gamestonk_terminal/prediction_techniques/knn_view.py @@ -21,6 +21,7 @@ from gamestonk_terminal.prediction_techniques.pred_helper import ( print_pretty_prediction, price_prediction_backtesting_color, print_prediction_kpis, + get_backtesting_data, ) from gamestonk_terminal.config_plot import PLOT_DPI @@ -108,34 +109,12 @@ def k_nearest_neighbors(other_args: List[str], s_ticker: str, df_stock: pd.DataF # BACKTESTING if ns_parser.s_end_date: - if ns_parser.s_end_date < df_stock.index[0]: - print( - "Backtesting not allowed, since End Date is older than Start Date of historical data\n" - ) - return - - if ns_parser.s_end_date < get_next_stock_market_days( - last_stock_day=df_stock.index[0], - n_next_days=ns_parser.n_inputs + ns_parser.n_days, - )[-1]: - print( - "Backtesting not allowed, since End Date is too close to Start Date to train model\n" - ) - return - - future_index = get_next_stock_market_days( - last_stock_day=ns_parser.s_end_date, n_next_days=ns_parser.n_days + df_future, df_stock, bt_flag = get_backtesting_data( + df_stock, ns_parser.s_end_date, ns_parser.n_days ) - - if future_index[-1] > datetime.datetime.now(): - print( - "Backtesting not allowed, since End Date + Prediction days is in the future\n" - ) + if not bt_flag: return - df_future = df_stock[future_index[0] : future_index[-1]] - df_stock = df_stock[: ns_parser.s_end_date] - # Split training data stock_x, stock_y = splitTrain.split_train( df_stock["5. adjusted close"].values, diff --git a/gamestonk_terminal/prediction_techniques/pred_helper.py b/gamestonk_terminal/prediction_techniques/pred_helper.py index 6cd4624863e..71168ecb188 100644 --- a/gamestonk_terminal/prediction_techniques/pred_helper.py +++ b/gamestonk_terminal/prediction_techniques/pred_helper.py @@ -13,6 +13,8 @@ from sklearn.metrics import ( r2_score, mean_squared_error, ) +from sklearn.model_selection import train_test_split + from gamestonk_terminal.helper_funcs import ( get_next_stock_market_days, patch_pandas_text_adjustment, @@ -24,6 +26,51 @@ from gamestonk_terminal.config_plot import PLOT_DPI register_matplotlib_converters() +def prepare_test_train_valid( + df_stock, parsed_end_date, n_input_days, n_predict_days, test_size +): + + dates = df_stock.index + prices = df_stock["5. adjusted close"].values + + input_dates = [] + input_prices = [] + next_n_day_prices = [] + next_n_day_dates = [] + + for idx in range(len(prices) - n_input_days - n_predict_days): + input_prices.append(prices[idx : idx + n_input_days]) + input_dates.append(dates[idx : idx + n_input_days]) + next_n_day_prices.append( + prices[idx + n_input_days : idx + n_input_days + n_predict_days] + ) + next_n_day_dates.append( + dates[idx + n_input_days : idx + n_input_days + n_predict_days] + ) + + input_dates = np.asarray(input_dates) + input_prices = np.asarray(input_prices) + next_n_day_prices = np.asarray(next_n_day_prices) + next_n_day_dates = np.asarray(next_n_day_dates) + + ( + X_train, + X_test, + y_train, + y_test, + X_dates_train, + X_dates_test, + y_dates_train, + y_dates_test, + ) = train_test_split( + input_prices, + next_n_day_prices, + input_dates, + next_n_day_dates, + test_size=test_size, + ) + + def get_backtesting_data( df_stock: pd.DataFrame, parsed_end_date: datetime, pred_days: int ) -> Tuple[pd.DataFrame, pd.DataFrame, bool]: @@ -84,19 +131,12 @@ def plot_pred( df_stock: pd.DataFrame, df_future: pd.DataFrame, df_pred: pd.DataFrame, + title: str, bt_flag: bool, ): plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) plt.plot(df_stock.index, df_stock["5. adjusted close"], lw=2) - # BACKTESTING - if ns_parser.s_end_date: - plt.title( - f"BACKTESTING: {ns_parser.n_length} Moving Average on {s_ticker} - {ns_parser.n_days} days prediction" - ) - else: - plt.title( - f"{ns_parser.n_length} Moving Average on {s_ticker} - {ns_parser.n_days} days prediction" - ) + plt.title(title) plt.xlim(df_stock.index[0], get_next_stock_market_days(df_pred.index[-1], 1)[-1]) plt.xlabel("Time") plt.ylabel("Share Price ($)") diff --git a/gamestonk_terminal/prediction_techniques/sma_view.py b/gamestonk_terminal/prediction_techniques/sma_view.py index 902939018a9..b59a4c5a7ac 100644 --- a/gamestonk_terminal/prediction_techniques/sma_view.py +++ b/gamestonk_terminal/prediction_techniques/sma_view.py @@ -139,8 +139,8 @@ def simple_moving_average(other_args: List[str], s_ticker: str, df_stock: pd.Dat plt.grid(b=True, which="major", color="#666666", linestyle="-") plt.minorticks_on() plt.grid(b=True, which="minor", color="#999999", linestyle="-", alpha=0.2) - df_ma = df_stock["5. adjusted close"].rolling(window=ns_parser.n_length).mean() - plt.plot(df_ma.index, df_ma, lw=2, linestyle="--", c="tab:orange") + # df_ma = df_stock["5. adjusted close"].rolling(window=ns_parser.n_length).mean() + # plt.plot(df_ma.index, df_ma, lw=2, linestyle="--", c="tab:orange") plt.plot( [df_stock.index[-1], df_pred.index[0]], [df_stock["5. adjusted close"].values[-1], df_pred.values[0]], From 51f4e9a695c433d5114c50003ef09cc321c8de89 Mon Sep 17 00:00:00 2001 From: Jamie Date: Thu, 6 May 2021 18:50:24 -0400 Subject: [PATCH 10/29] merge main + small changes --- .../prediction_techniques/knn_view.py | 13 +++++++-- .../neural_networks_helper.py | 12 --------- .../prediction_techniques/pred_helper.py | 27 +++++++++++++++---- 3 files changed, 33 insertions(+), 19 deletions(-) delete mode 100644 gamestonk_terminal/prediction_techniques/neural_networks_helper.py diff --git a/gamestonk_terminal/prediction_techniques/knn_view.py b/gamestonk_terminal/prediction_techniques/knn_view.py index 950c57b46d2..9b56b6eca4f 100644 --- a/gamestonk_terminal/prediction_techniques/knn_view.py +++ b/gamestonk_terminal/prediction_techniques/knn_view.py @@ -63,7 +63,7 @@ def k_nearest_neighbors(other_args: List[str], s_ticker: str, df_stock: pd.DataF dest="n_inputs", type=check_positive, default=40, - help="number of days to use for prediction.", + help="number of days to use as input for prediction.", ) parser.add_argument( "-d", @@ -99,7 +99,16 @@ def k_nearest_neighbors(other_args: List[str], s_ticker: str, df_stock: pd.DataF type=valid_date, dest="s_end_date", default=None, - help="The end date (format YYYY-MM-DD) to select - Backtesting", + help="The end date (format YYYY-MM-DD) to select for testing", + ) + + parser.add_argument( + "-t", + "--test_size", + default = .2, + dest="test_size", + type = float, + help = "Percentage of data to validate in sample" ) try: diff --git a/gamestonk_terminal/prediction_techniques/neural_networks_helper.py b/gamestonk_terminal/prediction_techniques/neural_networks_helper.py deleted file mode 100644 index 125105080ea..00000000000 --- a/gamestonk_terminal/prediction_techniques/neural_networks_helper.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Helper functions""" -__docformat__ = "numpy" -from typing import List, Tuple -from datetime import datetime -import pandas as pd -import numpy as np -from matplotlib import pyplot as plt -from gamestonk_terminal.helper_funcs import ( - get_next_stock_market_days, - patch_pandas_text_adjustment, - plot_autoscale, -) diff --git a/gamestonk_terminal/prediction_techniques/pred_helper.py b/gamestonk_terminal/prediction_techniques/pred_helper.py index 71168ecb188..7c14d9a449a 100644 --- a/gamestonk_terminal/prediction_techniques/pred_helper.py +++ b/gamestonk_terminal/prediction_techniques/pred_helper.py @@ -26,10 +26,12 @@ from gamestonk_terminal.config_plot import PLOT_DPI register_matplotlib_converters() -def prepare_test_train_valid( +def prepare_train_valid_test( df_stock, parsed_end_date, n_input_days, n_predict_days, test_size ): + df_test = df_stock[df_stock.index >= parsed_end_date] + df_stock = df_stock[df_stock.index < parsed_end_date] dates = df_stock.index prices = df_stock["5. adjusted close"].values @@ -38,7 +40,12 @@ def prepare_test_train_valid( next_n_day_prices = [] next_n_day_dates = [] + test_days = [] + test_inputs = [] + test_next_n_days = [] + for idx in range(len(prices) - n_input_days - n_predict_days): + input_prices.append(prices[idx : idx + n_input_days]) input_dates.append(dates[idx : idx + n_input_days]) next_n_day_prices.append( @@ -55,13 +62,13 @@ def prepare_test_train_valid( ( X_train, - X_test, + X_valid, y_train, - y_test, + y_valid, X_dates_train, - X_dates_test, + X_dates_valid, y_dates_train, - y_dates_test, + y_dates_valid, ) = train_test_split( input_prices, next_n_day_prices, @@ -69,6 +76,16 @@ def prepare_test_train_valid( next_n_day_dates, test_size=test_size, ) + return ( + X_train, + X_valid, + y_train, + y_valid, + X_dates_train, + X_dates_valid, + y_dates_train, + y_dates_valid, + ) def get_backtesting_data( From e92d844f925e4e675f22ba66161012da9fa7f058 Mon Sep 17 00:00:00 2001 From: Jamie Date: Thu, 6 May 2021 21:06:12 -0400 Subject: [PATCH 11/29] refactor train/valid data --- .../prediction_techniques/knn_view.py | 6 +- .../neural_networks_view.py | 118 +++++---- .../prediction_techniques/pred_helper.py | 227 ++++++------------ 3 files changed, 153 insertions(+), 198 deletions(-) diff --git a/gamestonk_terminal/prediction_techniques/knn_view.py b/gamestonk_terminal/prediction_techniques/knn_view.py index 9b56b6eca4f..07b48affe9f 100644 --- a/gamestonk_terminal/prediction_techniques/knn_view.py +++ b/gamestonk_terminal/prediction_techniques/knn_view.py @@ -105,10 +105,10 @@ def k_nearest_neighbors(other_args: List[str], s_ticker: str, df_stock: pd.DataF parser.add_argument( "-t", "--test_size", - default = .2, + default=0.2, dest="test_size", - type = float, - help = "Percentage of data to validate in sample" + type=float, + help="Percentage of data to validate in sample", ) try: diff --git a/gamestonk_terminal/prediction_techniques/neural_networks_view.py b/gamestonk_terminal/prediction_techniques/neural_networks_view.py index 41130ea932c..faafe3c0d51 100644 --- a/gamestonk_terminal/prediction_techniques/neural_networks_view.py +++ b/gamestonk_terminal/prediction_techniques/neural_networks_view.py @@ -29,6 +29,7 @@ from gamestonk_terminal.prediction_techniques.pred_helper import ( print_pretty_prediction_nn, price_prediction_backtesting_color, print_prediction_kpis, + prepare_scale_train_valid_test, ) from gamestonk_terminal import feature_flags as gtff @@ -143,7 +144,7 @@ def _parse_args(prog: str, description: str, other_args: List[str]): action="store", dest="n_epochs", type=check_positive, - default=200, + default=50, help="number of training epochs.", ) parser.add_argument( @@ -251,6 +252,14 @@ def _parse_args(prog: str, description: str, other_args: List[str]): default=1, help="number of loops to iterate and train models", ) + parser.add_argument( + "-v", + "--valid", + type=float, + dest="valid_split", + default=0.1, + help="Validation data split fraction", + ) try: ns_parser = parse_known_args_and_warn(parser, other_args) @@ -1016,59 +1025,80 @@ def lstm(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): return # Setup backtesting - df_stock, df_future = _setup_backtesting(df_stock, ns_parser) + # df_stock, df_future = _setup_backtesting(df_stock, ns_parser) # Pre-process data - scaler, stock_train_data, stock_x, stock_y = _preprocess_split( - df_stock, ns_parser - ) - stock_x = np.reshape(stock_x, (stock_x.shape[0], stock_x.shape[1], 1)) - stock_y = np.reshape(stock_y, (stock_y.shape[0], stock_y.shape[1], 1)) + # scaler, stock_train_data, stock_x, stock_y = _preprocess_split( + # df_stock, ns_parser + # ) + # stock_x = np.reshape(stock_x, (stock_x.shape[0], stock_x.shape[1], 1)) + # stock_y = np.reshape(stock_y, (stock_y.shape[0], stock_y.shape[1], 1)) + ( + X_train, + X_test, + y_train, + y_test, + X_dates_train, + X_dates_test, + y_dates_train, + y_dates_test, + test_data, + dates_test, + scaler, + ) = prepare_scale_train_valid_test(df_stock, ns_parser) # Build Neural Network model model = build_neural_network_model( cfg_nn_models.Long_Short_Term_Memory, ns_parser.n_inputs, ns_parser.n_days ) model.compile(optimizer=ns_parser.s_optimizer, loss=ns_parser.s_loss) - - for idx_loop in range(ns_parser.n_loops): - # Train our model - model.fit( - stock_x, - stock_y, - epochs=ns_parser.n_epochs, - batch_size=ns_parser.n_batch_size, - verbose=1, - ) - print("") - - print(model.summary()) - print("") - - # Prediction - yhat = model.predict( - stock_train_data[-ns_parser.n_inputs :].reshape( - 1, ns_parser.n_inputs, 1 - ), - verbose=0, - ) - - if idx_loop == 0: - # Re-scale the data back, plot, and print the results - df_pred = _rescale_data( - df_stock, ns_parser, scaler, yhat, idx_loop - ).to_frame() - else: - df_pred = df_pred.join( - _rescale_data( - df_stock, ns_parser, scaler, yhat, idx_loop - ).to_frame() - ) - - _plot_and_print_results( - df_stock, ns_parser, df_future, df_pred, "LSTM", s_ticker + print(X_train.shape, X_test.shape) + model.fit( + X_train.reshape(X_train.shape[0], X_train.shape[1], 1), + y_train, + epochs=ns_parser.n_epochs, + verbose=True, + validation_data=(X_test, y_test), ) + # for idx_loop in range(ns_parser.n_loops): + # Train our model + # model.fit( + # stock_x, + # stock_y, + # epochs=ns_parser.n_epochs, + # batch_size=ns_parser.n_batch_size, + # verbose=1, + # ) + # print("") + + # print(model.summary()) + # print("") + + # # Prediction + # yhat = model.predict( + # stock_train_data[-ns_parser.n_inputs :].reshape( + # 1, ns_parser.n_inputs, 1 + # ), + # verbose=0, + # ) + + # if idx_loop == 0: + # Re-scale the data back, plot, and print the results + # df_pred = _rescale_data( + # df_stock, ns_parser, scaler, yhat, idx_loop + # ).to_frame() + # else: + # df_pred = df_pred.join( + # _rescale_data( + # df_stock, ns_parser, scaler, yhat, idx_loop + # ).to_frame() + # ) + + # _plot_and_print_results( + # df_stock, ns_parser, df_future, df_pred, "LSTM", s_ticker + # ) + except Exception as e: print(e) traceback.print_exc() diff --git a/gamestonk_terminal/prediction_techniques/pred_helper.py b/gamestonk_terminal/prediction_techniques/pred_helper.py index 7c14d9a449a..72f7071d970 100644 --- a/gamestonk_terminal/prediction_techniques/pred_helper.py +++ b/gamestonk_terminal/prediction_techniques/pred_helper.py @@ -1,8 +1,9 @@ """ Prediction helper functions """ __docformat__ = "numpy" +import argparse from typing import Tuple -from datetime import datetime +from datetime import datetime, timedelta import numpy as np import pandas as pd from pandas.plotting import register_matplotlib_converters @@ -14,7 +15,7 @@ from sklearn.metrics import ( mean_squared_error, ) from sklearn.model_selection import train_test_split - +from sklearn.preprocessing import StandardScaler, MinMaxScaler, Normalizer from gamestonk_terminal.helper_funcs import ( get_next_stock_market_days, patch_pandas_text_adjustment, @@ -26,26 +27,72 @@ from gamestonk_terminal.config_plot import PLOT_DPI register_matplotlib_converters() -def prepare_train_valid_test( - df_stock, parsed_end_date, n_input_days, n_predict_days, test_size +def prepare_scale_train_valid_test( + df_stock: pd.DataFrame, ns_parser: argparse.Namespace ): + """ + Prepare and scale train, validate and test data. + Parameters + ---------- + df_stock: pd.DataFrame + Dataframe of stock prices + ns_parser: argparse.Namespace + Parsed arguments + Returns + ------- + X_train: np.ndarray + Array of training data. Shape (# samples, n_inputs, 1) + X_test: np.ndarray + Array of validation data. Shape (#samples, n_inputs, 1) + y_train: np.ndarray + Array of training outputs. Shape (#samples, n_days) + y_test: np.ndarray + Array of validation outputs. Shape (#samples, n_days) + X_dates_train: np.ndarray + Array of dates for X_train + X_dates_test: np.ndarray + Array of dates for X_test + y_dates_train: np.ndarray + Array of dates for y_train + y_dates_test: np.ndarray + Array of dates for y)test + test_data: np.ndarray + Array of prices after the specified end date + dates_test: np.ndarray + Array of dates after specified end date + scaler: + Fitted preprocesser + """ + parsed_end_date = ns_parser.s_end_date or df_stock.index[-1]-timedelta(days=3) + n_input_days = ns_parser.n_inputs + n_predict_days = ns_parser.n_days + test_size = ns_parser.valid_split - df_test = df_stock[df_stock.index >= parsed_end_date] - df_stock = df_stock[df_stock.index < parsed_end_date] - dates = df_stock.index - prices = df_stock["5. adjusted close"].values + # Pre-process data + if ns_parser.s_preprocessing == "standardization": + scaler = StandardScaler() + + elif ns_parser.s_preprocessing == "minmax": + scaler = MinMaxScaler() + + elif ns_parser.s_preprocessing == "normalization": + scaler = Normalizer() + + test_data = df_stock[df_stock.index > parsed_end_date] + train_data = df_stock[df_stock.index < parsed_end_date] + dates = train_data.index + dates_test = test_data.index + scaler = scaler + train_data = scaler.fit_transform(train_data.values.reshape(-1, 1)) + test_data = scaler.transform(test_data.values.reshape(-1, 1)) + prices = train_data input_dates = [] input_prices = [] next_n_day_prices = [] next_n_day_dates = [] - test_days = [] - test_inputs = [] - test_next_n_days = [] - for idx in range(len(prices) - n_input_days - n_predict_days): - input_prices.append(prices[idx : idx + n_input_days]) input_dates.append(dates[idx : idx + n_input_days]) next_n_day_prices.append( @@ -55,20 +102,20 @@ def prepare_train_valid_test( dates[idx + n_input_days : idx + n_input_days + n_predict_days] ) - input_dates = np.asarray(input_dates) - input_prices = np.asarray(input_prices) - next_n_day_prices = np.asarray(next_n_day_prices) - next_n_day_dates = np.asarray(next_n_day_dates) + input_dates = np.array(input_dates) + input_prices = np.array(input_prices) + next_n_day_prices = np.array(next_n_day_prices) + next_n_day_dates = np.array(next_n_day_dates) ( X_train, - X_valid, + X_test, y_train, - y_valid, + y_test, X_dates_train, - X_dates_valid, + X_dates_test, y_dates_train, - y_dates_valid, + y_dates_test, ) = train_test_split( input_prices, next_n_day_prices, @@ -76,15 +123,19 @@ def prepare_train_valid_test( next_n_day_dates, test_size=test_size, ) + return ( X_train, - X_valid, + X_test, y_train, - y_valid, + y_test, X_dates_train, - X_dates_valid, + X_dates_test, y_dates_train, - y_dates_valid, + y_dates_test, + test_data, + dates_test, + scaler, ) @@ -144,132 +195,6 @@ def get_backtesting_data( return df_future, df_stock, True -def plot_pred( - df_stock: pd.DataFrame, - df_future: pd.DataFrame, - df_pred: pd.DataFrame, - title: str, - bt_flag: bool, -): - plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) - plt.plot(df_stock.index, df_stock["5. adjusted close"], lw=2) - plt.title(title) - plt.xlim(df_stock.index[0], get_next_stock_market_days(df_pred.index[-1], 1)[-1]) - plt.xlabel("Time") - plt.ylabel("Share Price ($)") - plt.grid(b=True, which="major", color="#666666", linestyle="-") - plt.minorticks_on() - plt.grid(b=True, which="minor", color="#999999", linestyle="-", alpha=0.2) - df_ma = df_stock["5. adjusted close"].rolling(window=ns_parser.n_length).mean() - plt.plot(df_ma.index, df_ma, lw=2, linestyle="--", c="tab:orange") - plt.plot( - [df_stock.index[-1], df_pred.index[0]], - [df_stock["5. adjusted close"].values[-1], df_pred.values[0]], - lw=1, - c="tab:green", - linestyle="--", - ) - plt.plot(df_pred.index, df_pred, lw=2, c="tab:green") - plt.axvspan( - df_stock.index[-1], df_pred.index[-1], facecolor="tab:orange", alpha=0.2 - ) - _, _, ymin, ymax = plt.axis() - plt.vlines(df_stock.index[-1], ymin, ymax, linewidth=1, linestyle="--", color="k") - - if bt_flag: - plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) - plt.subplot(211) - plt.plot( - df_future.index, - df_future["5. adjusted close"], - lw=2, - c="tab:blue", - ls="--", - ) - plt.plot(df_pred.index, df_pred, lw=2, c="green") - plt.scatter(df_future.index, df_future["5. adjusted close"], c="tab:blue", lw=3) - plt.plot( - [df_stock.index[-1], df_future.index[0]], - [ - df_stock["5. adjusted close"].values[-1], - df_future["5. adjusted close"].values[0], - ], - lw=2, - c="tab:blue", - ls="--", - ) - plt.scatter(df_pred.index, df_pred, c="green", lw=3) - plt.plot( - [df_stock.index[-1], df_pred.index[0]], - [df_stock["5. adjusted close"].values[-1], df_pred.values[0]], - lw=2, - c="green", - ls="--", - ) - plt.title("BACKTESTING: Real data price versus Prediction") - plt.xlim(df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)) - plt.xticks( - [df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)], - visible=True, - ) - plt.ylabel("Share Price ($)") - plt.grid(b=True, which="major", color="#666666", linestyle="-") - plt.minorticks_on() - plt.grid(b=True, which="minor", color="#999999", linestyle="-", alpha=0.2) - plt.legend(["Real data", "Prediction data"]) - plt.xticks([]) - - plt.subplot(212) - plt.axhline(y=0, color="k", linestyle="--", linewidth=2) - plt.plot( - df_future.index, - 100 - * (df_pred.values - df_future["5. adjusted close"].values) - / df_future["5. adjusted close"].values, - lw=2, - c="red", - ) - plt.scatter( - df_future.index, - 100 - * (df_pred.values - df_future["5. adjusted close"].values) - / df_future["5. adjusted close"].values, - c="red", - lw=5, - ) - plt.title("BACKTESTING: Error between Real data and Prediction [%]") - plt.plot( - [df_stock.index[-1], df_future.index[0]], - [ - 0, - 100 - * (df_pred.values[0] - df_future["5. adjusted close"].values[0]) - / df_future["5. adjusted close"].values[0], - ], - lw=2, - ls="--", - c="red", - ) - plt.xlim(df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)) - plt.xticks( - [df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)], - visible=True, - ) - plt.xlabel("Time") - plt.ylabel("Prediction Error (%)") - plt.grid(b=True, which="major", color="#666666", linestyle="-") - plt.minorticks_on() - plt.grid(b=True, which="minor", color="#999999", linestyle="-", alpha=0.2) - plt.legend(["Real data", "Prediction data"]) - - if gtff.USE_ION: - plt.ion() - - plt.show() - - pass - - def price_prediction_color(val: float, last_val: float) -> str: """Set prediction to be a colored string""" if float(val) > last_val: From 6706ace1c8b75e5c25dc657ee305f5fbb17c0cdd Mon Sep 17 00:00:00 2001 From: Jamie Date: Fri, 7 May 2021 15:41:10 -0400 Subject: [PATCH 12/29] refactor train/test --- .../prediction_techniques/neural_networks_view.py | 10 ++++++++-- .../prediction_techniques/pred_helper.py | 5 ++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/gamestonk_terminal/prediction_techniques/neural_networks_view.py b/gamestonk_terminal/prediction_techniques/neural_networks_view.py index faafe3c0d51..5dcdbc0cb49 100644 --- a/gamestonk_terminal/prediction_techniques/neural_networks_view.py +++ b/gamestonk_terminal/prediction_techniques/neural_networks_view.py @@ -1023,7 +1023,6 @@ def lstm(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): ) if not ns_parser: return - # Setup backtesting # df_stock, df_future = _setup_backtesting(df_stock, ns_parser) @@ -1046,7 +1045,7 @@ def lstm(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): test_data, dates_test, scaler, - ) = prepare_scale_train_valid_test(df_stock, ns_parser) + ) = prepare_scale_train_valid_test(df_stock["5. adjusted close"], ns_parser) # Build Neural Network model model = build_neural_network_model( cfg_nn_models.Long_Short_Term_Memory, ns_parser.n_inputs, ns_parser.n_days @@ -1061,6 +1060,13 @@ def lstm(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): validation_data=(X_test, y_test), ) + preds = model.predict(X_test.reshape(X_test.shape[0], X_test.shape[1], 1)) + + plt.scatter(df_stock.index, df_stock["5. adjusted close"].values, s= 3) + for i in range(len(y_test)): + plt.plot(y_dates_test[i], scaler.inverse_transform(preds[i].reshape(-1,1)), "r", lw=3) + plt.fill_between(y_dates_test[i]) + plt.show() # for idx_loop in range(ns_parser.n_loops): # Train our model # model.fit( diff --git a/gamestonk_terminal/prediction_techniques/pred_helper.py b/gamestonk_terminal/prediction_techniques/pred_helper.py index 72f7071d970..5f5757d2b1d 100644 --- a/gamestonk_terminal/prediction_techniques/pred_helper.py +++ b/gamestonk_terminal/prediction_techniques/pred_helper.py @@ -102,10 +102,10 @@ def prepare_scale_train_valid_test( dates[idx + n_input_days : idx + n_input_days + n_predict_days] ) - input_dates = np.array(input_dates) + input_dates = np.asarray(input_dates) input_prices = np.array(input_prices) next_n_day_prices = np.array(next_n_day_prices) - next_n_day_dates = np.array(next_n_day_dates) + next_n_day_dates = np.asarray(next_n_day_dates) ( X_train, @@ -123,7 +123,6 @@ def prepare_scale_train_valid_test( next_n_day_dates, test_size=test_size, ) - return ( X_train, X_test, From 4691e4db70de63b69c81598708cd619d89dbf4cb Mon Sep 17 00:00:00 2001 From: Jamie Date: Sat, 8 May 2021 12:08:02 -0400 Subject: [PATCH 13/29] stopping point --- .../neural_networks_view.py | 35 +++++++------------ .../prediction_techniques/pred_helper.py | 20 ++++++----- 2 files changed, 24 insertions(+), 31 deletions(-) diff --git a/gamestonk_terminal/prediction_techniques/neural_networks_view.py b/gamestonk_terminal/prediction_techniques/neural_networks_view.py index 5dcdbc0cb49..2d149a5b7d2 100644 --- a/gamestonk_terminal/prediction_techniques/neural_networks_view.py +++ b/gamestonk_terminal/prediction_techniques/neural_networks_view.py @@ -1023,27 +1023,17 @@ def lstm(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): ) if not ns_parser: return - # Setup backtesting - # df_stock, df_future = _setup_backtesting(df_stock, ns_parser) - - # Pre-process data - # scaler, stock_train_data, stock_x, stock_y = _preprocess_split( - # df_stock, ns_parser - # ) - # stock_x = np.reshape(stock_x, (stock_x.shape[0], stock_x.shape[1], 1)) - # stock_y = np.reshape(stock_y, (stock_y.shape[0], stock_y.shape[1], 1)) - ( X_train, - X_test, + X_valid, y_train, - y_test, + y_valid, X_dates_train, - X_dates_test, + X_dates_valid, y_dates_train, - y_dates_test, - test_data, - dates_test, + y_dates_valid, + forecast_data, + dates_valid, scaler, ) = prepare_scale_train_valid_test(df_stock["5. adjusted close"], ns_parser) # Build Neural Network model @@ -1051,21 +1041,22 @@ def lstm(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): cfg_nn_models.Long_Short_Term_Memory, ns_parser.n_inputs, ns_parser.n_days ) model.compile(optimizer=ns_parser.s_optimizer, loss=ns_parser.s_loss) - print(X_train.shape, X_test.shape) + print(X_train.shape, X_valid.shape) model.fit( X_train.reshape(X_train.shape[0], X_train.shape[1], 1), y_train, epochs=ns_parser.n_epochs, verbose=True, - validation_data=(X_test, y_test), + validation_data=(X_valid, y_valid), ) - preds = model.predict(X_test.reshape(X_test.shape[0], X_test.shape[1], 1)) + preds = model.predict(X_valid.reshape(X_valid.shape[0], X_valid.shape[1], 1)) plt.scatter(df_stock.index, df_stock["5. adjusted close"].values, s= 3) - for i in range(len(y_test)): - plt.plot(y_dates_test[i], scaler.inverse_transform(preds[i].reshape(-1,1)), "r", lw=3) - plt.fill_between(y_dates_test[i]) + for i in range(len(y_valid)): + plt.plot(y_dates_valid[i], scaler.inverse_transform(preds[i].reshape(-1,1)), "r", lw=3) + plt.fill_between(y_dates_valid[i], scaler.inverse_transform(preds[i].reshape(-1,1)), + scaler.inverse_transform(y_valid[i].reshape(-1,1))) plt.show() # for idx_loop in range(ns_parser.n_loops): # Train our model diff --git a/gamestonk_terminal/prediction_techniques/pred_helper.py b/gamestonk_terminal/prediction_techniques/pred_helper.py index 5f5757d2b1d..4258863570e 100644 --- a/gamestonk_terminal/prediction_techniques/pred_helper.py +++ b/gamestonk_terminal/prediction_techniques/pred_helper.py @@ -63,9 +63,10 @@ def prepare_scale_train_valid_test( scaler: Fitted preprocesser """ - parsed_end_date = ns_parser.s_end_date or df_stock.index[-1]-timedelta(days=3) + n_input_days = ns_parser.n_inputs n_predict_days = ns_parser.n_days + parsed_end_date = (ns_parser.s_end_date or df_stock.index[-1]) - timedelta(days= n_input_days) test_size = ns_parser.valid_split # Pre-process data @@ -78,6 +79,7 @@ def prepare_scale_train_valid_test( elif ns_parser.s_preprocessing == "normalization": scaler = Normalizer() + forecast_input = 1 test_data = df_stock[df_stock.index > parsed_end_date] train_data = df_stock[df_stock.index < parsed_end_date] dates = train_data.index @@ -109,13 +111,13 @@ def prepare_scale_train_valid_test( ( X_train, - X_test, + X_valid, y_train, - y_test, + y_valid, X_dates_train, - X_dates_test, + X_dates_valid, y_dates_train, - y_dates_test, + y_dates_valid, ) = train_test_split( input_prices, next_n_day_prices, @@ -125,13 +127,13 @@ def prepare_scale_train_valid_test( ) return ( X_train, - X_test, + X_valid, y_train, - y_test, + y_valid, X_dates_train, - X_dates_test, + X_dates_valid, y_dates_train, - y_dates_test, + y_dates_valid, test_data, dates_test, scaler, From a8223abb5a997d4c8e836fdec526b4fe2ef74303 Mon Sep 17 00:00:00 2001 From: Jamie Date: Sun, 9 May 2021 20:00:10 -0400 Subject: [PATCH 14/29] update helper functions and reintroduce loops --- .../neural_networks_view.py | 99 +++++++++---------- .../prediction_techniques/pred_helper.py | 97 +++++++++++++++++- 2 files changed, 136 insertions(+), 60 deletions(-) diff --git a/gamestonk_terminal/prediction_techniques/neural_networks_view.py b/gamestonk_terminal/prediction_techniques/neural_networks_view.py index 2d149a5b7d2..9a606154f4a 100644 --- a/gamestonk_terminal/prediction_techniques/neural_networks_view.py +++ b/gamestonk_terminal/prediction_techniques/neural_networks_view.py @@ -30,6 +30,8 @@ from gamestonk_terminal.prediction_techniques.pred_helper import ( price_prediction_backtesting_color, print_prediction_kpis, prepare_scale_train_valid_test, + forecast, + plot_data_predictions, ) from gamestonk_terminal import feature_flags as gtff @@ -1032,69 +1034,56 @@ def lstm(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): X_dates_valid, y_dates_train, y_dates_valid, - forecast_data, - dates_valid, + forecast_data_input, + dates_forecast_input, scaler, ) = prepare_scale_train_valid_test(df_stock["5. adjusted close"], ns_parser) - # Build Neural Network model - model = build_neural_network_model( - cfg_nn_models.Long_Short_Term_Memory, ns_parser.n_inputs, ns_parser.n_days + print( + f"Training on {X_train.shape[0]} samples. Using {X_valid.shape[0]} samples for validation. {ns_parser.n_loops} loops" ) - model.compile(optimizer=ns_parser.s_optimizer, loss=ns_parser.s_loss) - print(X_train.shape, X_valid.shape) - model.fit( - X_train.reshape(X_train.shape[0], X_train.shape[1], 1), - y_train, - epochs=ns_parser.n_epochs, - verbose=True, - validation_data=(X_valid, y_valid), + future_dates = get_next_stock_market_days( + dates_forecast_input[-1], n_next_days=ns_parser.n_days ) - preds = model.predict(X_valid.reshape(X_valid.shape[0], X_valid.shape[1], 1)) + preds = np.zeros((ns_parser.n_loops, X_valid.shape[0], ns_parser.n_days)) + forecast_data = np.zeros((ns_parser.n_loops, ns_parser.n_days)) + for i in range(ns_parser.n_loops): + # Build Neural Network model + model = build_neural_network_model( + cfg_nn_models.Long_Short_Term_Memory, + ns_parser.n_inputs, + ns_parser.n_days, + ) + model.compile(optimizer=ns_parser.s_optimizer, loss=ns_parser.s_loss) - plt.scatter(df_stock.index, df_stock["5. adjusted close"].values, s= 3) - for i in range(len(y_valid)): - plt.plot(y_dates_valid[i], scaler.inverse_transform(preds[i].reshape(-1,1)), "r", lw=3) - plt.fill_between(y_dates_valid[i], scaler.inverse_transform(preds[i].reshape(-1,1)), - scaler.inverse_transform(y_valid[i].reshape(-1,1))) - plt.show() - # for idx_loop in range(ns_parser.n_loops): - # Train our model - # model.fit( - # stock_x, - # stock_y, - # epochs=ns_parser.n_epochs, - # batch_size=ns_parser.n_batch_size, - # verbose=1, - # ) - # print("") + model.fit( + X_train.reshape(X_train.shape[0], X_train.shape[1], 1), + y_train, + epochs=ns_parser.n_epochs, + verbose=True, + batch_size=ns_parser.n_batch_size, + validation_data=(X_valid, y_valid), + ) - # print(model.summary()) - # print("") + preds[i] = model.predict( + X_valid.reshape(X_valid.shape[0], X_valid.shape[1], 1) + ).reshape(X_valid.shape[0], ns_parser.n_days) + forecast_data[i] = forecast( + forecast_data_input, future_dates, model, scaler + ).values.flat - # # Prediction - # yhat = model.predict( - # stock_train_data[-ns_parser.n_inputs :].reshape( - # 1, ns_parser.n_inputs, 1 - # ), - # verbose=0, - # ) - - # if idx_loop == 0: - # Re-scale the data back, plot, and print the results - # df_pred = _rescale_data( - # df_stock, ns_parser, scaler, yhat, idx_loop - # ).to_frame() - # else: - # df_pred = df_pred.join( - # _rescale_data( - # df_stock, ns_parser, scaler, yhat, idx_loop - # ).to_frame() - # ) - - # _plot_and_print_results( - # df_stock, ns_parser, df_future, df_pred, "LSTM", s_ticker - # ) + forecast_data_df = pd.DataFrame(forecast_data.T, index=future_dates) + print(forecast_data_df) + plot_data_predictions( + df_stock, + preds.median(axis=0), + y_valid, + y_dates_valid, + scaler, + f"LSTM Model on {s_ticker}", + forecast_data_df, + ) + print("") except Exception as e: print(e) diff --git a/gamestonk_terminal/prediction_techniques/pred_helper.py b/gamestonk_terminal/prediction_techniques/pred_helper.py index 4258863570e..e7f42abc582 100644 --- a/gamestonk_terminal/prediction_techniques/pred_helper.py +++ b/gamestonk_terminal/prediction_techniques/pred_helper.py @@ -66,7 +66,6 @@ def prepare_scale_train_valid_test( n_input_days = ns_parser.n_inputs n_predict_days = ns_parser.n_days - parsed_end_date = (ns_parser.s_end_date or df_stock.index[-1]) - timedelta(days= n_input_days) test_size = ns_parser.valid_split # Pre-process data @@ -79,9 +78,17 @@ def prepare_scale_train_valid_test( elif ns_parser.s_preprocessing == "normalization": scaler = Normalizer() - forecast_input = 1 - test_data = df_stock[df_stock.index > parsed_end_date] - train_data = df_stock[df_stock.index < parsed_end_date] + # Test data is used for forcasting. Takes the last n_input_days data points. + # These points are not fed into training + + if ns_parser.s_end_date: + df_stock = df_stock[df_stock.index <= ns_parser.s_end_date] + if n_input_days + n_predict_days < df_stock.shape[0]: + print("Cannot train enough input days to predict with loaded dataframe") + return + + test_data = df_stock.iloc[-n_input_days:] + train_data = df_stock.iloc[:-n_input_days] dates = train_data.index dates_test = test_data.index scaler = scaler @@ -140,6 +147,86 @@ def prepare_scale_train_valid_test( ) +def forecast(input_values, future_dates, model, scaler): + + future_values = scaler.inverse_transform( + model.predict(input_values.reshape(1, -1, 1)).reshape(-1, 1) + ) + df_future = pd.DataFrame( + future_values, index=future_dates, columns=["Predicted Price"] + ) + return df_future + + +def plot_data_predictions( + df_stock, preds, y_valid, y_dates_valid, scaler, title, forecast_data +): + + plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) + plt.plot( + df_stock.index, + df_stock["5. adjusted close"].values, + "-ob", + lw=1, + ms=2, + label="Prices", + ) + for i in range(len(y_valid) - 1): + plt.plot( + y_dates_valid[i], + scaler.inverse_transform(preds[i].reshape(-1, 1)), + "r", + lw=3, + ) + plt.fill_between( + y_dates_valid[i], + scaler.inverse_transform(preds[i].reshape(-1, 1)).ravel(), + scaler.inverse_transform(y_valid[i].reshape(-1, 1)).ravel(), + color="k", + alpha=0.2, + ) + plt.plot( + y_dates_valid[-1], + scaler.inverse_transform(preds[-1].reshape(-1, 1)), + "r", + lw=3, + label="Predicions", + ) + plt.fill_between( + y_dates_valid[-1], + scaler.inverse_transform(preds[-1].reshape(-1, 1)).ravel(), + scaler.inverse_transform(y_valid[-1].reshape(-1, 1)).ravel(), + color="k", + alpha=0.2, + ) + plt.axvspan( + df_stock.index[-1], forecast_data.index[-1], facecolor="tab:orange", alpha=0.2 + ) + _, _, ymin, ymax = plt.axis() + plt.vlines( + df_stock.index[-1], + ymin, + ymax, + colors="k", + linewidth=3, + linestyle="--", + color="k", + ) + plt.plot(forecast_data.index, forecast_data.values, "-ok", ms=5) + plt.legend(loc=0) + plt.xlim(df_stock.index[0], forecast_data.index[-1] + timedelta(days=1)) + plt.xlabel("Time") + plt.ylabel("Share Price ($)") + plt.grid(b=True, which="major", color="#666666", linestyle="-") + plt.minorticks_on() + plt.grid(b=True, which="minor", color="#999999", linestyle="-", alpha=0.2) + plt.title(title) + if gtff.USE_ION: + plt.ion() + plt.show() + print("") + + def get_backtesting_data( df_stock: pd.DataFrame, parsed_end_date: datetime, pred_days: int ) -> Tuple[pd.DataFrame, pd.DataFrame, bool]: @@ -150,7 +237,7 @@ def get_backtesting_data( df_stock: pd.DataFrame Dataframe of stock prices parsed_end_date: datetime - Date that seperates test from prediction + Date that separates test from prediction pred_days: int Number of days to predict From 0218cdd3681710df2eeab1d53803fb1fbee63367 Mon Sep 17 00:00:00 2001 From: Jamie Date: Mon, 10 May 2021 11:15:32 -0400 Subject: [PATCH 15/29] add conv1d model --- .../config_neural_network_models.py | 8 ++++ .../neural_networks_view.py | 43 ++++++++++++++++--- .../prediction_techniques/pred_controller.py | 2 + 3 files changed, 48 insertions(+), 5 deletions(-) diff --git a/gamestonk_terminal/config_neural_network_models.py b/gamestonk_terminal/config_neural_network_models.py index 9d788f9aa59..d0705ac2f54 100644 --- a/gamestonk_terminal/config_neural_network_models.py +++ b/gamestonk_terminal/config_neural_network_models.py @@ -39,3 +39,11 @@ Long_Short_Term_Memory = [ {"LSTM": {"units": 15, "activation": "tanh", "return_sequences": False}}, {"Dense": {"activation": "relu"}}, ] + +Convolutional = [ + {"Conv1D": {"filters": 20, "kernel_size": 10, "activation": "relu"}}, + {"MaxPool1D": {"pool_size": 2}}, + {"Conv1D": {"filters": 10, "kernel_size": 5, "activation": "relu"}}, + {"MaxPool1D": {"pool_size": 2}}, + {"Dense": {"activation": "relu"}}, +] diff --git a/gamestonk_terminal/prediction_techniques/neural_networks_view.py b/gamestonk_terminal/prediction_techniques/neural_networks_view.py index 9a606154f4a..57a6122ee41 100644 --- a/gamestonk_terminal/prediction_techniques/neural_networks_view.py +++ b/gamestonk_terminal/prediction_techniques/neural_networks_view.py @@ -14,7 +14,15 @@ from pandas.plotting import register_matplotlib_converters from TimeSeriesCrossValidation import splitTrain from sklearn.preprocessing import MinMaxScaler, StandardScaler, Normalizer from tensorflow.keras.models import Sequential -from tensorflow.keras.layers import LSTM, SimpleRNN, Dense, Dropout +from tensorflow.keras.layers import ( + LSTM, + SimpleRNN, + Dense, + Dropout, + Conv1D, + MaxPool1D, + AvgPool1D, +) from gamestonk_terminal.helper_funcs import ( check_positive, @@ -90,6 +98,18 @@ def build_neural_network_model( else: model.add(Dense(**d_layer["Dense"])) + # Conv1D Layer + elif str(*d_layer) == "Conv1D": + if idx_layer == 0: + model.add(Conv1D(**d_layer["Conv1D"], input_shape=(n_inputs, 1))) + else: + model.add(Conv1D(**d_layer["Conv1D"])) + # Max Pooling Layer for after Conv Layer + elif str(*d_layer) == "MaxPool1D": + model.add(MaxPool1D(**d_layer["MaxPool1D"])) + # Allow for if user wants to do average pooling + elif str(*d_layer) == "AvgPool1D": + model.add(MaxPool1D(**d_layer["AvgPool1D"])) # Dropout (Regularization) elif str(*d_layer) == "Dropout": model.add(Dropout(**d_layer["Dropout"])) @@ -1013,9 +1033,6 @@ def lstm(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): df_stock: pd.DataFrame Dataframe of stock prices - Returns - ------- - """ try: ns_parser = _parse_args( @@ -1039,7 +1056,8 @@ def lstm(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): scaler, ) = prepare_scale_train_valid_test(df_stock["5. adjusted close"], ns_parser) print( - f"Training on {X_train.shape[0]} samples. Using {X_valid.shape[0]} samples for validation. {ns_parser.n_loops} loops" + f"Training on {X_train.shape[0]} samples. Using {X_valid.shape[0]} samples for validation." + f" {ns_parser.n_loops} loops" ) future_dates = get_next_stock_market_days( dates_forecast_input[-1], n_next_days=ns_parser.n_days @@ -1092,3 +1110,18 @@ def lstm(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): finally: _restore_env() + + +def conv1d(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): + """ + Train a Long-Short-Term-Memory Neural Net (lstm) + Parameters + ---------- + other_args:List[str] + Argparse arguments + s_ticker: str + Stock ticker + df_stock: pd.DataFrame + Dataframe of stock prices + """ + pass diff --git a/gamestonk_terminal/prediction_techniques/pred_controller.py b/gamestonk_terminal/prediction_techniques/pred_controller.py index b0df8b0cf02..9c491f6f2f8 100644 --- a/gamestonk_terminal/prediction_techniques/pred_controller.py +++ b/gamestonk_terminal/prediction_techniques/pred_controller.py @@ -40,6 +40,7 @@ class PredictionTechniquesController: "mlp", "rnn", "lstm", + "conv1d", ] if gtff.ENABLE_FBPROPHET: @@ -91,6 +92,7 @@ class PredictionTechniquesController: print(" mlp MultiLayer Perceptron") print(" rnn Recurrent Neural Network") print(" lstm Long-Short Term Memory") + print(" conv1d 1D Convolutional Neural Network") if gtff.ENABLE_FBPROPHET: print(" prophet Facebook's prophet prediction") print("") From cafbb058b69489690b189809b97076e67f624e84 Mon Sep 17 00:00:00 2001 From: Jamie Date: Mon, 10 May 2021 12:40:52 -0400 Subject: [PATCH 16/29] Finish first refactor for LSTM model including loops and double checking the set end date. --- .../neural_networks_view.py | 6 ++- .../prediction_techniques/pred_helper.py | 50 ++++++++++++++----- 2 files changed, 43 insertions(+), 13 deletions(-) diff --git a/gamestonk_terminal/prediction_techniques/neural_networks_view.py b/gamestonk_terminal/prediction_techniques/neural_networks_view.py index 57a6122ee41..a4afc318bec 100644 --- a/gamestonk_terminal/prediction_techniques/neural_networks_view.py +++ b/gamestonk_terminal/prediction_techniques/neural_networks_view.py @@ -1054,7 +1054,10 @@ def lstm(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): forecast_data_input, dates_forecast_input, scaler, + is_error ) = prepare_scale_train_valid_test(df_stock["5. adjusted close"], ns_parser) + if is_error: + return print( f"Training on {X_train.shape[0]} samples. Using {X_valid.shape[0]} samples for validation." f" {ns_parser.n_loops} loops" @@ -1094,12 +1097,13 @@ def lstm(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): print(forecast_data_df) plot_data_predictions( df_stock, - preds.median(axis=0), + np.median(preds,axis=0), y_valid, y_dates_valid, scaler, f"LSTM Model on {s_ticker}", forecast_data_df, + ns_parser.n_loops ) print("") diff --git a/gamestonk_terminal/prediction_techniques/pred_helper.py b/gamestonk_terminal/prediction_techniques/pred_helper.py index e7f42abc582..e10c5625e9a 100644 --- a/gamestonk_terminal/prediction_techniques/pred_helper.py +++ b/gamestonk_terminal/prediction_techniques/pred_helper.py @@ -83,9 +83,11 @@ def prepare_scale_train_valid_test( if ns_parser.s_end_date: df_stock = df_stock[df_stock.index <= ns_parser.s_end_date] - if n_input_days + n_predict_days < df_stock.shape[0]: - print("Cannot train enough input days to predict with loaded dataframe") - return + print(df_stock.shape) + print(n_input_days + n_predict_days) + if n_input_days + n_predict_days > df_stock.shape[0]: + print("Cannot train enough input days to predict with loaded dataframe\n") + return None,None,None,None,None,None,None,None,None,None,None,True test_data = df_stock.iloc[-n_input_days:] train_data = df_stock.iloc[:-n_input_days] @@ -144,6 +146,7 @@ def prepare_scale_train_valid_test( test_data, dates_test, scaler, + False ) @@ -159,7 +162,7 @@ def forecast(input_values, future_dates, model, scaler): def plot_data_predictions( - df_stock, preds, y_valid, y_dates_valid, scaler, title, forecast_data + df_stock, preds, y_valid, y_dates_valid, scaler, title, forecast_data, n_loops ): plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) @@ -172,19 +175,34 @@ def plot_data_predictions( label="Prices", ) for i in range(len(y_valid) - 1): + + y_pred = scaler.inverse_transform(preds[i].reshape(-1, 1)).ravel() + y_act = scaler.inverse_transform(y_valid[i].reshape(-1, 1)).ravel() + plt.plot( y_dates_valid[i], - scaler.inverse_transform(preds[i].reshape(-1, 1)), + y_pred, "r", lw=3, ) plt.fill_between( y_dates_valid[i], - scaler.inverse_transform(preds[i].reshape(-1, 1)).ravel(), - scaler.inverse_transform(y_valid[i].reshape(-1, 1)).ravel(), - color="k", + y_pred, + y_act, + where=(y_pred < y_act), + color="r", alpha=0.2, ) + plt.fill_between( + y_dates_valid[i], + y_pred, + y_act, + where=(y_pred > y_act), + color="g", + alpha=0.2, + ) + + # Leave this one out of the loop so that the legend doesnt get overpopulated with "Predictions" plt.plot( y_dates_valid[-1], scaler.inverse_transform(preds[-1].reshape(-1, 1)), @@ -200,11 +218,11 @@ def plot_data_predictions( alpha=0.2, ) plt.axvspan( - df_stock.index[-1], forecast_data.index[-1], facecolor="tab:orange", alpha=0.2 + forecast_data.index[0]-timedelta(days=1), forecast_data.index[-1], facecolor="tab:orange", alpha=0.2 ) _, _, ymin, ymax = plt.axis() plt.vlines( - df_stock.index[-1], + forecast_data.index[0]-timedelta(days=1), ymin, ymax, colors="k", @@ -212,7 +230,15 @@ def plot_data_predictions( linestyle="--", color="k", ) - plt.plot(forecast_data.index, forecast_data.values, "-ok", ms=5) + if n_loops == 1: + plt.plot(forecast_data.index, forecast_data.values, "-ok", ms=5, label="Forecast") + else: + plt.plot(forecast_data.index, forecast_data.median(axis=1).values, "-ok", ms=5, label="Forecast") + plt.fill_between(forecast_data.index, + forecast_data.quantile(.25,axis=1).values, + forecast_data.quantile(.75,axis=1).values, + color="c", + alpha = .3) plt.legend(loc=0) plt.xlim(df_stock.index[0], forecast_data.index[-1] + timedelta(days=1)) plt.xlabel("Time") @@ -226,7 +252,7 @@ def plot_data_predictions( plt.show() print("") - +# Deprecated def get_backtesting_data( df_stock: pd.DataFrame, parsed_end_date: datetime, pred_days: int ) -> Tuple[pd.DataFrame, pd.DataFrame, bool]: From 1573f2d558658593213d5b35f5de97579552bc22 Mon Sep 17 00:00:00 2001 From: Jamie Date: Mon, 10 May 2021 16:46:01 -0400 Subject: [PATCH 17/29] Added 1d cnn model to prediction options. Rearranged functions from view to helper. --- .../config_neural_network_models.py | 1 + .../neural_networks_view.py | 1266 ++++------------- .../prediction_techniques/pred_controller.py | 4 + .../prediction_techniques/pred_helper.py | 358 ++++- 4 files changed, 600 insertions(+), 1029 deletions(-) diff --git a/gamestonk_terminal/config_neural_network_models.py b/gamestonk_terminal/config_neural_network_models.py index d0705ac2f54..28c7181e870 100644 --- a/gamestonk_terminal/config_neural_network_models.py +++ b/gamestonk_terminal/config_neural_network_models.py @@ -45,5 +45,6 @@ Convolutional = [ {"MaxPool1D": {"pool_size": 2}}, {"Conv1D": {"filters": 10, "kernel_size": 5, "activation": "relu"}}, {"MaxPool1D": {"pool_size": 2}}, + {"Flatten": {}}, {"Dense": {"activation": "relu"}}, ] diff --git a/gamestonk_terminal/prediction_techniques/neural_networks_view.py b/gamestonk_terminal/prediction_techniques/neural_networks_view.py index a4afc318bec..c7420bee22b 100644 --- a/gamestonk_terminal/prediction_techniques/neural_networks_view.py +++ b/gamestonk_terminal/prediction_techniques/neural_networks_view.py @@ -1,18 +1,11 @@ """ Neural Networks View""" __docformat__ = "numpy" -import argparse -from typing import List, Any, Union -import datetime -import os +from typing import List, Any import traceback -from warnings import simplefilter import numpy as np -import matplotlib.pyplot as plt import pandas as pd -from pandas.plotting import register_matplotlib_converters -from TimeSeriesCrossValidation import splitTrain -from sklearn.preprocessing import MinMaxScaler, StandardScaler, Normalizer +from keras.callbacks import EarlyStopping from tensorflow.keras.models import Sequential from tensorflow.keras.layers import ( LSTM, @@ -22,46 +15,40 @@ from tensorflow.keras.layers import ( Conv1D, MaxPool1D, AvgPool1D, + Flatten, ) - -from gamestonk_terminal.helper_funcs import ( - check_positive, - parse_known_args_and_warn, - valid_date, - patch_pandas_text_adjustment, - get_next_stock_market_days, - plot_autoscale, -) - +from gamestonk_terminal.helper_funcs import get_next_stock_market_days from gamestonk_terminal.prediction_techniques.pred_helper import ( - print_pretty_prediction_nn, - price_prediction_backtesting_color, - print_prediction_kpis, prepare_scale_train_valid_test, forecast, plot_data_predictions, + parse_args, + restore_env, ) - -from gamestonk_terminal import feature_flags as gtff - -from gamestonk_terminal.config_plot import PLOT_DPI from gamestonk_terminal import config_neural_network_models as cfg_nn_models - -register_matplotlib_converters() - -os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" - -simplefilter(action="ignore", category=FutureWarning) - -# store the user's TensorFlow environment variables -ORIGINAL_TF_XLA_FLAGS = os.environ.get("TF_XLA_FLAGS") -ORIGINAL_TF_FORCE_GPU_ALLOW_GROWTH = os.environ.get("TF_FORCE_GPU_ALLOW_GROWTH") - +es = EarlyStopping(monitor="val_loss", patience=10) def build_neural_network_model( Recurrent_Neural_Network: List[Any], n_inputs: int, n_days: int ): + """ + Builds neural netfrom config_neural_network_models.py + Parameters + ---------- + Recurrent_Neural_Network: List[Any] + List of layers with parameters as a dictionary in the file + n_inputs: int + Number of days that will be fed into the NN + n_days: int + Number of days the NN wants to predict + + Returns + ------- + model: + Keras sequential model with layers from the file + + """ model = Sequential() for idx_layer, d_layer in enumerate(Recurrent_Neural_Network): @@ -109,746 +96,19 @@ def build_neural_network_model( model.add(MaxPool1D(**d_layer["MaxPool1D"])) # Allow for if user wants to do average pooling elif str(*d_layer) == "AvgPool1D": - model.add(MaxPool1D(**d_layer["AvgPool1D"])) + model.add(AvgPool1D(**d_layer["AvgPool1D"])) # Dropout (Regularization) elif str(*d_layer) == "Dropout": model.add(Dropout(**d_layer["Dropout"])) - + # Flatten layer for Convolutions + elif str(*d_layer) == "Flatten": + model.add(Flatten()) else: print(f"Incorrect neuron type: {str(*d_layer)}") return model -def _parse_args(prog: str, description: str, other_args: List[str]): - """ - Create an argparser and parse other_args. Will print help if user requests it. - Parameters - ---------- - prog: str - Program for argparser - description: str - Description for argparser - other_args - Argparse arguments to pass - Returns - ------- - ns_parser: argparse.Namespace - Parsed argument parser - """ - - parser = argparse.ArgumentParser( - prog=prog, - description=description, - add_help=False, - formatter_class=argparse.RawTextHelpFormatter, # enable multiline help messages - ) - parser.add_argument( - "-d", - "--days", - action="store", - dest="n_days", - type=check_positive, - default=5, - help="prediction days.", - ) - parser.add_argument( - "-i", - "--input", - action="store", - dest="n_inputs", - type=check_positive, - default=40, - help="number of days to use for prediction.", - ) - parser.add_argument( - "--epochs", - action="store", - dest="n_epochs", - type=check_positive, - default=50, - help="number of training epochs.", - ) - parser.add_argument( - "-j", - "--jumps", - action="store", - dest="n_jumps", - type=check_positive, - default=1, - help="number of jumps in training data.", - ) - parser.add_argument( - "-p", - "--pp", - action="store", - dest="s_preprocessing", - default="minmax", - choices=["normalization", "standardization", "minmax", "none"], - help="pre-processing data.", - ) - parser.add_argument( - "-o", - "--optimizer", - action="store", - dest="s_optimizer", - default="adam", - choices=[ - "adam", - "adagrad", - "adadelta", - "adamax", - "ftrl", - "nadam", - "rmsprop", - "sgd", - ], - help="optimization technique (see https://www.tensorflow.org/api_docs/python/tf/keras/optimizers)", - ) - parser.add_argument( - "-l", - "--loss", - action="store", - dest="s_loss", - default="mae", - choices=[ - "mae", - "mape", - "mse", - "msle", - "poisson", - "logcosh", - "kld", - "hinge", - "squared_hinge", - "huber", - ], - help="loss function (see https://www.tensorflow.org/api_docs/python/tf/keras/losses)", - ) - parser.add_argument( - "-e", - "--end", - action="store", - type=valid_date, - dest="s_end_date", - default=None, - help="The end date (format YYYY-MM-DD) to select - Backtesting", - ) - parser.add_argument( - "--batch_size", - action="store", - dest="n_batch_size", - type=check_positive, - default=None, - help="batch size for model fitting (use a power of 2)", - ) - parser.add_argument( - "--xla_cpu", - action="store_true", - dest="b_xla_cpu", - default=False, - help="enable XLA for CPU (see https://www.tensorflow.org/xla)", - ) - parser.add_argument( - "--xla_gpu", - action="store_true", - dest="b_xla_gpu", - default=False, - help="enable XLA for GPU (see https://www.tensorflow.org/xla)", - ) - parser.add_argument( - "--force_gpu_allow_growth", - action="store", - dest="s_force_gpu_allow_growth", - default="true", - choices=["true", "false", "default"], - help="true: GPU memory will grow as needed. \n" - "false: TensorFlow will allocate 100%% of GPU memory. \n" - "default: usually the same as false, uses env/TensorFlow default", - ) - parser.add_argument( - "--loops", - action="store", - dest="n_loops", - type=check_positive, - default=1, - help="number of loops to iterate and train models", - ) - parser.add_argument( - "-v", - "--valid", - type=float, - dest="valid_split", - default=0.1, - help="Validation data split fraction", - ) - - try: - ns_parser = parse_known_args_and_warn(parser, other_args) - if not ns_parser: - return None - - # set xla flags if requested - xla_flags = ( - set(ORIGINAL_TF_XLA_FLAGS.split(" ")) if ORIGINAL_TF_XLA_FLAGS else set() - ) - if ns_parser.b_xla_cpu or ns_parser.b_xla_gpu: - xla_flags.add("--tf_xla_enable_xla_devices") - if ns_parser.b_xla_cpu: - xla_flags.add("--tf_xla_cpu_global_jit") - if ns_parser.b_xla_gpu: - xla_flags.add("--tf_xla_auto_jit=2") - os.environ["TF_XLA_FLAGS"] = " ".join(xla_flags) - - # set GPU memory growth flag - if ns_parser.s_force_gpu_allow_growth == "true": - os.environ["TF_FORCE_GPU_ALLOW_GROWTH"] = "true" - elif ns_parser.s_force_gpu_allow_growth == "false": - os.environ["TF_FORCE_GPU_ALLOW_GROWTH"] = "false" - - return ns_parser - - except SystemExit: - print("") - return None - - -def _restore_env(): - """Restore environment variables to original values""" - - def restore(key, value): - if value is None: - if key in os.environ: - del os.environ[key] - else: - os.environ[key] = value - - restore("TF_XLA_FLAGS", ORIGINAL_TF_XLA_FLAGS) - restore("TF_FORCE_GPU_ALLOW_GROWTH", ORIGINAL_TF_FORCE_GPU_ALLOW_GROWTH) - - -def _setup_backtesting(df_stock: pd.DataFrame, ns_parser: argparse.Namespace): - """ - Setup backtesting when end data is specified - Parameters - ---------- - df_stock: pd.DataFrame - Stock data - ns_parser: argparse.Namespace - Parsed arguments - Returns - ------- - df_stock: pd.DataFrame - Data before end date - df_future: pd.DataFrame - Data after end date - Raises - ------- - Exception - If configuration is invalid - """ - - df_future = None - if ns_parser.s_end_date: - if ns_parser.s_end_date < df_stock.index[0]: - raise Exception( - "Backtesting not allowed, since End Date is older than Start Date of historical data" - ) - - if ns_parser.s_end_date < get_next_stock_market_days( - last_stock_day=df_stock.index[0], - n_next_days=ns_parser.n_inputs + ns_parser.n_days, - )[-1]: - raise Exception( - "Backtesting not allowed, since End Date is too close to Start Date to train model" - ) - - future_index = get_next_stock_market_days( - last_stock_day=ns_parser.s_end_date, n_next_days=ns_parser.n_days - ) - - if future_index[-1] > datetime.datetime.now(): - raise Exception( - "Backtesting not allowed, since End Date + Prediction days is in the future" - ) - - df_future = df_stock[future_index[0] : future_index[-1]] - df_stock = df_stock[: ns_parser.s_end_date] - return df_stock, df_future - - -def _preprocess_split(df_stock: pd.DataFrame, ns_parser: argparse.Namespace): - """ - Preprocess and split dstock data - Parameters - ---------- - df_stock: pd.DataFrame - Dataframe of stock prices - ns_parser: argparse.Namespace - Parsed argparse arguments - - Returns - ------- - scaler: Union[MinMaxScaler, StandardScaler, Normalizer] - sklearn scaler used - stock_train_data: np.ndarray - All training data - stock_x: np.ndarray - Inputs for model - stock_y: np.ndarray - Outputs for model - Raises - ------- - Exception - Not enough training data is provided - """ - - # Pre-process data - if ns_parser.s_preprocessing == "standardization": - scaler = StandardScaler() - stock_train_data = scaler.fit_transform( - np.array(df_stock["5. adjusted close"].values.reshape(-1, 1)) - ) - elif ns_parser.s_preprocessing == "minmax": - scaler = MinMaxScaler() - stock_train_data = scaler.fit_transform( - np.array(df_stock["5. adjusted close"].values.reshape(-1, 1)) - ) - elif ns_parser.s_preprocessing == "normalization": - scaler = Normalizer() - stock_train_data = scaler.fit_transform( - np.array(df_stock["5. adjusted close"].values.reshape(-1, 1)) - ) - else: # No pre-processing - stock_train_data = np.array(df_stock["5. adjusted close"].values.reshape(-1, 1)) - - # Split training data for the neural network - stock_x, stock_y = splitTrain.split_train( - stock_train_data, - ns_parser.n_inputs, - ns_parser.n_days, - numJumps=ns_parser.n_jumps, - ) - if not stock_x: - raise Exception("Given the model parameters more training data is needed.") - - stock_x = np.array(stock_x) - stock_y = np.array(stock_y) - - return scaler, stock_train_data, stock_x, stock_y - - -def _rescale_data( - df_stock: pd.DataFrame, - ns_parser: argparse.Namespace, - scaler: Union[MinMaxScaler, StandardScaler, Normalizer], - yhat: np.ndarray, - idx_loop: int, -): - """ - Re-scale the data back and return the prediction dataframe. - Parameters - ---------- - df_stock: pd.DataFrame - Dataframe of stock prices - ns_parser: argparse.Namespace - Parsed argument parser - scaler: - One of the predefined sklearn scalers - yhat: np.ndarry - Predicted values - idx_loop: int - Loop index - - Returns - ------- - df_pred: pd.Series - Series of resclaled predictions - """ - if ( - (ns_parser.s_preprocessing == "standardization") - or (ns_parser.s_preprocessing == "normalization") - or (ns_parser.s_preprocessing == "minmax") - ): - y_pred_test_t = scaler.inverse_transform(yhat.tolist()) - else: - y_pred_test_t = yhat - - l_pred_days = get_next_stock_market_days( - last_stock_day=df_stock["5. adjusted close"].index[-1], - n_next_days=ns_parser.n_days, - ) - column_name = f"Price [{idx_loop+1}]" - df_pred = pd.Series(y_pred_test_t[0].tolist(), index=l_pred_days, name=column_name) - return df_pred - - -def _plot_and_print_results( - df_stock: pd.DataFrame, - ns_parser: argparse.Namespace, - df_future: pd.DataFrame, - df_pred: pd.DataFrame, - model_name: str, - s_ticker: str, -): - """ - Plot and print results - Parameters - ---------- - df_stock: pd.DataFrame: - Dataframe of stock prices - ns_parser: argparse.Namespace - Parsed argument parser - df_future: pd.DataFrame - Dataframe of future prices - df_pred: pd.DataFrame - Dataframe of predicted prices - model_name: str - Model Name - s_ticker: str - Ticker - - Returns - ------- - - """ - - # Plotting - plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) - plt.plot(df_stock.index, df_stock["5. adjusted close"], lw=3) - - # BACKTESTING - if ns_parser.n_loops == 1: - if ns_parser.s_end_date: - plt.title( - f"BACKTESTING: {model_name} on {s_ticker} - {ns_parser.n_days} days prediction" - ) - else: - plt.title( - f"{model_name} on {s_ticker} - {ns_parser.n_days} days prediction" - ) - else: - if ns_parser.s_end_date: - plt.title( - f"{ns_parser.n_loops} loops - BACKTESTING: {model_name} on {s_ticker}" - f" - {ns_parser.n_days} days prediction" - ) - else: - plt.title( - f"{ns_parser.n_loops} loops - {model_name} on {s_ticker} - {ns_parser.n_days} days prediction" - ) - - plt.xlim(df_stock.index[0], get_next_stock_market_days(df_pred.index[-1], 1)[-1]) - plt.xlabel("Time") - plt.ylabel("Share Price ($)") - plt.grid(b=True, which="major", color="#666666", linestyle="-") - plt.minorticks_on() - plt.grid(b=True, which="minor", color="#999999", linestyle="-", alpha=0.2) - - if ns_parser.n_loops == 1: - plt.plot( - [df_stock.index[-1], df_pred.index[0]], - [ - df_stock["5. adjusted close"].values[-1], - df_pred[df_pred.columns[0]].values[0], - ], - lw=1, - c="tab:green", - linestyle="--", - ) - plt.plot(df_pred.index, df_pred[df_pred.columns[0]], lw=2, c="tab:green") - else: - df_quantiles = pd.DataFrame() - df_quantiles["Quantile 10%"] = df_pred.quantile(0.1, axis=1) - df_quantiles["Median"] = df_pred.quantile(0.5, axis=1) - df_quantiles["Quantile 90%"] = df_pred.quantile(0.9, axis=1) - - plt.plot(df_pred.index, df_quantiles["Median"], lw=2, c="tab:green") - plt.fill_between( - df_pred.index, - df_quantiles["Quantile 10%"], - df_quantiles["Quantile 90%"], - alpha=0.30, - color="tab:green", - interpolate=True, - ) - plt.fill_between( - [df_stock.index[-1], df_pred.index[0]], - [ - df_stock["5. adjusted close"].values[-1], - df_quantiles["Quantile 10%"].values[0], - ], - [ - df_stock["5. adjusted close"].values[-1], - df_quantiles["Quantile 90%"].values[0], - ], - alpha=0.30, - color="tab:green", - interpolate=True, - ) - - plt.axvspan( - df_stock.index[-1], df_pred.index[-1], facecolor="tab:orange", alpha=0.2 - ) - _, _, ymin, ymax = plt.axis() - plt.vlines( - df_stock.index[-1], - ymin, - ymax, - colors="k", - linewidth=3, - linestyle="--", - color="k", - ) - - # BACKTESTING - if ns_parser.s_end_date: - plt.plot( - df_future.index, - df_future["5. adjusted close"], - lw=2, - c="tab:blue", - ls="--", - ) - plt.plot( - [df_stock.index[-1], df_future.index[0]], - [ - df_stock["5. adjusted close"].values[-1], - df_future["5. adjusted close"].values[0], - ], - lw=1, - c="tab:blue", - linestyle="--", - ) - - if gtff.USE_ION: - plt.ion() - - plt.show() - - # BACKTESTING - if ns_parser.s_end_date: - plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) - plt.subplot(211) - plt.plot( - df_future.index, - df_future["5. adjusted close"], - lw=2, - c="tab:blue", - ls="--", - ) - if ns_parser.n_loops == 1: - plt.plot(df_pred.index, df_pred, lw=2, c="green") - else: - plt.plot( - df_quantiles["Median"].index, df_quantiles["Median"], lw=2, c="green" - ) - - plt.scatter(df_future.index, df_future["5. adjusted close"], c="tab:blue", lw=3) - plt.plot( - [df_stock.index[-1], df_future.index[0]], - [ - df_stock["5. adjusted close"].values[-1], - df_future["5. adjusted close"].values[0], - ], - lw=2, - c="tab:blue", - ls="--", - ) - if ns_parser.n_loops == 1: - plt.scatter(df_pred.index, df_pred, c="green", lw=3) - plt.plot( - [df_stock.index[-1], df_pred.index[0]], - [df_stock["5. adjusted close"].values[-1], df_pred.values[0]], - lw=2, - c="green", - ls="--", - ) - else: - plt.scatter( - df_quantiles["Median"].index, df_quantiles["Median"], lw=3, c="green" - ) - plt.plot( - [df_stock.index[-1], df_quantiles["Median"].index[0]], - [ - df_stock["5. adjusted close"].values[-1], - df_quantiles["Median"].values[0], - ], - lw=2, - c="green", - ls="--", - ) - plt.fill_between( - df_pred.index, - df_quantiles["Quantile 10%"], - df_quantiles["Quantile 90%"], - alpha=0.30, - color="tab:green", - interpolate=True, - ) - - plt.title("BACKTESTING: Real data price versus Prediction") - plt.xlim(df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)) - plt.xticks( - [df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)], - visible=True, - ) - plt.ylabel("Share Price ($)") - plt.grid(b=True, which="major", color="#666666", linestyle="-") - plt.minorticks_on() - plt.grid(b=True, which="minor", color="#999999", linestyle="-", alpha=0.2) - plt.legend(["Real data", "Prediction data"]) - plt.xticks([]) - - plt.subplot(212) - plt.axhline(y=0, color="k", linestyle="--", linewidth=2) - if ns_parser.n_loops == 1: - plt.plot( - df_future.index, - 100 - * ( - df_pred[df_pred.columns[0]].values - - df_future["5. adjusted close"].values - ) - / df_future["5. adjusted close"].values, - lw=2, - c="red", - ) - plt.scatter( - df_future.index, - 100 - * ( - df_pred[df_pred.columns[0]].values - - df_future["5. adjusted close"].values - ) - / df_future["5. adjusted close"].values, - c="red", - lw=5, - ) - plt.plot( - [df_stock.index[-1], df_future.index[0]], - [ - 0, - 100 - * ( - df_pred[df_pred.columns[0]].values[0] - - df_future["5. adjusted close"].values[0] - ) - / df_future["5. adjusted close"].values[0], - ], - lw=2, - ls="--", - c="red", - ) - else: - plt.plot( - df_future.index, - 100 - * ( - df_quantiles["Median"].values - - df_future["5. adjusted close"].values - ) - / df_future["5. adjusted close"].values, - lw=2, - c="red", - ) - plt.scatter( - df_future.index, - 100 - * ( - df_quantiles["Median"].values - - df_future["5. adjusted close"].values - ) - / df_future["5. adjusted close"].values, - c="red", - lw=5, - ) - plt.plot( - [df_stock.index[-1], df_future.index[0]], - [ - 0, - 100 - * ( - df_quantiles["Median"].values[0] - - df_future["5. adjusted close"].values[0] - ) - / df_future["5. adjusted close"].values[0], - ], - lw=2, - ls="--", - c="red", - ) - plt.fill_between( - df_pred.index, - 100 - * ( - df_quantiles["Quantile 10%"].values - - df_future["5. adjusted close"].values - ) - / df_future["5. adjusted close"].values, - 100 - * ( - df_quantiles["Quantile 90%"].values - - df_future["5. adjusted close"].values - ) - / df_future["5. adjusted close"].values, - alpha=0.30, - color="red", - interpolate=True, - ) - plt.title("BACKTESTING: Error between Real data and Prediction [%]") - - plt.xlim(df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)) - plt.xticks( - [df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)], - visible=True, - ) - plt.xlabel("Time") - plt.ylabel("Prediction Error (%)") - plt.grid(b=True, which="major", color="#666666", linestyle="-") - plt.minorticks_on() - plt.grid(b=True, which="minor", color="#999999", linestyle="-", alpha=0.2) - plt.legend(["Real data", "Prediction data"]) - - if gtff.USE_ION: - plt.ion() - - plt.show() - - # Refactor prediction dataframe for backtesting print - if ns_parser.n_loops == 1: - df_pred.rename(columns={df_pred.columns[0]: "Prediction"}, inplace=True) - - else: - df_pred = pd.DataFrame() - df_pred["Prediction"] = df_quantiles["Median"] - - df_pred["Real"] = df_future["5. adjusted close"] - - if gtff.USE_COLOR: - patch_pandas_text_adjustment() - - print("Time Real [$] x Prediction [$]") - print(df_pred.apply(price_prediction_backtesting_color, axis=1).to_string()) - else: - print(df_pred[["Real", "Prediction"]].round(2).to_string()) - - print("") - print_prediction_kpis(df_pred["Real"].values, df_pred["Prediction"].values) - print("") - - else: - patch_pandas_text_adjustment() - # Print prediction data - print_pretty_prediction_nn(df_pred, df_stock["5. adjusted close"].values[-1]) - print("") - - if ns_parser.n_loops > 1: - print("Prediction Stats:") - print(df_quantiles.round(2).to_string()) - print("") - - def mlp(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): """ Train a multi-layer perceptron model @@ -861,183 +121,11 @@ def mlp(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): df_stock: pd.DataFrame Loaded stock dataframe - Returns - ------- - """ try: - ns_parser = _parse_args( - prog="mlp", description="""Multilayer Perceptron. """, other_args=other_args - ) - if not ns_parser: - return - - # Setup backtesting - df_stock, df_future = _setup_backtesting(df_stock, ns_parser) - - # Pre-process data - scaler, stock_train_data, stock_x, stock_y = _preprocess_split( - df_stock, ns_parser - ) - stock_x = np.reshape(stock_x, (stock_x.shape[0], stock_x.shape[1])) - stock_y = np.reshape(stock_y, (stock_y.shape[0], stock_y.shape[1])) - - # Build Neural Network model - model = build_neural_network_model( - cfg_nn_models.MultiLayer_Perceptron, ns_parser.n_inputs, ns_parser.n_days - ) - model.compile(optimizer=ns_parser.s_optimizer, loss=ns_parser.s_loss) - - for idx_loop in range(ns_parser.n_loops): - # Train our model - model.fit( - stock_x, - stock_y, - epochs=ns_parser.n_epochs, - batch_size=ns_parser.n_batch_size, - verbose=1, - ) - print("") - - print(model.summary()) - print("") - - # Prediction - yhat = model.predict( - stock_train_data[-ns_parser.n_inputs :].reshape(1, ns_parser.n_inputs), - verbose=0, - ) - - if idx_loop == 0: - # Re-scale the data back, plot, and print the results - df_pred = _rescale_data( - df_stock, ns_parser, scaler, yhat, idx_loop - ).to_frame() - else: - df_pred = df_pred.join( - _rescale_data( - df_stock, ns_parser, scaler, yhat, idx_loop - ).to_frame() - ) - - _plot_and_print_results( - df_stock, ns_parser, df_future, df_pred, "MLP", s_ticker - ) - - except Exception as e: - print(e) - traceback.print_exc() - print("") - - finally: - _restore_env() - - -def rnn(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): - """ - Train a Recurrent Neural Network (rnn) - Parameters - ---------- - other_args:List[str] - Argparse arguments - s_ticker: str - Stock ticker - df_stock: pd.DataFrame - Dataframe of stock prices - - Returns - ------- - - """ - try: - ns_parser = _parse_args( - prog="rnn", - description="""Recurrent Neural Network. """, - other_args=other_args, - ) - if not ns_parser: - return - - # Setup backtesting - df_stock, df_future = _setup_backtesting(df_stock, ns_parser) - - # Pre-process data - scaler, stock_train_data, stock_x, stock_y = _preprocess_split( - df_stock, ns_parser - ) - stock_x = np.reshape(stock_x, (stock_x.shape[0], stock_x.shape[1], 1)) - stock_y = np.reshape(stock_y, (stock_y.shape[0], stock_y.shape[1], 1)) - - # Build Neural Network model - model = build_neural_network_model( - cfg_nn_models.Recurrent_Neural_Network, ns_parser.n_inputs, ns_parser.n_days - ) - model.compile(optimizer=ns_parser.s_optimizer, loss=ns_parser.s_loss) - - for idx_loop in range(ns_parser.n_loops): - # Train our model - model.fit( - stock_x, - stock_y, - epochs=ns_parser.n_epochs, - batch_size=ns_parser.n_batch_size, - verbose=1, - ) - print("") - - print(model.summary()) - print("") - - # Prediction - yhat = model.predict( - stock_train_data[-ns_parser.n_inputs :].reshape( - 1, ns_parser.n_inputs, 1 - ), - verbose=0, - ) - - if idx_loop == 0: - # Re-scale the data back, plot, and print the results - df_pred = _rescale_data( - df_stock, ns_parser, scaler, yhat, idx_loop - ).to_frame() - else: - df_pred = df_pred.join( - _rescale_data( - df_stock, ns_parser, scaler, yhat, idx_loop - ).to_frame() - ) - - _plot_and_print_results( - df_stock, ns_parser, df_future, df_pred, "RNN", s_ticker - ) - - except Exception as e: - print(e) - traceback.print_exc() - print("") - - finally: - _restore_env() - - -def lstm(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): - """ - Train a Long-Short-Term-Memory Neural Net (lstm) - Parameters - ---------- - other_args:List[str] - Argparse arguments - s_ticker: str - Stock ticker - df_stock: pd.DataFrame - Dataframe of stock prices - - """ - try: - ns_parser = _parse_args( - prog="lstm", - description="""Long-Short Term Memory. """, + ns_parser = parse_args( + prog="mlp", + description="""Multi-Layered-Perceptron. """, other_args=other_args, ) if not ns_parser: @@ -1047,14 +135,14 @@ def lstm(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): X_valid, y_train, y_valid, - X_dates_train, - X_dates_valid, - y_dates_train, + _, + _, + _, y_dates_valid, forecast_data_input, dates_forecast_input, scaler, - is_error + is_error, ) = prepare_scale_train_valid_test(df_stock["5. adjusted close"], ns_parser) if is_error: return @@ -1084,6 +172,7 @@ def lstm(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): verbose=True, batch_size=ns_parser.n_batch_size, validation_data=(X_valid, y_valid), + callbacks=[es], ) preds[i] = model.predict( @@ -1097,13 +186,13 @@ def lstm(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): print(forecast_data_df) plot_data_predictions( df_stock, - np.median(preds,axis=0), + np.median(preds, axis=0), y_valid, y_dates_valid, scaler, - f"LSTM Model on {s_ticker}", + f"MLP Model on {s_ticker}", forecast_data_df, - ns_parser.n_loops + ns_parser.n_loops, ) print("") @@ -1113,10 +202,105 @@ def lstm(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): print("") finally: - _restore_env() + restore_env() -def conv1d(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): +def rnn(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): + """ + Train a Recurrent Neural Network (rnn) + Parameters + ---------- + other_args:List[str] + Argparse arguments + s_ticker: str + Stock ticker + df_stock: pd.DataFrame + Dataframe of stock prices + + """ + try: + ns_parser = parse_args( + prog="lstm", + description="""Long-Short Term Memory. """, + other_args=other_args, + ) + if not ns_parser: + return + ( + X_train, + X_valid, + y_train, + y_valid, + _, + _, + _, + y_dates_valid, + forecast_data_input, + dates_forecast_input, + scaler, + is_error, + ) = prepare_scale_train_valid_test(df_stock["5. adjusted close"], ns_parser) + if is_error: + return + print( + f"Training on {X_train.shape[0]} samples. Using {X_valid.shape[0]} samples for validation." + f" {ns_parser.n_loops} loops" + ) + future_dates = get_next_stock_market_days( + dates_forecast_input[-1], n_next_days=ns_parser.n_days + ) + + preds = np.zeros((ns_parser.n_loops, X_valid.shape[0], ns_parser.n_days)) + forecast_data = np.zeros((ns_parser.n_loops, ns_parser.n_days)) + for i in range(ns_parser.n_loops): + # Build Neural Network model + model = build_neural_network_model( + cfg_nn_models.Long_Short_Term_Memory, + ns_parser.n_inputs, + ns_parser.n_days, + ) + model.compile(optimizer=ns_parser.s_optimizer, loss=ns_parser.s_loss) + + model.fit( + X_train.reshape(X_train.shape[0], X_train.shape[1], 1), + y_train, + epochs=ns_parser.n_epochs, + verbose=True, + batch_size=ns_parser.n_batch_size, + validation_data=(X_valid, y_valid), + callbacks=[es], + ) + + preds[i] = model.predict( + X_valid.reshape(X_valid.shape[0], X_valid.shape[1], 1) + ).reshape(X_valid.shape[0], ns_parser.n_days) + forecast_data[i] = forecast( + forecast_data_input, future_dates, model, scaler + ).values.flat + + forecast_data_df = pd.DataFrame(forecast_data.T, index=future_dates) + print(forecast_data_df) + plot_data_predictions( + df_stock, + np.median(preds, axis=0), + y_valid, + y_dates_valid, + scaler, + f"RNN Model on {s_ticker}", + forecast_data_df, + ns_parser.n_loops, + ) + print("") + except Exception as e: + print(e) + traceback.print_exc() + print("") + + finally: + restore_env() + + +def lstm(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): """ Train a Long-Short-Term-Memory Neural Net (lstm) Parameters @@ -1128,4 +312,180 @@ def conv1d(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): df_stock: pd.DataFrame Dataframe of stock prices """ - pass + try: + ns_parser = parse_args( + prog="lstm", + description="""Long-Short Term Memory. """, + other_args=other_args, + ) + if not ns_parser: + return + ( + X_train, + X_valid, + y_train, + y_valid, + _, + _, + _, + y_dates_valid, + forecast_data_input, + dates_forecast_input, + scaler, + is_error, + ) = prepare_scale_train_valid_test(df_stock["5. adjusted close"], ns_parser) + if is_error: + return + print( + f"Training on {X_train.shape[0]} samples. Using {X_valid.shape[0]} samples for validation." + f" {ns_parser.n_loops} loops" + ) + future_dates = get_next_stock_market_days( + dates_forecast_input[-1], n_next_days=ns_parser.n_days + ) + + preds = np.zeros((ns_parser.n_loops, X_valid.shape[0], ns_parser.n_days)) + forecast_data = np.zeros((ns_parser.n_loops, ns_parser.n_days)) + for i in range(ns_parser.n_loops): + # Build Neural Network model + model = build_neural_network_model( + cfg_nn_models.Long_Short_Term_Memory, + ns_parser.n_inputs, + ns_parser.n_days, + ) + model.compile(optimizer=ns_parser.s_optimizer, loss=ns_parser.s_loss) + + model.fit( + X_train.reshape(X_train.shape[0], X_train.shape[1], 1), + y_train, + epochs=ns_parser.n_epochs, + verbose=True, + batch_size=ns_parser.n_batch_size, + validation_data=(X_valid, y_valid), + callbacks=[es], + ) + + preds[i] = model.predict( + X_valid.reshape(X_valid.shape[0], X_valid.shape[1], 1) + ).reshape(X_valid.shape[0], ns_parser.n_days) + + forecast_data[i] = forecast( + forecast_data_input, future_dates, model, scaler + ).values.flat + + forecast_data_df = pd.DataFrame(forecast_data.T, index=future_dates) + print(forecast_data_df) + plot_data_predictions( + df_stock, + np.median(preds, axis=0), + y_valid, + y_dates_valid, + scaler, + f"LSTM Model on {s_ticker}", + forecast_data_df, + ns_parser.n_loops, + ) + print("") + + except Exception as e: + print(e) + traceback.print_exc() + print("") + + finally: + restore_env() + + +def conv1d(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): + """ + Train a 1D Convolutional Neural Net (1D CNN) + Parameters + ---------- + other_args:List[str] + Argparse arguments + s_ticker: str + Stock ticker + df_stock: pd.DataFrame + Dataframe of stock prices + """ + try: + ns_parser = parse_args( + prog="conv1d", + description="""1D CNN.""", + other_args=other_args, + ) + if not ns_parser: + return + ( + X_train, + X_valid, + y_train, + y_valid, + _, + _, + _, + y_dates_valid, + forecast_data_input, + dates_forecast_input, + scaler, + is_error, + ) = prepare_scale_train_valid_test(df_stock["5. adjusted close"], ns_parser) + if is_error: + return + print( + f"Training on {X_train.shape[0]} samples. Using {X_valid.shape[0]} samples for validation." + f" {ns_parser.n_loops} loops" + ) + future_dates = get_next_stock_market_days( + dates_forecast_input[-1], n_next_days=ns_parser.n_days + ) + + preds = np.zeros((ns_parser.n_loops, X_valid.shape[0], ns_parser.n_days)) + forecast_data = np.zeros((ns_parser.n_loops, ns_parser.n_days)) + for i in range(ns_parser.n_loops): + # Build Neural Network model + model = build_neural_network_model( + cfg_nn_models.Convolutional, + ns_parser.n_inputs, + ns_parser.n_days, + ) + model.compile(optimizer=ns_parser.s_optimizer, loss=ns_parser.s_loss) + + model.fit( + X_train.reshape(X_train.shape[0], X_train.shape[1], 1), + y_train, + epochs=ns_parser.n_epochs, + verbose=True, + batch_size=ns_parser.n_batch_size, + validation_data=(X_valid, y_valid), + callbacks = [es] + ) + + preds[i] = model.predict( + X_valid.reshape(X_valid.shape[0], X_valid.shape[1], 1) + ).reshape(X_valid.shape[0], ns_parser.n_days) + forecast_data[i] = forecast( + forecast_data_input, future_dates, model, scaler + ).values.flat + + forecast_data_df = pd.DataFrame(forecast_data.T, index=future_dates) + print(forecast_data_df) + plot_data_predictions( + df_stock, + np.median(preds, axis=0), + y_valid, + y_dates_valid, + scaler, + f"Conv1D Model on {s_ticker}", + forecast_data_df, + ns_parser.n_loops, + ) + print("") + + except Exception as e: + print(e) + traceback.print_exc() + print("") + + finally: + restore_env() diff --git a/gamestonk_terminal/prediction_techniques/pred_controller.py b/gamestonk_terminal/prediction_techniques/pred_controller.py index 9c491f6f2f8..38f209c47b8 100644 --- a/gamestonk_terminal/prediction_techniques/pred_controller.py +++ b/gamestonk_terminal/prediction_techniques/pred_controller.py @@ -177,6 +177,10 @@ class PredictionTechniquesController: """Process lstm command""" neural_networks_view.lstm(other_args, self.ticker, self.stock) + def call_conv1d(self, other_args: List[str]): + """Process conv1d command""" + neural_networks_view.conv1d(other_args, self.ticker, self.stock) + if gtff.ENABLE_FBPROPHET: def call_prophet(self, other_args: List[str]): diff --git a/gamestonk_terminal/prediction_techniques/pred_helper.py b/gamestonk_terminal/prediction_techniques/pred_helper.py index e10c5625e9a..29cbeb1e7b2 100644 --- a/gamestonk_terminal/prediction_techniques/pred_helper.py +++ b/gamestonk_terminal/prediction_techniques/pred_helper.py @@ -2,8 +2,10 @@ __docformat__ = "numpy" import argparse -from typing import Tuple -from datetime import datetime, timedelta +from typing import List +import os +from warnings import simplefilter +from datetime import timedelta import numpy as np import pandas as pd from pandas.plotting import register_matplotlib_converters @@ -17,15 +19,233 @@ from sklearn.metrics import ( from sklearn.model_selection import train_test_split from sklearn.preprocessing import StandardScaler, MinMaxScaler, Normalizer from gamestonk_terminal.helper_funcs import ( - get_next_stock_market_days, - patch_pandas_text_adjustment, + check_positive, + parse_known_args_and_warn, + valid_date, plot_autoscale, ) from gamestonk_terminal import feature_flags as gtff from gamestonk_terminal.config_plot import PLOT_DPI +def get_backtesting_data(): + pass register_matplotlib_converters() +os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" + +simplefilter(action="ignore", category=FutureWarning) + +# store the user's TensorFlow environment variables +ORIGINAL_TF_XLA_FLAGS = os.environ.get("TF_XLA_FLAGS") +ORIGINAL_TF_FORCE_GPU_ALLOW_GROWTH = os.environ.get("TF_FORCE_GPU_ALLOW_GROWTH") + + +def restore_env(): + """Restore environment variables to original values""" + + def restore(key, value): + if value is None: + if key in os.environ: + del os.environ[key] + else: + os.environ[key] = value + + restore("TF_XLA_FLAGS", ORIGINAL_TF_XLA_FLAGS) + restore("TF_FORCE_GPU_ALLOW_GROWTH", ORIGINAL_TF_FORCE_GPU_ALLOW_GROWTH) + + +def parse_args(prog: str, description: str, other_args: List[str]): + """ + Create an argparser and parse other_args. Will print help if user requests it. + Parameters + ---------- + prog: str + Program for argparser + description: str + Description for argparser + other_args + Argparse arguments to pass + Returns + ------- + ns_parser: argparse.Namespace + Parsed argument parser + """ + + parser = argparse.ArgumentParser( + prog=prog, + description=description, + add_help=False, + formatter_class=argparse.RawTextHelpFormatter, # enable multiline help messages + ) + parser.add_argument( + "-d", + "--days", + action="store", + dest="n_days", + type=check_positive, + default=5, + help="prediction days.", + ) + parser.add_argument( + "-i", + "--input", + action="store", + dest="n_inputs", + type=check_positive, + default=40, + help="number of days to use for prediction.", + ) + parser.add_argument( + "--epochs", + action="store", + dest="n_epochs", + type=check_positive, + default=50, + help="number of training epochs.", + ) + parser.add_argument( + "-j", + "--jumps", + action="store", + dest="n_jumps", + type=check_positive, + default=1, + help="number of jumps in training data.", + ) + parser.add_argument( + "-p", + "--pp", + action="store", + dest="s_preprocessing", + default="minmax", + choices=["normalization", "standardization", "minmax", "none"], + help="pre-processing data.", + ) + parser.add_argument( + "-o", + "--optimizer", + action="store", + dest="s_optimizer", + default="adam", + choices=[ + "adam", + "adagrad", + "adadelta", + "adamax", + "ftrl", + "nadam", + "rmsprop", + "sgd", + ], + help="optimization technique (see https://www.tensorflow.org/api_docs/python/tf/keras/optimizers)", + ) + parser.add_argument( + "-l", + "--loss", + action="store", + dest="s_loss", + default="mae", + choices=[ + "mae", + "mape", + "mse", + "msle", + "poisson", + "logcosh", + "kld", + "hinge", + "squared_hinge", + "huber", + ], + help="loss function (see https://www.tensorflow.org/api_docs/python/tf/keras/losses)", + ) + parser.add_argument( + "-e", + "--end", + action="store", + type=valid_date, + dest="s_end_date", + default=None, + help="The end date (format YYYY-MM-DD) to select - Backtesting", + ) + parser.add_argument( + "--batch_size", + action="store", + dest="n_batch_size", + type=check_positive, + default=None, + help="batch size for model fitting (use a power of 2)", + ) + parser.add_argument( + "--xla_cpu", + action="store_true", + dest="b_xla_cpu", + default=False, + help="enable XLA for CPU (see https://www.tensorflow.org/xla)", + ) + parser.add_argument( + "--xla_gpu", + action="store_true", + dest="b_xla_gpu", + default=False, + help="enable XLA for GPU (see https://www.tensorflow.org/xla)", + ) + parser.add_argument( + "--force_gpu_allow_growth", + action="store", + dest="s_force_gpu_allow_growth", + default="true", + choices=["true", "false", "default"], + help="true: GPU memory will grow as needed. \n" + "false: TensorFlow will allocate 100%% of GPU memory. \n" + "default: usually the same as false, uses env/TensorFlow default", + ) + parser.add_argument( + "--loops", + action="store", + dest="n_loops", + type=check_positive, + default=1, + help="number of loops to iterate and train models", + ) + parser.add_argument( + "-v", + "--valid", + type=float, + dest="valid_split", + default=0.1, + help="Validation data split fraction", + ) + + try: + ns_parser = parse_known_args_and_warn(parser, other_args) + if not ns_parser: + return None + + # set xla flags if requested + xla_flags = ( + set(ORIGINAL_TF_XLA_FLAGS.split(" ")) if ORIGINAL_TF_XLA_FLAGS else set() + ) + if ns_parser.b_xla_cpu or ns_parser.b_xla_gpu: + xla_flags.add("--tf_xla_enable_xla_devices") + if ns_parser.b_xla_cpu: + xla_flags.add("--tf_xla_cpu_global_jit") + if ns_parser.b_xla_gpu: + xla_flags.add("--tf_xla_auto_jit=2") + os.environ["TF_XLA_FLAGS"] = " ".join(xla_flags) + + # set GPU memory growth flag + if ns_parser.s_force_gpu_allow_growth == "true": + os.environ["TF_FORCE_GPU_ALLOW_GROWTH"] = "true" + elif ns_parser.s_force_gpu_allow_growth == "false": + os.environ["TF_FORCE_GPU_ALLOW_GROWTH"] = "false" + + return ns_parser + + except SystemExit: + print("") + return None + def prepare_scale_train_valid_test( df_stock: pd.DataFrame, ns_parser: argparse.Namespace @@ -78,7 +298,7 @@ def prepare_scale_train_valid_test( elif ns_parser.s_preprocessing == "normalization": scaler = Normalizer() - # Test data is used for forcasting. Takes the last n_input_days data points. + # Test data is used for forecasting. Takes the last n_input_days data points. # These points are not fed into training if ns_parser.s_end_date: @@ -87,13 +307,25 @@ def prepare_scale_train_valid_test( print(n_input_days + n_predict_days) if n_input_days + n_predict_days > df_stock.shape[0]: print("Cannot train enough input days to predict with loaded dataframe\n") - return None,None,None,None,None,None,None,None,None,None,None,True + return ( + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + True, + ) test_data = df_stock.iloc[-n_input_days:] train_data = df_stock.iloc[:-n_input_days] dates = train_data.index dates_test = test_data.index - scaler = scaler train_data = scaler.fit_transform(train_data.values.reshape(-1, 1)) test_data = scaler.transform(test_data.values.reshape(-1, 1)) prices = train_data @@ -146,12 +378,29 @@ def prepare_scale_train_valid_test( test_data, dates_test, scaler, - False + False, ) -def forecast(input_values, future_dates, model, scaler): +def forecast(input_values:np.ndarray, future_dates:List, model, scaler)-> pd.DataFrame: + """ + Forecast the stock movement over future days and rescale + Parameters + ---------- + input_values: np.ndarray + Array of values to be fed into the model + future_dates: List + List of future dates + model: keras sequential + Pretrained model + scaler: + Fit scaler to be used to 'unscale' the data + Returns + ------- + df_future: pd.DataFrame + Dataframe of predicted values + """ future_values = scaler.inverse_transform( model.predict(input_values.reshape(1, -1, 1)).reshape(-1, 1) ) @@ -183,7 +432,7 @@ def plot_data_predictions( y_dates_valid[i], y_pred, "r", - lw=3, + lw=1, ) plt.fill_between( y_dates_valid[i], @@ -207,7 +456,7 @@ def plot_data_predictions( y_dates_valid[-1], scaler.inverse_transform(preds[-1].reshape(-1, 1)), "r", - lw=3, + lw=2, label="Predicions", ) plt.fill_between( @@ -218,11 +467,14 @@ def plot_data_predictions( alpha=0.2, ) plt.axvspan( - forecast_data.index[0]-timedelta(days=1), forecast_data.index[-1], facecolor="tab:orange", alpha=0.2 + forecast_data.index[0] - timedelta(days=1), + forecast_data.index[-1], + facecolor="tab:orange", + alpha=0.2, ) _, _, ymin, ymax = plt.axis() plt.vlines( - forecast_data.index[0]-timedelta(days=1), + forecast_data.index[0] - timedelta(days=1), ymin, ymax, colors="k", @@ -231,14 +483,24 @@ def plot_data_predictions( color="k", ) if n_loops == 1: - plt.plot(forecast_data.index, forecast_data.values, "-ok", ms=5, label="Forecast") + plt.plot( + forecast_data.index, forecast_data.values, "-ok", ms=5, label="Forecast" + ) else: - plt.plot(forecast_data.index, forecast_data.median(axis=1).values, "-ok", ms=5, label="Forecast") - plt.fill_between(forecast_data.index, - forecast_data.quantile(.25,axis=1).values, - forecast_data.quantile(.75,axis=1).values, - color="c", - alpha = .3) + plt.plot( + forecast_data.index, + forecast_data.median(axis=1).values, + "-ok", + ms=5, + label="Forecast", + ) + plt.fill_between( + forecast_data.index, + forecast_data.quantile(0.25, axis=1).values, + forecast_data.quantile(0.75, axis=1).values, + color="c", + alpha=0.3, + ) plt.legend(loc=0) plt.xlim(df_stock.index[0], forecast_data.index[-1] + timedelta(days=1)) plt.xlabel("Time") @@ -252,62 +514,6 @@ def plot_data_predictions( plt.show() print("") -# Deprecated -def get_backtesting_data( - df_stock: pd.DataFrame, parsed_end_date: datetime, pred_days: int -) -> Tuple[pd.DataFrame, pd.DataFrame, bool]: - """ - - Parameters - ---------- - df_stock: pd.DataFrame - Dataframe of stock prices - parsed_end_date: datetime - Date that separates test from prediction - pred_days: int - Number of days to predict - - Returns - ------- - df_future: pd.DataFrame - Dataframe of future prices - df_stock: pd.DataFrame - Dataframe of past prices - Flag: bool - Returns True if backtesting meets requirements, False if there are errors - - """ - if parsed_end_date < df_stock.index[0]: - print( - "Backtesting not allowed, since End Date is older than Start Date of historical data\n" - ) - return pd.DataFrame(), pd.DataFrame(), False - - if ( - parsed_end_date - < get_next_stock_market_days( - last_stock_day=df_stock.index[0], n_next_days=5 + pred_days - )[-1] - ): - print( - "Backtesting not allowed, since End Date is too close to Start Date to train model\n" - ) - return pd.DataFrame(), pd.DataFrame(), False - - future_index = get_next_stock_market_days( - last_stock_day=parsed_end_date, n_next_days=pred_days - ) - - if future_index[-1] > datetime.now(): - print( - "Backtesting not allowed, since End Date + Prediction days is in the future\n" - ) - return pd.DataFrame(), pd.DataFrame(), False - - df_future = df_stock[future_index[0] : future_index[-1]] - df_stock = df_stock[:parsed_end_date] - return df_future, df_stock, True - def price_prediction_color(val: float, last_val: float) -> str: """Set prediction to be a colored string""" From ab2841bd013032851b01047bc1a0765c6f6b85ef Mon Sep 17 00:00:00 2001 From: Jamie Date: Mon, 10 May 2021 21:38:38 -0400 Subject: [PATCH 18/29] add back print pretty predictions and add n_loops logic for plotting --- .../neural_networks_view.py | 66 ++++++++++++++++--- .../prediction_techniques/pred_helper.py | 24 ++++--- 2 files changed, 72 insertions(+), 18 deletions(-) diff --git a/gamestonk_terminal/prediction_techniques/neural_networks_view.py b/gamestonk_terminal/prediction_techniques/neural_networks_view.py index c7420bee22b..ba41c4988e8 100644 --- a/gamestonk_terminal/prediction_techniques/neural_networks_view.py +++ b/gamestonk_terminal/prediction_techniques/neural_networks_view.py @@ -24,16 +24,18 @@ from gamestonk_terminal.prediction_techniques.pred_helper import ( plot_data_predictions, parse_args, restore_env, + print_pretty_prediction, ) from gamestonk_terminal import config_neural_network_models as cfg_nn_models es = EarlyStopping(monitor="val_loss", patience=10) + def build_neural_network_model( Recurrent_Neural_Network: List[Any], n_inputs: int, n_days: int ): """ - Builds neural netfrom config_neural_network_models.py + Builds neural net from config_neural_network_models.py Parameters ---------- Recurrent_Neural_Network: List[Any] @@ -171,7 +173,10 @@ def mlp(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): epochs=ns_parser.n_epochs, verbose=True, batch_size=ns_parser.n_batch_size, - validation_data=(X_valid, y_valid), + validation_data=( + X_valid.reshape(X_valid.shape[0], X_valid.shape[1], 1), + y_valid, + ), callbacks=[es], ) @@ -183,7 +188,15 @@ def mlp(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): ).values.flat forecast_data_df = pd.DataFrame(forecast_data.T, index=future_dates) - print(forecast_data_df) + if ns_parser.n_loops > 1: + forecast_data_df["Median"] = forecast_data_df.median(axis=1) + print_pretty_prediction( + forecast_data_df["Median"], df_stock["5. adjusted close"].values[-1] + ) + else: + print_pretty_prediction( + forecast_data_df[0], df_stock["5. adjusted close"].values[-1] + ) plot_data_predictions( df_stock, np.median(preds, axis=0), @@ -267,7 +280,10 @@ def rnn(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): epochs=ns_parser.n_epochs, verbose=True, batch_size=ns_parser.n_batch_size, - validation_data=(X_valid, y_valid), + validation_data=( + X_valid.reshape(X_valid.shape[0], X_valid.shape[1], 1), + y_valid, + ), callbacks=[es], ) @@ -279,7 +295,15 @@ def rnn(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): ).values.flat forecast_data_df = pd.DataFrame(forecast_data.T, index=future_dates) - print(forecast_data_df) + if ns_parser.n_loops > 1: + forecast_data_df["Median"] = forecast_data_df.median(axis=1) + print_pretty_prediction( + forecast_data_df["Median"], df_stock["5. adjusted close"].values[-1] + ) + else: + print_pretty_prediction( + forecast_data_df[0], df_stock["5. adjusted close"].values[-1] + ) plot_data_predictions( df_stock, np.median(preds, axis=0), @@ -361,7 +385,10 @@ def lstm(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): epochs=ns_parser.n_epochs, verbose=True, batch_size=ns_parser.n_batch_size, - validation_data=(X_valid, y_valid), + validation_data=( + X_valid.reshape(X_valid.shape[0], X_valid.shape[1], 1), + y_valid, + ), callbacks=[es], ) @@ -374,7 +401,15 @@ def lstm(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): ).values.flat forecast_data_df = pd.DataFrame(forecast_data.T, index=future_dates) - print(forecast_data_df) + if ns_parser.n_loops > 1: + forecast_data_df["Median"] = forecast_data_df.median(axis=1) + print_pretty_prediction( + forecast_data_df["Median"], df_stock["5. adjusted close"].values[-1] + ) + else: + print_pretty_prediction( + forecast_data_df[0], df_stock["5. adjusted close"].values[-1] + ) plot_data_predictions( df_stock, np.median(preds, axis=0), @@ -457,8 +492,11 @@ def conv1d(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): epochs=ns_parser.n_epochs, verbose=True, batch_size=ns_parser.n_batch_size, - validation_data=(X_valid, y_valid), - callbacks = [es] + validation_data=( + X_valid.reshape(X_valid.shape[0], X_valid.shape[1], 1), + y_valid, + ), + callbacks=[es], ) preds[i] = model.predict( @@ -469,7 +507,15 @@ def conv1d(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): ).values.flat forecast_data_df = pd.DataFrame(forecast_data.T, index=future_dates) - print(forecast_data_df) + if ns_parser.n_loops > 1: + forecast_data_df["Median"] = forecast_data_df.median(axis=1) + print_pretty_prediction( + forecast_data_df["Median"], df_stock["5. adjusted close"].values[-1] + ) + else: + print_pretty_prediction( + forecast_data_df[0], df_stock["5. adjusted close"].values[-1] + ) plot_data_predictions( df_stock, np.median(preds, axis=0), diff --git a/gamestonk_terminal/prediction_techniques/pred_helper.py b/gamestonk_terminal/prediction_techniques/pred_helper.py index 29cbeb1e7b2..0922a463da1 100644 --- a/gamestonk_terminal/prediction_techniques/pred_helper.py +++ b/gamestonk_terminal/prediction_techniques/pred_helper.py @@ -27,8 +27,11 @@ from gamestonk_terminal.helper_funcs import ( from gamestonk_terminal import feature_flags as gtff from gamestonk_terminal.config_plot import PLOT_DPI + def get_backtesting_data(): pass + + register_matplotlib_converters() os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" @@ -303,8 +306,6 @@ def prepare_scale_train_valid_test( if ns_parser.s_end_date: df_stock = df_stock[df_stock.index <= ns_parser.s_end_date] - print(df_stock.shape) - print(n_input_days + n_predict_days) if n_input_days + n_predict_days > df_stock.shape[0]: print("Cannot train enough input days to predict with loaded dataframe\n") return ( @@ -324,9 +325,10 @@ def prepare_scale_train_valid_test( test_data = df_stock.iloc[-n_input_days:] train_data = df_stock.iloc[:-n_input_days] - dates = train_data.index + + dates = df_stock.index dates_test = test_data.index - train_data = scaler.fit_transform(train_data.values.reshape(-1, 1)) + train_data = scaler.fit_transform(df_stock.values.reshape(-1, 1)) test_data = scaler.transform(test_data.values.reshape(-1, 1)) prices = train_data @@ -382,7 +384,9 @@ def prepare_scale_train_valid_test( ) -def forecast(input_values:np.ndarray, future_dates:List, model, scaler)-> pd.DataFrame: +def forecast( + input_values: np.ndarray, future_dates: List, model, scaler +) -> pd.DataFrame: """ Forecast the stock movement over future days and rescale Parameters @@ -457,7 +461,7 @@ def plot_data_predictions( scaler.inverse_transform(preds[-1].reshape(-1, 1)), "r", lw=2, - label="Predicions", + label="Predictions", ) plt.fill_between( y_dates_valid[-1], @@ -484,14 +488,18 @@ def plot_data_predictions( ) if n_loops == 1: plt.plot( - forecast_data.index, forecast_data.values, "-ok", ms=5, label="Forecast" + forecast_data.index, + forecast_data.values, + "-ok", + ms=3, + label="Forecast", ) else: plt.plot( forecast_data.index, forecast_data.median(axis=1).values, "-ok", - ms=5, + ms=3, label="Forecast", ) plt.fill_between( From aa1b9032fb5950ef06171e90f6b53b1ab49fccb5 Mon Sep 17 00:00:00 2001 From: Jamie Date: Mon, 10 May 2021 21:42:39 -0400 Subject: [PATCH 19/29] add early stopping in config --- gamestonk_terminal/config_neural_network_models.py | 3 +++ .../prediction_techniques/neural_networks_view.py | 7 +++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/gamestonk_terminal/config_neural_network_models.py b/gamestonk_terminal/config_neural_network_models.py index 28c7181e870..30b8af07831 100644 --- a/gamestonk_terminal/config_neural_network_models.py +++ b/gamestonk_terminal/config_neural_network_models.py @@ -48,3 +48,6 @@ Convolutional = [ {"Flatten": {}}, {"Dense": {"activation": "relu"}}, ] + +# Add early stopping to save time training when validation is plateaued +Early_Stop_Patience = 10 \ No newline at end of file diff --git a/gamestonk_terminal/prediction_techniques/neural_networks_view.py b/gamestonk_terminal/prediction_techniques/neural_networks_view.py index ba41c4988e8..f12d159d63c 100644 --- a/gamestonk_terminal/prediction_techniques/neural_networks_view.py +++ b/gamestonk_terminal/prediction_techniques/neural_networks_view.py @@ -28,8 +28,11 @@ from gamestonk_terminal.prediction_techniques.pred_helper import ( ) from gamestonk_terminal import config_neural_network_models as cfg_nn_models -es = EarlyStopping(monitor="val_loss", patience=10) - +if cfg_nn_models.Early_Stop_Patience: + es = EarlyStopping(monitor="val_loss", patience=cfg_nn_models.Early_Stop_Patience) +else: + #Set patience to very long value + es = EarlyStopping(monitor="val_loss", patience=1000) def build_neural_network_model( Recurrent_Neural_Network: List[Any], n_inputs: int, n_days: int From 825db5af27cc8c389816aa262347e6bc42c7b48b Mon Sep 17 00:00:00 2001 From: Jamie Date: Mon, 10 May 2021 21:52:17 -0400 Subject: [PATCH 20/29] add type hints for keras --- .../config_neural_network_models.py | 1 + .../prediction_techniques/neural_networks_view.py | 4 ++-- .../prediction_techniques/pred_helper.py | 15 ++++++++++++--- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/gamestonk_terminal/config_neural_network_models.py b/gamestonk_terminal/config_neural_network_models.py index 30b8af07831..4c8eb6af487 100644 --- a/gamestonk_terminal/config_neural_network_models.py +++ b/gamestonk_terminal/config_neural_network_models.py @@ -50,4 +50,5 @@ Convolutional = [ ] # Add early stopping to save time training when validation is plateaued +# Can be set to large number or None to ignore Early_Stop_Patience = 10 \ No newline at end of file diff --git a/gamestonk_terminal/prediction_techniques/neural_networks_view.py b/gamestonk_terminal/prediction_techniques/neural_networks_view.py index f12d159d63c..751f52cc383 100644 --- a/gamestonk_terminal/prediction_techniques/neural_networks_view.py +++ b/gamestonk_terminal/prediction_techniques/neural_networks_view.py @@ -36,7 +36,7 @@ else: def build_neural_network_model( Recurrent_Neural_Network: List[Any], n_inputs: int, n_days: int -): +)->Sequential: """ Builds neural net from config_neural_network_models.py Parameters @@ -50,7 +50,7 @@ def build_neural_network_model( Returns ------- - model: + model: Sequential Keras sequential model with layers from the file """ diff --git a/gamestonk_terminal/prediction_techniques/pred_helper.py b/gamestonk_terminal/prediction_techniques/pred_helper.py index 0922a463da1..565ba07202c 100644 --- a/gamestonk_terminal/prediction_techniques/pred_helper.py +++ b/gamestonk_terminal/prediction_techniques/pred_helper.py @@ -18,6 +18,7 @@ from sklearn.metrics import ( ) from sklearn.model_selection import train_test_split from sklearn.preprocessing import StandardScaler, MinMaxScaler, Normalizer +from keras.models import Sequential from gamestonk_terminal.helper_funcs import ( check_positive, parse_known_args_and_warn, @@ -214,12 +215,20 @@ def parse_args(prog: str, description: str, other_args: List[str]): parser.add_argument( "-v", "--valid", - type=float, + type=check_positive, dest="valid_split", default=0.1, help="Validation data split fraction", ) + parser.add_argument( + "--lr", + type = check_positive, + dest = "lr", + default=None, + help="Specify learning rate for optimizer." + ) + try: ns_parser = parse_known_args_and_warn(parser, other_args) if not ns_parser: @@ -385,7 +394,7 @@ def prepare_scale_train_valid_test( def forecast( - input_values: np.ndarray, future_dates: List, model, scaler + input_values: np.ndarray, future_dates: List, model:Sequential, scaler ) -> pd.DataFrame: """ Forecast the stock movement over future days and rescale @@ -395,7 +404,7 @@ def forecast( Array of values to be fed into the model future_dates: List List of future dates - model: keras sequential + model: Sequential Pretrained model scaler: Fit scaler to be used to 'unscale' the data From 62b074b48f7a547343809b7f11da29311148b3ea Mon Sep 17 00:00:00 2001 From: Jamie Date: Fri, 14 May 2021 14:46:55 -0400 Subject: [PATCH 21/29] updated KNN. Added shuffle flag for train/validation split. --- .../config_neural_network_models.py | 2 +- .../prediction_techniques/knn_view.py | 267 ++++-------------- .../neural_networks_view.py | 24 +- .../prediction_techniques/pred_helper.py | 69 +++-- 4 files changed, 117 insertions(+), 245 deletions(-) diff --git a/gamestonk_terminal/config_neural_network_models.py b/gamestonk_terminal/config_neural_network_models.py index 4c8eb6af487..48da79b2630 100644 --- a/gamestonk_terminal/config_neural_network_models.py +++ b/gamestonk_terminal/config_neural_network_models.py @@ -51,4 +51,4 @@ Convolutional = [ # Add early stopping to save time training when validation is plateaued # Can be set to large number or None to ignore -Early_Stop_Patience = 10 \ No newline at end of file +Early_Stop_Patience = 10 diff --git a/gamestonk_terminal/prediction_techniques/knn_view.py b/gamestonk_terminal/prediction_techniques/knn_view.py index 07b48affe9f..f1c191fb53e 100644 --- a/gamestonk_terminal/prediction_techniques/knn_view.py +++ b/gamestonk_terminal/prediction_techniques/knn_view.py @@ -3,30 +3,21 @@ __docformat__ = "numpy" import argparse from typing import List -import datetime -import matplotlib.pyplot as plt import pandas as pd from pandas.plotting import register_matplotlib_converters from sklearn import neighbors -from TimeSeriesCrossValidation import splitTrain from gamestonk_terminal.helper_funcs import ( check_positive, parse_known_args_and_warn, valid_date, - patch_pandas_text_adjustment, get_next_stock_market_days, - plot_autoscale, ) from gamestonk_terminal.prediction_techniques.pred_helper import ( print_pretty_prediction, - price_prediction_backtesting_color, - print_prediction_kpis, - get_backtesting_data, + prepare_scale_train_valid_test, + plot_data_predictions, ) -from gamestonk_terminal.config_plot import PLOT_DPI -from gamestonk_terminal import feature_flags as gtff - register_matplotlib_converters() @@ -106,224 +97,74 @@ def k_nearest_neighbors(other_args: List[str], s_ticker: str, df_stock: pd.DataF "-t", "--test_size", default=0.2, - dest="test_size", + dest="valid_split", type=float, help="Percentage of data to validate in sample", ) + parser.add_argument( + "-p", + "--pp", + action="store", + dest="s_preprocessing", + default="none", + choices=["normalization", "standardization", "minmax", "none"], + help="pre-processing data.", + ) try: ns_parser = parse_known_args_and_warn(parser, other_args) if not ns_parser: return - # BACKTESTING - if ns_parser.s_end_date: - df_future, df_stock, bt_flag = get_backtesting_data( - df_stock, ns_parser.s_end_date, ns_parser.n_days - ) - if not bt_flag: - return - - # Split training data - stock_x, stock_y = splitTrain.split_train( - df_stock["5. adjusted close"].values, - ns_parser.n_inputs, - ns_parser.n_days, - ns_parser.n_jumps, - ) - - if not stock_x: - print("Given the model parameters more training data is needed.\n") + ( + X_train, + X_valid, + y_train, + y_valid, + _, + _, + _, + y_dates_valid, + forecast_data_input, + dates_forecast_input, + scaler, + is_error, + ) = prepare_scale_train_valid_test(df_stock["5. adjusted close"], ns_parser) + if is_error: + print("Error preparing data") return + print( + f"Training on {X_train.shape[0]} sequences of length {X_train.shape[1]}. Using {X_valid.shape[0]} sequences " + f" of length {X_valid.shape[1]} for validation" + ) + future_dates = get_next_stock_market_days( + dates_forecast_input[-1], n_next_days=ns_parser.n_days + ) # Machine Learning model knn = neighbors.KNeighborsRegressor(n_neighbors=ns_parser.n_neighbors) - knn.fit(stock_x, stock_y) - - # Prediction data - l_predictions = knn.predict( - df_stock["5. adjusted close"].values[-ns_parser.n_inputs :].reshape(1, -1) - )[0] - l_pred_days = get_next_stock_market_days( - last_stock_day=df_stock["5. adjusted close"].index[-1], - n_next_days=ns_parser.n_days, - ) - df_pred = pd.Series(l_predictions, index=l_pred_days, name="Price") - - # Plotting - plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) - plt.plot(df_stock.index, df_stock["5. adjusted close"], lw=2) - s_knn = f"{ns_parser.n_neighbors}-Nearest Neighbors on {s_ticker}" - # BACKTESTING - if ns_parser.s_end_date: - plt.title(f"BACKTESTING: {s_knn} - {ns_parser.n_days} days prediction") - else: - plt.title(f"{s_knn} - {ns_parser.n_days} days prediction") - plt.xlim( - df_stock.index[0], get_next_stock_market_days(df_pred.index[-1], 1)[-1] - ) - plt.xlabel("Time") - plt.ylabel("Share Price ($)") - plt.grid(b=True, which="major", color="#666666", linestyle="-") - plt.minorticks_on() - plt.grid(b=True, which="minor", color="#999999", linestyle="-", alpha=0.2) - plt.plot( - [df_stock.index[-1], df_pred.index[0]], - [df_stock["5. adjusted close"].values[-1], df_pred.values[0]], - lw=1, - c="tab:green", - linestyle="--", - ) - plt.plot(df_pred.index, df_pred, lw=2, c="tab:green") - plt.axvspan( - df_stock.index[-1], df_pred.index[-1], facecolor="tab:orange", alpha=0.2 - ) - _, _, ymin, ymax = plt.axis() - plt.vlines( - df_stock.index[-1], ymin, ymax, linewidth=1, linestyle="--", color="k" + knn.fit( + X_train.reshape(X_train.shape[0], X_train.shape[1]), + y_train.reshape(y_train.shape[0], y_train.shape[1]), ) - # BACKTESTING - if ns_parser.s_end_date: - plt.plot( - df_future.index, - df_future["5. adjusted close"], - lw=2, - c="tab:blue", - ls="--", - ) - plt.plot( - [df_stock.index[-1], df_future.index[0]], - [ - df_stock["5. adjusted close"].values[-1], - df_future["5. adjusted close"].values[0], - ], - lw=1, - c="tab:blue", - linestyle="--", - ) + preds = knn.predict(X_valid.reshape(X_valid.shape[0], X_valid.shape[1])) + forecast_data = knn.predict(forecast_data_input.reshape(1, -1)) - if gtff.USE_ION: - plt.ion() - - plt.show() - - # BACKTESTING - if ns_parser.s_end_date: - plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) - plt.subplot(211) - plt.plot( - df_future.index, - df_future["5. adjusted close"], - lw=2, - c="tab:blue", - ls="--", - ) - plt.plot(df_pred.index, df_pred, lw=2, c="green") - plt.scatter( - df_future.index, df_future["5. adjusted close"], c="tab:blue", lw=3 - ) - plt.plot( - [df_stock.index[-1], df_future.index[0]], - [ - df_stock["5. adjusted close"].values[-1], - df_future["5. adjusted close"].values[0], - ], - lw=2, - c="tab:blue", - ls="--", - ) - plt.scatter(df_pred.index, df_pred, c="green", lw=3) - plt.plot( - [df_stock.index[-1], df_pred.index[0]], - [df_stock["5. adjusted close"].values[-1], df_pred.values[0]], - lw=2, - c="green", - ls="--", - ) - plt.title("BACKTESTING: Real data price versus Prediction") - plt.xlim(df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)) - plt.xticks( - [df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)] - ) - plt.ylabel("Share Price ($)") - plt.grid(b=True, which="major", color="#666666", linestyle="-") - plt.minorticks_on() - plt.grid(b=True, which="minor", color="#999999", linestyle="-", alpha=0.2) - plt.legend(["Real data", "Prediction data"]) - plt.xticks([]) - - plt.subplot(212) - plt.axhline(y=0, color="k", linestyle="--", linewidth=2) - plt.plot( - df_future.index, - 100 - * (df_pred.values - df_future["5. adjusted close"].values) - / df_future["5. adjusted close"].values, - lw=2, - c="red", - ) - plt.scatter( - df_future.index, - 100 - * (df_pred.values - df_future["5. adjusted close"].values) - / df_future["5. adjusted close"].values, - c="red", - lw=5, - ) - plt.title("BACKTESTING: Error between Real data and Prediction [%]") - plt.plot( - [df_stock.index[-1], df_future.index[0]], - [ - 0, - 100 - * (df_pred.values[0] - df_future["5. adjusted close"].values[0]) - / df_future["5. adjusted close"].values[0], - ], - lw=2, - ls="--", - c="red", - ) - plt.xlim(df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)) - plt.xticks( - [df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)] - ) - plt.xlabel("Time") - plt.ylabel("Prediction Error (%)") - plt.grid(b=True, which="major", color="#666666", linestyle="-") - plt.minorticks_on() - plt.grid(b=True, which="minor", color="#999999", linestyle="-", alpha=0.2) - plt.legend(["Real data", "Prediction data"]) - - if gtff.USE_ION: - plt.ion() - - plt.show() - - # Refactor prediction dataframe for backtesting print - df_pred.name = "Prediction" - df_pred = df_pred.to_frame() - df_pred["Real"] = df_future["5. adjusted close"] - - if gtff.USE_COLOR: - patch_pandas_text_adjustment() - - print("Time Real [$] x Prediction [$]") - print( - df_pred.apply( - price_prediction_backtesting_color, axis=1 - ).to_string() - ) - else: - print(df_pred[["Real", "Prediction"]].round(2).to_string()) - - print("") - print_prediction_kpis(df_pred["Real"].values, df_pred["Prediction"].values) - - else: - # Print prediction data - print_pretty_prediction(df_pred, df_stock["5. adjusted close"].values[-1]) - print("") + forecast_data_df = pd.DataFrame(forecast_data.T, index=future_dates) + print_pretty_prediction( + forecast_data_df[0], df_stock["5. adjusted close"].values[-1] + ) + plot_data_predictions( + df_stock, + preds, + y_valid, + y_dates_valid, + scaler, + f"KNN Model with {ns_parser.n_neighbors} Neighbors on {s_ticker}", + forecast_data_df, + 1, + ) except Exception as e: print(e) diff --git a/gamestonk_terminal/prediction_techniques/neural_networks_view.py b/gamestonk_terminal/prediction_techniques/neural_networks_view.py index 751f52cc383..c15723e8e97 100644 --- a/gamestonk_terminal/prediction_techniques/neural_networks_view.py +++ b/gamestonk_terminal/prediction_techniques/neural_networks_view.py @@ -5,7 +5,7 @@ from typing import List, Any import traceback import numpy as np import pandas as pd -from keras.callbacks import EarlyStopping +from tensorflow.keras.callbacks import EarlyStopping from tensorflow.keras.models import Sequential from tensorflow.keras.layers import ( LSTM, @@ -31,12 +31,13 @@ from gamestonk_terminal import config_neural_network_models as cfg_nn_models if cfg_nn_models.Early_Stop_Patience: es = EarlyStopping(monitor="val_loss", patience=cfg_nn_models.Early_Stop_Patience) else: - #Set patience to very long value + # Set patience to very long value es = EarlyStopping(monitor="val_loss", patience=1000) + def build_neural_network_model( Recurrent_Neural_Network: List[Any], n_inputs: int, n_days: int -)->Sequential: +) -> Sequential: """ Builds neural net from config_neural_network_models.py Parameters @@ -152,8 +153,8 @@ def mlp(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): if is_error: return print( - f"Training on {X_train.shape[0]} samples. Using {X_valid.shape[0]} samples for validation." - f" {ns_parser.n_loops} loops" + f"Training on {X_train.shape[0]} sequences of length {X_train.shape[1]}. Using {X_valid.shape[0]} sequences " + f" of length {X_valid.shape[1]} for validation. Model will run {ns_parser.n_loops} loops" ) future_dates = get_next_stock_market_days( dates_forecast_input[-1], n_next_days=ns_parser.n_days @@ -259,8 +260,8 @@ def rnn(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): if is_error: return print( - f"Training on {X_train.shape[0]} samples. Using {X_valid.shape[0]} samples for validation." - f" {ns_parser.n_loops} loops" + f"Training on {X_train.shape[0]} sequences of length {X_train.shape[1]}. Using {X_valid.shape[0]} sequences " + f" of length {X_valid.shape[1]} for validation. Model will run {ns_parser.n_loops} loops" ) future_dates = get_next_stock_market_days( dates_forecast_input[-1], n_next_days=ns_parser.n_days @@ -364,8 +365,8 @@ def lstm(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): if is_error: return print( - f"Training on {X_train.shape[0]} samples. Using {X_valid.shape[0]} samples for validation." - f" {ns_parser.n_loops} loops" + f"Training on {X_train.shape[0]} sequences of length {X_train.shape[1]}. Using {X_valid.shape[0]} sequences " + f" of length {X_valid.shape[1]} for validation. Model will run {ns_parser.n_loops} loops" ) future_dates = get_next_stock_market_days( dates_forecast_input[-1], n_next_days=ns_parser.n_days @@ -471,8 +472,8 @@ def conv1d(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): if is_error: return print( - f"Training on {X_train.shape[0]} samples. Using {X_valid.shape[0]} samples for validation." - f" {ns_parser.n_loops} loops" + f"Training on {X_train.shape[0]} sequences of length {X_train.shape[1]}. Using {X_valid.shape[0]} sequences " + f" of length {X_valid.shape[1]} for validation. Model will run {ns_parser.n_loops} loops" ) future_dates = get_next_stock_market_days( dates_forecast_input[-1], n_next_days=ns_parser.n_days @@ -487,6 +488,7 @@ def conv1d(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): ns_parser.n_inputs, ns_parser.n_days, ) + model.compile(optimizer=ns_parser.s_optimizer, loss=ns_parser.s_loss) model.fit( diff --git a/gamestonk_terminal/prediction_techniques/pred_helper.py b/gamestonk_terminal/prediction_techniques/pred_helper.py index 565ba07202c..8fe7683106c 100644 --- a/gamestonk_terminal/prediction_techniques/pred_helper.py +++ b/gamestonk_terminal/prediction_techniques/pred_helper.py @@ -18,7 +18,7 @@ from sklearn.metrics import ( ) from sklearn.model_selection import train_test_split from sklearn.preprocessing import StandardScaler, MinMaxScaler, Normalizer -from keras.models import Sequential +from tensorflow.keras.models import Sequential from gamestonk_terminal.helper_funcs import ( check_positive, parse_known_args_and_warn, @@ -215,7 +215,7 @@ def parse_args(prog: str, description: str, other_args: List[str]): parser.add_argument( "-v", "--valid", - type=check_positive, + type=float, dest="valid_split", default=0.1, help="Validation data split fraction", @@ -223,10 +223,18 @@ def parse_args(prog: str, description: str, other_args: List[str]): parser.add_argument( "--lr", - type = check_positive, - dest = "lr", + type=check_positive, + dest="lr", default=None, - help="Specify learning rate for optimizer." + help="Specify learning rate for optimizer.", + ) + + parser.add_argument( + "--no_shuffle", + action="store_false", + dest="no_shuffle", + default=True, + help="Specify if shuffling validation inputs.", ) try: @@ -275,11 +283,11 @@ def prepare_scale_train_valid_test( X_train: np.ndarray Array of training data. Shape (# samples, n_inputs, 1) X_test: np.ndarray - Array of validation data. Shape (#samples, n_inputs, 1) + Array of validation data. Shape (totoal sequences - #samples, n_inputs, 1) y_train: np.ndarray Array of training outputs. Shape (#samples, n_days) y_test: np.ndarray - Array of validation outputs. Shape (#samples, n_days) + Array of validation outputs. Shape (total sequences -#samples, n_days) X_dates_train: np.ndarray Array of dates for X_train X_dates_test: np.ndarray @@ -287,7 +295,7 @@ def prepare_scale_train_valid_test( y_dates_train: np.ndarray Array of dates for y_train y_dates_test: np.ndarray - Array of dates for y)test + Array of dates for y_test test_data: np.ndarray Array of prices after the specified end date dates_test: np.ndarray @@ -310,6 +318,8 @@ def prepare_scale_train_valid_test( elif ns_parser.s_preprocessing == "normalization": scaler = Normalizer() + elif ns_parser.s_preprocessing == "none": + scaler = None # Test data is used for forecasting. Takes the last n_input_days data points. # These points are not fed into training @@ -337,8 +347,13 @@ def prepare_scale_train_valid_test( dates = df_stock.index dates_test = test_data.index - train_data = scaler.fit_transform(df_stock.values.reshape(-1, 1)) - test_data = scaler.transform(test_data.values.reshape(-1, 1)) + if scaler: + train_data = scaler.fit_transform(df_stock.values.reshape(-1, 1)) + test_data = scaler.transform(test_data.values.reshape(-1, 1)) + else: + train_data = df_stock.values.reshape(-1, 1) + test_data = test_data.values.reshape(-1, 1) + prices = train_data input_dates = [] @@ -376,6 +391,7 @@ def prepare_scale_train_valid_test( input_dates, next_n_day_dates, test_size=test_size, + shuffle=ns_parser.no_shuffle, ) return ( X_train, @@ -394,7 +410,7 @@ def prepare_scale_train_valid_test( def forecast( - input_values: np.ndarray, future_dates: List, model:Sequential, scaler + input_values: np.ndarray, future_dates: List, model: Sequential, scaler ) -> pd.DataFrame: """ Forecast the stock movement over future days and rescale @@ -414,9 +430,13 @@ def forecast( df_future: pd.DataFrame Dataframe of predicted values """ - future_values = scaler.inverse_transform( - model.predict(input_values.reshape(1, -1, 1)).reshape(-1, 1) - ) + if scaler: + future_values = scaler.inverse_transform( + model.predict(input_values.reshape(1, -1, 1)).reshape(-1, 1) + ) + else: + future_values = model.predict(input_values.reshape(1, -1, 1)).reshape(-1, 1) + df_future = pd.DataFrame( future_values, index=future_dates, columns=["Predicted Price"] ) @@ -438,9 +458,12 @@ def plot_data_predictions( ) for i in range(len(y_valid) - 1): - y_pred = scaler.inverse_transform(preds[i].reshape(-1, 1)).ravel() - y_act = scaler.inverse_transform(y_valid[i].reshape(-1, 1)).ravel() - + if scaler: + y_pred = scaler.inverse_transform(preds[i].reshape(-1, 1)).ravel() + y_act = scaler.inverse_transform(y_valid[i].reshape(-1, 1)).ravel() + else: + y_pred = preds[i].ravel() + y_act = y_valid[i].ravel() plt.plot( y_dates_valid[i], y_pred, @@ -465,17 +488,23 @@ def plot_data_predictions( ) # Leave this one out of the loop so that the legend doesnt get overpopulated with "Predictions" + if scaler: + final_pred = scaler.inverse_transform(preds[-1].reshape(-1, 1)).ravel() + final_valid = scaler.inverse_transform(y_valid[-1].reshape(-1, 1)).ravel() + else: + final_pred = preds[-1].reshape(-1, 1).ravel() + final_valid = y_valid[-1].reshape(-1, 1).ravel() plt.plot( y_dates_valid[-1], - scaler.inverse_transform(preds[-1].reshape(-1, 1)), + final_pred, "r", lw=2, label="Predictions", ) plt.fill_between( y_dates_valid[-1], - scaler.inverse_transform(preds[-1].reshape(-1, 1)).ravel(), - scaler.inverse_transform(y_valid[-1].reshape(-1, 1)).ravel(), + final_pred, + final_valid, color="k", alpha=0.2, ) From 637a75661a1d32afbd2c15ab68ab482aebddce6a Mon Sep 17 00:00:00 2001 From: Jamie Date: Fri, 14 May 2021 14:57:19 -0400 Subject: [PATCH 22/29] clean up menu and docstrings --- .../prediction_techniques/arima_view.py | 3 - .../prediction_techniques/ets_view.py | 3 - .../prediction_techniques/pred_controller.py | 7 - .../prediction_techniques/pred_helper.py | 4 - .../prediction_techniques/regression_view.py | 3 - .../prediction_techniques/sma_view.py | 307 ------------------ 6 files changed, 327 deletions(-) delete mode 100644 gamestonk_terminal/prediction_techniques/sma_view.py diff --git a/gamestonk_terminal/prediction_techniques/arima_view.py b/gamestonk_terminal/prediction_techniques/arima_view.py index 930003ef9d7..e0153ef5927 100644 --- a/gamestonk_terminal/prediction_techniques/arima_view.py +++ b/gamestonk_terminal/prediction_techniques/arima_view.py @@ -41,9 +41,6 @@ def arima(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): df_stock: pd.DataFrame Dataframe of prices - Returns - ------- - """ parser = argparse.ArgumentParser( add_help=False, diff --git a/gamestonk_terminal/prediction_techniques/ets_view.py b/gamestonk_terminal/prediction_techniques/ets_view.py index 96238826ba9..7b3e02f1136 100644 --- a/gamestonk_terminal/prediction_techniques/ets_view.py +++ b/gamestonk_terminal/prediction_techniques/ets_view.py @@ -61,9 +61,6 @@ def exponential_smoothing(other_args: List[str], s_ticker: str, df_stock: pd.Dat df_stock: pd.DataFrame Loaded stock dataframe - Returns - ------- - """ parser = argparse.ArgumentParser( add_help=False, diff --git a/gamestonk_terminal/prediction_techniques/pred_controller.py b/gamestonk_terminal/prediction_techniques/pred_controller.py index 38f209c47b8..293d06c4553 100644 --- a/gamestonk_terminal/prediction_techniques/pred_controller.py +++ b/gamestonk_terminal/prediction_techniques/pred_controller.py @@ -17,7 +17,6 @@ from gamestonk_terminal.prediction_techniques import ( knn_view, neural_networks_view, regression_view, - sma_view, ) @@ -29,7 +28,6 @@ class PredictionTechniquesController: "help", "q", "quit", - "sma", "ets", "knn", "linear", @@ -81,7 +79,6 @@ class PredictionTechniquesController: print(" q quit this menu, and shows back to main menu") print(" quit quit to abandon program") print("") - print(" sma simple moving average") print(" ets exponential smoothing (e.g. Holt-Winters)") print(" knn k-Nearest Neighbors") print(" linear linear regression (polynomial 1)") @@ -125,10 +122,6 @@ class PredictionTechniquesController: """Process Quit command - quit the program""" return True - def call_sma(self, other_args: List[str]): - """Process sma command""" - sma_view.simple_moving_average(other_args, self.ticker, self.stock) - def call_ets(self, other_args: List[str]): """Process ets command""" ets_view.exponential_smoothing(other_args, self.ticker, self.stock) diff --git a/gamestonk_terminal/prediction_techniques/pred_helper.py b/gamestonk_terminal/prediction_techniques/pred_helper.py index 8fe7683106c..a2b2f005139 100644 --- a/gamestonk_terminal/prediction_techniques/pred_helper.py +++ b/gamestonk_terminal/prediction_techniques/pred_helper.py @@ -29,10 +29,6 @@ from gamestonk_terminal import feature_flags as gtff from gamestonk_terminal.config_plot import PLOT_DPI -def get_backtesting_data(): - pass - - register_matplotlib_converters() os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" diff --git a/gamestonk_terminal/prediction_techniques/regression_view.py b/gamestonk_terminal/prediction_techniques/regression_view.py index 6f81805f5a0..2746fc291b1 100644 --- a/gamestonk_terminal/prediction_techniques/regression_view.py +++ b/gamestonk_terminal/prediction_techniques/regression_view.py @@ -54,9 +54,6 @@ def regression( polynomial: int Order of polynomial - Returns - ------- - """ parser = argparse.ArgumentParser( add_help=False, diff --git a/gamestonk_terminal/prediction_techniques/sma_view.py b/gamestonk_terminal/prediction_techniques/sma_view.py deleted file mode 100644 index b59a4c5a7ac..00000000000 --- a/gamestonk_terminal/prediction_techniques/sma_view.py +++ /dev/null @@ -1,307 +0,0 @@ -""" SMA View""" -__docformat__ = "numpy" - -import argparse -from typing import List -import datetime -import numpy as np -import matplotlib.pyplot as plt -import pandas as pd -from pandas.plotting import register_matplotlib_converters - -from gamestonk_terminal.helper_funcs import ( - check_positive, - parse_known_args_and_warn, - valid_date, - patch_pandas_text_adjustment, - get_next_stock_market_days, - plot_autoscale, -) - -from gamestonk_terminal.prediction_techniques.pred_helper import ( - print_pretty_prediction, - price_prediction_backtesting_color, - print_prediction_kpis, - get_backtesting_data, -) - -from gamestonk_terminal.config_plot import PLOT_DPI -from gamestonk_terminal import feature_flags as gtff - -register_matplotlib_converters() - - -def simple_moving_average(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): - """ - Run an SMA model - Parameters - ---------- - other_args: List[str] - Argparse arguments - s_ticker: str - Stock ticker - df_stock: pd.DataFrame - Dataframe of stock prices - - Returns - ------- - - """ - parser = argparse.ArgumentParser( - add_help=False, - prog="sma", - description=""" - Moving Averages are used to smooth the data in an array to - help eliminate noise and identify trends. The Simple Moving Average is literally - the simplest form of a moving average. Each output value is the average of the - previous n values. In a Simple Moving Average, each value in the time period carries - equal weight, and values outside of the time period are not included in the average. - This makes it less responsive to recent changes in the data, which can be useful for - filtering out those changes. - """, - ) - - parser.add_argument( - "-l", - "--length", - action="store", - dest="n_length", - type=check_positive, - default=20, - help="length of SMA window.", - ) - parser.add_argument( - "-d", - "--days", - action="store", - dest="n_days", - type=check_positive, - default=5, - help="prediction days.", - ) - parser.add_argument( - "-e", - "--end", - action="store", - type=valid_date, - dest="s_end_date", - default=None, - help="The end date (format YYYY-MM-DD) to select - Backtesting", - ) - - try: - ns_parser = parse_known_args_and_warn(parser, other_args) - if not ns_parser: - return - - # BACKTESTING - if ns_parser.s_end_date: - df_future, df_stock, bt_flag = get_backtesting_data( - df_stock, ns_parser.s_end_date, ns_parser.n_days - ) - if not bt_flag: - return - - # Prediction data - l_predictions = list() # type: List - for pred_day in range(ns_parser.n_days): - if pred_day < ns_parser.n_length: - l_ma_stock = df_stock["5. adjusted close"].values[ - -ns_parser.n_length + pred_day : - ] - else: - l_ma_stock = list() - l_predictions.append(np.mean(np.append(l_ma_stock, l_predictions))) - - l_pred_days = get_next_stock_market_days( - last_stock_day=df_stock["5. adjusted close"].index[-1], - n_next_days=ns_parser.n_days, - ) - df_pred = pd.Series(l_predictions, index=l_pred_days, name="Price") - - # Plotting - plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) - plt.plot(df_stock.index, df_stock["5. adjusted close"], lw=2) - # BACKTESTING - if ns_parser.s_end_date: - plt.title( - f"BACKTESTING: {ns_parser.n_length} Moving Average on {s_ticker} - {ns_parser.n_days} days prediction" - ) - else: - plt.title( - f"{ns_parser.n_length} Moving Average on {s_ticker} - {ns_parser.n_days} days prediction" - ) - plt.xlim( - df_stock.index[0], get_next_stock_market_days(df_pred.index[-1], 1)[-1] - ) - plt.xlabel("Time") - plt.ylabel("Share Price ($)") - plt.grid(b=True, which="major", color="#666666", linestyle="-") - plt.minorticks_on() - plt.grid(b=True, which="minor", color="#999999", linestyle="-", alpha=0.2) - # df_ma = df_stock["5. adjusted close"].rolling(window=ns_parser.n_length).mean() - # plt.plot(df_ma.index, df_ma, lw=2, linestyle="--", c="tab:orange") - plt.plot( - [df_stock.index[-1], df_pred.index[0]], - [df_stock["5. adjusted close"].values[-1], df_pred.values[0]], - lw=1, - c="tab:green", - linestyle="--", - ) - plt.plot(df_pred.index, df_pred, lw=2, c="tab:green") - plt.axvspan( - df_stock.index[-1], df_pred.index[-1], facecolor="tab:orange", alpha=0.2 - ) - _, _, ymin, ymax = plt.axis() - plt.vlines( - df_stock.index[-1], ymin, ymax, linewidth=1, linestyle="--", color="k" - ) - - # BACKTESTING - if ns_parser.s_end_date: - plt.plot( - df_future.index, - df_future["5. adjusted close"], - lw=2, - c="tab:blue", - ls="--", - ) - plt.plot( - [df_stock.index[-1], df_future.index[0]], - [ - df_stock["5. adjusted close"].values[-1], - df_future["5. adjusted close"].values[0], - ], - lw=1, - c="tab:blue", - linestyle="--", - ) - - if gtff.USE_ION: - plt.ion() - - plt.show() - - # BACKTESTING - if ns_parser.s_end_date: - plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) - plt.subplot(211) - plt.plot( - df_future.index, - df_future["5. adjusted close"], - lw=2, - c="tab:blue", - ls="--", - ) - plt.plot(df_pred.index, df_pred, lw=2, c="green") - plt.scatter( - df_future.index, df_future["5. adjusted close"], c="tab:blue", lw=3 - ) - plt.plot( - [df_stock.index[-1], df_future.index[0]], - [ - df_stock["5. adjusted close"].values[-1], - df_future["5. adjusted close"].values[0], - ], - lw=2, - c="tab:blue", - ls="--", - ) - plt.scatter(df_pred.index, df_pred, c="green", lw=3) - plt.plot( - [df_stock.index[-1], df_pred.index[0]], - [df_stock["5. adjusted close"].values[-1], df_pred.values[0]], - lw=2, - c="green", - ls="--", - ) - plt.title("BACKTESTING: Real data price versus Prediction") - plt.xlim(df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)) - plt.xticks( - [df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)], - visible=True, - ) - plt.ylabel("Share Price ($)") - plt.grid(b=True, which="major", color="#666666", linestyle="-") - plt.minorticks_on() - plt.grid(b=True, which="minor", color="#999999", linestyle="-", alpha=0.2) - plt.legend(["Real data", "Prediction data"]) - plt.xticks([]) - - plt.subplot(212) - plt.axhline(y=0, color="k", linestyle="--", linewidth=2) - plt.plot( - df_future.index, - 100 - * (df_pred.values - df_future["5. adjusted close"].values) - / df_future["5. adjusted close"].values, - lw=2, - c="red", - ) - plt.scatter( - df_future.index, - 100 - * (df_pred.values - df_future["5. adjusted close"].values) - / df_future["5. adjusted close"].values, - c="red", - lw=5, - ) - plt.title("BACKTESTING: Error between Real data and Prediction [%]") - plt.plot( - [df_stock.index[-1], df_future.index[0]], - [ - 0, - 100 - * (df_pred.values[0] - df_future["5. adjusted close"].values[0]) - / df_future["5. adjusted close"].values[0], - ], - lw=2, - ls="--", - c="red", - ) - plt.xlim(df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)) - plt.xticks( - [df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)], - visible=True, - ) - plt.xlabel("Time") - plt.ylabel("Prediction Error (%)") - plt.grid(b=True, which="major", color="#666666", linestyle="-") - plt.minorticks_on() - plt.grid(b=True, which="minor", color="#999999", linestyle="-", alpha=0.2) - plt.legend(["Real data", "Prediction data"]) - - if gtff.USE_ION: - plt.ion() - - plt.show() - - # Refactor prediction dataframe for backtesting print - df_pred.name = "Prediction" - df_pred = df_pred.to_frame() - df_pred["Real"] = df_future["5. adjusted close"] - - if gtff.USE_COLOR: - - patch_pandas_text_adjustment() - - print("Time Real [$] x Prediction [$]") - print( - df_pred.apply( - price_prediction_backtesting_color, axis=1 - ).to_string() - ) - else: - print(df_pred[["Real", "Prediction"]].round(2).to_string()) - - print("") - print_prediction_kpis(df_pred["Real"].values, df_pred["Prediction"].values) - - else: - # Print prediction data - print_pretty_prediction(df_pred, df_stock["5. adjusted close"].values[-1]) - print("") - - except Exception as e: - print(e) - print("") From ecb55b15c38a377db91351ad2acbbf9a594904f0 Mon Sep 17 00:00:00 2001 From: Jamie Date: Fri, 14 May 2021 15:17:56 -0400 Subject: [PATCH 23/29] edit optimizers and allow for lr through CL --- .../config_neural_network_models.py | 4 +++ .../neural_networks_view.py | 35 ++++++++++++++++--- .../prediction_techniques/pred_helper.py | 29 +++++---------- 3 files changed, 43 insertions(+), 25 deletions(-) diff --git a/gamestonk_terminal/config_neural_network_models.py b/gamestonk_terminal/config_neural_network_models.py index 48da79b2630..5408539bb13 100644 --- a/gamestonk_terminal/config_neural_network_models.py +++ b/gamestonk_terminal/config_neural_network_models.py @@ -49,6 +49,10 @@ Convolutional = [ {"Dense": {"activation": "relu"}}, ] +# Optimizers : https://www.tensorflow.org/api_docs/python/tf/keras/optimizers +# Adam, Adagrad, Adadelta, Adamax,Ftrl, Nadam, Rmsprop, Sgd +Optimizer = "Adam" + # Add early stopping to save time training when validation is plateaued # Can be set to large number or None to ignore Early_Stop_Patience = 10 diff --git a/gamestonk_terminal/prediction_techniques/neural_networks_view.py b/gamestonk_terminal/prediction_techniques/neural_networks_view.py index c15723e8e97..6a510ab6291 100644 --- a/gamestonk_terminal/prediction_techniques/neural_networks_view.py +++ b/gamestonk_terminal/prediction_techniques/neural_networks_view.py @@ -17,6 +17,16 @@ from tensorflow.keras.layers import ( AvgPool1D, Flatten, ) +from tensorflow.keras.optimizers import ( + Adam, + Adamax, + Adagrad, + Adadelta, + Ftrl, + Nadam, + RMSprop, + SGD, +) from gamestonk_terminal.helper_funcs import get_next_stock_market_days from gamestonk_terminal.prediction_techniques.pred_helper import ( prepare_scale_train_valid_test, @@ -28,6 +38,18 @@ from gamestonk_terminal.prediction_techniques.pred_helper import ( ) from gamestonk_terminal import config_neural_network_models as cfg_nn_models +optimizers = { + "Adam": Adam, + "Adagrad": Adagrad, + "Adadelta": Adadelta, + "Adamax": Adamax, + "Ftrl": Ftrl, + "Nadam": Nadam, + "Rmsprop": RMSprop, + "Ggd": SGD, +} + + if cfg_nn_models.Early_Stop_Patience: es = EarlyStopping(monitor="val_loss", patience=cfg_nn_models.Early_Stop_Patience) else: @@ -169,7 +191,8 @@ def mlp(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): ns_parser.n_inputs, ns_parser.n_days, ) - model.compile(optimizer=ns_parser.s_optimizer, loss=ns_parser.s_loss) + model.compile(optimizer=optimizers[cfg_nn_models.Optimizer](lr=ns_parser.lr), + loss=ns_parser.s_loss) model.fit( X_train.reshape(X_train.shape[0], X_train.shape[1], 1), @@ -276,8 +299,8 @@ def rnn(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): ns_parser.n_inputs, ns_parser.n_days, ) - model.compile(optimizer=ns_parser.s_optimizer, loss=ns_parser.s_loss) - + model.compile(optimizer=optimizers[cfg_nn_models.Optimizer](lr=ns_parser.lr), + loss=ns_parser.s_loss) model.fit( X_train.reshape(X_train.shape[0], X_train.shape[1], 1), y_train, @@ -381,7 +404,8 @@ def lstm(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): ns_parser.n_inputs, ns_parser.n_days, ) - model.compile(optimizer=ns_parser.s_optimizer, loss=ns_parser.s_loss) + model.compile(optimizer=optimizers[cfg_nn_models.Optimizer](lr=ns_parser.lr), + loss=ns_parser.s_loss) model.fit( X_train.reshape(X_train.shape[0], X_train.shape[1], 1), @@ -489,7 +513,8 @@ def conv1d(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): ns_parser.n_days, ) - model.compile(optimizer=ns_parser.s_optimizer, loss=ns_parser.s_loss) + model.compile(optimizer=optimizers[cfg_nn_models.Optimizer](lr=ns_parser.lr), + loss=ns_parser.s_loss) model.fit( X_train.reshape(X_train.shape[0], X_train.shape[1], 1), diff --git a/gamestonk_terminal/prediction_techniques/pred_helper.py b/gamestonk_terminal/prediction_techniques/pred_helper.py index a2b2f005139..88410d33d75 100644 --- a/gamestonk_terminal/prediction_techniques/pred_helper.py +++ b/gamestonk_terminal/prediction_techniques/pred_helper.py @@ -40,6 +40,13 @@ ORIGINAL_TF_XLA_FLAGS = os.environ.get("TF_XLA_FLAGS") ORIGINAL_TF_FORCE_GPU_ALLOW_GROWTH = os.environ.get("TF_FORCE_GPU_ALLOW_GROWTH") +def check_valid_frac(num) -> float: + + if (num > 0) and (num <1): + return num + else: + raise argparse.ArgumentTypeError(f"{num} is an invalid percentage") + def restore_env(): """Restore environment variables to original values""" @@ -121,24 +128,6 @@ def parse_args(prog: str, description: str, other_args: List[str]): choices=["normalization", "standardization", "minmax", "none"], help="pre-processing data.", ) - parser.add_argument( - "-o", - "--optimizer", - action="store", - dest="s_optimizer", - default="adam", - choices=[ - "adam", - "adagrad", - "adadelta", - "adamax", - "ftrl", - "nadam", - "rmsprop", - "sgd", - ], - help="optimization technique (see https://www.tensorflow.org/api_docs/python/tf/keras/optimizers)", - ) parser.add_argument( "-l", "--loss", @@ -211,7 +200,7 @@ def parse_args(prog: str, description: str, other_args: List[str]): parser.add_argument( "-v", "--valid", - type=float, + type=check_valid_frac, dest="valid_split", default=0.1, help="Validation data split fraction", @@ -221,7 +210,7 @@ def parse_args(prog: str, description: str, other_args: List[str]): "--lr", type=check_positive, dest="lr", - default=None, + default=0.01, help="Specify learning rate for optimizer.", ) From b3f863228bdc735e9eb5c8d4740fb9a0e575b449 Mon Sep 17 00:00:00 2001 From: Jamie Date: Fri, 14 May 2021 15:35:33 -0400 Subject: [PATCH 24/29] update readme --- .../FUNE-TUNING-NN-models.md | 24 ++++++-- .../prediction_techniques/README.md | 56 ++++++++++++------- 2 files changed, 56 insertions(+), 24 deletions(-) diff --git a/gamestonk_terminal/prediction_techniques/FUNE-TUNING-NN-models.md b/gamestonk_terminal/prediction_techniques/FUNE-TUNING-NN-models.md index ad4ab6bab15..e3189b8163c 100644 --- a/gamestonk_terminal/prediction_techniques/FUNE-TUNING-NN-models.md +++ b/gamestonk_terminal/prediction_techniques/FUNE-TUNING-NN-models.md @@ -16,7 +16,17 @@ load MSFT -s 2014-01-01 3. **Out of this training data, select which part will be used for validation** -Since this is a timeseries, and the data has a sequence, the data at the end of the timeseries is the one to be used for validation. As a rule of thumb I usually try to "steal" the least amount of data from the training set, since the more recent the data the more important it is for training. Hence, one could set the end date of training data to be the current day - days of predictions. +Anyone reading this, note this is quite a discussion between the devs. Since this is a time series, there is an inherent +ordering of the data. However, since returns are random, it is hard to predict, and ordering data may not have any meaning. + +To this end, we allow users to specify how much validation data to use and if it should be shuffled. By default, we will split +into 10% validation data with a random shuffle. This will allow the user to see how the model performs at various times in your sequence. + +``` +-v/--valid : validation split of data. Deafult 0.1 (10%) +--no_shuffle : Flag that will order the validation data so that the last (-v) percent of data is the validation. +``` +To try "backtesting" to a certain data, you can specify the end date. This will forecast and compare the true data from that time. ``` -e/--end : end date (format YYYY-MM-DD) of the stock - Backtesting. Default None. ``` @@ -25,7 +35,7 @@ Since this is a timeseries, and the data has a sequence, the data at the end of Decide whether you want to pre-process your share price data or not. I.e. do you want to use these values between 18 and 245, or perhaps normalize these between 0 and 1? The later may improve performance. ``` --p/--pp : pre-processing data. Default normalization. +-p/--pp : pre-processing data. Default minmax. ``` 5. **Select input, prediction days and jumps in training data** @@ -91,14 +101,20 @@ This model is set in [config_neural_network_models.py](/config_neural_network_mo -l/--loss : loss function. Default mae. ``` -**Optimizer technique**: Adaptive Moment Estimation (_adam_) is usually the default choice for this type of problems. See https://www.tensorflow.org/api_docs/python/tf/keras/optimizers. +**Optimizer technique**:Adaptive Moment Estimation (_adam_) is usually the default choice for this type of problems. See https://www.tensorflow.org/api_docs/python/tf/keras/optimizers. + +The optimizer selection has been moved to teh config_neural_network_models.py file and the new argument is the learning rate. +This tells your model how "fast" to train by adjusting the factor that gets multiplied to the loss function for updating weights ``` --o/--optimizer : optimization technique. Default adam. +--lr : learning rate. Default 0.01. ``` 9. **Training epochs** Select the number of epochs to train your model. The bigger number of epochs, the longer the training will take, and also increase chances of overfitting. + +An early stopping has been implemented that will (if desired) stop training once the validation loss has plateaued. the patience (number of +epochs with no improvement) can be found in the config file. ``` --epochs : number of training epochs. Default 200. ``` diff --git a/gamestonk_terminal/prediction_techniques/README.md b/gamestonk_terminal/prediction_techniques/README.md index 4646140a780..cd4ddda2b02 100644 --- a/gamestonk_terminal/prediction_techniques/README.md +++ b/gamestonk_terminal/prediction_techniques/README.md @@ -2,8 +2,6 @@ This menu aims to predict the share price of a pre-loaded stock, and the usage of the following commands along with an example will be exploited below. See [How to fune-tuning a NN model](FUNE-TUNING-NN-models.md). - * [sma](#sma) - - simple moving average * [ets](#ets) - Exponential Smoothing (e.g. Holt-Winters) * [knn](#knn) @@ -28,6 +26,8 @@ This menu aims to predict the share price of a pre-loaded stock, and the usage o * [lstm](#lstm) - Long-Short Term Memory - Contains a [backtesting example](#backtesting) + * [conv1d](#conv1d) + - 1D Convolutional Neural Net **Note:** _Use this at your own discretion. All of these prediciton techniques rely solely on the closing price of the stock. This means that there are several factors that the models aren't aware of at the time of prediction, and may - drastically - move the price up or down. Examples are: news, analyst price targets, reddit post, tweets from Elon Musk, and so on._ @@ -35,17 +35,6 @@ This menu aims to predict the share price of a pre-loaded stock, and the usage o -## sma -``` -usage: sma [-l N_LENGTH] [-d N_DAYS] -``` -Simple Moving Average: - * -l : length of SMA window. Default 20. - * -d : prediciton days. Default 5. - * -e : end date (format YYYY-MM-DD) of the stock - Backtesting. Default None. - -![sma](https://user-images.githubusercontent.com/25267873/108604945-d29aea80-73a8-11eb-8dac-6a545b9c52b9.png) - ## ets ``` usage: ets [-t TREND] [-s SEASONAL] [-p SEASONAL_PERIODS] [-d N_DAYS] @@ -154,7 +143,7 @@ Facebook's Prophet: ## mlp ``` usage: mlp [-d N_DAYS] [-i N_INPUTS] [-j N_JUMPS] [--epochs N_EPOCHS] [-p {minmax, normalization,standardization,none}] -[-o {adam,adagrad,adadelta,adamax,ftrl,nadam,optimizer,rmsprop,sgd}] [-l {mae,mape,mse,msle}] [-e S_END_DATE] [--loops N_LOOPS] + [-l {mae,mape,mse,msle}] [-e S_END_DATE] [--loops N_LOOPS] [-v VALID] [--lr LEARNING_RATE] [--no_shuffle] ``` MulitLayer Perceptron: * -d : prediciton days. Default 5. @@ -162,14 +151,16 @@ MulitLayer Perceptron: * -j : number of jumps in training data. Default 1. * --epochs : number of training epochs. Default 200. * -p : pre-processing data. Default minmax. - * -o : optimization technique. Default adam. + * -v : validation split. Default 0.1 * -l : loss function. Default mae. * -e : end date (format YYYY-MM-DD) of the stock - Backtesting. Default None. * --xla_cpu: if present, will enable XLA for CPU (overrides environment variables during run). * --xla_gpu: if present, will enable XLA for GPU (overrides environment variables during run). * --force_allow_gpu_growth: if true, will force TensorFlow to allow GPU memory usage to grow as needed. Otherwise will allocate 100% of available GPU memory when CUDA is set up. Default true. * --batch_size: batch size for model training, should not be used unless advanced user. Default None. + * --lr : learning rate for optimizer * --loops: number of loops to iterate and train models. Default 1. + * --no_shuffle : split validation data in time_ordered way instead of random. Due to the complexity of defining a model through command line, one can define it in: [config_neural_network_models.txt](/config_neural_network_models.py) ``` @@ -191,7 +182,7 @@ MultiLayer_Perceptron \ ## rnn ``` usage: rnn [-d N_DAYS] [-i N_INPUTS] [-j N_JUMPS] [--epochs N_EPOCHS] [-p {normalization,standardization,none}] -[-o {adam,adagrad,adadelta,adamax,ftrl,nadam,optimizer,rmsprop,sgd}] [-l {mae,mape,mse,msle}] [-e S_END_DATE] [--loops N_LOOPS] + [-l {mae,mape,mse,msle}] [-e S_END_DATE] [--loops N_LOOPS] [-v VALID] [--lr LEARNING_RATE] [--no_shuffle] ``` Recurrent Neural Network: * -d : prediciton days. Default 5. @@ -199,7 +190,7 @@ Recurrent Neural Network: * -j : number of jumps in training data. Default 1. * --epochs : number of training epochs. Default 200. * -p : pre-processing data. Default normalization. - * -o : optimization technique. Default adam. + * -v : validation split. Default 0.1 * -l : loss function. Default mae. * -e : end date (format YYYY-MM-DD) of the stock - Backtesting. Default None. * --xla_cpu: if present, will enable XLA for CPU (overrides environment variables during run). @@ -207,6 +198,8 @@ Recurrent Neural Network: * --force_allow_gpu_growth: if true, will force TensorFlow to allow GPU memory usage to grow as needed. Otherwise will allocate 100% of available GPU memory when CUDA is set up. Default true. * --batch_size: batch size for model training, should not be used unless advanced user. Default None. * --loops: number of loops to iterate and train models. Default 1. + * --lr : learning rate for optimizer + * --no_shuffle : split validation data in time_ordered way instead of random. Due to the complexity of defining a model through command line, one can define it in: [config_neural_network_models.txt](/config_neural_network_models.py) ``` @@ -235,7 +228,7 @@ Recurrent_Neural_Network \ ## lstm ``` usage: lstm [-d N_DAYS] [-i N_INPUTS] [-j N_JUMPS] [--epochs N_EPOCHS] [-p {normalization,standardization,none}] -[-o {adam,adagrad,adadelta,adamax,ftrl,nadam,optimizer,rmsprop,sgd}] [-l {mae,mape,mse,msle}] [-e S_END_DATE] [--loops N_LOOPS] + [-l {mae,mape,mse,msle}] [-e S_END_DATE] [--loops N_LOOPS] [-v VALID] [--lr LEARNING_RATE] [--no_shuffle] ``` Long-Short Term Memory: * -d : prediciton days. Default 5. @@ -243,14 +236,16 @@ Long-Short Term Memory: * -j : number of jumps in training data. Default 1. * --epochs : number of training epochs. Default 200. * -p : pre-processing data. Default normalization. - * -o : optimization technique. Default adam. + * -v : validation split. Default 0.1 * -l : loss function. Default mae. * -e : end date (format YYYY-MM-DD) of the stock - Backtesting. Default None. * --xla_cpu: if present, will enable XLA for CPU (overrides environment variables during run). * --xla_gpu: if present, will enable XLA for GPU (overrides environment variables during run). * --force_allow_gpu_growth: if true, will force TensorFlow to allow GPU memory usage to grow as needed. Otherwise will allocate 100% of available GPU memory when CUDA is set up. Default true. * --batch_size: batch size for model training, should not be used unless advanced user. Default None. - * --loops: number of loops to iterate and train models. Default 1. + * --loops: number of loops to iterate and train models. Default 1 + * --lr : learning rate for optimizer + * --no_shuffle : split validation data in time_ordered way instead of random. Due to the complexity of defining a model through command line, one can define it in: [config_neural_network_models.py](/config_neural_network_models.py) ``` @@ -273,3 +268,24 @@ Long_Short_Term_Memory \ Captura de ecrã 2021-03-14, às 00 06 51 +## conv1d +``` +usage: lstm [-d N_DAYS] [-i N_INPUTS] [-j N_JUMPS] [--epochs N_EPOCHS] [-p {normalization,standardization,minmax,none}] + [-l {mae,mape,mse,msle}] [-e S_END_DATE] [--loops N_LOOPS] [-v VALID] [--lr LEARNING_RATE] [--no_shuffle] +``` +1D Convolutional Neural Net: + * -d : prediction days. Default 5. + * -i : number of days to use for prediction. Default 40. + * -j : number of jumps in training data. Default 1. + * --epochs : number of training epochs. Default 50. + * -p : pre-processing data. Default minmax. + * -v : validation split. Default 0.1 + * -l : loss function. Default mae. + * -e : end date (format YYYY-MM-DD) of the stock - Backtesting. Default None. + * --xla_cpu: if present, will enable XLA for CPU (overrides environment variables during run). + * --xla_gpu: if present, will enable XLA for GPU (overrides environment variables during run). + * --force_allow_gpu_growth: if true, will force TensorFlow to allow GPU memory usage to grow as needed. Otherwise will allocate 100% of available GPU memory when CUDA is set up. Default true. + * --batch_size: batch size for model training, should not be used unless advanced user. Default None. + * --loops: number of loops to iterate and train models. Default 1. + * --lr : learning rate for optimizer + * --no_shuffle : split validation data in time_ordered way instead of random. \ No newline at end of file From 461f705694081f67b987912c549a0aa657adf30b Mon Sep 17 00:00:00 2001 From: Jamie Date: Fri, 14 May 2021 15:38:00 -0400 Subject: [PATCH 25/29] more readme and reset flags --- gamestonk_terminal/README.md | 3 +-- .../neural_networks_view.py | 24 ++++++++++++------- .../prediction_techniques/pred_helper.py | 3 ++- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/gamestonk_terminal/README.md b/gamestonk_terminal/README.md index a1d86689ddb..b5f1857b160 100644 --- a/gamestonk_terminal/README.md +++ b/gamestonk_terminal/README.md @@ -242,7 +242,6 @@ Command|Description|Source Command|Technique|Sources ------ | ------------|--- -`sma` |simple moving average | [Wikipedia](https://en.wikipedia.org/wiki/Moving_average#Simple_moving_average), [Investopedia](https://www.investopedia.com/terms/s/sma.asp) `knn` |k-Nearest Neighbors | [Wikipedia](https://en.wikipedia.org/wiki/K-nearest_neighbors_algorithm) `linear` |linear regression (polynomial 1) | [Wikipedia](https://en.wikipedia.org/wiki/Linear_regression), [Investopedia](https://www.investopedia.com/terms/r/regression.asp) `quadratic` |quadratic regression (polynomial 2) | [Wikipedia](https://en.wikipedia.org/wiki/Polynomial_regression), [Investopedia](https://www.investopedia.com/terms/r/regression.asp) @@ -253,7 +252,7 @@ Command|Technique|Sources `mlp` |MultiLayer Perceptron | [Wikipedia](https://en.wikipedia.org/wiki/Multilayer_perceptron) `rnn` |Recurrent Neural Network | [Wikipedia](https://en.wikipedia.org/wiki/Recurrent_neural_network) `lstm` |Long Short-Term Memory | [Wikipedia](https://en.wikipedia.org/wiki/Long_short-term_memory), [Details](https://colah.github.io/posts/2015-08-Understanding-LSTMs/) - +`conv1d` |1D Convolution Neural Net| [Wikipedia](https://en.wikipedia.org/wiki/Convolutional_neural_network)   ## Portfolio Analysis [»](portfolio/README.md) diff --git a/gamestonk_terminal/prediction_techniques/neural_networks_view.py b/gamestonk_terminal/prediction_techniques/neural_networks_view.py index 6a510ab6291..30357106fb8 100644 --- a/gamestonk_terminal/prediction_techniques/neural_networks_view.py +++ b/gamestonk_terminal/prediction_techniques/neural_networks_view.py @@ -191,8 +191,10 @@ def mlp(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): ns_parser.n_inputs, ns_parser.n_days, ) - model.compile(optimizer=optimizers[cfg_nn_models.Optimizer](lr=ns_parser.lr), - loss=ns_parser.s_loss) + model.compile( + optimizer=optimizers[cfg_nn_models.Optimizer](lr=ns_parser.lr), + loss=ns_parser.s_loss, + ) model.fit( X_train.reshape(X_train.shape[0], X_train.shape[1], 1), @@ -299,8 +301,10 @@ def rnn(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): ns_parser.n_inputs, ns_parser.n_days, ) - model.compile(optimizer=optimizers[cfg_nn_models.Optimizer](lr=ns_parser.lr), - loss=ns_parser.s_loss) + model.compile( + optimizer=optimizers[cfg_nn_models.Optimizer](lr=ns_parser.lr), + loss=ns_parser.s_loss, + ) model.fit( X_train.reshape(X_train.shape[0], X_train.shape[1], 1), y_train, @@ -404,8 +408,10 @@ def lstm(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): ns_parser.n_inputs, ns_parser.n_days, ) - model.compile(optimizer=optimizers[cfg_nn_models.Optimizer](lr=ns_parser.lr), - loss=ns_parser.s_loss) + model.compile( + optimizer=optimizers[cfg_nn_models.Optimizer](lr=ns_parser.lr), + loss=ns_parser.s_loss, + ) model.fit( X_train.reshape(X_train.shape[0], X_train.shape[1], 1), @@ -513,8 +519,10 @@ def conv1d(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): ns_parser.n_days, ) - model.compile(optimizer=optimizers[cfg_nn_models.Optimizer](lr=ns_parser.lr), - loss=ns_parser.s_loss) + model.compile( + optimizer=optimizers[cfg_nn_models.Optimizer](lr=ns_parser.lr), + loss=ns_parser.s_loss, + ) model.fit( X_train.reshape(X_train.shape[0], X_train.shape[1], 1), diff --git a/gamestonk_terminal/prediction_techniques/pred_helper.py b/gamestonk_terminal/prediction_techniques/pred_helper.py index 88410d33d75..de8616e7ec7 100644 --- a/gamestonk_terminal/prediction_techniques/pred_helper.py +++ b/gamestonk_terminal/prediction_techniques/pred_helper.py @@ -42,11 +42,12 @@ ORIGINAL_TF_FORCE_GPU_ALLOW_GROWTH = os.environ.get("TF_FORCE_GPU_ALLOW_GROWTH") def check_valid_frac(num) -> float: - if (num > 0) and (num <1): + if (num > 0) and (num < 1): return num else: raise argparse.ArgumentTypeError(f"{num} is an invalid percentage") + def restore_env(): """Restore environment variables to original values""" From 0032d794ed13de4d97eed33a0859cef7530188f4 Mon Sep 17 00:00:00 2001 From: Jamie Date: Fri, 14 May 2021 15:41:10 -0400 Subject: [PATCH 26/29] codespell --- .../prediction_techniques/FUNE-TUNING-NN-models.md | 4 ++-- gamestonk_terminal/prediction_techniques/pred_helper.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/gamestonk_terminal/prediction_techniques/FUNE-TUNING-NN-models.md b/gamestonk_terminal/prediction_techniques/FUNE-TUNING-NN-models.md index e3189b8163c..3e6ffb67384 100644 --- a/gamestonk_terminal/prediction_techniques/FUNE-TUNING-NN-models.md +++ b/gamestonk_terminal/prediction_techniques/FUNE-TUNING-NN-models.md @@ -23,7 +23,7 @@ To this end, we allow users to specify how much validation data to use and if it into 10% validation data with a random shuffle. This will allow the user to see how the model performs at various times in your sequence. ``` --v/--valid : validation split of data. Deafult 0.1 (10%) +-v/--valid : validation split of data. Default 0.1 (10%) --no_shuffle : Flag that will order the validation data so that the last (-v) percent of data is the validation. ``` To try "backtesting" to a certain data, you can specify the end date. This will forecast and compare the true data from that time. @@ -103,7 +103,7 @@ This model is set in [config_neural_network_models.py](/config_neural_network_mo **Optimizer technique**:Adaptive Moment Estimation (_adam_) is usually the default choice for this type of problems. See https://www.tensorflow.org/api_docs/python/tf/keras/optimizers. -The optimizer selection has been moved to teh config_neural_network_models.py file and the new argument is the learning rate. +The optimizer selection has been moved to the config_neural_network_models.py file and the new argument is the learning rate. This tells your model how "fast" to train by adjusting the factor that gets multiplied to the loss function for updating weights ``` --lr : learning rate. Default 0.01. diff --git a/gamestonk_terminal/prediction_techniques/pred_helper.py b/gamestonk_terminal/prediction_techniques/pred_helper.py index de8616e7ec7..effd1ef8429 100644 --- a/gamestonk_terminal/prediction_techniques/pred_helper.py +++ b/gamestonk_terminal/prediction_techniques/pred_helper.py @@ -473,7 +473,7 @@ def plot_data_predictions( alpha=0.2, ) - # Leave this one out of the loop so that the legend doesnt get overpopulated with "Predictions" + # Leave this one out of the loop so that the legend doesn't get overpopulated with "Predictions" if scaler: final_pred = scaler.inverse_transform(preds[-1].reshape(-1, 1)).ravel() final_valid = scaler.inverse_transform(y_valid[-1].reshape(-1, 1)).ravel() From 4e4e9588a3f6d299e0f7233137de2ca1ce0895e8 Mon Sep 17 00:00:00 2001 From: Jamie Date: Fri, 14 May 2021 15:45:19 -0400 Subject: [PATCH 27/29] linter --- gamestonk_terminal/prediction_techniques/pred_helper.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/gamestonk_terminal/prediction_techniques/pred_helper.py b/gamestonk_terminal/prediction_techniques/pred_helper.py index effd1ef8429..9ce767451e6 100644 --- a/gamestonk_terminal/prediction_techniques/pred_helper.py +++ b/gamestonk_terminal/prediction_techniques/pred_helper.py @@ -41,11 +41,9 @@ ORIGINAL_TF_FORCE_GPU_ALLOW_GROWTH = os.environ.get("TF_FORCE_GPU_ALLOW_GROWTH") def check_valid_frac(num) -> float: - - if (num > 0) and (num < 1): - return num - else: + if (num < 0) or (num > 1): raise argparse.ArgumentTypeError(f"{num} is an invalid percentage") + return num def restore_env(): From f5d53417e5ed81d988a2474363ecd2c50b7d13ed Mon Sep 17 00:00:00 2001 From: Jamie Date: Mon, 17 May 2021 11:53:44 -0400 Subject: [PATCH 28/29] Put loss and preprocesser in cfg file --- .../config_neural_network_models.py | 12 +++++ .../neural_networks_view.py | 8 +-- .../prediction_techniques/pred_helper.py | 49 +++---------------- 3 files changed, 23 insertions(+), 46 deletions(-) diff --git a/gamestonk_terminal/config_neural_network_models.py b/gamestonk_terminal/config_neural_network_models.py index 5408539bb13..38ef798e18e 100644 --- a/gamestonk_terminal/config_neural_network_models.py +++ b/gamestonk_terminal/config_neural_network_models.py @@ -56,3 +56,15 @@ Optimizer = "Adam" # Add early stopping to save time training when validation is plateaued # Can be set to large number or None to ignore Early_Stop_Patience = 10 + +# Losses https://www.tensorflow.org/api_docs/python/tf/keras/losses +# mae, mape, mse, msle, poisson, logcosh, kld, hinge, squared_hinge, huber +Loss = "mae" + +# Preprocessing technique to use : https://scikit-learn.org/stable/modules/preprocessing.html +# Options are: +# minmax - Adjusts data to be between 0 and 1 - Most Used for Stocks +# standardization - Standardizes data so that it has mean 0 and standard deviation = 1 +# normalization - Normalizes data so that L2 norm (sum x_i^2) = 1 +# none - No preprocessing +Preprocess = "minmax" diff --git a/gamestonk_terminal/prediction_techniques/neural_networks_view.py b/gamestonk_terminal/prediction_techniques/neural_networks_view.py index 30357106fb8..e88b59a679d 100644 --- a/gamestonk_terminal/prediction_techniques/neural_networks_view.py +++ b/gamestonk_terminal/prediction_techniques/neural_networks_view.py @@ -193,7 +193,7 @@ def mlp(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): ) model.compile( optimizer=optimizers[cfg_nn_models.Optimizer](lr=ns_parser.lr), - loss=ns_parser.s_loss, + loss=cfg_nn_models.Loss, ) model.fit( @@ -303,7 +303,7 @@ def rnn(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): ) model.compile( optimizer=optimizers[cfg_nn_models.Optimizer](lr=ns_parser.lr), - loss=ns_parser.s_loss, + loss=cfg_nn_models.Loss, ) model.fit( X_train.reshape(X_train.shape[0], X_train.shape[1], 1), @@ -410,7 +410,7 @@ def lstm(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): ) model.compile( optimizer=optimizers[cfg_nn_models.Optimizer](lr=ns_parser.lr), - loss=ns_parser.s_loss, + loss=cfg_nn_models.Loss, ) model.fit( @@ -521,7 +521,7 @@ def conv1d(other_args: List[str], s_ticker: str, df_stock: pd.DataFrame): model.compile( optimizer=optimizers[cfg_nn_models.Optimizer](lr=ns_parser.lr), - loss=ns_parser.s_loss, + loss=cfg_nn_models.Loss, ) model.fit( diff --git a/gamestonk_terminal/prediction_techniques/pred_helper.py b/gamestonk_terminal/prediction_techniques/pred_helper.py index 9ce767451e6..197a5c16e2a 100644 --- a/gamestonk_terminal/prediction_techniques/pred_helper.py +++ b/gamestonk_terminal/prediction_techniques/pred_helper.py @@ -25,6 +25,7 @@ from gamestonk_terminal.helper_funcs import ( valid_date, plot_autoscale, ) +from gamestonk_terminal import config_neural_network_models as cfg from gamestonk_terminal import feature_flags as gtff from gamestonk_terminal.config_plot import PLOT_DPI @@ -39,6 +40,8 @@ simplefilter(action="ignore", category=FutureWarning) ORIGINAL_TF_XLA_FLAGS = os.environ.get("TF_XLA_FLAGS") ORIGINAL_TF_FORCE_GPU_ALLOW_GROWTH = os.environ.get("TF_FORCE_GPU_ALLOW_GROWTH") +PREPROCESSER = cfg.Preprocess + def check_valid_frac(num) -> float: if (num < 0) or (num > 1): @@ -109,44 +112,6 @@ def parse_args(prog: str, description: str, other_args: List[str]): default=50, help="number of training epochs.", ) - parser.add_argument( - "-j", - "--jumps", - action="store", - dest="n_jumps", - type=check_positive, - default=1, - help="number of jumps in training data.", - ) - parser.add_argument( - "-p", - "--pp", - action="store", - dest="s_preprocessing", - default="minmax", - choices=["normalization", "standardization", "minmax", "none"], - help="pre-processing data.", - ) - parser.add_argument( - "-l", - "--loss", - action="store", - dest="s_loss", - default="mae", - choices=[ - "mae", - "mape", - "mse", - "msle", - "poisson", - "logcosh", - "kld", - "hinge", - "squared_hinge", - "huber", - ], - help="loss function (see https://www.tensorflow.org/api_docs/python/tf/keras/losses)", - ) parser.add_argument( "-e", "--end", @@ -293,16 +258,16 @@ def prepare_scale_train_valid_test( test_size = ns_parser.valid_split # Pre-process data - if ns_parser.s_preprocessing == "standardization": + if PREPROCESSER == "standardization": scaler = StandardScaler() - elif ns_parser.s_preprocessing == "minmax": + elif PREPROCESSER == "minmax": scaler = MinMaxScaler() - elif ns_parser.s_preprocessing == "normalization": + elif PREPROCESSER == "normalization": scaler = Normalizer() - elif ns_parser.s_preprocessing == "none": + elif (PREPROCESSER == "none") or (PREPROCESSER is None): scaler = None # Test data is used for forecasting. Takes the last n_input_days data points. # These points are not fed into training From 3e62704efca626557b17a25dd373f3ffacdb03f0 Mon Sep 17 00:00:00 2001 From: Jamie Date: Mon, 17 May 2021 12:01:21 -0400 Subject: [PATCH 29/29] Update readmes --- .../FUNE-TUNING-NN-models.md | 13 ++-- .../prediction_techniques/README.md | 63 +++++++------------ 2 files changed, 27 insertions(+), 49 deletions(-) diff --git a/gamestonk_terminal/prediction_techniques/FUNE-TUNING-NN-models.md b/gamestonk_terminal/prediction_techniques/FUNE-TUNING-NN-models.md index 3e6ffb67384..2468e6f8c43 100644 --- a/gamestonk_terminal/prediction_techniques/FUNE-TUNING-NN-models.md +++ b/gamestonk_terminal/prediction_techniques/FUNE-TUNING-NN-models.md @@ -35,8 +35,9 @@ To try "backtesting" to a certain data, you can specify the end date. This will Decide whether you want to pre-process your share price data or not. I.e. do you want to use these values between 18 and 245, or perhaps normalize these between 0 and 1? The later may improve performance. ``` --p/--pp : pre-processing data. Default minmax. +Preprocess = "minmax" ``` +This is defined in the config file. 5. **Select input, prediction days and jumps in training data** @@ -54,13 +55,6 @@ Personally, I recommend to set this for a really short window. This is because t -d/--days : prediciton days. Default 5. ``` -The **jumps in the training data** corresponds to the number of days we want to jump between training loops. - -Let's say we start training our Neural Network from 2021-01-04, with 10 inputs (days) and 5 prediction days. If jump is set to 1, it means that the after optimizing our model for input data from 2021-01-04 to 2021-01-15, and predicting from 2021-01-18 to 2021-01-22, we will optimize the model for input data from 2021-01-05 to 2021-01-18, and predicting from 2021-01-19 to 2021-01-25. On the other hand, if we set the jump to 5, it means that we are identifying a "weekly" frequency in the data, and therefore, our next model optimization would use input data from 2021-01-11 to 2021-01-22, and predicting from 2021-01-25 to 2021-01-29. -``` --j/--jumps : number of jumps in training data. Default 1. -``` - 6. **Select NN architecture.** TL;DR: LSTM > RNN > MLP **MLP**: The neurons don't have any memory. @@ -98,8 +92,9 @@ This model is set in [config_neural_network_models.py](/config_neural_network_mo **Loss function**: Often chosen having the activation functions of the hidden layers in mind. See https://machinelearningmastery.com/how-to-choose-loss-functions-when-training-deep-learning-neural-networks/, and https://www.tensorflow.org/api_docs/python/tf/keras/losses. ``` --l/--loss : loss function. Default mae. +Loss = "mae" ``` +This is defined in the config file. **Optimizer technique**:Adaptive Moment Estimation (_adam_) is usually the default choice for this type of problems. See https://www.tensorflow.org/api_docs/python/tf/keras/optimizers. diff --git a/gamestonk_terminal/prediction_techniques/README.md b/gamestonk_terminal/prediction_techniques/README.md index cd4ddda2b02..cf73839b70a 100644 --- a/gamestonk_terminal/prediction_techniques/README.md +++ b/gamestonk_terminal/prediction_techniques/README.md @@ -43,19 +43,18 @@ Exponential Smoothing (based on trend+seasonality, see https://otexts.com/fpp2/t * -t : trend component: N: None, A: Additive, Ad: Additive Damped. Default N. * -s : seasonality component: N: None, A: Additive, M: Multiplicative. Default N. * -p : seasonal periods. Default 5. - * -d : prediciton days. Default 5. + * -d : prediction days. Default 5. * -e : end date (format YYYY-MM-DD) of the stock - Backtesting. Default None. ![ets_pltr](https://user-images.githubusercontent.com/25267873/110266847-97a6d280-7fb6-11eb-997e-0b598abc713b.png) ## knn ``` -usage: knn [-i N_INPUTS] [-d N_DAYS] [-j N_JUMPS] [-n N_NEIGHBORS] +usage: knn [-i N_INPUTS] [-d N_DAYS] [-n N_NEIGHBORS] ``` k-Nearest Neighbors: * -i : number of days to use for prediction. Default 40. - * -d : prediciton days. Default 5. - * -j : number of jumps in training data. Default 1. + * -d : prediction days. Default 5. * -n : number of neighbors to use on the algorithm. Default 20. * -e : end date (format YYYY-MM-DD) of the stock - Backtesting. Default None. @@ -63,51 +62,47 @@ k-Nearest Neighbors: ## linear ``` -usage: linear [-i N_INPUTS] [-d N_DAYS] [-j N_JUMPS] -usage: regression -p 1 [-i N_INPUTS] [-d N_DAYS] [-j N_JUMPS] +usage: linear [-i N_INPUTS] [-d N_DAYS] +usage: regression -p 1 [-i N_INPUTS] [-d N_DAYS] ``` Linear Regression (p=1): * -i : number of days to use for prediction. Default 40. - * -d : prediciton days. Default 5. - * -j : number of jumps in training data. Default 1. + * -d : prediction days. Default 5. * -e : end date (format YYYY-MM-DD) of the stock - Backtesting. Default None. ![linear](https://user-images.githubusercontent.com/25267873/108604948-d3cc1780-73a8-11eb-860f-49274a34038b.png) ## quadratic ``` -usage: quadratic [-i N_INPUTS] [-d N_DAYS] [-j N_JUMPS] -usage: regression -p 2 [-i N_INPUTS] [-d N_DAYS] [-j N_JUMPS] +usage: quadratic [-i N_INPUTS] [-d N_DAYS] +usage: regression -p 2 [-i N_INPUTS] [-d N_DAYS] ``` Quadratic Regression (p=2): * -i : number of days to use for prediction. Default 40. - * -d : prediciton days. Default 5. - * -j : number of jumps in training data. Default 1. + * -d : prediction days. Default 5. * -e : end date (format YYYY-MM-DD) of the stock - Backtesting. Default None. ![quadratic](https://user-images.githubusercontent.com/25267873/108604935-cca50980-73a8-11eb-9af1-bba807203cc6.png) ## cubic ``` -usage: cubic [-i N_INPUTS] [-d N_DAYS] [-j N_JUMPS] -usage: regression -p 3 [-i N_INPUTS] [-d N_DAYS] [-j N_JUMPS] +usage: cubic [-i N_INPUTS] [-d N_DAYS] +usage: regression -p 3 [-i N_INPUTS] [-d N_DAYS] ``` Cubic Regression (p=3): * -i : number of days to use for prediction. Default 40. - * -d : prediciton days. Default 5. - * -j : number of jumps in training data. Default 1. + * -d : prediction days. Default 5. * -e : end date (format YYYY-MM-DD) of the stock - Backtesting. Default None. ![cubic](https://user-images.githubusercontent.com/25267873/108604941-d169bd80-73a8-11eb-9220-84a7013e1283.png) ## regression ``` -usage: regression -p N_POLYNOMIAL [-i N_INPUTS] [-d N_DAYS] [-j N_JUMPS] +usage: regression -p N_POLYNOMIAL [-i N_INPUTS] [-d N_DAYS] ``` Regression: * -i : number of days to use for prediction. Default 40. - * -d : prediciton days. Default 5. - * -j : number of jumps in training data. Default 1. + * -d : prediction days. Default 5. * -p : polynomial associated with regression. Required. * -e : end date (format YYYY-MM-DD) of the stock - Backtesting. Default None. @@ -135,24 +130,21 @@ Auto-Regressive Integrated Moving Average: usage: fbprophet [-d N_DAYS] ``` Facebook's Prophet: - * -d : prediciton days. Default 5. + * -d : prediction days. Default 5. * -e : end date (format YYYY-MM-DD) of the stock - Backtesting. Default None. ![prophet](https://user-images.githubusercontent.com/25267873/108604938-cf9ffa00-73a8-11eb-973b-0affb343e2f6.png) ## mlp ``` -usage: mlp [-d N_DAYS] [-i N_INPUTS] [-j N_JUMPS] [--epochs N_EPOCHS] [-p {minmax, normalization,standardization,none}] - [-l {mae,mape,mse,msle}] [-e S_END_DATE] [--loops N_LOOPS] [-v VALID] [--lr LEARNING_RATE] [--no_shuffle] +usage: mlp [-d N_DAYS] [-i N_INPUTS] [--epochs N_EPOCHS] [-e S_END_DATE] [--loops N_LOOPS] [-v VALID] + [--lr LEARNING_RATE] [--no_shuffle] ``` MulitLayer Perceptron: * -d : prediciton days. Default 5. * -i : number of days to use for prediction. Default 40. - * -j : number of jumps in training data. Default 1. * --epochs : number of training epochs. Default 200. - * -p : pre-processing data. Default minmax. * -v : validation split. Default 0.1 - * -l : loss function. Default mae. * -e : end date (format YYYY-MM-DD) of the stock - Backtesting. Default None. * --xla_cpu: if present, will enable XLA for CPU (overrides environment variables during run). * --xla_gpu: if present, will enable XLA for GPU (overrides environment variables during run). @@ -181,17 +173,14 @@ MultiLayer_Perceptron \ ## rnn ``` -usage: rnn [-d N_DAYS] [-i N_INPUTS] [-j N_JUMPS] [--epochs N_EPOCHS] [-p {normalization,standardization,none}] +usage: rnn [-d N_DAYS] [-i N_INPUTS] [--epochs N_EPOCHS] [-p {normalization,standardization,none}] [-l {mae,mape,mse,msle}] [-e S_END_DATE] [--loops N_LOOPS] [-v VALID] [--lr LEARNING_RATE] [--no_shuffle] ``` Recurrent Neural Network: * -d : prediciton days. Default 5. * -i : number of days to use for prediction. Default 40. - * -j : number of jumps in training data. Default 1. * --epochs : number of training epochs. Default 200. - * -p : pre-processing data. Default normalization. * -v : validation split. Default 0.1 - * -l : loss function. Default mae. * -e : end date (format YYYY-MM-DD) of the stock - Backtesting. Default None. * --xla_cpu: if present, will enable XLA for CPU (overrides environment variables during run). * --xla_gpu: if present, will enable XLA for GPU (overrides environment variables during run). @@ -227,17 +216,14 @@ Recurrent_Neural_Network \ ## lstm ``` -usage: lstm [-d N_DAYS] [-i N_INPUTS] [-j N_JUMPS] [--epochs N_EPOCHS] [-p {normalization,standardization,none}] - [-l {mae,mape,mse,msle}] [-e S_END_DATE] [--loops N_LOOPS] [-v VALID] [--lr LEARNING_RATE] [--no_shuffle] +usage: lstm [-d N_DAYS] [-i N_INPUTS] [--epochs N_EPOCHS] [-e S_END_DATE] [--loops N_LOOPS] [-v VALID] + [--lr LEARNING_RATE] [--no_shuffle] ``` Long-Short Term Memory: - * -d : prediciton days. Default 5. + * -d : prediction days. Default 5. * -i : number of days to use for prediction. Default 40. - * -j : number of jumps in training data. Default 1. * --epochs : number of training epochs. Default 200. - * -p : pre-processing data. Default normalization. * -v : validation split. Default 0.1 - * -l : loss function. Default mae. * -e : end date (format YYYY-MM-DD) of the stock - Backtesting. Default None. * --xla_cpu: if present, will enable XLA for CPU (overrides environment variables during run). * --xla_gpu: if present, will enable XLA for GPU (overrides environment variables during run). @@ -270,17 +256,14 @@ Long_Short_Term_Memory \ ## conv1d ``` -usage: lstm [-d N_DAYS] [-i N_INPUTS] [-j N_JUMPS] [--epochs N_EPOCHS] [-p {normalization,standardization,minmax,none}] - [-l {mae,mape,mse,msle}] [-e S_END_DATE] [--loops N_LOOPS] [-v VALID] [--lr LEARNING_RATE] [--no_shuffle] +usage: lstm [-d N_DAYS] [-i N_INPUTS] [--epochs N_EPOCHS] [-e S_END_DATE] [--loops N_LOOPS] [-v VALID] +[--lr LEARNING_RATE] [--no_shuffle] ``` 1D Convolutional Neural Net: * -d : prediction days. Default 5. * -i : number of days to use for prediction. Default 40. - * -j : number of jumps in training data. Default 1. * --epochs : number of training epochs. Default 50. - * -p : pre-processing data. Default minmax. * -v : validation split. Default 0.1 - * -l : loss function. Default mae. * -e : end date (format YYYY-MM-DD) of the stock - Backtesting. Default None. * --xla_cpu: if present, will enable XLA for CPU (overrides environment variables during run). * --xla_gpu: if present, will enable XLA for GPU (overrides environment variables during run).