From 077162a275e83f29a2929ce285b1fca1d0e0319f Mon Sep 17 00:00:00 2001 From: Laurent Andriamifidy Date: Mon, 15 Mar 2021 12:10:03 +0100 Subject: [PATCH 01/64] removed rogue GME ticker in cell --- notebooks/templates/due_diligence.ipynb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/notebooks/templates/due_diligence.ipynb b/notebooks/templates/due_diligence.ipynb index 139a074132b..8d6a30e8fd1 100644 --- a/notebooks/templates/due_diligence.ipynb +++ b/notebooks/templates/due_diligence.ipynb @@ -35,9 +35,7 @@ ] }, "outputs": [], - "source": [ - "ticker=\"GME\"" - ] + "source": [] }, { "cell_type": "code", From 3edd8e3cc70dd676500187778aac290f0314fb4e Mon Sep 17 00:00:00 2001 From: Laurent Andriamifidy Date: Tue, 16 Mar 2021 14:09:59 +0100 Subject: [PATCH 02/64] removed unused cell w "datetime.datetime.now()" --- notebooks/templates/due_diligence.ipynb | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/notebooks/templates/due_diligence.ipynb b/notebooks/templates/due_diligence.ipynb index 8d6a30e8fd1..bfaa7aeb4cc 100644 --- a/notebooks/templates/due_diligence.ipynb +++ b/notebooks/templates/due_diligence.ipynb @@ -15,16 +15,6 @@ " sys.path.append(module_path)" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "exact-price", - "metadata": {}, - "outputs": [], - "source": [ - "datetime.datetime.now()" - ] - }, { "cell_type": "code", "execution_count": null, From c5888c6cc488591e42c87c3a4750fef264457a49 Mon Sep 17 00:00:00 2001 From: Laurent Andriamifidy Date: Tue, 16 Mar 2021 15:02:12 +0100 Subject: [PATCH 03/64] added options features to roadmap --- ROADMAP.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/ROADMAP.md b/ROADMAP.md index 74e5d1e9828..7e37722daa4 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -19,7 +19,7 @@ * [Frontend](#Frontend) * [Developer Experience](#Developer-Experience) * [User Experience](#User-Experience) - +* [Options](#Options) --- ## Discovery @@ -203,3 +203,17 @@ ### Long term - [ ] Add a WHL installation + + +--- + +## Options + +### Short term + +- [ ] Add Volume graph +- [ ] Add Open Interest graph + +### Long term + +- [ ] Add max pain graph From b9254093987e7d9c1bad7a67770a0c097c72b293 Mon Sep 17 00:00:00 2001 From: Laurent Andriamifidy Date: Tue, 16 Mar 2021 15:02:44 +0100 Subject: [PATCH 04/64] assigned options @lolrenx --- ROADMAP.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 7e37722daa4..170d2fba538 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -211,8 +211,8 @@ ### Short term -- [ ] Add Volume graph -- [ ] Add Open Interest graph +- [ ] Add Volume graph (@lolrenx) +- [ ] Add Open Interest graph (@lolrenx) ### Long term From 9320436f14503d6713e04f60126ed19ecf7907d0 Mon Sep 17 00:00:00 2001 From: Laurent Andriamifidy Date: Tue, 16 Mar 2021 18:30:11 +0100 Subject: [PATCH 05/64] added options readme --- gamestonk_terminal/options/README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 gamestonk_terminal/options/README.md diff --git a/gamestonk_terminal/options/README.md b/gamestonk_terminal/options/README.md new file mode 100644 index 00000000000..da19584da2a --- /dev/null +++ b/gamestonk_terminal/options/README.md @@ -0,0 +1,18 @@ +# OPTIONS DATA + +This menu aims to give insight on options. options can cause significant share price movement, and option pricing gives info about market sentiment for a ticker. + +* [volume](#volume) + * Show traded options for a given expiry [Yahoo Finance] + + + +## volume + +```text +usage: volume -d 'yyyy-mm-dd' +``` + +Display volume graph for a date. [Source: Yahoo Finance] + +* -d : expiration date. From e29405283e1b737ed81b084415e5f5dc57e600c8 Mon Sep 17 00:00:00 2001 From: Jamie Date: Tue, 16 Mar 2021 14:44:02 -0400 Subject: [PATCH 06/64] Added a fred page and added the ability to look at GDP data. --- gamestonk_terminal/config_terminal.py | 3 ++ gamestonk_terminal/fred/README.md | 24 ++++++++++ gamestonk_terminal/fred/__init__.py | 0 gamestonk_terminal/fred/fred_api.py | 66 +++++++++++++++++++++++++++ gamestonk_terminal/fred/fred_menu.py | 64 ++++++++++++++++++++++++++ gamestonk_terminal/main_helper.py | 4 ++ terminal.py | 6 ++- 7 files changed, 166 insertions(+), 1 deletion(-) create mode 100644 gamestonk_terminal/fred/README.md create mode 100644 gamestonk_terminal/fred/__init__.py create mode 100644 gamestonk_terminal/fred/fred_api.py create mode 100644 gamestonk_terminal/fred/fred_menu.py diff --git a/gamestonk_terminal/config_terminal.py b/gamestonk_terminal/config_terminal.py index f1b8757890d..8aa5b4b9437 100644 --- a/gamestonk_terminal/config_terminal.py +++ b/gamestonk_terminal/config_terminal.py @@ -25,3 +25,6 @@ API_POLYGON_KEY = os.getenv("GT_API_POLYGON_KEY") or "REPLACE_ME" API_TWITTER_KEY = os.getenv("GT_API_TWITTER_KEY") or "REPLACE_ME" API_TWITTER_SECRET_KEY = os.getenv("GT_API_TWITTER_SECRET_KEY") or "REPLACE_ME" API_TWITTER_BEARER_TOKEN = os.getenv("GT_API_TWITTER_BEARER_TOKEN") or "REPLACE_ME" + +# fred api ket +FRED_API_KEY = os.getenv("GT_FRED_API_KEY") or "REPLACE_ME" \ No newline at end of file diff --git a/gamestonk_terminal/fred/README.md b/gamestonk_terminal/fred/README.md new file mode 100644 index 00000000000..5ccf25df3ac --- /dev/null +++ b/gamestonk_terminal/fred/README.md @@ -0,0 +1,24 @@ +# fred + +The purpose of fred is to have the ability to look at economic data. + +* [GDP](#GDP) + * Get GDP data + +## GDP +```text +usage: GDP [-n N_TO_GET] [-s START_DATE] [-p PLOT]] +``` +Gets GDP data. Data is released quarterly +* -n : Number of points to display. If unspecified, it will display all available. +* -s : Start Date. Format m/d/y. First point to get +* -p : Option to plot. Defaults to False + +To get the last 20 reported GDP values: +```` +GDP -n 20 +```` +To plot all GDP data since 1/1/2010: +```` +GDP -s 1/1/2010 -p True +```` \ No newline at end of file diff --git a/gamestonk_terminal/fred/__init__.py b/gamestonk_terminal/fred/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/gamestonk_terminal/fred/fred_api.py b/gamestonk_terminal/fred/fred_api.py new file mode 100644 index 00000000000..9c3844cf758 --- /dev/null +++ b/gamestonk_terminal/fred/fred_api.py @@ -0,0 +1,66 @@ +from fredapi import Fred +import argparse +import matplotlib.pyplot as plt +from gamestonk_terminal.helper_funcs import ( + parse_known_args_and_warn, +) +from gamestonk_terminal.config_terminal import FRED_API_KEY + +def get_GDP(l_args): + fred = Fred(api_key=FRED_API_KEY) + parser = argparse.ArgumentParser( + add_help=False, + prog="GDP", + description=""" + GDP + """, + ) + parser.add_argument( + "-n", + dest='n_to_get', + type=int, + default=-1, + required=False, + help="Number of GDP Values to Grab", + ) + + parser.add_argument("-s", + dest = "start_date", + type = str, + default='1/1/2020', + required=False, + help='Date to Start') + + parser.add_argument("-p", + dest = "plot_", + type = bool, + default = False, + required=False, + help = 'Plot GDP') + + try: + ns_parser = parse_known_args_and_warn(parser, l_args) + + if not ns_parser: + return + + #gdp = fred.get_series_latest_release('GDP') + gdp = fred.get_series('GDP',ns_parser.start_date) + if int(ns_parser.n_to_get) > 0: + lastn = gdp.tail(int(ns_parser.n_to_get)) + else: + lastn = gdp + for date, val in lastn.iteritems(): + print(f'Date: {date.strftime("%m-%d-%Y")}, GDP: {val} ') + if ns_parser.plot_: + plt.figure() + lastn.plot(style = 'ok') + plt.xlabel('Time') + plt.ylabel('GDP') + plt.show() + except Exception as e: + print(e) + print("") + return + + diff --git a/gamestonk_terminal/fred/fred_menu.py b/gamestonk_terminal/fred/fred_menu.py new file mode 100644 index 00000000000..3563b3c3596 --- /dev/null +++ b/gamestonk_terminal/fred/fred_menu.py @@ -0,0 +1,64 @@ +import argparse + +from gamestonk_terminal import feature_flags as gtff +from gamestonk_terminal.helper_funcs import get_flair +from gamestonk_terminal.menu import session +from prompt_toolkit.completion import NestedCompleter +from gamestonk_terminal.fred import fred_api + + +def print_fred(): + """ Print help """ + + print("\nFred Economic Data:") # https://github.com/JerBouma/FundamentalAnalysis + print(" help show this menu again") + print(" q quit this menu, and shows back to main menu") + print(" quit quit to abandon program") + print(" ") + print(" GDP get GDP") + return + +def fred_menu(): + fred_parser = argparse.ArgumentParser(prog="fa", add_help=False) + choices = [ + "help", + "q", + "quit", + "GDP"] + fred_parser.add_argument("cmd", choices=choices) + completer = NestedCompleter.from_nested_dict({c: None for c in choices}) + + should_print_help = True + while True: + if should_print_help: + print_fred() + should_print_help = False + + # Get input command from user + if session and gtff.USE_PROMPT_TOOLKIT: + as_input = session.prompt( + f"{get_flair()} (fred)> ", + completer=completer, + ) + else: + as_input = input(f"{get_flair()} (fred)> ") + try: + (ns_known_args, l_args) = fred_parser.parse_known_args(as_input.split()) + + except SystemExit: + print("The command selected doesn't exist\n") + continue + + if ns_known_args.cmd == "help": + should_print_help = True + + elif ns_known_args.cmd == "q": + # Just leave the FA menu + return False + + elif ns_known_args.cmd == "quit": + # Abandon the program + return True + + elif ns_known_args.cmd == "GDP": + fred_api.get_GDP(l_args) diff --git a/gamestonk_terminal/main_helper.py b/gamestonk_terminal/main_helper.py index 4e650481d3e..461af450570 100644 --- a/gamestonk_terminal/main_helper.py +++ b/gamestonk_terminal/main_helper.py @@ -52,6 +52,10 @@ def print_help(s_ticker, s_start, s_interval, b_is_market_open): print( " sen sentiment of the market, \t from: reddit, stocktwits, twitter" ) + print( + " fred Data from Federal Reserve Bank of St.Louis" + ) + if s_ticker: print( " res research web page, \t e.g.: macroaxis, yahoo finance, fool" diff --git a/terminal.py b/terminal.py index 0d114f028fa..897c417a496 100644 --- a/terminal.py +++ b/terminal.py @@ -20,7 +20,7 @@ from gamestonk_terminal.papermill import papermill_menu as mill from gamestonk_terminal.sentiment import sen_menu as sm from gamestonk_terminal.technical_analysis import ta_menu as tam from gamestonk_terminal.comparison_analysis import ca_menu as cam - +from gamestonk_terminal.fred import fred_menu as fm # import warnings # warnings.simplefilter("always") @@ -65,6 +65,7 @@ def main(): "dd", "pred", "ca", + "fred" ] menu_parser.add_argument("opt", choices=choices) completer = NestedCompleter.from_nested_dict({c: None for c in choices}) @@ -172,6 +173,9 @@ def main(): elif ns_known_args.opt == "dd": b_quit = ddm.dd_menu(df_stock, s_ticker, s_start, s_interval) + elif ns_known_args.opt == "fred": + b_quit = fm.fred_menu() + elif ns_known_args.opt == "pred": if not gtff.ENABLE_PREDICT: From db1a97f4b8125f81fad4ccd10cbc25ae0d565b4e Mon Sep 17 00:00:00 2001 From: Jamie Date: Tue, 16 Mar 2021 15:07:13 -0400 Subject: [PATCH 07/64] Added function to allow for custom fred data. --- gamestonk_terminal/fred/README.md | 26 +++++++++++- gamestonk_terminal/fred/fred_api.py | 63 ++++++++++++++++++++++++++++ gamestonk_terminal/fred/fred_menu.py | 7 +++- requirements.txt | 1 + 4 files changed, 94 insertions(+), 3 deletions(-) diff --git a/gamestonk_terminal/fred/README.md b/gamestonk_terminal/fred/README.md index 5ccf25df3ac..65e5a814155 100644 --- a/gamestonk_terminal/fred/README.md +++ b/gamestonk_terminal/fred/README.md @@ -4,7 +4,9 @@ The purpose of fred is to have the ability to look at economic data. * [GDP](#GDP) * Get GDP data - +* [cust](#cust) + * Get custom fred data + ## GDP ```text usage: GDP [-n N_TO_GET] [-s START_DATE] [-p PLOT]] @@ -21,4 +23,24 @@ GDP -n 20 To plot all GDP data since 1/1/2010: ```` GDP -s 1/1/2010 -p True -```` \ No newline at end of file +```` + + +## cust + +The fred database has many different datasets freely available. They each have their own custom SERIES_ID which can be obtained through the website. + +```` +usage: cust [-id] [-s START_DATE] [-p PLOT] [-disp DISP] +```` +* -id : Series ID for FRED data. Required argument +* -s : Start data to acquire data. Defaults to 1/1/2020 +* -p : Option to plot. Defaults to False +* -disp : Option to print data to console. Defaults to True. + +The GDP is specifically defined in this menu, but to reproduce it with the custom function, it would be +```` +cust -id GDP -s 1/1/2020 -p True -disp False +```` + +The plot flag allows for plotting and the disp flag tells whether or not to print data to the console. \ No newline at end of file diff --git a/gamestonk_terminal/fred/fred_api.py b/gamestonk_terminal/fred/fred_api.py index 9c3844cf758..dbad6692689 100644 --- a/gamestonk_terminal/fred/fred_api.py +++ b/gamestonk_terminal/fred/fred_api.py @@ -63,4 +63,67 @@ def get_GDP(l_args): print("") return +def custom_data(l_args): + fred = Fred(api_key=FRED_API_KEY) + + parser = argparse.ArgumentParser( + add_help=False, + prog="Custom", + description=""" + Custom Data + """, + ) + parser.add_argument('-id', + dest = 'series_id', + required = True, + type=str, + help = 'FRED Series ID') + parser.add_argument("-s", + dest="start_date", + type=str, + default='1/1/2020', + required=False, + help='Date to Start') + parser.add_argument("-p", + dest="plot_", + type=bool, + default=False, + required=False, + help='Plot data') + parser.add_argument("-disp", + dest="disp_", + type=bool, + default=True, + required=False, + help='Printdata') + + + try: + ns_parser = parse_known_args_and_warn(parser, l_args) + + if not ns_parser: + return + + data = fred.get_series(ns_parser.series_id,ns_parser.start_date) + if ns_parser.disp_: + for date, val in data.iteritems(): + print(f'Date: {date.strftime("%m-%d-%Y")}, DATA: {val} ') + try: + if ns_parser.plot_: + plt.figure() + data.plot(style='ok') + plt.xlabel('Time') + plt.ylabel('DATA') + plt.show() + except Exception as e: + print(e) + print("") + return + except Exception as e: + print(e) + print("") + return + + + diff --git a/gamestonk_terminal/fred/fred_menu.py b/gamestonk_terminal/fred/fred_menu.py index 3563b3c3596..96ed35f3130 100644 --- a/gamestonk_terminal/fred/fred_menu.py +++ b/gamestonk_terminal/fred/fred_menu.py @@ -16,6 +16,7 @@ def print_fred(): print(" quit quit to abandon program") print(" ") print(" GDP get GDP") + print(" cust User Specified FRED Data") return def fred_menu(): @@ -24,7 +25,8 @@ def fred_menu(): "help", "q", "quit", - "GDP"] + "GDP", + "cust"] fred_parser.add_argument("cmd", choices=choices) completer = NestedCompleter.from_nested_dict({c: None for c in choices}) @@ -62,3 +64,6 @@ def fred_menu(): elif ns_known_args.cmd == "GDP": fred_api.get_GDP(l_args) + + elif ns_known_args.cmd == "cust": + fred_api.custom_data(l_args) diff --git a/requirements.txt b/requirements.txt index a6b36ab22f8..e2b6c236cc2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,6 +25,7 @@ decorator==4.4.2; python_full_version >= "3.6.1" and python_version >= "3.6" and defusedxml==0.7.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" entrypoints==0.3; python_version >= "3.6" finviz==1.3.4 +fredapi==0.4.3 fundamentalanalysis==0.2.6 hijri-converter==2.1.1; python_version >= "3.6" holidays==0.10.5.2 From 287f3f1e0c21f8a6ed26662d3b9e8d6438372835 Mon Sep 17 00:00:00 2001 From: Jamie Date: Tue, 16 Mar 2021 15:32:59 -0400 Subject: [PATCH 08/64] poetry add fredapi --- gamestonk_terminal/config_terminal.py | 2 +- poetry.lock | 25 ++++++++++++++++++++++++- pyproject.toml | 1 + 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/gamestonk_terminal/config_terminal.py b/gamestonk_terminal/config_terminal.py index 8aa5b4b9437..ba3ca0764d7 100644 --- a/gamestonk_terminal/config_terminal.py +++ b/gamestonk_terminal/config_terminal.py @@ -26,5 +26,5 @@ API_TWITTER_KEY = os.getenv("GT_API_TWITTER_KEY") or "REPLACE_ME" API_TWITTER_SECRET_KEY = os.getenv("GT_API_TWITTER_SECRET_KEY") or "REPLACE_ME" API_TWITTER_BEARER_TOKEN = os.getenv("GT_API_TWITTER_BEARER_TOKEN") or "REPLACE_ME" -# fred api ket +# fred api key FRED_API_KEY = os.getenv("GT_FRED_API_KEY") or "REPLACE_ME" \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index 44353264ba9..f31b2b8df6f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -476,6 +476,17 @@ category = "main" optional = true python-versions = "*" +[[package]] +name = "fredapi" +version = "0.4.3" +description = "Python API for Federal Reserve Economic Data (FRED) from St. Louis Fed" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +pandas = "*" + [[package]] name = "ftfy" version = "5.9" @@ -2474,7 +2485,7 @@ prediction = ["fbprophet", "tensorflow", "pmdarima", "transformers", "flair", "t [metadata] lock-version = "1.1" python-versions = "^3.6.8" -content-hash = "cfb418b140b12dd94550b1f21431ceb8bf23df30ae4ec9735a98bc76edbfbb85" +content-hash = "184502c6b8d4b7db5baa4323e7f70b2585c6140c304583f0678e24681b8f9c9b" [metadata.files] absl-py = [ @@ -2778,6 +2789,10 @@ flatbuffers = [ {file = "flatbuffers-1.12-py2.py3-none-any.whl", hash = "sha256:9e9ef47fa92625c4721036e7c4124182668dc6021d9e7c73704edd395648deb9"}, {file = "flatbuffers-1.12.tar.gz", hash = "sha256:63bb9a722d5e373701913e226135b28a6f6ac200d5cc7b4d919fa38d73b44610"}, ] +fredapi = [ + {file = "fredapi-0.4.3-py3-none-any.whl", hash = "sha256:e06075592eabddadfe0635f3c59e9072c9f48647430328b4582171ea10f7df2f"}, + {file = "fredapi-0.4.3.tar.gz", hash = "sha256:d9b3194fb60541991bd33f019c710d4a9580ecfb5e47efbf2d2571888a2aac02"}, +] ftfy = [ {file = "ftfy-5.9.tar.gz", hash = "sha256:8c4fb2863c0b82eae2ab3cf353d9ade268dfbde863d322f78d6a9fd5cefb31e9"}, ] @@ -3610,18 +3625,26 @@ pyyaml = [ {file = "PyYAML-5.4.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185"}, {file = "PyYAML-5.4.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253"}, {file = "PyYAML-5.4.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc"}, + {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347"}, + {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_s390x.whl", hash = "sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541"}, {file = "PyYAML-5.4.1-cp36-cp36m-win32.whl", hash = "sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5"}, {file = "PyYAML-5.4.1-cp36-cp36m-win_amd64.whl", hash = "sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df"}, {file = "PyYAML-5.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018"}, {file = "PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63"}, + {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa"}, + {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_s390x.whl", hash = "sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0"}, {file = "PyYAML-5.4.1-cp37-cp37m-win32.whl", hash = "sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b"}, {file = "PyYAML-5.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf"}, {file = "PyYAML-5.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46"}, {file = "PyYAML-5.4.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb"}, + {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247"}, + {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_s390x.whl", hash = "sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc"}, {file = "PyYAML-5.4.1-cp38-cp38-win32.whl", hash = "sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc"}, {file = "PyYAML-5.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696"}, {file = "PyYAML-5.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77"}, {file = "PyYAML-5.4.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183"}, + {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122"}, + {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_s390x.whl", hash = "sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6"}, {file = "PyYAML-5.4.1-cp39-cp39-win32.whl", hash = "sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10"}, {file = "PyYAML-5.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db"}, {file = "PyYAML-5.4.1.tar.gz", hash = "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e"}, diff --git a/pyproject.toml b/pyproject.toml index 0eb5c36af5f..b11836c7ad0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,7 @@ jupyter = "^1.0.0" mplfinance = "^0.12.7-alpha.7" seaborn = "^0.11.0" scipy = "1.5.4" +fredapi = "^0.4.3" [tool.poetry.dev-dependencies] pytest = "^6.2.2" From 027e71bf0847dad2b759e49e6baa6e639a3a566a Mon Sep 17 00:00:00 2001 From: Jamie Date: Tue, 16 Mar 2021 15:36:20 -0400 Subject: [PATCH 09/64] delete commmented text --- gamestonk_terminal/config_terminal.py | 4 ++-- gamestonk_terminal/fred/fred_api.py | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/gamestonk_terminal/config_terminal.py b/gamestonk_terminal/config_terminal.py index ba3ca0764d7..21e62b02c93 100644 --- a/gamestonk_terminal/config_terminal.py +++ b/gamestonk_terminal/config_terminal.py @@ -26,5 +26,5 @@ API_TWITTER_KEY = os.getenv("GT_API_TWITTER_KEY") or "REPLACE_ME" API_TWITTER_SECRET_KEY = os.getenv("GT_API_TWITTER_SECRET_KEY") or "REPLACE_ME" API_TWITTER_BEARER_TOKEN = os.getenv("GT_API_TWITTER_BEARER_TOKEN") or "REPLACE_ME" -# fred api key -FRED_API_KEY = os.getenv("GT_FRED_API_KEY") or "REPLACE_ME" \ No newline at end of file +# https://fred.stlouisfed.org/docs/api/api_key.html +API_FRED_KEY = os.getenv("GT_FRED_API_KEY") or "REPLACE_ME" \ No newline at end of file diff --git a/gamestonk_terminal/fred/fred_api.py b/gamestonk_terminal/fred/fred_api.py index dbad6692689..e1f6d7d273a 100644 --- a/gamestonk_terminal/fred/fred_api.py +++ b/gamestonk_terminal/fred/fred_api.py @@ -4,7 +4,7 @@ import matplotlib.pyplot as plt from gamestonk_terminal.helper_funcs import ( parse_known_args_and_warn, ) -from gamestonk_terminal.config_terminal import FRED_API_KEY +from gamestonk_terminal.config_terminal import API_FRED_KEY def get_GDP(l_args): fred = Fred(api_key=FRED_API_KEY) @@ -44,14 +44,17 @@ def get_GDP(l_args): if not ns_parser: return - #gdp = fred.get_series_latest_release('GDP') gdp = fred.get_series('GDP',ns_parser.start_date) + if int(ns_parser.n_to_get) > 0: lastn = gdp.tail(int(ns_parser.n_to_get)) + else: lastn = gdp + for date, val in lastn.iteritems(): print(f'Date: {date.strftime("%m-%d-%Y")}, GDP: {val} ') + if ns_parser.plot_: plt.figure() lastn.plot(style = 'ok') From 005f73151889d11e1cd8464ab9cc791e2ee1627f Mon Sep 17 00:00:00 2001 From: Jamie Date: Tue, 16 Mar 2021 15:41:56 -0400 Subject: [PATCH 10/64] black gamestonk_terminal --- gamestonk_terminal/config_terminal.py | 2 +- gamestonk_terminal/fred/fred_api.py | 88 ++++++++++++--------------- gamestonk_terminal/fred/fred_menu.py | 8 +-- gamestonk_terminal/main_helper.py | 4 +- 4 files changed, 43 insertions(+), 59 deletions(-) diff --git a/gamestonk_terminal/config_terminal.py b/gamestonk_terminal/config_terminal.py index 21e62b02c93..6939d823cee 100644 --- a/gamestonk_terminal/config_terminal.py +++ b/gamestonk_terminal/config_terminal.py @@ -27,4 +27,4 @@ API_TWITTER_SECRET_KEY = os.getenv("GT_API_TWITTER_SECRET_KEY") or "REPLACE_ME" API_TWITTER_BEARER_TOKEN = os.getenv("GT_API_TWITTER_BEARER_TOKEN") or "REPLACE_ME" # https://fred.stlouisfed.org/docs/api/api_key.html -API_FRED_KEY = os.getenv("GT_FRED_API_KEY") or "REPLACE_ME" \ No newline at end of file +API_FRED_KEY = os.getenv("GT_FRED_API_KEY") or "REPLACE_ME" diff --git a/gamestonk_terminal/fred/fred_api.py b/gamestonk_terminal/fred/fred_api.py index e1f6d7d273a..fafdca5c129 100644 --- a/gamestonk_terminal/fred/fred_api.py +++ b/gamestonk_terminal/fred/fred_api.py @@ -6,6 +6,7 @@ from gamestonk_terminal.helper_funcs import ( ) from gamestonk_terminal.config_terminal import API_FRED_KEY + def get_GDP(l_args): fred = Fred(api_key=FRED_API_KEY) parser = argparse.ArgumentParser( @@ -17,26 +18,25 @@ def get_GDP(l_args): ) parser.add_argument( "-n", - dest='n_to_get', + dest="n_to_get", type=int, default=-1, required=False, help="Number of GDP Values to Grab", ) - parser.add_argument("-s", - dest = "start_date", - type = str, - default='1/1/2020', - required=False, - help='Date to Start') + parser.add_argument( + "-s", + dest="start_date", + type=str, + default="1/1/2020", + required=False, + help="Date to Start", + ) - parser.add_argument("-p", - dest = "plot_", - type = bool, - default = False, - required=False, - help = 'Plot GDP') + parser.add_argument( + "-p", dest="plot_", type=bool, default=False, required=False, help="Plot GDP" + ) try: ns_parser = parse_known_args_and_warn(parser, l_args) @@ -44,7 +44,7 @@ def get_GDP(l_args): if not ns_parser: return - gdp = fred.get_series('GDP',ns_parser.start_date) + gdp = fred.get_series("GDP", ns_parser.start_date) if int(ns_parser.n_to_get) > 0: lastn = gdp.tail(int(ns_parser.n_to_get)) @@ -57,15 +57,16 @@ def get_GDP(l_args): if ns_parser.plot_: plt.figure() - lastn.plot(style = 'ok') - plt.xlabel('Time') - plt.ylabel('GDP') + lastn.plot(style="ok") + plt.xlabel("Time") + plt.ylabel("GDP") plt.show() except Exception as e: print(e) print("") return + def custom_data(l_args): fred = Fred(api_key=FRED_API_KEY) @@ -76,30 +77,23 @@ def custom_data(l_args): Custom Data """, ) - parser.add_argument('-id', - dest = 'series_id', - required = True, - type=str, - help = 'FRED Series ID') - parser.add_argument("-s", - dest="start_date", - type=str, - default='1/1/2020', - required=False, - help='Date to Start') - parser.add_argument("-p", - dest="plot_", - type=bool, - default=False, - required=False, - help='Plot data') - parser.add_argument("-disp", - dest="disp_", - type=bool, - default=True, - required=False, - help='Printdata') - + parser.add_argument( + "-id", dest="series_id", required=True, type=str, help="FRED Series ID" + ) + parser.add_argument( + "-s", + dest="start_date", + type=str, + default="1/1/2020", + required=False, + help="Date to Start", + ) + parser.add_argument( + "-p", dest="plot_", type=bool, default=False, required=False, help="Plot data" + ) + parser.add_argument( + "-disp", dest="disp_", type=bool, default=True, required=False, help="Printdata" + ) try: ns_parser = parse_known_args_and_warn(parser, l_args) @@ -107,16 +101,16 @@ def custom_data(l_args): if not ns_parser: return - data = fred.get_series(ns_parser.series_id,ns_parser.start_date) + data = fred.get_series(ns_parser.series_id, ns_parser.start_date) if ns_parser.disp_: for date, val in data.iteritems(): print(f'Date: {date.strftime("%m-%d-%Y")}, DATA: {val} ') try: if ns_parser.plot_: plt.figure() - data.plot(style='ok') - plt.xlabel('Time') - plt.ylabel('DATA') + data.plot(style="ok") + plt.xlabel("Time") + plt.ylabel("DATA") plt.show() except Exception as e: print(e) @@ -126,7 +120,3 @@ def custom_data(l_args): print(e) print("") return - - - - diff --git a/gamestonk_terminal/fred/fred_menu.py b/gamestonk_terminal/fred/fred_menu.py index 96ed35f3130..9094870a8ec 100644 --- a/gamestonk_terminal/fred/fred_menu.py +++ b/gamestonk_terminal/fred/fred_menu.py @@ -19,14 +19,10 @@ def print_fred(): print(" cust User Specified FRED Data") return + def fred_menu(): fred_parser = argparse.ArgumentParser(prog="fa", add_help=False) - choices = [ - "help", - "q", - "quit", - "GDP", - "cust"] + choices = ["help", "q", "quit", "GDP", "cust"] fred_parser.add_argument("cmd", choices=choices) completer = NestedCompleter.from_nested_dict({c: None for c in choices}) diff --git a/gamestonk_terminal/main_helper.py b/gamestonk_terminal/main_helper.py index 461af450570..3f491e3835e 100644 --- a/gamestonk_terminal/main_helper.py +++ b/gamestonk_terminal/main_helper.py @@ -52,9 +52,7 @@ def print_help(s_ticker, s_start, s_interval, b_is_market_open): print( " sen sentiment of the market, \t from: reddit, stocktwits, twitter" ) - print( - " fred Data from Federal Reserve Bank of St.Louis" - ) + print(" fred Data from Federal Reserve Bank of St.Louis") if s_ticker: print( From ca3711d99b0c9089a6c31c940122ab22e283a605 Mon Sep 17 00:00:00 2001 From: Jamie Date: Tue, 16 Mar 2021 15:44:27 -0400 Subject: [PATCH 11/64] black terminal.py --- terminal.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/terminal.py b/terminal.py index 897c417a496..5e4ce05ae7d 100644 --- a/terminal.py +++ b/terminal.py @@ -21,6 +21,7 @@ from gamestonk_terminal.sentiment import sen_menu as sm from gamestonk_terminal.technical_analysis import ta_menu as tam from gamestonk_terminal.comparison_analysis import ca_menu as cam from gamestonk_terminal.fred import fred_menu as fm + # import warnings # warnings.simplefilter("always") @@ -65,7 +66,7 @@ def main(): "dd", "pred", "ca", - "fred" + "fred", ] menu_parser.add_argument("opt", choices=choices) completer = NestedCompleter.from_nested_dict({c: None for c in choices}) From eeb320231ae510bc24efe5c27d974ba6dd4be6d1 Mon Sep 17 00:00:00 2001 From: Jamie Date: Tue, 16 Mar 2021 15:46:29 -0400 Subject: [PATCH 12/64] fixed api key reference --- gamestonk_terminal/fred/fred_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gamestonk_terminal/fred/fred_api.py b/gamestonk_terminal/fred/fred_api.py index fafdca5c129..1246d592186 100644 --- a/gamestonk_terminal/fred/fred_api.py +++ b/gamestonk_terminal/fred/fred_api.py @@ -8,7 +8,7 @@ from gamestonk_terminal.config_terminal import API_FRED_KEY def get_GDP(l_args): - fred = Fred(api_key=FRED_API_KEY) + fred = Fred(api_key=API_FRED_KEY) parser = argparse.ArgumentParser( add_help=False, prog="GDP", @@ -68,7 +68,7 @@ def get_GDP(l_args): def custom_data(l_args): - fred = Fred(api_key=FRED_API_KEY) + fred = Fred(api_key=API_FRED_KEY) parser = argparse.ArgumentParser( add_help=False, From e52e5d43dcf45c18aa5395cdebce8f45cb5c975a Mon Sep 17 00:00:00 2001 From: Laurent Andriamifidy Date: Tue, 16 Mar 2021 20:58:53 +0100 Subject: [PATCH 13/64] generate options volume graph for ticker + expiration date --- gamestonk_terminal/options/volume.py | 39 ++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 gamestonk_terminal/options/volume.py diff --git a/gamestonk_terminal/options/volume.py b/gamestonk_terminal/options/volume.py new file mode 100644 index 00000000000..5731a36964e --- /dev/null +++ b/gamestonk_terminal/options/volume.py @@ -0,0 +1,39 @@ +import yfinance as yf +import seaborn as sns +import pandas as pd +import matplotlib.pyplot as plt +import plotly.express as px + + +def get_volume_graph(ticker_name, exp_date): + + opt = yf.Ticker(ticker_name).option_chain(exp_date) + # data available via: opt.calls, opt.puts + + volume_data = pd.concat( + [ + __volume_data(opt.calls, 'calls'), + __volume_data(opt.puts, 'puts') + ], + axis=0 + ) + fig = px.line( + volume_data, + x="strike", + y="volume", + title=f'{ticker_name} Volume for {exp_date}', + color='type' + ) + fig.show() + + +def __volume_data(opt_data, flag): + # get option chain for specific expiration + df = opt_data.pivot_table( + index='strike', + values=['volume', 'openInterest'], + aggfunc='sum') + df.reindex() + df['strike'] = df.index + df['type'] = flag + return df From ad7d546235df6bc9284debdbed62019406c8ec48 Mon Sep 17 00:00:00 2001 From: Laurent Andriamifidy Date: Tue, 16 Mar 2021 20:59:53 +0100 Subject: [PATCH 14/64] created options submenu --- gamestonk_terminal/options/option_menu.py | 77 +++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 gamestonk_terminal/options/option_menu.py diff --git a/gamestonk_terminal/options/option_menu.py b/gamestonk_terminal/options/option_menu.py new file mode 100644 index 00000000000..407b2b64090 --- /dev/null +++ b/gamestonk_terminal/options/option_menu.py @@ -0,0 +1,77 @@ +import argparse +import requests + +from gamestonk_terminal import config_terminal as cfg +from gamestonk_terminal.helper_funcs import get_flair, parse_known_args_and_warn +from gamestonk_terminal.options import get_volume_graph + +def volume(l_args, s_ticker): + """ Show traded options volume. [Source: Yahoo Finance] """ + parser = argparse.ArgumentParser( + add_help=False, + prog="volume", + description="""Display volume graph for a date. [Source: Yahoo Finance].""", + ) + + l_similar = [] + try: + ns_parser = parse_known_args_and_warn(parser, l_args) + if not ns_parser: + return + + except Exception as e: + print(e) + + print("") + return l_similar + +def opt_menu(df_stock, s_ticker, s_start, s_interval): + + # Add list of arguments that the options parser accepts + opt_parser = argparse.ArgumentParser(prog="opt", add_help=False) + choices = [ + "help", + "q", + "quit", + "volume" + ] + opt_parser.add_argument("cmd", choices=choices) + completer = NestedCompleter.from_nested_dict({c: None for c in choices}) + + print_technical_analysis(s_ticker, s_start, s_interval) + + # Loop forever and ever + while True: + # Get input command from user + if session and gtff.USE_PROMPT_TOOLKIT: + as_input = session.prompt( + f"{get_flair()} (opt)> ", + completer=completer, + ) + else: + as_input = input(f"{get_flair()} (opt)> ") + + # Images are non blocking - allows to close them if we type other command + plt.close() + + # Parse fundamental analysis command of the list of possible commands + try: + (ns_known_args, l_args) = opt_parser.parse_known_args(as_input.split()) + + except SystemExit: + print("The command selected doesn't exist\n") + continue + + if ns_known_args.cmd == "help": + print_technical_analysis(s_ticker, s_start, s_interval) + + elif ns_known_args.cmd == "q": + # Just leave the FA menu + return False + + elif ns_known_args.cmd == "quit": + # Abandon the program + return True + + else: + print("Command not recognized!") From 31f7d2b269d3338875c6a125d8e8341c57b85e59 Mon Sep 17 00:00:00 2001 From: Laurent Andriamifidy Date: Tue, 16 Mar 2021 21:05:25 +0100 Subject: [PATCH 15/64] renamed options menu --- gamestonk_terminal/options/{option_menu.py => op_menu.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename gamestonk_terminal/options/{option_menu.py => op_menu.py} (100%) diff --git a/gamestonk_terminal/options/option_menu.py b/gamestonk_terminal/options/op_menu.py similarity index 100% rename from gamestonk_terminal/options/option_menu.py rename to gamestonk_terminal/options/op_menu.py From c657c602010a199d1379abb87cc0ec134021f200 Mon Sep 17 00:00:00 2001 From: Laurent Andriamifidy Date: Tue, 16 Mar 2021 21:06:15 +0100 Subject: [PATCH 16/64] renamed main function --- gamestonk_terminal/options/op_menu.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gamestonk_terminal/options/op_menu.py b/gamestonk_terminal/options/op_menu.py index 407b2b64090..a223852f82c 100644 --- a/gamestonk_terminal/options/op_menu.py +++ b/gamestonk_terminal/options/op_menu.py @@ -5,7 +5,8 @@ from gamestonk_terminal import config_terminal as cfg from gamestonk_terminal.helper_funcs import get_flair, parse_known_args_and_warn from gamestonk_terminal.options import get_volume_graph -def volume(l_args, s_ticker): + +def print_volume(l_args, s_ticker): """ Show traded options volume. [Source: Yahoo Finance] """ parser = argparse.ArgumentParser( add_help=False, @@ -23,7 +24,8 @@ def volume(l_args, s_ticker): print(e) print("") - return l_similar + return get_volume_graph(s_ticker, expiration_date) + def opt_menu(df_stock, s_ticker, s_start, s_interval): From c320329b372c44f92d1492bc7b3a26d0db8f3c20 Mon Sep 17 00:00:00 2001 From: Jamie Date: Tue, 16 Mar 2021 17:10:05 -0400 Subject: [PATCH 17/64] minor formatting changes --- gamestonk_terminal/fred/README.md | 24 ++++++++++---------- gamestonk_terminal/fred/fred_api.py | 33 ++++++++++++++++++++++------ gamestonk_terminal/fred/fred_menu.py | 12 +++++----- gamestonk_terminal/main_helper.py | 2 +- 4 files changed, 45 insertions(+), 26 deletions(-) diff --git a/gamestonk_terminal/fred/README.md b/gamestonk_terminal/fred/README.md index 65e5a814155..ec0ef0791f1 100644 --- a/gamestonk_terminal/fred/README.md +++ b/gamestonk_terminal/fred/README.md @@ -1,28 +1,28 @@ -# fred +# FRED The purpose of fred is to have the ability to look at economic data. -* [GDP](#GDP) +* [gdp](#gdp) * Get GDP data * [cust](#cust) * Get custom fred data -## GDP +## gdp ```text -usage: GDP [-n N_TO_GET] [-s START_DATE] [-p PLOT]] +usage: gdp [-n N_TO_GET] [-s START_DATE] [-p PLOT]] ``` Gets GDP data. Data is released quarterly * -n : Number of points to display. If unspecified, it will display all available. * -s : Start Date. Format m/d/y. First point to get * -p : Option to plot. Defaults to False -To get the last 20 reported GDP values: +To get and plot the last 20 reported GDP values: ```` -GDP -n 20 +gdp -n 20 ```` -To plot all GDP data since 1/1/2010: +To show all GDP data since 1/1/2010, with no plot. ```` -GDP -s 1/1/2010 -p True +gdp -s 1/1/2010 -p True ```` @@ -35,12 +35,12 @@ usage: cust [-id] [-s START_DATE] [-p PLOT] [-disp DISP] ```` * -id : Series ID for FRED data. Required argument * -s : Start data to acquire data. Defaults to 1/1/2020 -* -p : Option to plot. Defaults to False -* -disp : Option to print data to console. Defaults to True. +* -p : Option to plot. Defaults to True +* -disp : Option to print data to console. Defaults to False. The GDP is specifically defined in this menu, but to reproduce it with the custom function, it would be ```` -cust -id GDP -s 1/1/2020 -p True -disp False +cust -id GDP -s 1/1/2020 -p True ```` -The plot flag allows for plotting and the disp flag tells whether or not to print data to the console. \ No newline at end of file +The plot flag allows for plotting and the disp flag tells whether to print data to the console. \ No newline at end of file diff --git a/gamestonk_terminal/fred/fred_api.py b/gamestonk_terminal/fred/fred_api.py index 1246d592186..394387e014e 100644 --- a/gamestonk_terminal/fred/fred_api.py +++ b/gamestonk_terminal/fred/fred_api.py @@ -35,7 +35,11 @@ def get_GDP(l_args): ) parser.add_argument( - "-p", dest="plot_", type=bool, default=False, required=False, help="Plot GDP" + "-p", dest="plot", + type=bool, + default=True, + required=False, + help="Plot GDP" ) try: @@ -55,12 +59,13 @@ def get_GDP(l_args): for date, val in lastn.iteritems(): print(f'Date: {date.strftime("%m-%d-%Y")}, GDP: {val} ') - if ns_parser.plot_: + if ns_parser.plot: plt.figure() lastn.plot(style="ok") plt.xlabel("Time") plt.ylabel("GDP") plt.show() + except Exception as e: print(e) print("") @@ -78,8 +83,12 @@ def custom_data(l_args): """, ) parser.add_argument( - "-id", dest="series_id", required=True, type=str, help="FRED Series ID" + "-id", dest="series_id", + required=True, + type=str, + help="FRED Series ID" ) + parser.add_argument( "-s", dest="start_date", @@ -89,10 +98,20 @@ def custom_data(l_args): help="Date to Start", ) parser.add_argument( - "-p", dest="plot_", type=bool, default=False, required=False, help="Plot data" + "-p", + dest="plot", + type=bool, + default=True, + required=False, + help="Plot data" ) parser.add_argument( - "-disp", dest="disp_", type=bool, default=True, required=False, help="Printdata" + "-disp", + dest="disp", + type=bool, + default=False, + required=False, + help="Print data" ) try: @@ -102,11 +121,11 @@ def custom_data(l_args): return data = fred.get_series(ns_parser.series_id, ns_parser.start_date) - if ns_parser.disp_: + if ns_parser.disp: for date, val in data.iteritems(): print(f'Date: {date.strftime("%m-%d-%Y")}, DATA: {val} ') try: - if ns_parser.plot_: + if ns_parser.plot: plt.figure() data.plot(style="ok") plt.xlabel("Time") diff --git a/gamestonk_terminal/fred/fred_menu.py b/gamestonk_terminal/fred/fred_menu.py index 9094870a8ec..f7f2d427fad 100644 --- a/gamestonk_terminal/fred/fred_menu.py +++ b/gamestonk_terminal/fred/fred_menu.py @@ -10,19 +10,19 @@ from gamestonk_terminal.fred import fred_api def print_fred(): """ Print help """ - print("\nFred Economic Data:") # https://github.com/JerBouma/FundamentalAnalysis + print("\nFred Economic Data:") print(" help show this menu again") print(" q quit this menu, and shows back to main menu") print(" quit quit to abandon program") print(" ") - print(" GDP get GDP") + print(" gdp get GDP") print(" cust User Specified FRED Data") return def fred_menu(): - fred_parser = argparse.ArgumentParser(prog="fa", add_help=False) - choices = ["help", "q", "quit", "GDP", "cust"] + fred_parser = argparse.ArgumentParser(prog="fred", add_help=False) + choices = ["help", "q", "quit", "gdp", "cust"] fred_parser.add_argument("cmd", choices=choices) completer = NestedCompleter.from_nested_dict({c: None for c in choices}) @@ -51,14 +51,14 @@ def fred_menu(): should_print_help = True elif ns_known_args.cmd == "q": - # Just leave the FA menu + # Leave the fred menu return False elif ns_known_args.cmd == "quit": # Abandon the program return True - elif ns_known_args.cmd == "GDP": + elif ns_known_args.cmd == "gdp": fred_api.get_GDP(l_args) elif ns_known_args.cmd == "cust": diff --git a/gamestonk_terminal/main_helper.py b/gamestonk_terminal/main_helper.py index 3f491e3835e..c0c0d5e6e89 100644 --- a/gamestonk_terminal/main_helper.py +++ b/gamestonk_terminal/main_helper.py @@ -52,7 +52,7 @@ def print_help(s_ticker, s_start, s_interval, b_is_market_open): print( " sen sentiment of the market, \t from: reddit, stocktwits, twitter" ) - print(" fred Data from Federal Reserve Bank of St.Louis") + print(" fred economic data, \t\t\t from: Federal Reserve Bank of St. Louis ") if s_ticker: print( From 5abd4aa5f22304c200c68fd38d43d03858ad915f Mon Sep 17 00:00:00 2001 From: Jamie Date: Tue, 16 Mar 2021 17:34:41 -0400 Subject: [PATCH 18/64] added helper function str to bool to fix issue where using -p False was not changing from the default. --- gamestonk_terminal/fred/README.md | 37 +++++++++++++++++++++++++---- gamestonk_terminal/fred/fred_api.py | 35 ++++++++++++++------------- gamestonk_terminal/helper_funcs.py | 9 +++++++ gamestonk_terminal/main_helper.py | 4 +++- 4 files changed, 62 insertions(+), 23 deletions(-) diff --git a/gamestonk_terminal/fred/README.md b/gamestonk_terminal/fred/README.md index ec0ef0791f1..a394a9b0ca6 100644 --- a/gamestonk_terminal/fred/README.md +++ b/gamestonk_terminal/fred/README.md @@ -13,16 +13,43 @@ usage: gdp [-n N_TO_GET] [-s START_DATE] [-p PLOT]] ``` Gets GDP data. Data is released quarterly * -n : Number of points to display. If unspecified, it will display all available. -* -s : Start Date. Format m/d/y. First point to get +* -s : Start Date. Format YYYY-mm-dd. First point to get. Defaults to Jan 1 2015 * -p : Option to plot. Defaults to False To get and plot the last 20 reported GDP values: ```` gdp -n 20 ```` -To show all GDP data since 1/1/2010, with no plot. +To show all GDP data since 1/1/2016, with no plot. ```` -gdp -s 1/1/2010 -p True +gdp -s 2016-01-01 -p False +```` +```` +Date: 01-01-2015, GDP: 18003.399 +Date: 04-01-2015, GDP: 18223.577 +Date: 07-01-2015, GDP: 18347.425 +Date: 10-01-2015, GDP: 18378.803 +Date: 01-01-2016, GDP: 18470.156 +Date: 04-01-2016, GDP: 18656.207 +Date: 07-01-2016, GDP: 18821.359 +Date: 10-01-2016, GDP: 19032.58 +Date: 01-01-2017, GDP: 19237.435 +Date: 04-01-2017, GDP: 19379.232 +Date: 07-01-2017, GDP: 19617.288 +Date: 10-01-2017, GDP: 19937.963 +Date: 01-01-2018, GDP: 20242.215 +Date: 04-01-2018, GDP: 20552.653 +Date: 07-01-2018, GDP: 20742.723 +Date: 10-01-2018, GDP: 20909.853 +Date: 01-01-2019, GDP: 21115.309 +Date: 04-01-2019, GDP: 21329.877 +Date: 07-01-2019, GDP: 21540.325 +Date: 10-01-2019, GDP: 21747.394 +Date: 01-01-2020, GDP: 21561.139 +Date: 04-01-2020, GDP: 19520.114 +Date: 07-01-2020, GDP: 21170.252 +Date: 10-01-2020, GDP: 21487.896 + ```` @@ -34,13 +61,13 @@ The fred database has many different datasets freely available. They each have usage: cust [-id] [-s START_DATE] [-p PLOT] [-disp DISP] ```` * -id : Series ID for FRED data. Required argument -* -s : Start data to acquire data. Defaults to 1/1/2020 +* -s : Start Date. Format YYYY-mm-dd. First point to get. Defaults to Jan 1 2015. * -p : Option to plot. Defaults to True * -disp : Option to print data to console. Defaults to False. The GDP is specifically defined in this menu, but to reproduce it with the custom function, it would be ```` -cust -id GDP -s 1/1/2020 -p True +cust -id GDP -s 2020-01-01 -p True ```` The plot flag allows for plotting and the disp flag tells whether to print data to the console. \ No newline at end of file diff --git a/gamestonk_terminal/fred/fred_api.py b/gamestonk_terminal/fred/fred_api.py index 394387e014e..b96aceb8a94 100644 --- a/gamestonk_terminal/fred/fred_api.py +++ b/gamestonk_terminal/fred/fred_api.py @@ -3,6 +3,8 @@ import argparse import matplotlib.pyplot as plt from gamestonk_terminal.helper_funcs import ( parse_known_args_and_warn, + valid_date, + str_to_bool, ) from gamestonk_terminal.config_terminal import API_FRED_KEY @@ -28,18 +30,19 @@ def get_GDP(l_args): parser.add_argument( "-s", dest="start_date", - type=str, - default="1/1/2020", + type=valid_date, + default="2015-01-01", required=False, help="Date to Start", ) parser.add_argument( - "-p", dest="plot", - type=bool, + "-p", + dest="plot", + type=str_to_bool, default=True, required=False, - help="Plot GDP" + help="Plot GDP", ) try: @@ -61,7 +64,7 @@ def get_GDP(l_args): if ns_parser.plot: plt.figure() - lastn.plot(style="ok") + lastn.plot(style="-ok") plt.xlabel("Time") plt.ylabel("GDP") plt.show() @@ -83,35 +86,32 @@ def custom_data(l_args): """, ) parser.add_argument( - "-id", dest="series_id", - required=True, - type=str, - help="FRED Series ID" + "-id", dest="series_id", required=True, type=str, help="FRED Series ID" ) parser.add_argument( "-s", dest="start_date", - type=str, - default="1/1/2020", + type=valid_date, + default="2015-01-01", required=False, help="Date to Start", ) parser.add_argument( "-p", dest="plot", - type=bool, + type=str_to_bool, default=True, required=False, - help="Plot data" + help="Plot data", ) parser.add_argument( "-disp", dest="disp", - type=bool, + type=str_to_bool, default=False, required=False, - help="Print data" + help="Print data", ) try: @@ -127,7 +127,7 @@ def custom_data(l_args): try: if ns_parser.plot: plt.figure() - data.plot(style="ok") + data.plot(style="-ok") plt.xlabel("Time") plt.ylabel("DATA") plt.show() @@ -135,6 +135,7 @@ def custom_data(l_args): print(e) print("") return + except Exception as e: print(e) print("") diff --git a/gamestonk_terminal/helper_funcs.py b/gamestonk_terminal/helper_funcs.py index cd1c92d2a25..f8653eb4a4a 100644 --- a/gamestonk_terminal/helper_funcs.py +++ b/gamestonk_terminal/helper_funcs.py @@ -353,3 +353,12 @@ def get_flair() -> str: return flair[gtff.USE_FLAIR] return "" + +def str_to_bool(value): + if isinstance(value, bool): + return value + if value.lower() in {"false", "f", "0", "no", "n"}: + return False + elif value.lower() in {"true", "t", "1", "yes", "y"}: + return True + raise ValueError(f"{value} is not a valid boolean value") diff --git a/gamestonk_terminal/main_helper.py b/gamestonk_terminal/main_helper.py index c0c0d5e6e89..180b4d73266 100644 --- a/gamestonk_terminal/main_helper.py +++ b/gamestonk_terminal/main_helper.py @@ -52,7 +52,9 @@ def print_help(s_ticker, s_start, s_interval, b_is_market_open): print( " sen sentiment of the market, \t from: reddit, stocktwits, twitter" ) - print(" fred economic data, \t\t\t from: Federal Reserve Bank of St. Louis ") + print( + " fred economic data, \t\t\t from: Federal Reserve Bank of St. Louis " + ) if s_ticker: print( From 97d15e2dd2a90c57a50545920a6e95c7d93ac9b9 Mon Sep 17 00:00:00 2001 From: Jamie Date: Tue, 16 Mar 2021 17:44:07 -0400 Subject: [PATCH 19/64] readme update --- gamestonk_terminal/fred/README.md | 40 ++++++++++++++++++++++++++---- gamestonk_terminal/helper_funcs.py | 1 + 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/gamestonk_terminal/fred/README.md b/gamestonk_terminal/fred/README.md index a394a9b0ca6..2343af81ae0 100644 --- a/gamestonk_terminal/fred/README.md +++ b/gamestonk_terminal/fred/README.md @@ -65,9 +65,39 @@ usage: cust [-id] [-s START_DATE] [-p PLOT] [-disp DISP] * -p : Option to plot. Defaults to True * -disp : Option to print data to console. Defaults to False. -The GDP is specifically defined in this menu, but to reproduce it with the custom function, it would be -```` -cust -id GDP -s 2020-01-01 -p True -```` +An example of custom data could be the unemployment rate. According to https://fred.stlouisfed.org/series/UNRATE , the series ID is UNRATE. So to plot this for 2019 to now -The plot flag allows for plotting and the disp flag tells whether to print data to the console. \ No newline at end of file +```` +cust -id UNRATE -s 2019-01-01 -disp True +```` +This will plot and display: + + +```` +Date: 01-01-2019, DATA: 4.0 +Date: 02-01-2019, DATA: 3.8 +Date: 03-01-2019, DATA: 3.8 +Date: 04-01-2019, DATA: 3.7 +Date: 05-01-2019, DATA: 3.7 +Date: 06-01-2019, DATA: 3.6 +Date: 07-01-2019, DATA: 3.6 +Date: 08-01-2019, DATA: 3.7 +Date: 09-01-2019, DATA: 3.5 +Date: 10-01-2019, DATA: 3.6 +Date: 11-01-2019, DATA: 3.6 +Date: 12-01-2019, DATA: 3.6 +Date: 01-01-2020, DATA: 3.5 +Date: 02-01-2020, DATA: 3.5 +Date: 03-01-2020, DATA: 4.4 +Date: 04-01-2020, DATA: 14.8 +Date: 05-01-2020, DATA: 13.3 +Date: 06-01-2020, DATA: 11.1 +Date: 07-01-2020, DATA: 10.2 +Date: 08-01-2020, DATA: 8.4 +Date: 09-01-2020, DATA: 7.8 +Date: 10-01-2020, DATA: 6.9 +Date: 11-01-2020, DATA: 6.7 +Date: 12-01-2020, DATA: 6.7 +Date: 01-01-2021, DATA: 6.3 +Date: 02-01-2021, DATA: 6.2 +```` \ No newline at end of file diff --git a/gamestonk_terminal/helper_funcs.py b/gamestonk_terminal/helper_funcs.py index f8653eb4a4a..b6cc6118132 100644 --- a/gamestonk_terminal/helper_funcs.py +++ b/gamestonk_terminal/helper_funcs.py @@ -354,6 +354,7 @@ def get_flair() -> str: return "" + def str_to_bool(value): if isinstance(value, bool): return value From 3ea287f8844d7b7c4f3aaf6680eab028ac71e629 Mon Sep 17 00:00:00 2001 From: Jamie Date: Tue, 16 Mar 2021 17:50:00 -0400 Subject: [PATCH 20/64] linting error --- gamestonk_terminal/helper_funcs.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/gamestonk_terminal/helper_funcs.py b/gamestonk_terminal/helper_funcs.py index b6cc6118132..deb9b4ed8af 100644 --- a/gamestonk_terminal/helper_funcs.py +++ b/gamestonk_terminal/helper_funcs.py @@ -358,8 +358,10 @@ def get_flair() -> str: def str_to_bool(value): if isinstance(value, bool): return value - if value.lower() in {"false", "f", "0", "no", "n"}: - return False - elif value.lower() in {"true", "t", "1", "yes", "y"}: - return True + else: + if value.lower() in {"false", "f", "0", "no", "n"}: + return False + else value.lower() in {"true", "t", "1", "yes", "y"}: + return True + raise ValueError(f"{value} is not a valid boolean value") From 6d1a7fd28b3a279369e2b277abdd7a069aa1afaa Mon Sep 17 00:00:00 2001 From: Laurent Andriamifidy Date: Tue, 16 Mar 2021 23:48:24 +0100 Subject: [PATCH 21/64] first running graph w plotly --- gamestonk_terminal/options/op_menu.py | 39 ++++++----------- gamestonk_terminal/options/volume.py | 61 +++++++++++++++++++-------- terminal.py | 7 ++- 3 files changed, 62 insertions(+), 45 deletions(-) diff --git a/gamestonk_terminal/options/op_menu.py b/gamestonk_terminal/options/op_menu.py index a223852f82c..60595320c63 100644 --- a/gamestonk_terminal/options/op_menu.py +++ b/gamestonk_terminal/options/op_menu.py @@ -2,30 +2,12 @@ import argparse import requests from gamestonk_terminal import config_terminal as cfg +from gamestonk_terminal import feature_flags as gtff from gamestonk_terminal.helper_funcs import get_flair, parse_known_args_and_warn -from gamestonk_terminal.options import get_volume_graph - - -def print_volume(l_args, s_ticker): - """ Show traded options volume. [Source: Yahoo Finance] """ - parser = argparse.ArgumentParser( - add_help=False, - prog="volume", - description="""Display volume graph for a date. [Source: Yahoo Finance].""", - ) - - l_similar = [] - try: - ns_parser = parse_known_args_and_warn(parser, l_args) - if not ns_parser: - return - - except Exception as e: - print(e) - - print("") - return get_volume_graph(s_ticker, expiration_date) +from gamestonk_terminal.options import volume as vol +from gamestonk_terminal.menu import session +from prompt_toolkit.completion import NestedCompleter def opt_menu(df_stock, s_ticker, s_start, s_interval): @@ -40,7 +22,7 @@ def opt_menu(df_stock, s_ticker, s_start, s_interval): opt_parser.add_argument("cmd", choices=choices) completer = NestedCompleter.from_nested_dict({c: None for c in choices}) - print_technical_analysis(s_ticker, s_start, s_interval) + #print_options(s_ticker, s_start, s_interval) # Loop forever and ever while True: @@ -53,8 +35,6 @@ def opt_menu(df_stock, s_ticker, s_start, s_interval): else: as_input = input(f"{get_flair()} (opt)> ") - # Images are non blocking - allows to close them if we type other command - plt.close() # Parse fundamental analysis command of the list of possible commands try: @@ -65,15 +45,20 @@ def opt_menu(df_stock, s_ticker, s_start, s_interval): continue if ns_known_args.cmd == "help": - print_technical_analysis(s_ticker, s_start, s_interval) + pass + #print_options(s_ticker, s_start, s_interval) elif ns_known_args.cmd == "q": - # Just leave the FA menu + # Just leave the options menu return False elif ns_known_args.cmd == "quit": # Abandon the program return True + elif ns_known_args.cmd == "volume": + # call the volume graph + vol.volume_graph(l_args, s_ticker) + else: print("Command not recognized!") diff --git a/gamestonk_terminal/options/volume.py b/gamestonk_terminal/options/volume.py index 5731a36964e..e539ff060ec 100644 --- a/gamestonk_terminal/options/volume.py +++ b/gamestonk_terminal/options/volume.py @@ -3,29 +3,56 @@ import seaborn as sns import pandas as pd import matplotlib.pyplot as plt import plotly.express as px +import argparse +import datetime +from gamestonk_terminal.helper_funcs import get_flair, parse_known_args_and_warn, valid_date -def get_volume_graph(ticker_name, exp_date): - opt = yf.Ticker(ticker_name).option_chain(exp_date) - # data available via: opt.calls, opt.puts - - volume_data = pd.concat( - [ - __volume_data(opt.calls, 'calls'), - __volume_data(opt.puts, 'puts') - ], - axis=0 +def volume_graph(l_args, s_ticker): + """ Show traded options volume. [Source: Yahoo Finance] """ + parser = argparse.ArgumentParser( + add_help=False, + prog="volume", + description="""Display volume graph for a date. [Source: Yahoo Finance].""", ) - fig = px.line( - volume_data, - x="strike", - y="volume", - title=f'{ticker_name} Volume for {exp_date}', - color='type' + parser.add_argument( + "-e", + "--expiry", + type=valid_date, + dest="s_expiry_date", + help="The expiry date (format YYYY-MM-DD) for the option chain", + required=True ) - fig.show() + l_similar = [] + try: + ns_parser = parse_known_args_and_warn(parser, l_args) + if not ns_parser: + return + opt = yf.Ticker(s_ticker).option_chain(ns_parser.s_expiry_date.strftime("%Y-%m-%d")) + # data available via: opt.calls, opt.puts + + volume_data = pd.concat( + [ + __volume_data(opt.calls, 'calls'), + __volume_data(opt.puts, 'puts') + ], + axis=0 + ) + fig = px.line( + volume_data, + x="strike", + y="volume", + title=f'{s_ticker} Volume for {ns_parser.s_expiry_date.strftime("%Y-%m-%d")}', + color='type' + ) + fig.show() + + except SystemExit: + print("") + except Exception as e: + print(e) def __volume_data(opt_data, flag): # get option chain for specific expiration diff --git a/terminal.py b/terminal.py index e30805c294c..1f34f27ba63 100644 --- a/terminal.py +++ b/terminal.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python + #!/usr/bin/env python import argparse @@ -20,6 +20,7 @@ from gamestonk_terminal.papermill import papermill_menu as mill from gamestonk_terminal.sentiment import sen_menu as sm from gamestonk_terminal.technical_analysis import ta_menu as tam from gamestonk_terminal.comparison_analysis import ca_menu as cam +from gamestonk_terminal.options import op_menu as opm # import warnings # warnings.simplefilter("always") @@ -65,6 +66,7 @@ def main(): "dd", "pred", "ca", + "op" ] menu_parser.add_argument("opt", choices=choices) completer = NestedCompleter.from_nested_dict({c: None for c in choices}) @@ -155,6 +157,9 @@ def main(): elif ns_known_args.opt == "dd": b_quit = ddm.dd_menu(df_stock, s_ticker, s_start, s_interval) + elif ns_known_args.opt == "op": + b_quit = opm.opt_menu(df_stock, s_ticker, s_start, s_interval) + elif ns_known_args.opt == "pred": if not gtff.ENABLE_PREDICT: From 289195fbe9b7a430ee88bb7c8f343113b3db3454 Mon Sep 17 00:00:00 2001 From: Laurent Andriamifidy Date: Tue, 16 Mar 2021 23:55:28 +0100 Subject: [PATCH 22/64] cleanup --- gamestonk_terminal/options/op_menu.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/gamestonk_terminal/options/op_menu.py b/gamestonk_terminal/options/op_menu.py index 60595320c63..e6358421538 100644 --- a/gamestonk_terminal/options/op_menu.py +++ b/gamestonk_terminal/options/op_menu.py @@ -1,13 +1,10 @@ import argparse -import requests - -from gamestonk_terminal import config_terminal as cfg -from gamestonk_terminal import feature_flags as gtff -from gamestonk_terminal.helper_funcs import get_flair, parse_known_args_and_warn -from gamestonk_terminal.options import volume as vol -from gamestonk_terminal.menu import session from prompt_toolkit.completion import NestedCompleter +from gamestonk_terminal import feature_flags as gtff +from gamestonk_terminal.helper_funcs import get_flair +from gamestonk_terminal.options import volume as vol +from gamestonk_terminal.menu import session def opt_menu(df_stock, s_ticker, s_start, s_interval): From ce8de70f8dd6fef0a0dabc3d56fa3b6e22587403 Mon Sep 17 00:00:00 2001 From: Jamie Date: Tue, 16 Mar 2021 21:10:39 -0400 Subject: [PATCH 23/64] catch exception if required argument not given --- gamestonk_terminal/fred/fred_api.py | 3 ++- gamestonk_terminal/helper_funcs.py | 11 +++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/gamestonk_terminal/fred/fred_api.py b/gamestonk_terminal/fred/fred_api.py index b96aceb8a94..bb3f87a31f0 100644 --- a/gamestonk_terminal/fred/fred_api.py +++ b/gamestonk_terminal/fred/fred_api.py @@ -135,7 +135,8 @@ def custom_data(l_args): print(e) print("") return - + except SystemExit: + print("") except Exception as e: print(e) print("") diff --git a/gamestonk_terminal/helper_funcs.py b/gamestonk_terminal/helper_funcs.py index deb9b4ed8af..9e243ff2b4a 100644 --- a/gamestonk_terminal/helper_funcs.py +++ b/gamestonk_terminal/helper_funcs.py @@ -358,10 +358,9 @@ def get_flair() -> str: def str_to_bool(value): if isinstance(value, bool): return value + elif value.lower() in {"false", "f", "0", "no", "n"}: + return False + elif value.lower() in {"true", "t", "1", "yes", "y"}: + return True else: - if value.lower() in {"false", "f", "0", "no", "n"}: - return False - else value.lower() in {"true", "t", "1", "yes", "y"}: - return True - - raise ValueError(f"{value} is not a valid boolean value") + raise ValueError(f"{value} is not a valid boolean value") From 76ebd3031d989137be251635d68c3f4fef09724e Mon Sep 17 00:00:00 2001 From: Jamie Date: Tue, 16 Mar 2021 21:25:35 -0400 Subject: [PATCH 24/64] lint --- gamestonk_terminal/fred/fred_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gamestonk_terminal/fred/fred_api.py b/gamestonk_terminal/fred/fred_api.py index bb3f87a31f0..1f4ac4479d7 100644 --- a/gamestonk_terminal/fred/fred_api.py +++ b/gamestonk_terminal/fred/fred_api.py @@ -136,7 +136,7 @@ def custom_data(l_args): print("") return except SystemExit: - print("") + print("") except Exception as e: print(e) print("") From df8ec83b3b91a237fe63c9bc1c64ae3bdffc4d1e Mon Sep 17 00:00:00 2001 From: Jamie Date: Tue, 16 Mar 2021 21:26:29 -0400 Subject: [PATCH 25/64] rebase + lint --- gamestonk_terminal/helper_funcs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/gamestonk_terminal/helper_funcs.py b/gamestonk_terminal/helper_funcs.py index f18c6f4f6af..e0c97197290 100644 --- a/gamestonk_terminal/helper_funcs.py +++ b/gamestonk_terminal/helper_funcs.py @@ -367,6 +367,7 @@ def str_to_bool(value): else: raise ValueError(f"{value} is not a valid boolean value") + def get_screeninfo(): screens = get_monitors() # Get all available monitors if len(screens) - 1 < cfgPlot.MONITOR: # Check to see if chosen monitor is detected From b27987606b3b8e3abd42afeb841a0046d167dfa4 Mon Sep 17 00:00:00 2001 From: Jamie Date: Tue, 16 Mar 2021 21:31:07 -0400 Subject: [PATCH 26/64] pylint --- gamestonk_terminal/helper_funcs.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/gamestonk_terminal/helper_funcs.py b/gamestonk_terminal/helper_funcs.py index e0c97197290..1f0ab039d11 100644 --- a/gamestonk_terminal/helper_funcs.py +++ b/gamestonk_terminal/helper_funcs.py @@ -360,12 +360,11 @@ def get_flair() -> str: def str_to_bool(value): if isinstance(value, bool): return value - elif value.lower() in {"false", "f", "0", "no", "n"}: + if value.lower() in {"false", "f", "0", "no", "n"}: return False - elif value.lower() in {"true", "t", "1", "yes", "y"}: + if value.lower() in {"true", "t", "1", "yes", "y"}: return True - else: - raise ValueError(f"{value} is not a valid boolean value") + raise ValueError(f"{value} is not a valid boolean value") def get_screeninfo(): From 05065ec540fb8a4db24a641525fa74a622635fab Mon Sep 17 00:00:00 2001 From: buzzCraft <67235613+buzzCraft@users.noreply.github.com> Date: Wed, 17 Mar 2021 10:51:45 +0100 Subject: [PATCH 27/64] Added scaling to arima, ets, fbprophet, knn, nn, regression, sma and historical yahoo_finance_api --- .../comparison_analysis/yahoo_finance_api.py | 6 +++--- gamestonk_terminal/prediction_techniques/arima.py | 6 ++++-- gamestonk_terminal/prediction_techniques/ets.py | 6 ++++-- .../prediction_techniques/fbprophet.py | 6 ++++-- gamestonk_terminal/prediction_techniques/knn.py | 7 ++++--- .../prediction_techniques/neural_networks.py | 15 ++++++++------- .../prediction_techniques/regression.py | 7 ++++--- gamestonk_terminal/prediction_techniques/sma.py | 7 ++++--- 8 files changed, 35 insertions(+), 25 deletions(-) diff --git a/gamestonk_terminal/comparison_analysis/yahoo_finance_api.py b/gamestonk_terminal/comparison_analysis/yahoo_finance_api.py index 31d2c662a21..edd3e726d4d 100644 --- a/gamestonk_terminal/comparison_analysis/yahoo_finance_api.py +++ b/gamestonk_terminal/comparison_analysis/yahoo_finance_api.py @@ -6,11 +6,11 @@ import matplotlib.pyplot as plt import yfinance as yf import seaborn as sns -from gamestonk_terminal.helper_funcs import parse_known_args_and_warn +from gamestonk_terminal.helper_funcs import parse_known_args_and_warn, plot_autoscale +from gamestonk_terminal.config_plot import PLOT_DPI register_matplotlib_converters() - d_candle_types = { "o": "Open", "h": "High", @@ -79,7 +79,7 @@ def historical(l_args, df_stock, s_ticker, s_start, s_interval, similar): similar += ns_parser.l_also - plt.figure() + plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) plt.title(f"Similar companies to {s_ticker}") df_stock = yf.download(s_ticker, start=s_start, progress=False) plt.plot( diff --git a/gamestonk_terminal/prediction_techniques/arima.py b/gamestonk_terminal/prediction_techniques/arima.py index 5ed5b4e45b7..e68ac8aad8b 100644 --- a/gamestonk_terminal/prediction_techniques/arima.py +++ b/gamestonk_terminal/prediction_techniques/arima.py @@ -11,6 +11,7 @@ from gamestonk_terminal.helper_funcs import ( valid_date, patch_pandas_text_adjustment, get_next_stock_market_days, + plot_autoscale, ) from gamestonk_terminal.prediction_techniques.pred_helper import ( print_pretty_prediction, @@ -18,6 +19,7 @@ from gamestonk_terminal.prediction_techniques.pred_helper import ( print_prediction_kpis, ) +from gamestonk_terminal.config_plot import PLOT_DPI from gamestonk_terminal import feature_flags as gtff register_matplotlib_converters() @@ -168,7 +170,7 @@ def arima(l_args, s_ticker, df_stock): print("") # Plotting - plt.figure() + plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) plt.plot(df_stock.index, df_stock["5. adjusted close"], lw=2) if ns_parser.s_order: # BACKTESTING @@ -241,7 +243,7 @@ def arima(l_args, s_ticker, df_stock): # BACKTESTING if ns_parser.s_end_date: - plt.figure() + plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) plt.subplot(211) plt.plot( df_future.index, diff --git a/gamestonk_terminal/prediction_techniques/ets.py b/gamestonk_terminal/prediction_techniques/ets.py index a802b0cc977..96e17306d78 100644 --- a/gamestonk_terminal/prediction_techniques/ets.py +++ b/gamestonk_terminal/prediction_techniques/ets.py @@ -11,6 +11,7 @@ from gamestonk_terminal.helper_funcs import ( valid_date, patch_pandas_text_adjustment, get_next_stock_market_days, + plot_autoscale, ) from gamestonk_terminal.prediction_techniques.pred_helper import ( print_pretty_prediction, @@ -18,6 +19,7 @@ from gamestonk_terminal.prediction_techniques.pred_helper import ( print_prediction_kpis, ) +from gamestonk_terminal.config_plot import PLOT_DPI from gamestonk_terminal import feature_flags as gtff register_matplotlib_converters() @@ -179,7 +181,7 @@ def exponential_smoothing(l_args, s_ticker, df_stock): print(f"SSE: {round(model.sse, 2)}\n") # Plotting - plt.figure() + plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) plt.plot(df_stock.index, df_stock["5. adjusted close"], lw=2) # BACKTESTING if ns_parser.s_end_date: @@ -249,7 +251,7 @@ def exponential_smoothing(l_args, s_ticker, df_stock): # BACKTESTING if ns_parser.s_end_date: - plt.figure() + plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) plt.subplot(211) plt.plot( df_future.index, diff --git a/gamestonk_terminal/prediction_techniques/fbprophet.py b/gamestonk_terminal/prediction_techniques/fbprophet.py index 21447c7d1ca..e2294791b6c 100644 --- a/gamestonk_terminal/prediction_techniques/fbprophet.py +++ b/gamestonk_terminal/prediction_techniques/fbprophet.py @@ -11,12 +11,14 @@ from gamestonk_terminal.helper_funcs import ( parse_known_args_and_warn, valid_date, patch_pandas_text_adjustment, + plot_autoscale, ) from gamestonk_terminal.prediction_techniques.pred_helper import ( price_prediction_backtesting_color, print_prediction_kpis, ) +from gamestonk_terminal.config_plot import PLOT_DPI from gamestonk_terminal import feature_flags as gtff register_matplotlib_converters() @@ -114,7 +116,7 @@ def fbprophet(l_args, s_ticker, df_stock): ] # .apply(lambda x: f"{x:.2f} $") df_pred.index = l_pred_days - _, ax = plt.subplots() + _, ax = plt.subplots(figsize=plot_autoscale(), dpi=PLOT_DPI) model.plot( forecast[: -ns_parser.n_days], ax=ax, @@ -177,7 +179,7 @@ def fbprophet(l_args, s_ticker, df_stock): # BACKTESTING if ns_parser.s_end_date: - plt.figure() + plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) plt.subplot(211) plt.plot( df_future.index, diff --git a/gamestonk_terminal/prediction_techniques/knn.py b/gamestonk_terminal/prediction_techniques/knn.py index 584c6f73d80..bd515d5ac45 100644 --- a/gamestonk_terminal/prediction_techniques/knn.py +++ b/gamestonk_terminal/prediction_techniques/knn.py @@ -11,6 +11,7 @@ from gamestonk_terminal.helper_funcs import ( valid_date, patch_pandas_text_adjustment, get_next_stock_market_days, + plot_autoscale, ) from gamestonk_terminal.prediction_techniques.pred_helper import ( print_pretty_prediction, @@ -18,7 +19,7 @@ from gamestonk_terminal.prediction_techniques.pred_helper import ( print_prediction_kpis, ) - +from gamestonk_terminal.config_plot import PLOT_DPI from gamestonk_terminal import feature_flags as gtff register_matplotlib_converters() @@ -143,7 +144,7 @@ def k_nearest_neighbors(l_args, s_ticker, df_stock): df_pred = pd.Series(l_predictions, index=l_pred_days, name="Price") # Plotting - plt.figure() + plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) plt.plot(df_stock.index, df_stock["5. adjusted close"], lw=2) # BACKTESTING if ns_parser.s_end_date: @@ -202,7 +203,7 @@ def k_nearest_neighbors(l_args, s_ticker, df_stock): # BACKTESTING if ns_parser.s_end_date: - plt.figure() + plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) plt.subplot(211) plt.plot( df_future.index, diff --git a/gamestonk_terminal/prediction_techniques/neural_networks.py b/gamestonk_terminal/prediction_techniques/neural_networks.py index 64f2c2f02c0..d8a44978b7e 100644 --- a/gamestonk_terminal/prediction_techniques/neural_networks.py +++ b/gamestonk_terminal/prediction_techniques/neural_networks.py @@ -17,6 +17,7 @@ from gamestonk_terminal.helper_funcs import ( valid_date, patch_pandas_text_adjustment, get_next_stock_market_days, + plot_autoscale, ) from gamestonk_terminal.prediction_techniques.pred_helper import ( @@ -25,7 +26,7 @@ from gamestonk_terminal.prediction_techniques.pred_helper import ( print_prediction_kpis, ) - +from gamestonk_terminal.config_plot import PLOT_DPI from gamestonk_terminal import feature_flags as gtff from gamestonk_terminal import config_neural_network_models as cfg_nn_models @@ -275,7 +276,7 @@ def mlp(l_args, s_ticker, df_stock): df_pred = pd.Series(y_pred_test_t[0].tolist(), index=l_pred_days, name="Price") # Plotting - plt.figure() + plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) plt.plot(df_stock.index, df_stock["5. adjusted close"], lw=3) # BACKTESTING if ns_parser.s_end_date: @@ -341,7 +342,7 @@ def mlp(l_args, s_ticker, df_stock): # BACKTESTING if ns_parser.s_end_date: - plt.figure() + plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) plt.subplot(211) plt.plot( df_future.index, @@ -654,7 +655,7 @@ def rnn(l_args, s_ticker, df_stock): df_pred = pd.Series(y_pred_test_t[0].tolist(), index=l_pred_days, name="Price") # Plotting - plt.figure() + plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) plt.plot(df_stock.index, df_stock["5. adjusted close"], lw=3) # BACKTESTING if ns_parser.s_end_date: @@ -720,7 +721,7 @@ def rnn(l_args, s_ticker, df_stock): # BACKTESTING if ns_parser.s_end_date: - plt.figure() + plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) plt.subplot(211) plt.plot( df_future.index, @@ -1033,7 +1034,7 @@ def lstm(l_args, s_ticker, df_stock): df_pred = pd.Series(y_pred_test_t[0].tolist(), index=l_pred_days, name="Price") # Plotting - plt.figure() + plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) plt.plot(df_stock.index, df_stock["5. adjusted close"], lw=3) # BACKTESTING if ns_parser.s_end_date: @@ -1099,7 +1100,7 @@ def lstm(l_args, s_ticker, df_stock): # BACKTESTING if ns_parser.s_end_date: - plt.figure() + plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) plt.subplot(211) plt.plot( df_future.index, diff --git a/gamestonk_terminal/prediction_techniques/regression.py b/gamestonk_terminal/prediction_techniques/regression.py index 947914ae496..ffdeb881cec 100644 --- a/gamestonk_terminal/prediction_techniques/regression.py +++ b/gamestonk_terminal/prediction_techniques/regression.py @@ -14,6 +14,7 @@ from gamestonk_terminal.helper_funcs import ( valid_date, patch_pandas_text_adjustment, get_next_stock_market_days, + plot_autoscale, ) from gamestonk_terminal.prediction_techniques.pred_helper import ( @@ -22,7 +23,7 @@ from gamestonk_terminal.prediction_techniques.pred_helper import ( print_prediction_kpis, ) - +from gamestonk_terminal.config_plot import PLOT_DPI from gamestonk_terminal import feature_flags as gtff register_matplotlib_converters() @@ -163,7 +164,7 @@ def regression(l_args, s_ticker, df_stock, polynomial): df_pred = pd.Series(l_predictions, index=l_pred_days, name="Price") # Plotting - plt.figure() + plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) plt.plot(df_stock.index, df_stock["5. adjusted close"], lw=2) # BACKTESTING if ns_parser.s_end_date: @@ -225,7 +226,7 @@ def regression(l_args, s_ticker, df_stock, polynomial): # BACKTESTING if ns_parser.s_end_date: - plt.figure() + plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) plt.subplot(211) plt.plot( df_future.index, diff --git a/gamestonk_terminal/prediction_techniques/sma.py b/gamestonk_terminal/prediction_techniques/sma.py index 6b0504c1794..b371553343c 100644 --- a/gamestonk_terminal/prediction_techniques/sma.py +++ b/gamestonk_terminal/prediction_techniques/sma.py @@ -11,6 +11,7 @@ from gamestonk_terminal.helper_funcs import ( valid_date, patch_pandas_text_adjustment, get_next_stock_market_days, + plot_autoscale, ) from gamestonk_terminal.prediction_techniques.pred_helper import ( @@ -19,7 +20,7 @@ from gamestonk_terminal.prediction_techniques.pred_helper import ( print_prediction_kpis, ) - +from gamestonk_terminal.config_plot import PLOT_DPI from gamestonk_terminal import feature_flags as gtff register_matplotlib_converters() @@ -125,7 +126,7 @@ def simple_moving_average(l_args, s_ticker, df_stock): df_pred = pd.Series(l_predictions, index=l_pred_days, name="Price") # Plotting - plt.figure() + plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) plt.plot(df_stock.index, df_stock["5. adjusted close"], lw=2) # BACKTESTING if ns_parser.s_end_date: @@ -189,7 +190,7 @@ def simple_moving_average(l_args, s_ticker, df_stock): # BACKTESTING if ns_parser.s_end_date: - plt.figure() + plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) plt.subplot(211) plt.plot( df_future.index, From fac2804f1380b7c0e374704815d0b5e3625e8757 Mon Sep 17 00:00:00 2001 From: buzzCraft <67235613+buzzCraft@users.noreply.github.com> Date: Wed, 17 Mar 2021 10:58:32 +0100 Subject: [PATCH 28/64] Merge conflict fix --- .../prediction_techniques/neural_networks.py | 1378 +++++------------ 1 file changed, 428 insertions(+), 950 deletions(-) diff --git a/gamestonk_terminal/prediction_techniques/neural_networks.py b/gamestonk_terminal/prediction_techniques/neural_networks.py index d8a44978b7e..236906bd4d7 100644 --- a/gamestonk_terminal/prediction_techniques/neural_networks.py +++ b/gamestonk_terminal/prediction_techniques/neural_networks.py @@ -1,6 +1,7 @@ import argparse import datetime import os +import traceback from warnings import simplefilter import numpy as np import matplotlib.pyplot as plt @@ -38,6 +39,10 @@ os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" simplefilter(action="ignore", category=FutureWarning) +# store the user's TensorFlow environment variables +ORIGINAL_TF_XLA_FLAGS = os.environ.get("TF_XLA_FLAGS") +ORIGINAL_TF_FORCE_GPU_ALLOW_GROWTH = os.environ.get("TF_FORCE_GPU_ALLOW_GROWTH") + def build_neural_network_model(Recurrent_Neural_Network, n_inputs, n_days): model = Sequential() @@ -86,11 +91,15 @@ def build_neural_network_model(Recurrent_Neural_Network, n_inputs, n_days): return model -def mlp(l_args, s_ticker, df_stock): +def _parse_args(prog, description, l_args): + """Create an argparser and parse l_args. Will print help if user requests it. + :return: ns_parser""" parser = argparse.ArgumentParser( - add_help=False, prog="mlp", description="""Multilayer Perceptron. """ + prog=prog, + description=description, + add_help=False, + formatter_class=argparse.RawTextHelpFormatter, # enable multiline help messages ) - parser.add_argument( "-d", "--days", @@ -148,11 +157,10 @@ def mlp(l_args, s_ticker, df_stock): "adamax", "ftrl", "nadam", - "optimizer", "rmsprop", "sgd", ], - help="optimization technique.", + help="optimization technique (see https://www.tensorflow.org/api_docs/python/tf/keras/optimizers)", ) parser.add_argument( "-l", @@ -160,8 +168,19 @@ def mlp(l_args, s_ticker, df_stock): action="store", dest="s_loss", default="mae", - choices=["mae", "mape", "mse", "msle"], - help="loss function.", + choices=[ + "mae", + "mape", + "mse", + "msle", + "poisson", + "logcosh", + "kld", + "hinge", + "squared_hinge", + "huber", + ], + help="loss function (see https://www.tensorflow.org/api_docs/python/tf/keras/losses)", ) parser.add_argument( "-e", @@ -172,74 +191,358 @@ def mlp(l_args, s_ticker, df_stock): default=None, help="The end date (format YYYY-MM-DD) to select - Backtesting", ) + parser.add_argument( + "--batch_size", + action="store", + dest="n_batch_size", + type=check_positive, + default=None, + help="batch size for model fitting (use a power of 2)", + ) + parser.add_argument( + "--xla_cpu", + action="store_true", + dest="b_xla_cpu", + default=False, + help="enable XLA for CPU (see https://www.tensorflow.org/xla)", + ) + parser.add_argument( + "--xla_gpu", + action="store_true", + dest="b_xla_gpu", + default=False, + help="enable XLA for GPU (see https://www.tensorflow.org/xla)", + ) + parser.add_argument( + "--force_gpu_allow_growth", + action="store", + dest="s_force_gpu_allow_growth", + default="true", + choices=["true", "false", "default"], + help="true: GPU memory will grow as needed. \n" + "false: TensorFlow will allocate 100%% of GPU memory. \n" + "default: usually the same as false, uses env/TensorFlow default", + ) + ns_parser = parse_known_args_and_warn(parser, l_args) + if not ns_parser: + return None + + # set xla flags if requested + xla_flags = ( + set(ORIGINAL_TF_XLA_FLAGS.split(" ")) if ORIGINAL_TF_XLA_FLAGS else set() + ) + if ns_parser.b_xla_cpu or ns_parser.b_xla_gpu: + xla_flags.add("--tf_xla_enable_xla_devices") + if ns_parser.b_xla_cpu: + xla_flags.add("--tf_xla_cpu_global_jit") + if ns_parser.b_xla_gpu: + xla_flags.add("--tf_xla_auto_jit=2") + os.environ["TF_XLA_FLAGS"] = " ".join(xla_flags) + + # set GPU memory growth flag + if ns_parser.s_force_gpu_allow_growth == "true": + os.environ["TF_FORCE_GPU_ALLOW_GROWTH"] = "true" + elif ns_parser.s_force_gpu_allow_growth == "false": + os.environ["TF_FORCE_GPU_ALLOW_GROWTH"] = "false" + + return ns_parser + + +def _restore_env(): + """Restore environment variables to original values""" + + def restore(key, value): + if value is None: + if key in os.environ: + del os.environ[key] + else: + os.environ[key] = value + + restore("TF_XLA_FLAGS", ORIGINAL_TF_XLA_FLAGS) + restore("TF_FORCE_GPU_ALLOW_GROWTH", ORIGINAL_TF_FORCE_GPU_ALLOW_GROWTH) + + +def _setup_backtesting(df_stock, ns_parser): + """Set up backtesting if enabled + :return: (df_stock, df_future), where df_future is None if s_end_date is not set. + :raises Exception: if configuration is invalid""" + df_future = None + if ns_parser.s_end_date: + if ns_parser.s_end_date < df_stock.index[0]: + raise Exception( + "Backtesting not allowed, since End Date is older than Start Date of historical data" + ) + + if ns_parser.s_end_date < get_next_stock_market_days( + last_stock_day=df_stock.index[0], + n_next_days=ns_parser.n_inputs + ns_parser.n_days, + )[-1]: + raise Exception( + "Backtesting not allowed, since End Date is too close to Start Date to train model" + ) + + future_index = get_next_stock_market_days( + last_stock_day=ns_parser.s_end_date, n_next_days=ns_parser.n_days + ) + + if future_index[-1] > datetime.datetime.now(): + raise Exception( + "Backtesting not allowed, since End Date + Prediction days is in the future" + ) + + df_future = df_stock[future_index[0] : future_index[-1]] + df_stock = df_stock[: ns_parser.s_end_date] + return df_stock, df_future + + +def _preprocess_split(df_stock, ns_parser): + """Preprocess and split training data. + :return: (scaler, stock_train_data, stock_x, stock_y) + :raises Exception: if more training data is needed.""" + # Pre-process data + if ns_parser.s_preprocessing == "standardization": + scaler = StandardScaler() + stock_train_data = scaler.fit_transform( + np.array(df_stock["5. adjusted close"].values.reshape(-1, 1)) + ) + elif ns_parser.s_preprocessing == "normalization": + scaler = MinMaxScaler() + stock_train_data = scaler.fit_transform( + np.array(df_stock["5. adjusted close"].values.reshape(-1, 1)) + ) + else: # No pre-processing + stock_train_data = np.array(df_stock["5. adjusted close"].values.reshape(-1, 1)) + + # Split training data for the neural network + stock_x, stock_y = splitTrain.split_train( + stock_train_data, + ns_parser.n_inputs, + ns_parser.n_days, + numJumps=ns_parser.n_jumps, + ) + if not stock_x: + raise Exception("Given the model parameters more training data is needed.") + stock_x = np.array(stock_x) + stock_y = np.array(stock_y) + return scaler, stock_train_data, stock_x, stock_y + + +def _rescale_data(df_stock, ns_parser, scaler, yhat): + """Re-scale the data back and return the prediction dataframe. """ + if (ns_parser.s_preprocessing == "standardization") or ( + ns_parser.s_preprocessing == "normalization" + ): + y_pred_test_t = scaler.inverse_transform(yhat.tolist()) + else: + y_pred_test_t = yhat + + l_pred_days = get_next_stock_market_days( + last_stock_day=df_stock["5. adjusted close"].index[-1], + n_next_days=ns_parser.n_days, + ) + df_pred = pd.Series(y_pred_test_t[0].tolist(), index=l_pred_days, name="Price") + return df_pred + + +def _plot_and_print_results( + df_stock, ns_parser, df_future, df_pred, model_name, s_ticker +): + """Plot and print the results. """ + # Plotting + plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) + plt.plot(df_stock.index, df_stock["5. adjusted close"], lw=3) + + # BACKTESTING + if ns_parser.s_end_date: + plt.title( + f"BACKTESTING: {model_name} on {s_ticker} - {ns_parser.n_days} days prediction" + ) + else: + plt.title(f"{model_name} on {s_ticker} - {ns_parser.n_days} days prediction") + plt.xlim(df_stock.index[0], get_next_stock_market_days(df_pred.index[-1], 1)[-1]) + plt.xlabel("Time") + plt.ylabel("Share Price ($)") + plt.grid(b=True, which="major", color="#666666", linestyle="-") + plt.minorticks_on() + plt.grid(b=True, which="minor", color="#999999", linestyle="-", alpha=0.2) + plt.plot( + [df_stock.index[-1], df_pred.index[0]], + [df_stock["5. adjusted close"].values[-1], df_pred.values[0]], + lw=1, + c="tab:green", + linestyle="--", + ) + plt.plot(df_pred.index, df_pred, lw=2, c="tab:green") + plt.axvspan( + df_stock.index[-1], df_pred.index[-1], facecolor="tab:orange", alpha=0.2 + ) + _, _, ymin, ymax = plt.axis() + plt.vlines( + df_stock.index[-1], + ymin, + ymax, + colors="k", + linewidth=3, + linestyle="--", + color="k", + ) + + # BACKTESTING + if ns_parser.s_end_date: + plt.plot( + df_future.index, + df_future["5. adjusted close"], + lw=2, + c="tab:blue", + ls="--", + ) + plt.plot( + [df_stock.index[-1], df_future.index[0]], + [ + df_stock["5. adjusted close"].values[-1], + df_future["5. adjusted close"].values[0], + ], + lw=1, + c="tab:blue", + linestyle="--", + ) + + if gtff.USE_ION: + plt.ion() + + plt.show() + + # BACKTESTING + if ns_parser.s_end_date: + plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) + plt.subplot(211) + plt.plot( + df_future.index, + df_future["5. adjusted close"], + lw=2, + c="tab:blue", + ls="--", + ) + plt.plot(df_pred.index, df_pred, lw=2, c="green") + plt.scatter(df_future.index, df_future["5. adjusted close"], c="tab:blue", lw=3) + plt.plot( + [df_stock.index[-1], df_future.index[0]], + [ + df_stock["5. adjusted close"].values[-1], + df_future["5. adjusted close"].values[0], + ], + lw=2, + c="tab:blue", + ls="--", + ) + plt.scatter(df_pred.index, df_pred, c="green", lw=3) + plt.plot( + [df_stock.index[-1], df_pred.index[0]], + [df_stock["5. adjusted close"].values[-1], df_pred.values[0]], + lw=2, + c="green", + ls="--", + ) + plt.title("BACKTESTING: Real data price versus Prediction") + plt.xlim(df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)) + plt.xticks( + [df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)], + visible=True, + ) + plt.ylabel("Share Price ($)") + plt.grid(b=True, which="major", color="#666666", linestyle="-") + plt.minorticks_on() + plt.grid(b=True, which="minor", color="#999999", linestyle="-", alpha=0.2) + plt.legend(["Real data", "Prediction data"]) + plt.xticks([]) + + plt.subplot(212) + plt.axhline(y=0, color="k", linestyle="--", linewidth=2) + plt.plot( + df_future.index, + 100 + * (df_pred.values - df_future["5. adjusted close"].values) + / df_future["5. adjusted close"].values, + lw=2, + c="red", + ) + plt.scatter( + df_future.index, + 100 + * (df_pred.values - df_future["5. adjusted close"].values) + / df_future["5. adjusted close"].values, + c="red", + lw=5, + ) + plt.title("BACKTESTING: Error between Real data and Prediction [%]") + plt.plot( + [df_stock.index[-1], df_future.index[0]], + [ + 0, + 100 + * (df_pred.values[0] - df_future["5. adjusted close"].values[0]) + / df_future["5. adjusted close"].values[0], + ], + lw=2, + ls="--", + c="red", + ) + plt.xlim(df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)) + plt.xticks( + [df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)], + visible=True, + ) + plt.xlabel("Time") + plt.ylabel("Prediction Error (%)") + plt.grid(b=True, which="major", color="#666666", linestyle="-") + plt.minorticks_on() + plt.grid(b=True, which="minor", color="#999999", linestyle="-", alpha=0.2) + plt.legend(["Real data", "Prediction data"]) + + if gtff.USE_ION: + plt.ion() + + plt.show() + + # Refactor prediction dataframe for backtesting print + df_pred.name = "Prediction" + df_pred = df_pred.to_frame() + df_pred["Real"] = df_future["5. adjusted close"] + + if gtff.USE_COLOR: + patch_pandas_text_adjustment() + + print("Time Real [$] x Prediction [$]") + print(df_pred.apply(price_prediction_backtesting_color, axis=1).to_string()) + else: + print(df_pred[["Real", "Prediction"]].round(2).to_string()) + + print("") + print_prediction_kpis(df_pred["Real"].values, df_pred["Prediction"].values) + + else: + # Print prediction data + print_pretty_prediction(df_pred, df_stock["5. adjusted close"].values[-1]) + print("") + + +def mlp(l_args, s_ticker, df_stock): try: - ns_parser = parse_known_args_and_warn(parser, l_args) + ns_parser = _parse_args( + prog="mlp", description="""Multilayer Perceptron. """, l_args=l_args + ) if not ns_parser: return - # BACKTESTING - if ns_parser.s_end_date: - - if ns_parser.s_end_date < df_stock.index[0]: - print( - "Backtesting not allowed, since End Date is older than Start Date of historical data\n" - ) - return - - if ns_parser.s_end_date < get_next_stock_market_days( - last_stock_day=df_stock.index[0], - n_next_days=ns_parser.n_inputs + ns_parser.n_days, - )[-1]: - print( - "Backtesting not allowed, since End Date is too close to Start Date to train model\n" - ) - return - - future_index = get_next_stock_market_days( - last_stock_day=ns_parser.s_end_date, n_next_days=ns_parser.n_days - ) - - if future_index[-1] > datetime.datetime.now(): - print( - "Backtesting not allowed, since End Date + Prediction days is in the future\n" - ) - return - - df_future = df_stock[future_index[0] : future_index[-1]] - df_stock = df_stock[: ns_parser.s_end_date] + # Setup backtesting + df_stock, df_future = _setup_backtesting(df_stock, ns_parser) # Pre-process data - if ns_parser.s_preprocessing == "standardization": - scaler = StandardScaler() - stock_train_data = scaler.fit_transform( - np.array(df_stock["5. adjusted close"].values.reshape(-1, 1)) - ) - elif ns_parser.s_preprocessing == "normalization": - scaler = MinMaxScaler() - stock_train_data = scaler.fit_transform( - np.array(df_stock["5. adjusted close"].values.reshape(-1, 1)) - ) - else: # No pre-processing - stock_train_data = np.array( - df_stock["5. adjusted close"].values.reshape(-1, 1) - ) - - # Split training data for the neural network - stock_x, stock_y = splitTrain.split_train( - stock_train_data, - ns_parser.n_inputs, - ns_parser.n_days, - numJumps=ns_parser.n_jumps, + scaler, stock_train_data, stock_x, stock_y = _preprocess_split( + df_stock, ns_parser ) - - if not stock_x: - print("Given the model parameters more training data is needed.\n") - return - - stock_x = np.array(stock_x) stock_x = np.reshape(stock_x, (stock_x.shape[0], stock_x.shape[1])) - stock_y = np.array(stock_y) stock_y = np.reshape(stock_y, (stock_y.shape[0], stock_y.shape[1])) # Build Neural Network model @@ -249,7 +552,13 @@ def mlp(l_args, s_ticker, df_stock): model.compile(optimizer=ns_parser.s_optimizer, loss=ns_parser.s_loss) # Train our model - model.fit(stock_x, stock_y, epochs=ns_parser.n_epochs, verbose=1) + model.fit( + stock_x, + stock_y, + epochs=ns_parser.n_epochs, + batch_size=ns_parser.n_batch_size, + verbose=1, + ) print("") print(model.summary()) @@ -261,364 +570,37 @@ def mlp(l_args, s_ticker, df_stock): verbose=0, ) - # Re-scale the data back - if (ns_parser.s_preprocessing == "standardization") or ( - ns_parser.s_preprocessing == "normalization" - ): - y_pred_test_t = scaler.inverse_transform(yhat.tolist()) - else: - y_pred_test_t = yhat - - l_pred_days = get_next_stock_market_days( - last_stock_day=df_stock["5. adjusted close"].index[-1], - n_next_days=ns_parser.n_days, + # Re-scale the data back, plot, and print the results + df_pred = _rescale_data(df_stock, ns_parser, scaler, yhat) + _plot_and_print_results( + df_stock, ns_parser, df_future, df_pred, "MLP", s_ticker ) - df_pred = pd.Series(y_pred_test_t[0].tolist(), index=l_pred_days, name="Price") - - # Plotting - plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) - plt.plot(df_stock.index, df_stock["5. adjusted close"], lw=3) - # BACKTESTING - if ns_parser.s_end_date: - plt.title( - f"BACKTESTING: MLP on {s_ticker} - {ns_parser.n_days} days prediction" - ) - else: - plt.title(f"MLP on {s_ticker} - {ns_parser.n_days} days prediction") - plt.xlim( - df_stock.index[0], get_next_stock_market_days(df_pred.index[-1], 1)[-1] - ) - plt.xlabel("Time") - plt.ylabel("Share Price ($)") - plt.grid(b=True, which="major", color="#666666", linestyle="-") - plt.minorticks_on() - plt.grid(b=True, which="minor", color="#999999", linestyle="-", alpha=0.2) - plt.plot( - [df_stock.index[-1], df_pred.index[0]], - [df_stock["5. adjusted close"].values[-1], df_pred.values[0]], - lw=1, - c="tab:green", - linestyle="--", - ) - plt.plot(df_pred.index, df_pred, lw=2, c="tab:green") - plt.axvspan( - df_stock.index[-1], df_pred.index[-1], facecolor="tab:orange", alpha=0.2 - ) - _, _, ymin, ymax = plt.axis() - plt.vlines( - df_stock.index[-1], - ymin, - ymax, - colors="k", - linewidth=3, - linestyle="--", - color="k", - ) - - # BACKTESTING - if ns_parser.s_end_date: - plt.plot( - df_future.index, - df_future["5. adjusted close"], - lw=2, - c="tab:blue", - ls="--", - ) - plt.plot( - [df_stock.index[-1], df_future.index[0]], - [ - df_stock["5. adjusted close"].values[-1], - df_future["5. adjusted close"].values[0], - ], - lw=1, - c="tab:blue", - linestyle="--", - ) - - if gtff.USE_ION: - plt.ion() - - plt.show() - - # BACKTESTING - if ns_parser.s_end_date: - plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) - plt.subplot(211) - plt.plot( - df_future.index, - df_future["5. adjusted close"], - lw=2, - c="tab:blue", - ls="--", - ) - plt.plot(df_pred.index, df_pred, lw=2, c="green") - plt.scatter( - df_future.index, df_future["5. adjusted close"], c="tab:blue", lw=3 - ) - plt.plot( - [df_stock.index[-1], df_future.index[0]], - [ - df_stock["5. adjusted close"].values[-1], - df_future["5. adjusted close"].values[0], - ], - lw=2, - c="tab:blue", - ls="--", - ) - plt.scatter(df_pred.index, df_pred, c="green", lw=3) - plt.plot( - [df_stock.index[-1], df_pred.index[0]], - [df_stock["5. adjusted close"].values[-1], df_pred.values[0]], - lw=2, - c="green", - ls="--", - ) - plt.title("BACKTESTING: Real data price versus Prediction") - plt.xlim(df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)) - plt.xticks( - [df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)], - visible=True, - ) - plt.ylabel("Share Price ($)") - plt.grid(b=True, which="major", color="#666666", linestyle="-") - plt.minorticks_on() - plt.grid(b=True, which="minor", color="#999999", linestyle="-", alpha=0.2) - plt.legend(["Real data", "Prediction data"]) - plt.xticks([]) - - plt.subplot(212) - plt.axhline(y=0, color="k", linestyle="--", linewidth=2) - plt.plot( - df_future.index, - 100 - * (df_pred.values - df_future["5. adjusted close"].values) - / df_future["5. adjusted close"].values, - lw=2, - c="red", - ) - plt.scatter( - df_future.index, - 100 - * (df_pred.values - df_future["5. adjusted close"].values) - / df_future["5. adjusted close"].values, - c="red", - lw=5, - ) - plt.title("BACKTESTING: Error between Real data and Prediction [%]") - plt.plot( - [df_stock.index[-1], df_future.index[0]], - [ - 0, - 100 - * (df_pred.values[0] - df_future["5. adjusted close"].values[0]) - / df_future["5. adjusted close"].values[0], - ], - lw=2, - ls="--", - c="red", - ) - plt.xlim(df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)) - plt.xticks( - [df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)], - visible=True, - ) - plt.xlabel("Time") - plt.ylabel("Prediction Error (%)") - plt.grid(b=True, which="major", color="#666666", linestyle="-") - plt.minorticks_on() - plt.grid(b=True, which="minor", color="#999999", linestyle="-", alpha=0.2) - plt.legend(["Real data", "Prediction data"]) - - if gtff.USE_ION: - plt.ion() - - plt.show() - - # Refactor prediction dataframe for backtesting print - df_pred.name = "Prediction" - df_pred = df_pred.to_frame() - df_pred["Real"] = df_future["5. adjusted close"] - - if gtff.USE_COLOR: - - patch_pandas_text_adjustment() - - print("Time Real [$] x Prediction [$]") - print( - df_pred.apply( - price_prediction_backtesting_color, axis=1 - ).to_string() - ) - else: - print(df_pred[["Real", "Prediction"]].round(2).to_string()) - - print("") - print_prediction_kpis(df_pred["Real"].values, df_pred["Prediction"].values) - - else: - # Print prediction data - print_pretty_prediction(df_pred, df_stock["5. adjusted close"].values[-1]) - print("") except Exception as e: print(e) + traceback.print_exc() print("") + finally: + _restore_env() + def rnn(l_args, s_ticker, df_stock): - parser = argparse.ArgumentParser( - add_help=False, prog="rnn", description="""Recurrent Neural Network. """ - ) - - parser.add_argument( - "-d", - "--days", - action="store", - dest="n_days", - type=check_positive, - default=5, - help="prediction days.", - ) - parser.add_argument( - "-i", - "--input", - action="store", - dest="n_inputs", - type=check_positive, - default=40, - help="number of days to use for prediction.", - ) - parser.add_argument( - "--epochs", - action="store", - dest="n_epochs", - type=check_positive, - default=200, - help="number of training epochs.", - ) - parser.add_argument( - "-j", - "--jumps", - action="store", - dest="n_jumps", - type=check_positive, - default=1, - help="number of jumps in training data.", - ) - parser.add_argument( - "-p", - "--pp", - action="store", - dest="s_preprocessing", - default="normalization", - choices=["normalization", "standardization", "none"], - help="pre-processing data.", - ) - parser.add_argument( - "-o", - "--optimizer", - action="store", - dest="s_optimizer", - default="adam", - help="optimizer technique", - choices=[ - "adam", - "adagrad", - "adadelta", - "adamax", - "ftrl", - "nadam", - "optimizer", - "rmsprop", - "sgd", - ], - ) - parser.add_argument( - "-l", - "--loss", - action="store", - dest="s_loss", - default="mae", - choices=["mae", "mape", "mse", "msle"], - help="loss function.", - ) - parser.add_argument( - "-e", - "--end", - action="store", - type=valid_date, - dest="s_end_date", - default=None, - help="The end date (format YYYY-MM-DD) to select - Backtesting", - ) - try: - ns_parser = parse_known_args_and_warn(parser, l_args) + ns_parser = _parse_args( + prog="rnn", description="""Recurrent Neural Network. """, l_args=l_args + ) if not ns_parser: return - # BACKTESTING - if ns_parser.s_end_date: - - if ns_parser.s_end_date < df_stock.index[0]: - print( - "Backtesting not allowed, since End Date is older than Start Date of historical data\n" - ) - return - - if ns_parser.s_end_date < get_next_stock_market_days( - last_stock_day=df_stock.index[0], - n_next_days=ns_parser.n_inputs + ns_parser.n_days, - )[-1]: - print( - "Backtesting not allowed, since End Date is too close to Start Date to train model\n" - ) - return - - future_index = get_next_stock_market_days( - last_stock_day=ns_parser.s_end_date, n_next_days=ns_parser.n_days - ) - - if future_index[-1] > datetime.datetime.now(): - print( - "Backtesting not allowed, since End Date + Prediction days is in the future\n" - ) - return - - df_future = df_stock[future_index[0] : future_index[-1]] - df_stock = df_stock[: ns_parser.s_end_date] + # Setup backtesting + df_stock, df_future = _setup_backtesting(df_stock, ns_parser) # Pre-process data - if ns_parser.s_preprocessing == "standardization": - scaler = StandardScaler() - stock_train_data = scaler.fit_transform( - np.array(df_stock["5. adjusted close"].values.reshape(-1, 1)) - ) - elif ns_parser.s_preprocessing == "normalization": - scaler = MinMaxScaler() - stock_train_data = scaler.fit_transform( - np.array(df_stock["5. adjusted close"].values.reshape(-1, 1)) - ) - else: # No pre-processing - stock_train_data = np.array( - df_stock["5. adjusted close"].values.reshape(-1, 1) - ) - - # Split training data for the neural network - stock_x, stock_y = splitTrain.split_train( - stock_train_data, - ns_parser.n_inputs, - ns_parser.n_days, - numJumps=ns_parser.n_jumps, + scaler, stock_train_data, stock_x, stock_y = _preprocess_split( + df_stock, ns_parser ) - - if not stock_x: - print("Given the model parameters more training data is needed.\n") - return - - stock_x = np.array(stock_x) stock_x = np.reshape(stock_x, (stock_x.shape[0], stock_x.shape[1], 1)) - stock_y = np.array(stock_y) stock_y = np.reshape(stock_y, (stock_y.shape[0], stock_y.shape[1], 1)) # Build Neural Network model @@ -628,7 +610,13 @@ def rnn(l_args, s_ticker, df_stock): model.compile(optimizer=ns_parser.s_optimizer, loss=ns_parser.s_loss) # Train our model - model.fit(stock_x, stock_y, epochs=ns_parser.n_epochs, verbose=1) + model.fit( + stock_x, + stock_y, + epochs=ns_parser.n_epochs, + batch_size=ns_parser.n_batch_size, + verbose=1, + ) print("") print(model.summary()) @@ -640,364 +628,37 @@ def rnn(l_args, s_ticker, df_stock): verbose=0, ) - # Re-scale the data back - if (ns_parser.s_preprocessing == "standardization") or ( - ns_parser.s_preprocessing == "normalization" - ): - y_pred_test_t = scaler.inverse_transform(yhat.tolist()) - else: - y_pred_test_t = yhat - - l_pred_days = get_next_stock_market_days( - last_stock_day=df_stock["5. adjusted close"].index[-1], - n_next_days=ns_parser.n_days, + # Re-scale the data back, plot, and print the results + df_pred = _rescale_data(df_stock, ns_parser, scaler, yhat) + _plot_and_print_results( + df_stock, ns_parser, df_future, df_pred, "RNN", s_ticker ) - df_pred = pd.Series(y_pred_test_t[0].tolist(), index=l_pred_days, name="Price") - - # Plotting - plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) - plt.plot(df_stock.index, df_stock["5. adjusted close"], lw=3) - # BACKTESTING - if ns_parser.s_end_date: - plt.title( - f"BACKTESTING: RNN on {s_ticker} - {ns_parser.n_days} days prediction" - ) - else: - plt.title(f"RNN on {s_ticker} - {ns_parser.n_days} days prediction") - plt.xlim( - df_stock.index[0], get_next_stock_market_days(df_pred.index[-1], 1)[-1] - ) - plt.xlabel("Time") - plt.ylabel("Share Price ($)") - plt.grid(b=True, which="major", color="#666666", linestyle="-") - plt.minorticks_on() - plt.grid(b=True, which="minor", color="#999999", linestyle="-", alpha=0.2) - plt.plot( - [df_stock.index[-1], df_pred.index[0]], - [df_stock["5. adjusted close"].values[-1], df_pred.values[0]], - lw=1, - c="tab:green", - linestyle="--", - ) - plt.plot(df_pred.index, df_pred, lw=2, c="tab:green") - plt.axvspan( - df_stock.index[-1], df_pred.index[-1], facecolor="tab:orange", alpha=0.2 - ) - _, _, ymin, ymax = plt.axis() - plt.vlines( - df_stock.index[-1], - ymin, - ymax, - colors="k", - linewidth=3, - linestyle="--", - color="k", - ) - - # BACKTESTING - if ns_parser.s_end_date: - plt.plot( - df_future.index, - df_future["5. adjusted close"], - lw=2, - c="tab:blue", - ls="--", - ) - plt.plot( - [df_stock.index[-1], df_future.index[0]], - [ - df_stock["5. adjusted close"].values[-1], - df_future["5. adjusted close"].values[0], - ], - lw=1, - c="tab:blue", - linestyle="--", - ) - - if gtff.USE_ION: - plt.ion() - - plt.show() - - # BACKTESTING - if ns_parser.s_end_date: - plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) - plt.subplot(211) - plt.plot( - df_future.index, - df_future["5. adjusted close"], - lw=2, - c="tab:blue", - ls="--", - ) - plt.plot(df_pred.index, df_pred, lw=2, c="green") - plt.scatter( - df_future.index, df_future["5. adjusted close"], c="tab:blue", lw=3 - ) - plt.plot( - [df_stock.index[-1], df_future.index[0]], - [ - df_stock["5. adjusted close"].values[-1], - df_future["5. adjusted close"].values[0], - ], - lw=2, - c="tab:blue", - ls="--", - ) - plt.scatter(df_pred.index, df_pred, c="green", lw=3) - plt.plot( - [df_stock.index[-1], df_pred.index[0]], - [df_stock["5. adjusted close"].values[-1], df_pred.values[0]], - lw=2, - c="green", - ls="--", - ) - plt.title("BACKTESTING: Real data price versus Prediction") - plt.xlim(df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)) - plt.xticks( - [df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)], - visible=True, - ) - plt.ylabel("Share Price ($)") - plt.grid(b=True, which="major", color="#666666", linestyle="-") - plt.minorticks_on() - plt.grid(b=True, which="minor", color="#999999", linestyle="-", alpha=0.2) - plt.legend(["Real data", "Prediction data"]) - plt.xticks([]) - - plt.subplot(212) - plt.axhline(y=0, color="k", linestyle="--", linewidth=2) - plt.plot( - df_future.index, - 100 - * (df_pred.values - df_future["5. adjusted close"].values) - / df_future["5. adjusted close"].values, - lw=2, - c="red", - ) - plt.scatter( - df_future.index, - 100 - * (df_pred.values - df_future["5. adjusted close"].values) - / df_future["5. adjusted close"].values, - c="red", - lw=5, - ) - plt.title("BACKTESTING: Error between Real data and Prediction [%]") - plt.plot( - [df_stock.index[-1], df_future.index[0]], - [ - 0, - 100 - * (df_pred.values[0] - df_future["5. adjusted close"].values[0]) - / df_future["5. adjusted close"].values[0], - ], - lw=2, - ls="--", - c="red", - ) - plt.xlim(df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)) - plt.xticks( - [df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)], - visible=True, - ) - plt.xlabel("Time") - plt.ylabel("Prediction Error (%)") - plt.grid(b=True, which="major", color="#666666", linestyle="-") - plt.minorticks_on() - plt.grid(b=True, which="minor", color="#999999", linestyle="-", alpha=0.2) - plt.legend(["Real data", "Prediction data"]) - - if gtff.USE_ION: - plt.ion() - - plt.show() - - # Refactor prediction dataframe for backtesting print - df_pred.name = "Prediction" - df_pred = df_pred.to_frame() - df_pred["Real"] = df_future["5. adjusted close"] - - if gtff.USE_COLOR: - - patch_pandas_text_adjustment() - - print("Time Real [$] x Prediction [$]") - print( - df_pred.apply( - price_prediction_backtesting_color, axis=1 - ).to_string() - ) - else: - print(df_pred[["Real", "Prediction"]].round(2).to_string()) - - print("") - print_prediction_kpis(df_pred["Real"].values, df_pred["Prediction"].values) - - else: - # Print prediction data - print_pretty_prediction(df_pred, df_stock["5. adjusted close"].values[-1]) - print("") except Exception as e: print(e) + traceback.print_exc() print("") + finally: + _restore_env() + def lstm(l_args, s_ticker, df_stock): - parser = argparse.ArgumentParser( - add_help=False, prog="lstm", description="""Long-Short Term Memory. """ - ) - - parser.add_argument( - "-d", - "--days", - action="store", - dest="n_days", - type=check_positive, - default=5, - help="prediction days", - ) - parser.add_argument( - "-i", - "--input", - action="store", - dest="n_inputs", - type=check_positive, - default=40, - help="number of days to use for prediction.", - ) - parser.add_argument( - "--epochs", - action="store", - dest="n_epochs", - type=check_positive, - default=200, - help="number of training epochs.", - ) - parser.add_argument( - "-j", - "--jumps", - action="store", - dest="n_jumps", - type=check_positive, - default=1, - help="number of jumps in training data.", - ) - parser.add_argument( - "-p", - "--pp", - action="store", - dest="s_preprocessing", - default="normalization", - choices=["normalization", "standardization", "none"], - help="pre-processing data.", - ) - parser.add_argument( - "-o", - "--optimizer", - action="store", - dest="s_optimizer", - default="adam", - help="optimization technique.", - choices=[ - "adam", - "adagrad", - "adadelta", - "adamax", - "ftrl", - "nadam", - "optimizer", - "rmsprop", - "sgd", - ], - ) - parser.add_argument( - "-l", - "--loss", - action="store", - dest="s_loss", - default="mae", - choices=["mae", "mape", "mse", "msle"], - help="loss function.", - ) - parser.add_argument( - "-e", - "--end", - action="store", - type=valid_date, - dest="s_end_date", - default=None, - help="The end date (format YYYY-MM-DD) to select - Backtesting", - ) - try: - ns_parser = parse_known_args_and_warn(parser, l_args) + ns_parser = _parse_args( + prog="lstm", description="""Long-Short Term Memory. """, l_args=l_args + ) if not ns_parser: return - # BACKTESTING - if ns_parser.s_end_date: - - if ns_parser.s_end_date < df_stock.index[0]: - print( - "Backtesting not allowed, since End Date is older than Start Date of historical data\n" - ) - return - - if ns_parser.s_end_date < get_next_stock_market_days( - last_stock_day=df_stock.index[0], - n_next_days=ns_parser.n_inputs + ns_parser.n_days, - )[-1]: - print( - "Backtesting not allowed, since End Date is too close to Start Date to train model\n" - ) - return - - future_index = get_next_stock_market_days( - last_stock_day=ns_parser.s_end_date, n_next_days=ns_parser.n_days - ) - - if future_index[-1] > datetime.datetime.now(): - print( - "Backtesting not allowed, since End Date + Prediction days is in the future\n" - ) - return - - df_future = df_stock[future_index[0] : future_index[-1]] - df_stock = df_stock[: ns_parser.s_end_date] + # Setup backtesting + df_stock, df_future = _setup_backtesting(df_stock, ns_parser) # Pre-process data - if ns_parser.s_preprocessing == "standardization": - scaler = StandardScaler() - stock_train_data = scaler.fit_transform( - np.array(df_stock["5. adjusted close"].values.reshape(-1, 1)) - ) - elif ns_parser.s_preprocessing == "normalization": - scaler = MinMaxScaler() - stock_train_data = scaler.fit_transform( - np.array(df_stock["5. adjusted close"].values.reshape(-1, 1)) - ) - else: # No pre-processing - stock_train_data = np.array( - df_stock["5. adjusted close"].values.reshape(-1, 1) - ) - - # Split training data for the neural network - stock_x, stock_y = splitTrain.split_train( - stock_train_data, - ns_parser.n_inputs, - ns_parser.n_days, - numJumps=ns_parser.n_jumps, + scaler, stock_train_data, stock_x, stock_y = _preprocess_split( + df_stock, ns_parser ) - - if not stock_x: - print("Given the model parameters more training data is needed.\n") - return - - stock_x = np.array(stock_x) stock_x = np.reshape(stock_x, (stock_x.shape[0], stock_x.shape[1], 1)) - stock_y = np.array(stock_y) stock_y = np.reshape(stock_y, (stock_y.shape[0], stock_y.shape[1], 1)) # Build Neural Network model @@ -1007,7 +668,13 @@ def lstm(l_args, s_ticker, df_stock): model.compile(optimizer=ns_parser.s_optimizer, loss=ns_parser.s_loss) # Train our model - model.fit(stock_x, stock_y, epochs=ns_parser.n_epochs, verbose=1) + model.fit( + stock_x, + stock_y, + epochs=ns_parser.n_epochs, + batch_size=ns_parser.n_batch_size, + verbose=1, + ) print("") print(model.summary()) @@ -1019,205 +686,16 @@ def lstm(l_args, s_ticker, df_stock): verbose=0, ) - # Re-scale the data back - if (ns_parser.s_preprocessing == "standardization") or ( - ns_parser.s_preprocessing == "normalization" - ): - y_pred_test_t = scaler.inverse_transform(yhat.tolist()) - else: - y_pred_test_t = yhat - - l_pred_days = get_next_stock_market_days( - last_stock_day=df_stock["5. adjusted close"].index[-1], - n_next_days=ns_parser.n_days, + # Re-scale the data back, plot, and print the results + df_pred = _rescale_data(df_stock, ns_parser, scaler, yhat) + _plot_and_print_results( + df_stock, ns_parser, df_future, df_pred, "LSTM", s_ticker ) - df_pred = pd.Series(y_pred_test_t[0].tolist(), index=l_pred_days, name="Price") - - # Plotting - plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) - plt.plot(df_stock.index, df_stock["5. adjusted close"], lw=3) - # BACKTESTING - if ns_parser.s_end_date: - plt.title( - f"BACKTESTING: LSTM on {s_ticker} - {ns_parser.n_days} days prediction" - ) - else: - plt.title(f"LSTM on {s_ticker} - {ns_parser.n_days} days prediction") - plt.xlim( - df_stock.index[0], get_next_stock_market_days(df_pred.index[-1], 1)[-1] - ) - plt.xlabel("Time") - plt.ylabel("Share Price ($)") - plt.grid(b=True, which="major", color="#666666", linestyle="-") - plt.minorticks_on() - plt.grid(b=True, which="minor", color="#999999", linestyle="-", alpha=0.2) - plt.plot( - [df_stock.index[-1], df_pred.index[0]], - [df_stock["5. adjusted close"].values[-1], df_pred.values[0]], - lw=1, - c="tab:green", - linestyle="--", - ) - plt.plot(df_pred.index, df_pred, lw=2, c="tab:green") - plt.axvspan( - df_stock.index[-1], df_pred.index[-1], facecolor="tab:orange", alpha=0.2 - ) - _, _, ymin, ymax = plt.axis() - plt.vlines( - df_stock.index[-1], - ymin, - ymax, - colors="k", - linewidth=3, - linestyle="--", - color="k", - ) - - # BACKTESTING - if ns_parser.s_end_date: - plt.plot( - df_future.index, - df_future["5. adjusted close"], - lw=2, - c="tab:blue", - ls="--", - ) - plt.plot( - [df_stock.index[-1], df_future.index[0]], - [ - df_stock["5. adjusted close"].values[-1], - df_future["5. adjusted close"].values[0], - ], - lw=1, - c="tab:blue", - linestyle="--", - ) - - if gtff.USE_ION: - plt.ion() - - plt.show() - - # BACKTESTING - if ns_parser.s_end_date: - plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) - plt.subplot(211) - plt.plot( - df_future.index, - df_future["5. adjusted close"], - lw=2, - c="tab:blue", - ls="--", - ) - plt.plot(df_pred.index, df_pred, lw=2, c="green") - plt.scatter( - df_future.index, df_future["5. adjusted close"], c="tab:blue", lw=3 - ) - plt.plot( - [df_stock.index[-1], df_future.index[0]], - [ - df_stock["5. adjusted close"].values[-1], - df_future["5. adjusted close"].values[0], - ], - lw=2, - c="tab:blue", - ls="--", - ) - plt.scatter(df_pred.index, df_pred, c="green", lw=3) - plt.plot( - [df_stock.index[-1], df_pred.index[0]], - [df_stock["5. adjusted close"].values[-1], df_pred.values[0]], - lw=2, - c="green", - ls="--", - ) - plt.title("BACKTESTING: Real data price versus Prediction") - plt.xlim(df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)) - plt.xticks( - [df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)], - visible=True, - ) - plt.ylabel("Share Price ($)") - plt.grid(b=True, which="major", color="#666666", linestyle="-") - plt.minorticks_on() - plt.grid(b=True, which="minor", color="#999999", linestyle="-", alpha=0.2) - plt.legend(["Real data", "Prediction data"]) - plt.xticks([]) - - plt.subplot(212) - plt.axhline(y=0, color="k", linestyle="--", linewidth=2) - plt.plot( - df_future.index, - 100 - * (df_pred.values - df_future["5. adjusted close"].values) - / df_future["5. adjusted close"].values, - lw=2, - c="red", - ) - plt.scatter( - df_future.index, - 100 - * (df_pred.values - df_future["5. adjusted close"].values) - / df_future["5. adjusted close"].values, - c="red", - lw=5, - ) - plt.title("BACKTESTING: Error between Real data and Prediction [%]") - plt.plot( - [df_stock.index[-1], df_future.index[0]], - [ - 0, - 100 - * (df_pred.values[0] - df_future["5. adjusted close"].values[0]) - / df_future["5. adjusted close"].values[0], - ], - lw=2, - ls="--", - c="red", - ) - plt.xlim(df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)) - plt.xticks( - [df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)], - visible=True, - ) - plt.xlabel("Time") - plt.ylabel("Prediction Error (%)") - plt.grid(b=True, which="major", color="#666666", linestyle="-") - plt.minorticks_on() - plt.grid(b=True, which="minor", color="#999999", linestyle="-", alpha=0.2) - plt.legend(["Real data", "Prediction data"]) - - if gtff.USE_ION: - plt.ion() - - plt.show() - - # Refactor prediction dataframe for backtesting print - df_pred.name = "Prediction" - df_pred = df_pred.to_frame() - df_pred["Real"] = df_future["5. adjusted close"] - - if gtff.USE_COLOR: - - patch_pandas_text_adjustment() - - print("Time Real [$] x Prediction [$]") - print( - df_pred.apply( - price_prediction_backtesting_color, axis=1 - ).to_string() - ) - else: - print(df_pred[["Real", "Prediction"]].round(2).to_string()) - - print("") - print_prediction_kpis(df_pred["Real"].values, df_pred["Prediction"].values) - - else: - # Print prediction data - print_pretty_prediction(df_pred, df_stock["5. adjusted close"].values[-1]) - print("") except Exception as e: print(e) + traceback.print_exc() print("") + + finally: + _restore_env() \ No newline at end of file From 656d09cc92045c003f8e57e05fb1e6e0c4f0aace Mon Sep 17 00:00:00 2001 From: buzzCraft <67235613+buzzCraft@users.noreply.github.com> Date: Wed, 17 Mar 2021 10:59:44 +0100 Subject: [PATCH 29/64] Revert "Merge conflict fix" This reverts commit fac2804f1380b7c0e374704815d0b5e3625e8757. --- .../prediction_techniques/neural_networks.py | 1378 ++++++++++++----- 1 file changed, 950 insertions(+), 428 deletions(-) diff --git a/gamestonk_terminal/prediction_techniques/neural_networks.py b/gamestonk_terminal/prediction_techniques/neural_networks.py index 236906bd4d7..d8a44978b7e 100644 --- a/gamestonk_terminal/prediction_techniques/neural_networks.py +++ b/gamestonk_terminal/prediction_techniques/neural_networks.py @@ -1,7 +1,6 @@ import argparse import datetime import os -import traceback from warnings import simplefilter import numpy as np import matplotlib.pyplot as plt @@ -39,10 +38,6 @@ os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" simplefilter(action="ignore", category=FutureWarning) -# store the user's TensorFlow environment variables -ORIGINAL_TF_XLA_FLAGS = os.environ.get("TF_XLA_FLAGS") -ORIGINAL_TF_FORCE_GPU_ALLOW_GROWTH = os.environ.get("TF_FORCE_GPU_ALLOW_GROWTH") - def build_neural_network_model(Recurrent_Neural_Network, n_inputs, n_days): model = Sequential() @@ -91,15 +86,11 @@ def build_neural_network_model(Recurrent_Neural_Network, n_inputs, n_days): return model -def _parse_args(prog, description, l_args): - """Create an argparser and parse l_args. Will print help if user requests it. - :return: ns_parser""" +def mlp(l_args, s_ticker, df_stock): parser = argparse.ArgumentParser( - prog=prog, - description=description, - add_help=False, - formatter_class=argparse.RawTextHelpFormatter, # enable multiline help messages + add_help=False, prog="mlp", description="""Multilayer Perceptron. """ ) + parser.add_argument( "-d", "--days", @@ -157,10 +148,11 @@ def _parse_args(prog, description, l_args): "adamax", "ftrl", "nadam", + "optimizer", "rmsprop", "sgd", ], - help="optimization technique (see https://www.tensorflow.org/api_docs/python/tf/keras/optimizers)", + help="optimization technique.", ) parser.add_argument( "-l", @@ -168,19 +160,8 @@ def _parse_args(prog, description, l_args): action="store", dest="s_loss", default="mae", - choices=[ - "mae", - "mape", - "mse", - "msle", - "poisson", - "logcosh", - "kld", - "hinge", - "squared_hinge", - "huber", - ], - help="loss function (see https://www.tensorflow.org/api_docs/python/tf/keras/losses)", + choices=["mae", "mape", "mse", "msle"], + help="loss function.", ) parser.add_argument( "-e", @@ -191,358 +172,74 @@ def _parse_args(prog, description, l_args): default=None, help="The end date (format YYYY-MM-DD) to select - Backtesting", ) - parser.add_argument( - "--batch_size", - action="store", - dest="n_batch_size", - type=check_positive, - default=None, - help="batch size for model fitting (use a power of 2)", - ) - parser.add_argument( - "--xla_cpu", - action="store_true", - dest="b_xla_cpu", - default=False, - help="enable XLA for CPU (see https://www.tensorflow.org/xla)", - ) - parser.add_argument( - "--xla_gpu", - action="store_true", - dest="b_xla_gpu", - default=False, - help="enable XLA for GPU (see https://www.tensorflow.org/xla)", - ) - parser.add_argument( - "--force_gpu_allow_growth", - action="store", - dest="s_force_gpu_allow_growth", - default="true", - choices=["true", "false", "default"], - help="true: GPU memory will grow as needed. \n" - "false: TensorFlow will allocate 100%% of GPU memory. \n" - "default: usually the same as false, uses env/TensorFlow default", - ) - ns_parser = parse_known_args_and_warn(parser, l_args) - if not ns_parser: - return None - - # set xla flags if requested - xla_flags = ( - set(ORIGINAL_TF_XLA_FLAGS.split(" ")) if ORIGINAL_TF_XLA_FLAGS else set() - ) - if ns_parser.b_xla_cpu or ns_parser.b_xla_gpu: - xla_flags.add("--tf_xla_enable_xla_devices") - if ns_parser.b_xla_cpu: - xla_flags.add("--tf_xla_cpu_global_jit") - if ns_parser.b_xla_gpu: - xla_flags.add("--tf_xla_auto_jit=2") - os.environ["TF_XLA_FLAGS"] = " ".join(xla_flags) - - # set GPU memory growth flag - if ns_parser.s_force_gpu_allow_growth == "true": - os.environ["TF_FORCE_GPU_ALLOW_GROWTH"] = "true" - elif ns_parser.s_force_gpu_allow_growth == "false": - os.environ["TF_FORCE_GPU_ALLOW_GROWTH"] = "false" - - return ns_parser - - -def _restore_env(): - """Restore environment variables to original values""" - - def restore(key, value): - if value is None: - if key in os.environ: - del os.environ[key] - else: - os.environ[key] = value - - restore("TF_XLA_FLAGS", ORIGINAL_TF_XLA_FLAGS) - restore("TF_FORCE_GPU_ALLOW_GROWTH", ORIGINAL_TF_FORCE_GPU_ALLOW_GROWTH) - - -def _setup_backtesting(df_stock, ns_parser): - """Set up backtesting if enabled - :return: (df_stock, df_future), where df_future is None if s_end_date is not set. - :raises Exception: if configuration is invalid""" - df_future = None - if ns_parser.s_end_date: - if ns_parser.s_end_date < df_stock.index[0]: - raise Exception( - "Backtesting not allowed, since End Date is older than Start Date of historical data" - ) - - if ns_parser.s_end_date < get_next_stock_market_days( - last_stock_day=df_stock.index[0], - n_next_days=ns_parser.n_inputs + ns_parser.n_days, - )[-1]: - raise Exception( - "Backtesting not allowed, since End Date is too close to Start Date to train model" - ) - - future_index = get_next_stock_market_days( - last_stock_day=ns_parser.s_end_date, n_next_days=ns_parser.n_days - ) - - if future_index[-1] > datetime.datetime.now(): - raise Exception( - "Backtesting not allowed, since End Date + Prediction days is in the future" - ) - - df_future = df_stock[future_index[0] : future_index[-1]] - df_stock = df_stock[: ns_parser.s_end_date] - return df_stock, df_future - - -def _preprocess_split(df_stock, ns_parser): - """Preprocess and split training data. - :return: (scaler, stock_train_data, stock_x, stock_y) - :raises Exception: if more training data is needed.""" - # Pre-process data - if ns_parser.s_preprocessing == "standardization": - scaler = StandardScaler() - stock_train_data = scaler.fit_transform( - np.array(df_stock["5. adjusted close"].values.reshape(-1, 1)) - ) - elif ns_parser.s_preprocessing == "normalization": - scaler = MinMaxScaler() - stock_train_data = scaler.fit_transform( - np.array(df_stock["5. adjusted close"].values.reshape(-1, 1)) - ) - else: # No pre-processing - stock_train_data = np.array(df_stock["5. adjusted close"].values.reshape(-1, 1)) - - # Split training data for the neural network - stock_x, stock_y = splitTrain.split_train( - stock_train_data, - ns_parser.n_inputs, - ns_parser.n_days, - numJumps=ns_parser.n_jumps, - ) - if not stock_x: - raise Exception("Given the model parameters more training data is needed.") - stock_x = np.array(stock_x) - stock_y = np.array(stock_y) - return scaler, stock_train_data, stock_x, stock_y - - -def _rescale_data(df_stock, ns_parser, scaler, yhat): - """Re-scale the data back and return the prediction dataframe. """ - if (ns_parser.s_preprocessing == "standardization") or ( - ns_parser.s_preprocessing == "normalization" - ): - y_pred_test_t = scaler.inverse_transform(yhat.tolist()) - else: - y_pred_test_t = yhat - - l_pred_days = get_next_stock_market_days( - last_stock_day=df_stock["5. adjusted close"].index[-1], - n_next_days=ns_parser.n_days, - ) - df_pred = pd.Series(y_pred_test_t[0].tolist(), index=l_pred_days, name="Price") - return df_pred - - -def _plot_and_print_results( - df_stock, ns_parser, df_future, df_pred, model_name, s_ticker -): - """Plot and print the results. """ - # Plotting - plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) - plt.plot(df_stock.index, df_stock["5. adjusted close"], lw=3) - - # BACKTESTING - if ns_parser.s_end_date: - plt.title( - f"BACKTESTING: {model_name} on {s_ticker} - {ns_parser.n_days} days prediction" - ) - else: - plt.title(f"{model_name} on {s_ticker} - {ns_parser.n_days} days prediction") - plt.xlim(df_stock.index[0], get_next_stock_market_days(df_pred.index[-1], 1)[-1]) - plt.xlabel("Time") - plt.ylabel("Share Price ($)") - plt.grid(b=True, which="major", color="#666666", linestyle="-") - plt.minorticks_on() - plt.grid(b=True, which="minor", color="#999999", linestyle="-", alpha=0.2) - plt.plot( - [df_stock.index[-1], df_pred.index[0]], - [df_stock["5. adjusted close"].values[-1], df_pred.values[0]], - lw=1, - c="tab:green", - linestyle="--", - ) - plt.plot(df_pred.index, df_pred, lw=2, c="tab:green") - plt.axvspan( - df_stock.index[-1], df_pred.index[-1], facecolor="tab:orange", alpha=0.2 - ) - _, _, ymin, ymax = plt.axis() - plt.vlines( - df_stock.index[-1], - ymin, - ymax, - colors="k", - linewidth=3, - linestyle="--", - color="k", - ) - - # BACKTESTING - if ns_parser.s_end_date: - plt.plot( - df_future.index, - df_future["5. adjusted close"], - lw=2, - c="tab:blue", - ls="--", - ) - plt.plot( - [df_stock.index[-1], df_future.index[0]], - [ - df_stock["5. adjusted close"].values[-1], - df_future["5. adjusted close"].values[0], - ], - lw=1, - c="tab:blue", - linestyle="--", - ) - - if gtff.USE_ION: - plt.ion() - - plt.show() - - # BACKTESTING - if ns_parser.s_end_date: - plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) - plt.subplot(211) - plt.plot( - df_future.index, - df_future["5. adjusted close"], - lw=2, - c="tab:blue", - ls="--", - ) - plt.plot(df_pred.index, df_pred, lw=2, c="green") - plt.scatter(df_future.index, df_future["5. adjusted close"], c="tab:blue", lw=3) - plt.plot( - [df_stock.index[-1], df_future.index[0]], - [ - df_stock["5. adjusted close"].values[-1], - df_future["5. adjusted close"].values[0], - ], - lw=2, - c="tab:blue", - ls="--", - ) - plt.scatter(df_pred.index, df_pred, c="green", lw=3) - plt.plot( - [df_stock.index[-1], df_pred.index[0]], - [df_stock["5. adjusted close"].values[-1], df_pred.values[0]], - lw=2, - c="green", - ls="--", - ) - plt.title("BACKTESTING: Real data price versus Prediction") - plt.xlim(df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)) - plt.xticks( - [df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)], - visible=True, - ) - plt.ylabel("Share Price ($)") - plt.grid(b=True, which="major", color="#666666", linestyle="-") - plt.minorticks_on() - plt.grid(b=True, which="minor", color="#999999", linestyle="-", alpha=0.2) - plt.legend(["Real data", "Prediction data"]) - plt.xticks([]) - - plt.subplot(212) - plt.axhline(y=0, color="k", linestyle="--", linewidth=2) - plt.plot( - df_future.index, - 100 - * (df_pred.values - df_future["5. adjusted close"].values) - / df_future["5. adjusted close"].values, - lw=2, - c="red", - ) - plt.scatter( - df_future.index, - 100 - * (df_pred.values - df_future["5. adjusted close"].values) - / df_future["5. adjusted close"].values, - c="red", - lw=5, - ) - plt.title("BACKTESTING: Error between Real data and Prediction [%]") - plt.plot( - [df_stock.index[-1], df_future.index[0]], - [ - 0, - 100 - * (df_pred.values[0] - df_future["5. adjusted close"].values[0]) - / df_future["5. adjusted close"].values[0], - ], - lw=2, - ls="--", - c="red", - ) - plt.xlim(df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)) - plt.xticks( - [df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)], - visible=True, - ) - plt.xlabel("Time") - plt.ylabel("Prediction Error (%)") - plt.grid(b=True, which="major", color="#666666", linestyle="-") - plt.minorticks_on() - plt.grid(b=True, which="minor", color="#999999", linestyle="-", alpha=0.2) - plt.legend(["Real data", "Prediction data"]) - - if gtff.USE_ION: - plt.ion() - - plt.show() - - # Refactor prediction dataframe for backtesting print - df_pred.name = "Prediction" - df_pred = df_pred.to_frame() - df_pred["Real"] = df_future["5. adjusted close"] - - if gtff.USE_COLOR: - patch_pandas_text_adjustment() - - print("Time Real [$] x Prediction [$]") - print(df_pred.apply(price_prediction_backtesting_color, axis=1).to_string()) - else: - print(df_pred[["Real", "Prediction"]].round(2).to_string()) - - print("") - print_prediction_kpis(df_pred["Real"].values, df_pred["Prediction"].values) - - else: - # Print prediction data - print_pretty_prediction(df_pred, df_stock["5. adjusted close"].values[-1]) - print("") - - -def mlp(l_args, s_ticker, df_stock): try: - ns_parser = _parse_args( - prog="mlp", description="""Multilayer Perceptron. """, l_args=l_args - ) + ns_parser = parse_known_args_and_warn(parser, l_args) if not ns_parser: return - # Setup backtesting - df_stock, df_future = _setup_backtesting(df_stock, ns_parser) + # BACKTESTING + if ns_parser.s_end_date: + + if ns_parser.s_end_date < df_stock.index[0]: + print( + "Backtesting not allowed, since End Date is older than Start Date of historical data\n" + ) + return + + if ns_parser.s_end_date < get_next_stock_market_days( + last_stock_day=df_stock.index[0], + n_next_days=ns_parser.n_inputs + ns_parser.n_days, + )[-1]: + print( + "Backtesting not allowed, since End Date is too close to Start Date to train model\n" + ) + return + + future_index = get_next_stock_market_days( + last_stock_day=ns_parser.s_end_date, n_next_days=ns_parser.n_days + ) + + if future_index[-1] > datetime.datetime.now(): + print( + "Backtesting not allowed, since End Date + Prediction days is in the future\n" + ) + return + + df_future = df_stock[future_index[0] : future_index[-1]] + df_stock = df_stock[: ns_parser.s_end_date] # Pre-process data - scaler, stock_train_data, stock_x, stock_y = _preprocess_split( - df_stock, ns_parser + if ns_parser.s_preprocessing == "standardization": + scaler = StandardScaler() + stock_train_data = scaler.fit_transform( + np.array(df_stock["5. adjusted close"].values.reshape(-1, 1)) + ) + elif ns_parser.s_preprocessing == "normalization": + scaler = MinMaxScaler() + stock_train_data = scaler.fit_transform( + np.array(df_stock["5. adjusted close"].values.reshape(-1, 1)) + ) + else: # No pre-processing + stock_train_data = np.array( + df_stock["5. adjusted close"].values.reshape(-1, 1) + ) + + # Split training data for the neural network + stock_x, stock_y = splitTrain.split_train( + stock_train_data, + ns_parser.n_inputs, + ns_parser.n_days, + numJumps=ns_parser.n_jumps, ) + + if not stock_x: + print("Given the model parameters more training data is needed.\n") + return + + stock_x = np.array(stock_x) stock_x = np.reshape(stock_x, (stock_x.shape[0], stock_x.shape[1])) + stock_y = np.array(stock_y) stock_y = np.reshape(stock_y, (stock_y.shape[0], stock_y.shape[1])) # Build Neural Network model @@ -552,13 +249,7 @@ def mlp(l_args, s_ticker, df_stock): model.compile(optimizer=ns_parser.s_optimizer, loss=ns_parser.s_loss) # Train our model - model.fit( - stock_x, - stock_y, - epochs=ns_parser.n_epochs, - batch_size=ns_parser.n_batch_size, - verbose=1, - ) + model.fit(stock_x, stock_y, epochs=ns_parser.n_epochs, verbose=1) print("") print(model.summary()) @@ -570,37 +261,364 @@ def mlp(l_args, s_ticker, df_stock): verbose=0, ) - # Re-scale the data back, plot, and print the results - df_pred = _rescale_data(df_stock, ns_parser, scaler, yhat) - _plot_and_print_results( - df_stock, ns_parser, df_future, df_pred, "MLP", s_ticker + # Re-scale the data back + if (ns_parser.s_preprocessing == "standardization") or ( + ns_parser.s_preprocessing == "normalization" + ): + y_pred_test_t = scaler.inverse_transform(yhat.tolist()) + else: + y_pred_test_t = yhat + + l_pred_days = get_next_stock_market_days( + last_stock_day=df_stock["5. adjusted close"].index[-1], + n_next_days=ns_parser.n_days, ) + df_pred = pd.Series(y_pred_test_t[0].tolist(), index=l_pred_days, name="Price") + + # Plotting + plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) + plt.plot(df_stock.index, df_stock["5. adjusted close"], lw=3) + # BACKTESTING + if ns_parser.s_end_date: + plt.title( + f"BACKTESTING: MLP on {s_ticker} - {ns_parser.n_days} days prediction" + ) + else: + plt.title(f"MLP on {s_ticker} - {ns_parser.n_days} days prediction") + plt.xlim( + df_stock.index[0], get_next_stock_market_days(df_pred.index[-1], 1)[-1] + ) + plt.xlabel("Time") + plt.ylabel("Share Price ($)") + plt.grid(b=True, which="major", color="#666666", linestyle="-") + plt.minorticks_on() + plt.grid(b=True, which="minor", color="#999999", linestyle="-", alpha=0.2) + plt.plot( + [df_stock.index[-1], df_pred.index[0]], + [df_stock["5. adjusted close"].values[-1], df_pred.values[0]], + lw=1, + c="tab:green", + linestyle="--", + ) + plt.plot(df_pred.index, df_pred, lw=2, c="tab:green") + plt.axvspan( + df_stock.index[-1], df_pred.index[-1], facecolor="tab:orange", alpha=0.2 + ) + _, _, ymin, ymax = plt.axis() + plt.vlines( + df_stock.index[-1], + ymin, + ymax, + colors="k", + linewidth=3, + linestyle="--", + color="k", + ) + + # BACKTESTING + if ns_parser.s_end_date: + plt.plot( + df_future.index, + df_future["5. adjusted close"], + lw=2, + c="tab:blue", + ls="--", + ) + plt.plot( + [df_stock.index[-1], df_future.index[0]], + [ + df_stock["5. adjusted close"].values[-1], + df_future["5. adjusted close"].values[0], + ], + lw=1, + c="tab:blue", + linestyle="--", + ) + + if gtff.USE_ION: + plt.ion() + + plt.show() + + # BACKTESTING + if ns_parser.s_end_date: + plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) + plt.subplot(211) + plt.plot( + df_future.index, + df_future["5. adjusted close"], + lw=2, + c="tab:blue", + ls="--", + ) + plt.plot(df_pred.index, df_pred, lw=2, c="green") + plt.scatter( + df_future.index, df_future["5. adjusted close"], c="tab:blue", lw=3 + ) + plt.plot( + [df_stock.index[-1], df_future.index[0]], + [ + df_stock["5. adjusted close"].values[-1], + df_future["5. adjusted close"].values[0], + ], + lw=2, + c="tab:blue", + ls="--", + ) + plt.scatter(df_pred.index, df_pred, c="green", lw=3) + plt.plot( + [df_stock.index[-1], df_pred.index[0]], + [df_stock["5. adjusted close"].values[-1], df_pred.values[0]], + lw=2, + c="green", + ls="--", + ) + plt.title("BACKTESTING: Real data price versus Prediction") + plt.xlim(df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)) + plt.xticks( + [df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)], + visible=True, + ) + plt.ylabel("Share Price ($)") + plt.grid(b=True, which="major", color="#666666", linestyle="-") + plt.minorticks_on() + plt.grid(b=True, which="minor", color="#999999", linestyle="-", alpha=0.2) + plt.legend(["Real data", "Prediction data"]) + plt.xticks([]) + + plt.subplot(212) + plt.axhline(y=0, color="k", linestyle="--", linewidth=2) + plt.plot( + df_future.index, + 100 + * (df_pred.values - df_future["5. adjusted close"].values) + / df_future["5. adjusted close"].values, + lw=2, + c="red", + ) + plt.scatter( + df_future.index, + 100 + * (df_pred.values - df_future["5. adjusted close"].values) + / df_future["5. adjusted close"].values, + c="red", + lw=5, + ) + plt.title("BACKTESTING: Error between Real data and Prediction [%]") + plt.plot( + [df_stock.index[-1], df_future.index[0]], + [ + 0, + 100 + * (df_pred.values[0] - df_future["5. adjusted close"].values[0]) + / df_future["5. adjusted close"].values[0], + ], + lw=2, + ls="--", + c="red", + ) + plt.xlim(df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)) + plt.xticks( + [df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)], + visible=True, + ) + plt.xlabel("Time") + plt.ylabel("Prediction Error (%)") + plt.grid(b=True, which="major", color="#666666", linestyle="-") + plt.minorticks_on() + plt.grid(b=True, which="minor", color="#999999", linestyle="-", alpha=0.2) + plt.legend(["Real data", "Prediction data"]) + + if gtff.USE_ION: + plt.ion() + + plt.show() + + # Refactor prediction dataframe for backtesting print + df_pred.name = "Prediction" + df_pred = df_pred.to_frame() + df_pred["Real"] = df_future["5. adjusted close"] + + if gtff.USE_COLOR: + + patch_pandas_text_adjustment() + + print("Time Real [$] x Prediction [$]") + print( + df_pred.apply( + price_prediction_backtesting_color, axis=1 + ).to_string() + ) + else: + print(df_pred[["Real", "Prediction"]].round(2).to_string()) + + print("") + print_prediction_kpis(df_pred["Real"].values, df_pred["Prediction"].values) + + else: + # Print prediction data + print_pretty_prediction(df_pred, df_stock["5. adjusted close"].values[-1]) + print("") except Exception as e: print(e) - traceback.print_exc() print("") - finally: - _restore_env() - def rnn(l_args, s_ticker, df_stock): + parser = argparse.ArgumentParser( + add_help=False, prog="rnn", description="""Recurrent Neural Network. """ + ) + + parser.add_argument( + "-d", + "--days", + action="store", + dest="n_days", + type=check_positive, + default=5, + help="prediction days.", + ) + parser.add_argument( + "-i", + "--input", + action="store", + dest="n_inputs", + type=check_positive, + default=40, + help="number of days to use for prediction.", + ) + parser.add_argument( + "--epochs", + action="store", + dest="n_epochs", + type=check_positive, + default=200, + help="number of training epochs.", + ) + parser.add_argument( + "-j", + "--jumps", + action="store", + dest="n_jumps", + type=check_positive, + default=1, + help="number of jumps in training data.", + ) + parser.add_argument( + "-p", + "--pp", + action="store", + dest="s_preprocessing", + default="normalization", + choices=["normalization", "standardization", "none"], + help="pre-processing data.", + ) + parser.add_argument( + "-o", + "--optimizer", + action="store", + dest="s_optimizer", + default="adam", + help="optimizer technique", + choices=[ + "adam", + "adagrad", + "adadelta", + "adamax", + "ftrl", + "nadam", + "optimizer", + "rmsprop", + "sgd", + ], + ) + parser.add_argument( + "-l", + "--loss", + action="store", + dest="s_loss", + default="mae", + choices=["mae", "mape", "mse", "msle"], + help="loss function.", + ) + parser.add_argument( + "-e", + "--end", + action="store", + type=valid_date, + dest="s_end_date", + default=None, + help="The end date (format YYYY-MM-DD) to select - Backtesting", + ) + try: - ns_parser = _parse_args( - prog="rnn", description="""Recurrent Neural Network. """, l_args=l_args - ) + ns_parser = parse_known_args_and_warn(parser, l_args) if not ns_parser: return - # Setup backtesting - df_stock, df_future = _setup_backtesting(df_stock, ns_parser) + # BACKTESTING + if ns_parser.s_end_date: + + if ns_parser.s_end_date < df_stock.index[0]: + print( + "Backtesting not allowed, since End Date is older than Start Date of historical data\n" + ) + return + + if ns_parser.s_end_date < get_next_stock_market_days( + last_stock_day=df_stock.index[0], + n_next_days=ns_parser.n_inputs + ns_parser.n_days, + )[-1]: + print( + "Backtesting not allowed, since End Date is too close to Start Date to train model\n" + ) + return + + future_index = get_next_stock_market_days( + last_stock_day=ns_parser.s_end_date, n_next_days=ns_parser.n_days + ) + + if future_index[-1] > datetime.datetime.now(): + print( + "Backtesting not allowed, since End Date + Prediction days is in the future\n" + ) + return + + df_future = df_stock[future_index[0] : future_index[-1]] + df_stock = df_stock[: ns_parser.s_end_date] # Pre-process data - scaler, stock_train_data, stock_x, stock_y = _preprocess_split( - df_stock, ns_parser + if ns_parser.s_preprocessing == "standardization": + scaler = StandardScaler() + stock_train_data = scaler.fit_transform( + np.array(df_stock["5. adjusted close"].values.reshape(-1, 1)) + ) + elif ns_parser.s_preprocessing == "normalization": + scaler = MinMaxScaler() + stock_train_data = scaler.fit_transform( + np.array(df_stock["5. adjusted close"].values.reshape(-1, 1)) + ) + else: # No pre-processing + stock_train_data = np.array( + df_stock["5. adjusted close"].values.reshape(-1, 1) + ) + + # Split training data for the neural network + stock_x, stock_y = splitTrain.split_train( + stock_train_data, + ns_parser.n_inputs, + ns_parser.n_days, + numJumps=ns_parser.n_jumps, ) + + if not stock_x: + print("Given the model parameters more training data is needed.\n") + return + + stock_x = np.array(stock_x) stock_x = np.reshape(stock_x, (stock_x.shape[0], stock_x.shape[1], 1)) + stock_y = np.array(stock_y) stock_y = np.reshape(stock_y, (stock_y.shape[0], stock_y.shape[1], 1)) # Build Neural Network model @@ -610,13 +628,7 @@ def rnn(l_args, s_ticker, df_stock): model.compile(optimizer=ns_parser.s_optimizer, loss=ns_parser.s_loss) # Train our model - model.fit( - stock_x, - stock_y, - epochs=ns_parser.n_epochs, - batch_size=ns_parser.n_batch_size, - verbose=1, - ) + model.fit(stock_x, stock_y, epochs=ns_parser.n_epochs, verbose=1) print("") print(model.summary()) @@ -628,37 +640,364 @@ def rnn(l_args, s_ticker, df_stock): verbose=0, ) - # Re-scale the data back, plot, and print the results - df_pred = _rescale_data(df_stock, ns_parser, scaler, yhat) - _plot_and_print_results( - df_stock, ns_parser, df_future, df_pred, "RNN", s_ticker + # Re-scale the data back + if (ns_parser.s_preprocessing == "standardization") or ( + ns_parser.s_preprocessing == "normalization" + ): + y_pred_test_t = scaler.inverse_transform(yhat.tolist()) + else: + y_pred_test_t = yhat + + l_pred_days = get_next_stock_market_days( + last_stock_day=df_stock["5. adjusted close"].index[-1], + n_next_days=ns_parser.n_days, ) + df_pred = pd.Series(y_pred_test_t[0].tolist(), index=l_pred_days, name="Price") + + # Plotting + plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) + plt.plot(df_stock.index, df_stock["5. adjusted close"], lw=3) + # BACKTESTING + if ns_parser.s_end_date: + plt.title( + f"BACKTESTING: RNN on {s_ticker} - {ns_parser.n_days} days prediction" + ) + else: + plt.title(f"RNN on {s_ticker} - {ns_parser.n_days} days prediction") + plt.xlim( + df_stock.index[0], get_next_stock_market_days(df_pred.index[-1], 1)[-1] + ) + plt.xlabel("Time") + plt.ylabel("Share Price ($)") + plt.grid(b=True, which="major", color="#666666", linestyle="-") + plt.minorticks_on() + plt.grid(b=True, which="minor", color="#999999", linestyle="-", alpha=0.2) + plt.plot( + [df_stock.index[-1], df_pred.index[0]], + [df_stock["5. adjusted close"].values[-1], df_pred.values[0]], + lw=1, + c="tab:green", + linestyle="--", + ) + plt.plot(df_pred.index, df_pred, lw=2, c="tab:green") + plt.axvspan( + df_stock.index[-1], df_pred.index[-1], facecolor="tab:orange", alpha=0.2 + ) + _, _, ymin, ymax = plt.axis() + plt.vlines( + df_stock.index[-1], + ymin, + ymax, + colors="k", + linewidth=3, + linestyle="--", + color="k", + ) + + # BACKTESTING + if ns_parser.s_end_date: + plt.plot( + df_future.index, + df_future["5. adjusted close"], + lw=2, + c="tab:blue", + ls="--", + ) + plt.plot( + [df_stock.index[-1], df_future.index[0]], + [ + df_stock["5. adjusted close"].values[-1], + df_future["5. adjusted close"].values[0], + ], + lw=1, + c="tab:blue", + linestyle="--", + ) + + if gtff.USE_ION: + plt.ion() + + plt.show() + + # BACKTESTING + if ns_parser.s_end_date: + plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) + plt.subplot(211) + plt.plot( + df_future.index, + df_future["5. adjusted close"], + lw=2, + c="tab:blue", + ls="--", + ) + plt.plot(df_pred.index, df_pred, lw=2, c="green") + plt.scatter( + df_future.index, df_future["5. adjusted close"], c="tab:blue", lw=3 + ) + plt.plot( + [df_stock.index[-1], df_future.index[0]], + [ + df_stock["5. adjusted close"].values[-1], + df_future["5. adjusted close"].values[0], + ], + lw=2, + c="tab:blue", + ls="--", + ) + plt.scatter(df_pred.index, df_pred, c="green", lw=3) + plt.plot( + [df_stock.index[-1], df_pred.index[0]], + [df_stock["5. adjusted close"].values[-1], df_pred.values[0]], + lw=2, + c="green", + ls="--", + ) + plt.title("BACKTESTING: Real data price versus Prediction") + plt.xlim(df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)) + plt.xticks( + [df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)], + visible=True, + ) + plt.ylabel("Share Price ($)") + plt.grid(b=True, which="major", color="#666666", linestyle="-") + plt.minorticks_on() + plt.grid(b=True, which="minor", color="#999999", linestyle="-", alpha=0.2) + plt.legend(["Real data", "Prediction data"]) + plt.xticks([]) + + plt.subplot(212) + plt.axhline(y=0, color="k", linestyle="--", linewidth=2) + plt.plot( + df_future.index, + 100 + * (df_pred.values - df_future["5. adjusted close"].values) + / df_future["5. adjusted close"].values, + lw=2, + c="red", + ) + plt.scatter( + df_future.index, + 100 + * (df_pred.values - df_future["5. adjusted close"].values) + / df_future["5. adjusted close"].values, + c="red", + lw=5, + ) + plt.title("BACKTESTING: Error between Real data and Prediction [%]") + plt.plot( + [df_stock.index[-1], df_future.index[0]], + [ + 0, + 100 + * (df_pred.values[0] - df_future["5. adjusted close"].values[0]) + / df_future["5. adjusted close"].values[0], + ], + lw=2, + ls="--", + c="red", + ) + plt.xlim(df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)) + plt.xticks( + [df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)], + visible=True, + ) + plt.xlabel("Time") + plt.ylabel("Prediction Error (%)") + plt.grid(b=True, which="major", color="#666666", linestyle="-") + plt.minorticks_on() + plt.grid(b=True, which="minor", color="#999999", linestyle="-", alpha=0.2) + plt.legend(["Real data", "Prediction data"]) + + if gtff.USE_ION: + plt.ion() + + plt.show() + + # Refactor prediction dataframe for backtesting print + df_pred.name = "Prediction" + df_pred = df_pred.to_frame() + df_pred["Real"] = df_future["5. adjusted close"] + + if gtff.USE_COLOR: + + patch_pandas_text_adjustment() + + print("Time Real [$] x Prediction [$]") + print( + df_pred.apply( + price_prediction_backtesting_color, axis=1 + ).to_string() + ) + else: + print(df_pred[["Real", "Prediction"]].round(2).to_string()) + + print("") + print_prediction_kpis(df_pred["Real"].values, df_pred["Prediction"].values) + + else: + # Print prediction data + print_pretty_prediction(df_pred, df_stock["5. adjusted close"].values[-1]) + print("") except Exception as e: print(e) - traceback.print_exc() print("") - finally: - _restore_env() - def lstm(l_args, s_ticker, df_stock): + parser = argparse.ArgumentParser( + add_help=False, prog="lstm", description="""Long-Short Term Memory. """ + ) + + parser.add_argument( + "-d", + "--days", + action="store", + dest="n_days", + type=check_positive, + default=5, + help="prediction days", + ) + parser.add_argument( + "-i", + "--input", + action="store", + dest="n_inputs", + type=check_positive, + default=40, + help="number of days to use for prediction.", + ) + parser.add_argument( + "--epochs", + action="store", + dest="n_epochs", + type=check_positive, + default=200, + help="number of training epochs.", + ) + parser.add_argument( + "-j", + "--jumps", + action="store", + dest="n_jumps", + type=check_positive, + default=1, + help="number of jumps in training data.", + ) + parser.add_argument( + "-p", + "--pp", + action="store", + dest="s_preprocessing", + default="normalization", + choices=["normalization", "standardization", "none"], + help="pre-processing data.", + ) + parser.add_argument( + "-o", + "--optimizer", + action="store", + dest="s_optimizer", + default="adam", + help="optimization technique.", + choices=[ + "adam", + "adagrad", + "adadelta", + "adamax", + "ftrl", + "nadam", + "optimizer", + "rmsprop", + "sgd", + ], + ) + parser.add_argument( + "-l", + "--loss", + action="store", + dest="s_loss", + default="mae", + choices=["mae", "mape", "mse", "msle"], + help="loss function.", + ) + parser.add_argument( + "-e", + "--end", + action="store", + type=valid_date, + dest="s_end_date", + default=None, + help="The end date (format YYYY-MM-DD) to select - Backtesting", + ) + try: - ns_parser = _parse_args( - prog="lstm", description="""Long-Short Term Memory. """, l_args=l_args - ) + ns_parser = parse_known_args_and_warn(parser, l_args) if not ns_parser: return - # Setup backtesting - df_stock, df_future = _setup_backtesting(df_stock, ns_parser) + # BACKTESTING + if ns_parser.s_end_date: + + if ns_parser.s_end_date < df_stock.index[0]: + print( + "Backtesting not allowed, since End Date is older than Start Date of historical data\n" + ) + return + + if ns_parser.s_end_date < get_next_stock_market_days( + last_stock_day=df_stock.index[0], + n_next_days=ns_parser.n_inputs + ns_parser.n_days, + )[-1]: + print( + "Backtesting not allowed, since End Date is too close to Start Date to train model\n" + ) + return + + future_index = get_next_stock_market_days( + last_stock_day=ns_parser.s_end_date, n_next_days=ns_parser.n_days + ) + + if future_index[-1] > datetime.datetime.now(): + print( + "Backtesting not allowed, since End Date + Prediction days is in the future\n" + ) + return + + df_future = df_stock[future_index[0] : future_index[-1]] + df_stock = df_stock[: ns_parser.s_end_date] # Pre-process data - scaler, stock_train_data, stock_x, stock_y = _preprocess_split( - df_stock, ns_parser + if ns_parser.s_preprocessing == "standardization": + scaler = StandardScaler() + stock_train_data = scaler.fit_transform( + np.array(df_stock["5. adjusted close"].values.reshape(-1, 1)) + ) + elif ns_parser.s_preprocessing == "normalization": + scaler = MinMaxScaler() + stock_train_data = scaler.fit_transform( + np.array(df_stock["5. adjusted close"].values.reshape(-1, 1)) + ) + else: # No pre-processing + stock_train_data = np.array( + df_stock["5. adjusted close"].values.reshape(-1, 1) + ) + + # Split training data for the neural network + stock_x, stock_y = splitTrain.split_train( + stock_train_data, + ns_parser.n_inputs, + ns_parser.n_days, + numJumps=ns_parser.n_jumps, ) + + if not stock_x: + print("Given the model parameters more training data is needed.\n") + return + + stock_x = np.array(stock_x) stock_x = np.reshape(stock_x, (stock_x.shape[0], stock_x.shape[1], 1)) + stock_y = np.array(stock_y) stock_y = np.reshape(stock_y, (stock_y.shape[0], stock_y.shape[1], 1)) # Build Neural Network model @@ -668,13 +1007,7 @@ def lstm(l_args, s_ticker, df_stock): model.compile(optimizer=ns_parser.s_optimizer, loss=ns_parser.s_loss) # Train our model - model.fit( - stock_x, - stock_y, - epochs=ns_parser.n_epochs, - batch_size=ns_parser.n_batch_size, - verbose=1, - ) + model.fit(stock_x, stock_y, epochs=ns_parser.n_epochs, verbose=1) print("") print(model.summary()) @@ -686,16 +1019,205 @@ def lstm(l_args, s_ticker, df_stock): verbose=0, ) - # Re-scale the data back, plot, and print the results - df_pred = _rescale_data(df_stock, ns_parser, scaler, yhat) - _plot_and_print_results( - df_stock, ns_parser, df_future, df_pred, "LSTM", s_ticker + # Re-scale the data back + if (ns_parser.s_preprocessing == "standardization") or ( + ns_parser.s_preprocessing == "normalization" + ): + y_pred_test_t = scaler.inverse_transform(yhat.tolist()) + else: + y_pred_test_t = yhat + + l_pred_days = get_next_stock_market_days( + last_stock_day=df_stock["5. adjusted close"].index[-1], + n_next_days=ns_parser.n_days, ) + df_pred = pd.Series(y_pred_test_t[0].tolist(), index=l_pred_days, name="Price") + + # Plotting + plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) + plt.plot(df_stock.index, df_stock["5. adjusted close"], lw=3) + # BACKTESTING + if ns_parser.s_end_date: + plt.title( + f"BACKTESTING: LSTM on {s_ticker} - {ns_parser.n_days} days prediction" + ) + else: + plt.title(f"LSTM on {s_ticker} - {ns_parser.n_days} days prediction") + plt.xlim( + df_stock.index[0], get_next_stock_market_days(df_pred.index[-1], 1)[-1] + ) + plt.xlabel("Time") + plt.ylabel("Share Price ($)") + plt.grid(b=True, which="major", color="#666666", linestyle="-") + plt.minorticks_on() + plt.grid(b=True, which="minor", color="#999999", linestyle="-", alpha=0.2) + plt.plot( + [df_stock.index[-1], df_pred.index[0]], + [df_stock["5. adjusted close"].values[-1], df_pred.values[0]], + lw=1, + c="tab:green", + linestyle="--", + ) + plt.plot(df_pred.index, df_pred, lw=2, c="tab:green") + plt.axvspan( + df_stock.index[-1], df_pred.index[-1], facecolor="tab:orange", alpha=0.2 + ) + _, _, ymin, ymax = plt.axis() + plt.vlines( + df_stock.index[-1], + ymin, + ymax, + colors="k", + linewidth=3, + linestyle="--", + color="k", + ) + + # BACKTESTING + if ns_parser.s_end_date: + plt.plot( + df_future.index, + df_future["5. adjusted close"], + lw=2, + c="tab:blue", + ls="--", + ) + plt.plot( + [df_stock.index[-1], df_future.index[0]], + [ + df_stock["5. adjusted close"].values[-1], + df_future["5. adjusted close"].values[0], + ], + lw=1, + c="tab:blue", + linestyle="--", + ) + + if gtff.USE_ION: + plt.ion() + + plt.show() + + # BACKTESTING + if ns_parser.s_end_date: + plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) + plt.subplot(211) + plt.plot( + df_future.index, + df_future["5. adjusted close"], + lw=2, + c="tab:blue", + ls="--", + ) + plt.plot(df_pred.index, df_pred, lw=2, c="green") + plt.scatter( + df_future.index, df_future["5. adjusted close"], c="tab:blue", lw=3 + ) + plt.plot( + [df_stock.index[-1], df_future.index[0]], + [ + df_stock["5. adjusted close"].values[-1], + df_future["5. adjusted close"].values[0], + ], + lw=2, + c="tab:blue", + ls="--", + ) + plt.scatter(df_pred.index, df_pred, c="green", lw=3) + plt.plot( + [df_stock.index[-1], df_pred.index[0]], + [df_stock["5. adjusted close"].values[-1], df_pred.values[0]], + lw=2, + c="green", + ls="--", + ) + plt.title("BACKTESTING: Real data price versus Prediction") + plt.xlim(df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)) + plt.xticks( + [df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)], + visible=True, + ) + plt.ylabel("Share Price ($)") + plt.grid(b=True, which="major", color="#666666", linestyle="-") + plt.minorticks_on() + plt.grid(b=True, which="minor", color="#999999", linestyle="-", alpha=0.2) + plt.legend(["Real data", "Prediction data"]) + plt.xticks([]) + + plt.subplot(212) + plt.axhline(y=0, color="k", linestyle="--", linewidth=2) + plt.plot( + df_future.index, + 100 + * (df_pred.values - df_future["5. adjusted close"].values) + / df_future["5. adjusted close"].values, + lw=2, + c="red", + ) + plt.scatter( + df_future.index, + 100 + * (df_pred.values - df_future["5. adjusted close"].values) + / df_future["5. adjusted close"].values, + c="red", + lw=5, + ) + plt.title("BACKTESTING: Error between Real data and Prediction [%]") + plt.plot( + [df_stock.index[-1], df_future.index[0]], + [ + 0, + 100 + * (df_pred.values[0] - df_future["5. adjusted close"].values[0]) + / df_future["5. adjusted close"].values[0], + ], + lw=2, + ls="--", + c="red", + ) + plt.xlim(df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)) + plt.xticks( + [df_stock.index[-1], df_pred.index[-1] + datetime.timedelta(days=1)], + visible=True, + ) + plt.xlabel("Time") + plt.ylabel("Prediction Error (%)") + plt.grid(b=True, which="major", color="#666666", linestyle="-") + plt.minorticks_on() + plt.grid(b=True, which="minor", color="#999999", linestyle="-", alpha=0.2) + plt.legend(["Real data", "Prediction data"]) + + if gtff.USE_ION: + plt.ion() + + plt.show() + + # Refactor prediction dataframe for backtesting print + df_pred.name = "Prediction" + df_pred = df_pred.to_frame() + df_pred["Real"] = df_future["5. adjusted close"] + + if gtff.USE_COLOR: + + patch_pandas_text_adjustment() + + print("Time Real [$] x Prediction [$]") + print( + df_pred.apply( + price_prediction_backtesting_color, axis=1 + ).to_string() + ) + else: + print(df_pred[["Real", "Prediction"]].round(2).to_string()) + + print("") + print_prediction_kpis(df_pred["Real"].values, df_pred["Prediction"].values) + + else: + # Print prediction data + print_pretty_prediction(df_pred, df_stock["5. adjusted close"].values[-1]) + print("") except Exception as e: print(e) - traceback.print_exc() print("") - - finally: - _restore_env() \ No newline at end of file From 5db13087771b2b6483f192d8efb17d5067d6f3a1 Mon Sep 17 00:00:00 2001 From: Laurent Andriamifidy Date: Wed, 17 Mar 2021 11:29:09 +0100 Subject: [PATCH 30/64] cleaned up data generation graph w seaborn ok graph w plotly ready (pending) --- gamestonk_terminal/options/volume.py | 88 +++++++++++++++++++++------- 1 file changed, 66 insertions(+), 22 deletions(-) diff --git a/gamestonk_terminal/options/volume.py b/gamestonk_terminal/options/volume.py index e539ff060ec..14d71ced9ab 100644 --- a/gamestonk_terminal/options/volume.py +++ b/gamestonk_terminal/options/volume.py @@ -30,37 +30,81 @@ def volume_graph(l_args, s_ticker): if not ns_parser: return - opt = yf.Ticker(s_ticker).option_chain(ns_parser.s_expiry_date.strftime("%Y-%m-%d")) - # data available via: opt.calls, opt.puts - - volume_data = pd.concat( - [ - __volume_data(opt.calls, 'calls'), - __volume_data(opt.puts, 'puts') - ], - axis=0 - ) - fig = px.line( - volume_data, - x="strike", - y="volume", - title=f'{s_ticker} Volume for {ns_parser.s_expiry_date.strftime("%Y-%m-%d")}', - color='type' - ) - fig.show() + __get_volume_graph(s_ticker, ns_parser.s_expiry_date.strftime("%Y-%m-%d")) except SystemExit: print("") except Exception as e: print(e) -def __volume_data(opt_data, flag): - # get option chain for specific expiration - df = opt_data.pivot_table( + + +def __get_volume_graph(ticker_name, exp_date): + df = __get_volume_data(ticker_name, exp_date) + __generate_graph_sns(df, ticker_name, exp_date) + #__generate_graph_plotly(df, ticker_name, exp_date) + +def __pull_call_put_data(call_put, flag): + df = call_put.pivot_table( index='strike', - values=['volume', 'openInterest'], + values = ['volume','openInterest'], aggfunc='sum') + df.reindex() + df['strike'] = df.index df['type'] = flag + return df + +def __get_volume_data(ticker_name, exp_date): + + option_chain = yf.Ticker(ticker_name).option_chain(exp_date) + + calls = __pull_call_put_data( + option_chain.calls, + 'calls' + ) + + puts = __pull_call_put_data( + option_chain.puts, + 'puts' + ) + + volume_data = pd.concat( + [ + calls, + puts + ], + axis = 0 + ) + #dataframe + return volume_data + +def __generate_graph_plotly(df, ticker_name, exp_date): + #version with plotly express + fig = px.line( + df, + x="strike", + y="volume", + title=f'{ticker_name} options volume for {exp_date}', + color= 'type' + ) + fig.show() + + return + +def __generate_graph_sns(df, ticker_name, exp_date): + #version with seaborn express + plt.figure(figsize=(12,6)) + fig = sns.lineplot( + data = df, + x = 'strike', + y = 'volume', + hue = 'type', + palette= ['limegreen', 'tomato']) + + plt.title(f'{ticker_name} options volume for {exp_date}') + + plt.show() + return From 998aff3e4799cd08032b9cb8545af4776889ba20 Mon Sep 17 00:00:00 2001 From: David Volm Date: Wed, 17 Mar 2021 07:43:55 -0500 Subject: [PATCH 31/64] Added Coinpaprika API to ROADMAP --- ROADMAP.md | 1 + 1 file changed, 1 insertion(+) diff --git a/ROADMAP.md b/ROADMAP.md index a18f1c02854..ff7bfc21eeb 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -152,6 +152,7 @@ ### Short term - [ ] Add Coingecko +- [ ] Add [Coinpaprika](https://coinpaprika.com/api/) - [ ] Add crypto charts ### Long term From 78a9b4b2922f4ccefe87a36bf8bfd79c1f4a16b3 Mon Sep 17 00:00:00 2001 From: buzzCraft <67235613+buzzCraft@users.noreply.github.com> Date: Wed, 17 Mar 2021 13:51:09 +0100 Subject: [PATCH 32/64] Resolving merge conflicts --- gamestonk_terminal/prediction_techniques/neural_networks.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/gamestonk_terminal/prediction_techniques/neural_networks.py b/gamestonk_terminal/prediction_techniques/neural_networks.py index cd96610d3eb..6fd0a356abd 100644 --- a/gamestonk_terminal/prediction_techniques/neural_networks.py +++ b/gamestonk_terminal/prediction_techniques/neural_networks.py @@ -29,6 +29,7 @@ from gamestonk_terminal.prediction_techniques.pred_helper import ( from gamestonk_terminal import feature_flags as gtff +from gamestonk_terminal.config_plot import PLOT_DPI from gamestonk_terminal import config_neural_network_models as cfg_nn_models @@ -349,7 +350,7 @@ def _plot_and_print_results( ): """Plot and print the results. """ # Plotting - plt.figure() + plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) plt.plot(df_stock.index, df_stock["5. adjusted close"], lw=3) # BACKTESTING @@ -414,7 +415,7 @@ def _plot_and_print_results( # BACKTESTING if ns_parser.s_end_date: - plt.figure() + plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) plt.subplot(211) plt.plot( df_future.index, From 91ba7681056ec928495bb82141574bd58935b00f Mon Sep 17 00:00:00 2001 From: Laurent Andriamifidy Date: Wed, 17 Mar 2021 15:11:16 +0100 Subject: [PATCH 33/64] cleanup added option help main menu --- gamestonk_terminal/main_helper.py | 3 +++ gamestonk_terminal/options/op_menu.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/gamestonk_terminal/main_helper.py b/gamestonk_terminal/main_helper.py index 065b77325fd..143895e1e2b 100644 --- a/gamestonk_terminal/main_helper.py +++ b/gamestonk_terminal/main_helper.py @@ -71,6 +71,9 @@ def print_help(s_ticker, s_start, s_interval, b_is_market_open): print( " pred prediction techniques, \t e.g.: regression, arima, rnn, lstm, prophet" ) + print( + " op options info, \t e.g.: volume" + ) print("") diff --git a/gamestonk_terminal/options/op_menu.py b/gamestonk_terminal/options/op_menu.py index e6358421538..0522a9b9a5a 100644 --- a/gamestonk_terminal/options/op_menu.py +++ b/gamestonk_terminal/options/op_menu.py @@ -43,7 +43,7 @@ def opt_menu(df_stock, s_ticker, s_start, s_interval): if ns_known_args.cmd == "help": pass - #print_options(s_ticker, s_start, s_interval) + #print_options(s_ticker) elif ns_known_args.cmd == "q": # Just leave the options menu From 4ff3a0cd39d5a9b9a53209606e6e1fdf866efa99 Mon Sep 17 00:00:00 2001 From: Jamie Date: Wed, 17 Mar 2021 12:25:36 -0400 Subject: [PATCH 34/64] Plotting changes, flag updates. --- gamestonk_terminal/fred/README.md | 119 ++++++++++++++------------- gamestonk_terminal/fred/fred_api.py | 109 +++++++++++++++--------- gamestonk_terminal/fred/fred_menu.py | 4 +- 3 files changed, 131 insertions(+), 101 deletions(-) diff --git a/gamestonk_terminal/fred/README.md b/gamestonk_terminal/fred/README.md index 2343af81ae0..504412b7457 100644 --- a/gamestonk_terminal/fred/README.md +++ b/gamestonk_terminal/fred/README.md @@ -9,12 +9,13 @@ The purpose of fred is to have the ability to look at economic data. ## gdp ```text -usage: gdp [-n N_TO_GET] [-s START_DATE] [-p PLOT]] +usage: gdp [-n N_TO_GET] [-s START_DATE] [--noplot] [--hidedata] ``` Gets GDP data. Data is released quarterly * -n : Number of points to display. If unspecified, it will display all available. * -s : Start Date. Format YYYY-mm-dd. First point to get. Defaults to Jan 1 2015 -* -p : Option to plot. Defaults to False +* --noplot : Flag to hide plot +* --hidedata : Flag to suppress printing output To get and plot the last 20 reported GDP values: ```` @@ -22,34 +23,32 @@ gdp -n 20 ```` To show all GDP data since 1/1/2016, with no plot. ```` -gdp -s 2016-01-01 -p False +gdp -s 2016-01-01 --noplot ```` +Outputs: ```` -Date: 01-01-2015, GDP: 18003.399 -Date: 04-01-2015, GDP: 18223.577 -Date: 07-01-2015, GDP: 18347.425 -Date: 10-01-2015, GDP: 18378.803 -Date: 01-01-2016, GDP: 18470.156 -Date: 04-01-2016, GDP: 18656.207 -Date: 07-01-2016, GDP: 18821.359 -Date: 10-01-2016, GDP: 19032.58 -Date: 01-01-2017, GDP: 19237.435 -Date: 04-01-2017, GDP: 19379.232 -Date: 07-01-2017, GDP: 19617.288 -Date: 10-01-2017, GDP: 19937.963 -Date: 01-01-2018, GDP: 20242.215 -Date: 04-01-2018, GDP: 20552.653 -Date: 07-01-2018, GDP: 20742.723 -Date: 10-01-2018, GDP: 20909.853 -Date: 01-01-2019, GDP: 21115.309 -Date: 04-01-2019, GDP: 21329.877 -Date: 07-01-2019, GDP: 21540.325 -Date: 10-01-2019, GDP: 21747.394 -Date: 01-01-2020, GDP: 21561.139 -Date: 04-01-2020, GDP: 19520.114 -Date: 07-01-2020, GDP: 21170.252 -Date: 10-01-2020, GDP: 21487.896 - + GDP +Date +2016-01-01 18470.156 +2016-04-01 18656.207 +2016-07-01 18821.359 +2016-10-01 19032.580 +2017-01-01 19237.435 +2017-04-01 19379.232 +2017-07-01 19617.288 +2017-10-01 19937.963 +2018-01-01 20242.215 +2018-04-01 20552.653 +2018-07-01 20742.723 +2018-10-01 20909.853 +2019-01-01 21115.309 +2019-04-01 21329.877 +2019-07-01 21540.325 +2019-10-01 21747.394 +2020-01-01 21561.139 +2020-04-01 19520.114 +2020-07-01 21170.252 +2020-10-01 21487.896 ```` @@ -58,46 +57,48 @@ Date: 10-01-2020, GDP: 21487.896 The fred database has many different datasets freely available. They each have their own custom SERIES_ID which can be obtained through the website. ```` -usage: cust [-id] [-s START_DATE] [-p PLOT] [-disp DISP] +usage: cust [-i --id] [-s START_DATE] [--noplot] [--hidedata] ```` -* -id : Series ID for FRED data. Required argument +* -i/--id : Series ID for FRED data. Required argument * -s : Start Date. Format YYYY-mm-dd. First point to get. Defaults to Jan 1 2015. -* -p : Option to plot. Defaults to True -* -disp : Option to print data to console. Defaults to False. +* --noplot : Flag to suppress output plot. +* --hidedata : Flag to suppress data output An example of custom data could be the unemployment rate. According to https://fred.stlouisfed.org/series/UNRATE , the series ID is UNRATE. So to plot this for 2019 to now ```` -cust -id UNRATE -s 2019-01-01 -disp True +cust --id UNRATE -s 2019-01-01 ```` This will plot and display: ```` -Date: 01-01-2019, DATA: 4.0 -Date: 02-01-2019, DATA: 3.8 -Date: 03-01-2019, DATA: 3.8 -Date: 04-01-2019, DATA: 3.7 -Date: 05-01-2019, DATA: 3.7 -Date: 06-01-2019, DATA: 3.6 -Date: 07-01-2019, DATA: 3.6 -Date: 08-01-2019, DATA: 3.7 -Date: 09-01-2019, DATA: 3.5 -Date: 10-01-2019, DATA: 3.6 -Date: 11-01-2019, DATA: 3.6 -Date: 12-01-2019, DATA: 3.6 -Date: 01-01-2020, DATA: 3.5 -Date: 02-01-2020, DATA: 3.5 -Date: 03-01-2020, DATA: 4.4 -Date: 04-01-2020, DATA: 14.8 -Date: 05-01-2020, DATA: 13.3 -Date: 06-01-2020, DATA: 11.1 -Date: 07-01-2020, DATA: 10.2 -Date: 08-01-2020, DATA: 8.4 -Date: 09-01-2020, DATA: 7.8 -Date: 10-01-2020, DATA: 6.9 -Date: 11-01-2020, DATA: 6.7 -Date: 12-01-2020, DATA: 6.7 -Date: 01-01-2021, DATA: 6.3 -Date: 02-01-2021, DATA: 6.2 + UNRATE +Date +2019-01-01 4.0 +2019-02-01 3.8 +2019-03-01 3.8 +2019-04-01 3.7 +2019-05-01 3.7 +2019-06-01 3.6 +2019-07-01 3.6 +2019-08-01 3.7 +2019-09-01 3.5 +2019-10-01 3.6 +2019-11-01 3.6 +2019-12-01 3.6 +2020-01-01 3.5 +2020-02-01 3.5 +2020-03-01 4.4 +2020-04-01 14.8 +2020-05-01 13.3 +2020-06-01 11.1 +2020-07-01 10.2 +2020-08-01 8.4 +2020-09-01 7.8 +2020-10-01 6.9 +2020-11-01 6.7 +2020-12-01 6.7 +2021-01-01 6.3 +2021-02-01 6.2 ```` \ No newline at end of file diff --git a/gamestonk_terminal/fred/fred_api.py b/gamestonk_terminal/fred/fred_api.py index 1f4ac4479d7..e5fce3a148a 100644 --- a/gamestonk_terminal/fred/fred_api.py +++ b/gamestonk_terminal/fred/fred_api.py @@ -1,12 +1,18 @@ -from fredapi import Fred import argparse +import pandas as pd +from pandas.plotting import register_matplotlib_converters import matplotlib.pyplot as plt +from fredapi import Fred from gamestonk_terminal.helper_funcs import ( parse_known_args_and_warn, valid_date, - str_to_bool, + plot_autoscale, ) +from gamestonk_terminal.config_plot import PLOT_DPI from gamestonk_terminal.config_terminal import API_FRED_KEY +from gamestonk_terminal import feature_flags as gtff + +register_matplotlib_converters() def get_GDP(l_args): @@ -37,12 +43,19 @@ def get_GDP(l_args): ) parser.add_argument( - "-p", - dest="plot", - type=str_to_bool, + "--noplot", + action="store_false", default=True, - required=False, - help="Plot GDP", + dest="noplot", + help="Suppress output plot", + ) + + parser.add_argument( + "--hidedata", + action="store_false", + default=True, + dest="hidedata", + help="Suppress data display plot", ) try: @@ -52,22 +65,30 @@ def get_GDP(l_args): return gdp = fred.get_series("GDP", ns_parser.start_date) - + gdp = pd.DataFrame(gdp, columns=["GDP"]) + gdp.index.name = "Date" if int(ns_parser.n_to_get) > 0: lastn = gdp.tail(int(ns_parser.n_to_get)) else: lastn = gdp - for date, val in lastn.iteritems(): - print(f'Date: {date.strftime("%m-%d-%Y")}, GDP: {val} ') + if ns_parser.hidedata: + print(lastn) + print("") - if ns_parser.plot: - plt.figure() - lastn.plot(style="-ok") + if ns_parser.noplot: + plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) + plt.plot(lastn.index, lastn.GDP, "-ok") plt.xlabel("Time") - plt.ylabel("GDP") + plt.xlim(lastn.index[0], lastn.index[-1]) + plt.ylabel("GDP (bn $)") + plt.title("FRED GDP Data (in Billions of USD)") + plt.grid(b=True, which="major", color="#666666", linestyle="-") + plt.minorticks_on() + plt.grid(b=True, which="minor", color="#999999", linestyle="-", alpha=0.2) plt.show() + print("") except Exception as e: print(e) @@ -86,7 +107,7 @@ def custom_data(l_args): """, ) parser.add_argument( - "-id", dest="series_id", required=True, type=str, help="FRED Series ID" + "-i", "--id", dest="series_id", required=True, type=str, help="FRED Series ID" ) parser.add_argument( @@ -95,23 +116,23 @@ def custom_data(l_args): type=valid_date, default="2015-01-01", required=False, - help="Date to Start", + help="Starting date (YYYY-MM-DD) of data", ) + parser.add_argument( - "-p", - dest="plot", - type=str_to_bool, + "--noplot", + action="store_false", default=True, - required=False, - help="Plot data", + dest="noplot", + help="Suppress output plot", ) + parser.add_argument( - "-disp", - dest="disp", - type=str_to_bool, - default=False, - required=False, - help="Print data", + "--hidedata", + action="store_false", + default=True, + dest="hidedata", + help="Suppress data display plot", ) try: @@ -121,20 +142,28 @@ def custom_data(l_args): return data = fred.get_series(ns_parser.series_id, ns_parser.start_date) - if ns_parser.disp: - for date, val in data.iteritems(): - print(f'Date: {date.strftime("%m-%d-%Y")}, DATA: {val} ') - try: - if ns_parser.plot: - plt.figure() - data.plot(style="-ok") - plt.xlabel("Time") - plt.ylabel("DATA") - plt.show() - except Exception as e: - print(e) + + data = pd.DataFrame(data, columns=[f"{ns_parser.series_id}"]) + data.index.name = "Date" + if ns_parser.hidedata: + print(data) print("") - return + if ns_parser.noplot: + plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) + plt.plot(data.index, data.iloc[:, 0], "-ok") + plt.xlabel("Time") + plt.xlim(data.index[0], data.index[-1]) + plt.ylabel(f"{ns_parser.series_id}") + plt.grid(b=True, which="major", color="#666666", linestyle="-") + plt.minorticks_on() + plt.grid(b=True, which="minor", color="#999999", linestyle="-", alpha=0.2) + plt.title(f"FRED {ns_parser.series_id} Series") + plt.show() + print("") + + if gtff.USE_ION: + plt.ion() + except SystemExit: print("") except Exception as e: diff --git a/gamestonk_terminal/fred/fred_menu.py b/gamestonk_terminal/fred/fred_menu.py index f7f2d427fad..6c554fd0322 100644 --- a/gamestonk_terminal/fred/fred_menu.py +++ b/gamestonk_terminal/fred/fred_menu.py @@ -1,9 +1,8 @@ import argparse - +from prompt_toolkit.completion import NestedCompleter from gamestonk_terminal import feature_flags as gtff from gamestonk_terminal.helper_funcs import get_flair from gamestonk_terminal.menu import session -from prompt_toolkit.completion import NestedCompleter from gamestonk_terminal.fred import fred_api @@ -17,6 +16,7 @@ def print_fred(): print(" ") print(" gdp get GDP") print(" cust User Specified FRED Data") + print("") return From 1d4ca8065115ac65ffd207b8c3513edcee1fe8d3 Mon Sep 17 00:00:00 2001 From: Jamie Date: Wed, 17 Mar 2021 13:57:01 -0400 Subject: [PATCH 35/64] added more options for available series - more user friendly? --- gamestonk_terminal/fred/README.md | 109 +++++++++++++++------------ gamestonk_terminal/fred/fred_api.py | 75 +++++++++++------- gamestonk_terminal/fred/fred_menu.py | 31 ++++++-- 3 files changed, 134 insertions(+), 81 deletions(-) diff --git a/gamestonk_terminal/fred/README.md b/gamestonk_terminal/fred/README.md index 504412b7457..51b71a957fb 100644 --- a/gamestonk_terminal/fred/README.md +++ b/gamestonk_terminal/fred/README.md @@ -1,31 +1,41 @@ # FRED -The purpose of fred is to have the ability to look at economic data. +https://fred.stlouisfed.org -* [gdp](#gdp) - * Get GDP data +The purpose of fred is to have the ability to look at economic data. This page offers 2 options: + +* [predefined](#gdp) + * Get predefined data * [cust](#cust) * Get custom fred data -## gdp -```text -usage: gdp [-n N_TO_GET] [-s START_DATE] [--noplot] [--hidedata] -``` -Gets GDP data. Data is released quarterly -* -n : Number of points to display. If unspecified, it will display all available. -* -s : Start Date. Format YYYY-mm-dd. First point to get. Defaults to Jan 1 2015 -* --noplot : Flag to hide plot -* --hidedata : Flag to suppress printing output +The current predefined options are +* GDP +* 1,5,10,30 Year Treasury Rates +* 30 Year Mortgage Rates +* Unemployment Rate +* 3 Month LIBOR +* Moody's AAA Corporate Bond -To get and plot the last 20 reported GDP values: +## predefined +```text +usage: selected [-n N_TO_GET] [-s START_DATE] [--noplot] [--hidedata] +``` +Gets predefined data. Arguments +* selected : Key from the predefined list. +* -s : Start Date. Format YYYY-mm-dd. First point to get. Defaults to Jan 1 2019 +* --noplot : Flag to hide plot +* --hidedata : Flag to suppress printing output. Useful for data released daily/weekly. + +To plot the 30-Year Mortgage rates since 2018-01-01 suppressing the output. ```` -gdp -n 20 +mort30 -s 2018-01-01 --hidedata ```` To show all GDP data since 1/1/2016, with no plot. ```` gdp -s 2016-01-01 --noplot ```` -Outputs: +Output: ```` GDP Date @@ -64,41 +74,46 @@ usage: cust [-i --id] [-s START_DATE] [--noplot] [--hidedata] * --noplot : Flag to suppress output plot. * --hidedata : Flag to suppress data output -An example of custom data could be the unemployment rate. According to https://fred.stlouisfed.org/series/UNRATE , the series ID is UNRATE. So to plot this for 2019 to now +An example of custom data could be the USD/EURO exchange rate. +According to https://fred.stlouisfed.org/series/DEXUSEU , the series ID is DEXUSEU. So to plot this from the start of February 2021 to now (3/17) ```` -cust --id UNRATE -s 2019-01-01 +cust --id UNRATE -s 2021-02-01 ```` This will plot and display: - ```` - UNRATE -Date -2019-01-01 4.0 -2019-02-01 3.8 -2019-03-01 3.8 -2019-04-01 3.7 -2019-05-01 3.7 -2019-06-01 3.6 -2019-07-01 3.6 -2019-08-01 3.7 -2019-09-01 3.5 -2019-10-01 3.6 -2019-11-01 3.6 -2019-12-01 3.6 -2020-01-01 3.5 -2020-02-01 3.5 -2020-03-01 4.4 -2020-04-01 14.8 -2020-05-01 13.3 -2020-06-01 11.1 -2020-07-01 10.2 -2020-08-01 8.4 -2020-09-01 7.8 -2020-10-01 6.9 -2020-11-01 6.7 -2020-12-01 6.7 -2021-01-01 6.3 -2021-02-01 6.2 -```` \ No newline at end of file + DEXUSEU +Date +2021-02-01 1.2070 +2021-02-02 1.2020 +2021-02-03 1.2025 +2021-02-04 1.1974 +2021-02-05 1.2035 +2021-02-08 1.2045 +2021-02-09 1.2106 +2021-02-10 1.2132 +2021-02-11 1.2127 +2021-02-12 1.2126 +2021-02-15 NaN +2021-02-16 1.2107 +2021-02-17 1.2042 +2021-02-18 1.2078 +2021-02-19 1.2136 +2021-02-22 1.2155 +2021-02-23 1.2142 +2021-02-24 1.2143 +2021-02-25 1.2229 +2021-02-26 1.2093 +2021-03-01 1.2054 +2021-03-02 1.2079 +2021-03-03 1.2073 +2021-03-04 1.2045 +2021-03-05 1.1914 +2021-03-08 1.1849 +2021-03-09 1.1885 +2021-03-10 1.1900 +2021-03-11 1.1978 +```` + +Note the NaN on 2-15, which corresponds to President's Day. \ No newline at end of file diff --git a/gamestonk_terminal/fred/fred_api.py b/gamestonk_terminal/fred/fred_api.py index e5fce3a148a..490c152c7f8 100644 --- a/gamestonk_terminal/fred/fred_api.py +++ b/gamestonk_terminal/fred/fred_api.py @@ -14,32 +14,48 @@ from gamestonk_terminal import feature_flags as gtff register_matplotlib_converters() +api_map = { + "gdp": "GDP", + "t10": "DGS10", + "t01": "DGS1", + "t05": "DGS5", + "t30": "DGS30", + "mort30": "MORTGAGE30US", + "libor3m": "USD3MTD156N", + "moodAAA": "AAA", + "unemp": "UNRATE", +} +title_map = { + "gdp": "GDP", + "t10": "10-Year Treasury Constant Maturity Rate", + "t01": "1-Year Treasury Constant Maturity Rate", + "t05": "5-Year Treasury Constant Maturity Rate", + "t30": "30-Year Treasury Constant Maturity Rate", + "mort30": "30-Year Treasury Constant Maturity Rate", + "libor3m": "3-Month LIBOR, based on U.S. Dollar", + "moodAAA": "Moody's Seasoned AAA Corporate Bond Yield", + "unemp": "Unemployment Rate", +} -def get_GDP(l_args): + +def get_fred_data(l_args, choice): fred = Fred(api_key=API_FRED_KEY) + parser = argparse.ArgumentParser( add_help=False, - prog="GDP", + prog="Custom", description=""" - GDP - """, - ) - parser.add_argument( - "-n", - dest="n_to_get", - type=int, - default=-1, - required=False, - help="Number of GDP Values to Grab", + Custom Data + """, ) parser.add_argument( "-s", dest="start_date", type=valid_date, - default="2015-01-01", + default="2019-01-01", required=False, - help="Date to Start", + help="Starting date (YYYY-MM-DD) of data", ) parser.add_argument( @@ -64,32 +80,33 @@ def get_GDP(l_args): if not ns_parser: return - gdp = fred.get_series("GDP", ns_parser.start_date) - gdp = pd.DataFrame(gdp, columns=["GDP"]) - gdp.index.name = "Date" - if int(ns_parser.n_to_get) > 0: - lastn = gdp.tail(int(ns_parser.n_to_get)) - - else: - lastn = gdp + string_to_get = api_map[choice] + title = title_map[choice] + data = fred.get_series(string_to_get, ns_parser.start_date) + data = pd.DataFrame(data, columns=[f"{string_to_get}"]) + data.index.name = "Date" if ns_parser.hidedata: - print(lastn) + print(data) print("") - if ns_parser.noplot: plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) - plt.plot(lastn.index, lastn.GDP, "-ok") + plt.plot(data.index, data.iloc[:, 0], "-ok") plt.xlabel("Time") - plt.xlim(lastn.index[0], lastn.index[-1]) - plt.ylabel("GDP (bn $)") - plt.title("FRED GDP Data (in Billions of USD)") + plt.xlim(data.index[0], data.index[-1]) + plt.ylabel(f"{string_to_get}") plt.grid(b=True, which="major", color="#666666", linestyle="-") plt.minorticks_on() plt.grid(b=True, which="minor", color="#999999", linestyle="-", alpha=0.2) + plt.title(title) plt.show() print("") + if gtff.USE_ION: + plt.ion() + + except SystemExit: + print("") except Exception as e: print(e) print("") @@ -114,7 +131,7 @@ def custom_data(l_args): "-s", dest="start_date", type=valid_date, - default="2015-01-01", + default="2019-01-01", required=False, help="Starting date (YYYY-MM-DD) of data", ) diff --git a/gamestonk_terminal/fred/fred_menu.py b/gamestonk_terminal/fred/fred_menu.py index 6c554fd0322..1c0137d740b 100644 --- a/gamestonk_terminal/fred/fred_menu.py +++ b/gamestonk_terminal/fred/fred_menu.py @@ -14,15 +14,36 @@ def print_fred(): print(" q quit this menu, and shows back to main menu") print(" quit quit to abandon program") print(" ") - print(" gdp get GDP") - print(" cust User Specified FRED Data") + print(" gdp GDP") + print(" unemp Unemployment Rate") + print(" t01 1-Year Treasury Constant Maturity Rate") + print(" t05 5-Year Treasury Constant Maturity Rate") + print(" t10 10-Year Treasury Constant Maturity Rate") + print(" t30 30-Year Treasury Constant Maturity Rate") + print(" mort30 30-Year Fixed Rate Mortgage Average") + print(" libor3m 3-Month LIBOR, based on U.S. Dollar") + print(" moodAAA Moody's Seasoned AAA Corporate Bond Yield") + print("") + print(" cust User Specified FRED Data - Please Specify --id") print("") return def fred_menu(): fred_parser = argparse.ArgumentParser(prog="fred", add_help=False) - choices = ["help", "q", "quit", "gdp", "cust"] + defined_choices = [ + "gdp", + "unemp", + "t01", + "t05", + "t10", + "t30", + "mort30", + "libor3m", + "moodAAA", + ] + choices = ["help", "q", "quit", "cust"] + defined_choices + fred_parser.add_argument("cmd", choices=choices) completer = NestedCompleter.from_nested_dict({c: None for c in choices}) @@ -58,8 +79,8 @@ def fred_menu(): # Abandon the program return True - elif ns_known_args.cmd == "gdp": - fred_api.get_GDP(l_args) + elif ns_known_args.cmd in defined_choices: + fred_api.get_fred_data(l_args, ns_known_args.cmd) elif ns_known_args.cmd == "cust": fred_api.custom_data(l_args) From b862b2e853c1b9d036b0771ad0415e36e16c6a19 Mon Sep 17 00:00:00 2001 From: Laurent Andriamifidy Date: Wed, 17 Mar 2021 19:22:39 +0100 Subject: [PATCH 36/64] open interest start open interest work graph function added menu command added --- gamestonk_terminal/options/op_menu.py | 6 +++- gamestonk_terminal/options/volume.py | 45 +++++++++++++++++++++++---- 2 files changed, 44 insertions(+), 7 deletions(-) diff --git a/gamestonk_terminal/options/op_menu.py b/gamestonk_terminal/options/op_menu.py index 0522a9b9a5a..3ecda13fd2d 100644 --- a/gamestonk_terminal/options/op_menu.py +++ b/gamestonk_terminal/options/op_menu.py @@ -14,7 +14,8 @@ def opt_menu(df_stock, s_ticker, s_start, s_interval): "help", "q", "quit", - "volume" + "volume", + "oi" ] opt_parser.add_argument("cmd", choices=choices) completer = NestedCompleter.from_nested_dict({c: None for c in choices}) @@ -57,5 +58,8 @@ def opt_menu(df_stock, s_ticker, s_start, s_interval): # call the volume graph vol.volume_graph(l_args, s_ticker) + elif ns_known_args.cmd == "oi": + # call the volume graph + vol.open_interest_graph(l_args, s_ticker) else: print("Command not recognized!") diff --git a/gamestonk_terminal/options/volume.py b/gamestonk_terminal/options/volume.py index 14d71ced9ab..95fbef54822 100644 --- a/gamestonk_terminal/options/volume.py +++ b/gamestonk_terminal/options/volume.py @@ -7,7 +7,33 @@ import argparse import datetime from gamestonk_terminal.helper_funcs import get_flair, parse_known_args_and_warn, valid_date +def open_interest_graph(l_args, s_ticker): + """ Show traded options volume. [Source: Yahoo Finance] """ + parser = argparse.ArgumentParser( + add_help=False, + prog="volume", + description="""Display volume graph for a date. [Source: Yahoo Finance].""", + ) + parser.add_argument( + "-e", + "--expiry", + type=valid_date, + dest="s_expiry_date", + help="The expiry date (format YYYY-MM-DD) for the option chain", + required=True + ) + l_similar = [] + try: + ns_parser = parse_known_args_and_warn(parser, l_args) + if not ns_parser: + return + __get_open_interest_graph(s_ticker, ns_parser.s_expiry_date.strftime("%Y-%m-%d")) + + except SystemExit: + print("") + except Exception as e: + print(e) def volume_graph(l_args, s_ticker): """ Show traded options volume. [Source: Yahoo Finance] """ @@ -37,6 +63,10 @@ def volume_graph(l_args, s_ticker): except Exception as e: print(e) +def __get_open_interest_graph(ticker_name, exp_date): + df = __get_volume_data(ticker_name, exp_date) + __generate_graph_sns(df, ticker_name, exp_date, True) + #__generate_graph_plotly(df, ticker_name, exp_date) def __get_volume_graph(ticker_name, exp_date): @@ -81,30 +111,33 @@ def __get_volume_data(ticker_name, exp_date): #dataframe return volume_data -def __generate_graph_plotly(df, ticker_name, exp_date): +def __generate_graph_plotly(df, ticker_name, exp_date, for_open_interest = False): #version with plotly express + op_type = 'openInterest'if for_open_interest else 'volume' + fig = px.line( df, x="strike", - y="volume", - title=f'{ticker_name} options volume for {exp_date}', + y=op_type, + title=f'{ticker_name} options {op_type} for {exp_date}', color= 'type' ) fig.show() return -def __generate_graph_sns(df, ticker_name, exp_date): +def __generate_graph_sns(df, ticker_name, exp_date, for_open_interest = False): #version with seaborn express + op_type = 'openInterest'if for_open_interest else 'volume' plt.figure(figsize=(12,6)) fig = sns.lineplot( data = df, x = 'strike', - y = 'volume', + y = 'op_type', hue = 'type', palette= ['limegreen', 'tomato']) - plt.title(f'{ticker_name} options volume for {exp_date}') + plt.title(f'{ticker_name} options {op_type} for {exp_date}') plt.show() return From 2d3e5a38dcebac9639da88e9944282c30517e1c4 Mon Sep 17 00:00:00 2001 From: Elisha Hollander Date: Wed, 17 Mar 2021 20:34:31 +0200 Subject: [PATCH 37/64] Remove unnecessary variable b_quit is never used in this case, since the function has already returned --- terminal.py | 1 - 1 file changed, 1 deletion(-) diff --git a/terminal.py b/terminal.py index 4969ab67c9d..48c7e7cab05 100644 --- a/terminal.py +++ b/terminal.py @@ -224,7 +224,6 @@ def main(): except Exception as e: print(e) print("Either the ticker or the API_KEY are invalids. Try again!") - b_quit = False return else: From 22ddc12f4017deebd00e08a322ab3be7b9804938 Mon Sep 17 00:00:00 2001 From: Elisha Hollander Date: Wed, 17 Mar 2021 20:37:03 +0200 Subject: [PATCH 38/64] Remove unnecessary variable n_next_idx is defined as 0 and right afterwards it becomes the first value of an enumerated object --- gamestonk_terminal/sentiment/twitter_api.py | 1 - 1 file changed, 1 deletion(-) diff --git a/gamestonk_terminal/sentiment/twitter_api.py b/gamestonk_terminal/sentiment/twitter_api.py index dcad4037e8d..ca31ee737a7 100644 --- a/gamestonk_terminal/sentiment/twitter_api.py +++ b/gamestonk_terminal/sentiment/twitter_api.py @@ -357,7 +357,6 @@ def sentiment(l_args, s_ticker): df_tweets["created_at"].values[0], "%Y-%m-%d %H:%M:%S" ).day n_idx = 0 - n_next_idx = 0 for n_next_idx, dt_created in enumerate(df_tweets["created_at"]): if datetime.strptime(dt_created, "%Y-%m-%d %H:%M:%S").day > n_day: l_xticks.append(n_next_idx) From b78e2cbda8e648047dad306ef74742b997ce4d04 Mon Sep 17 00:00:00 2001 From: Laurent Andriamifidy Date: Wed, 17 Mar 2021 19:47:15 +0100 Subject: [PATCH 39/64] Update volume.py cleanup --- gamestonk_terminal/options/volume.py | 1 - 1 file changed, 1 deletion(-) diff --git a/gamestonk_terminal/options/volume.py b/gamestonk_terminal/options/volume.py index 95fbef54822..2456e199199 100644 --- a/gamestonk_terminal/options/volume.py +++ b/gamestonk_terminal/options/volume.py @@ -68,7 +68,6 @@ def __get_open_interest_graph(ticker_name, exp_date): __generate_graph_sns(df, ticker_name, exp_date, True) #__generate_graph_plotly(df, ticker_name, exp_date) - def __get_volume_graph(ticker_name, exp_date): df = __get_volume_data(ticker_name, exp_date) __generate_graph_sns(df, ticker_name, exp_date) From 69745672605f841fef97268c40da32bbc45d9575 Mon Sep 17 00:00:00 2001 From: Jamie Date: Wed, 17 Mar 2021 14:50:05 -0400 Subject: [PATCH 40/64] change options + README --- gamestonk_terminal/fred/README.md | 9 +++++++-- gamestonk_terminal/fred/fred_api.py | 18 ++++++++++-------- gamestonk_terminal/fred/fred_menu.py | 16 ++++++++++------ 3 files changed, 27 insertions(+), 16 deletions(-) diff --git a/gamestonk_terminal/fred/README.md b/gamestonk_terminal/fred/README.md index 51b71a957fb..f781a1ac1e4 100644 --- a/gamestonk_terminal/fred/README.md +++ b/gamestonk_terminal/fred/README.md @@ -14,8 +14,9 @@ The current predefined options are * 1,5,10,30 Year Treasury Rates * 30 Year Mortgage Rates * Unemployment Rate -* 3 Month LIBOR +* Effective Federal Funds rate * Moody's AAA Corporate Bond +* CAD to USD Conversion. ## predefined ```text @@ -31,6 +32,9 @@ To plot the 30-Year Mortgage rates since 2018-01-01 suppressing the output. ```` mort30 -s 2018-01-01 --hidedata ```` + +![mort30](https://user-images.githubusercontent.com/18151143/111521185-3d80dc80-872f-11eb-989d-c2bb256aa213.png) + To show all GDP data since 1/1/2016, with no plot. ```` gdp -s 2016-01-01 --noplot @@ -78,9 +82,10 @@ An example of custom data could be the USD/EURO exchange rate. According to https://fred.stlouisfed.org/series/DEXUSEU , the series ID is DEXUSEU. So to plot this from the start of February 2021 to now (3/17) ```` -cust --id UNRATE -s 2021-02-01 +cust --id DEXUSEU -s 2021-02-01 ```` This will plot and display: +![cust](https://user-images.githubusercontent.com/18151143/111521481-92bcee00-872f-11eb-8a02-e7346e56e462.png) ```` DEXUSEU diff --git a/gamestonk_terminal/fred/fred_api.py b/gamestonk_terminal/fred/fred_api.py index 490c152c7f8..1b90ca10e87 100644 --- a/gamestonk_terminal/fred/fred_api.py +++ b/gamestonk_terminal/fred/fred_api.py @@ -17,23 +17,25 @@ register_matplotlib_converters() api_map = { "gdp": "GDP", "t10": "DGS10", - "t01": "DGS1", - "t05": "DGS5", + "t1": "DGS1", + "t5": "DGS5", "t30": "DGS30", "mort30": "MORTGAGE30US", - "libor3m": "USD3MTD156N", + "fedrate": "FEDFUNDS", "moodAAA": "AAA", + "usdcad": "DEXCAUS", "unemp": "UNRATE", } title_map = { - "gdp": "GDP", + "gdp": "Gross Domestic Product", "t10": "10-Year Treasury Constant Maturity Rate", - "t01": "1-Year Treasury Constant Maturity Rate", - "t05": "5-Year Treasury Constant Maturity Rate", + "t1": "1-Year Treasury Constant Maturity Rate", + "t5": "5-Year Treasury Constant Maturity Rate", "t30": "30-Year Treasury Constant Maturity Rate", - "mort30": "30-Year Treasury Constant Maturity Rate", - "libor3m": "3-Month LIBOR, based on U.S. Dollar", + "mort30": "30-Year Mortgage Rate", + "fedrate": "Effective Federal Funds Rate", "moodAAA": "Moody's Seasoned AAA Corporate Bond Yield", + "usdcad": "Canada / U.S. Foreign Exchange Rate", "unemp": "Unemployment Rate", } diff --git a/gamestonk_terminal/fred/fred_menu.py b/gamestonk_terminal/fred/fred_menu.py index 1c0137d740b..5a7734835be 100644 --- a/gamestonk_terminal/fred/fred_menu.py +++ b/gamestonk_terminal/fred/fred_menu.py @@ -1,4 +1,5 @@ import argparse +from matplotlib import pyplot as plt from prompt_toolkit.completion import NestedCompleter from gamestonk_terminal import feature_flags as gtff from gamestonk_terminal.helper_funcs import get_flair @@ -16,13 +17,14 @@ def print_fred(): print(" ") print(" gdp GDP") print(" unemp Unemployment Rate") - print(" t01 1-Year Treasury Constant Maturity Rate") - print(" t05 5-Year Treasury Constant Maturity Rate") + print(" t1 1-Year Treasury Constant Maturity Rate") + print(" t5 5-Year Treasury Constant Maturity Rate") print(" t10 10-Year Treasury Constant Maturity Rate") print(" t30 30-Year Treasury Constant Maturity Rate") print(" mort30 30-Year Fixed Rate Mortgage Average") - print(" libor3m 3-Month LIBOR, based on U.S. Dollar") + print(" fedrate Effective Federal Funds Rate") print(" moodAAA Moody's Seasoned AAA Corporate Bond Yield") + print(" usdcad Canada / U.S. Foreign Exchange Rate (CAD per 1 USD)") print("") print(" cust User Specified FRED Data - Please Specify --id") print("") @@ -30,17 +32,19 @@ def print_fred(): def fred_menu(): + plt.close("all") fred_parser = argparse.ArgumentParser(prog="fred", add_help=False) defined_choices = [ "gdp", "unemp", - "t01", - "t05", + "t1", + "t5", "t10", "t30", "mort30", - "libor3m", + "fedrate", "moodAAA", + "usdcad", ] choices = ["help", "q", "quit", "cust"] + defined_choices From ccf4ce6c5d83b2a2717ec019049a218a6ff92b6f Mon Sep 17 00:00:00 2001 From: Laurent Andriamifidy Date: Wed, 17 Mar 2021 19:51:29 +0100 Subject: [PATCH 41/64] housekeeping --- gamestonk_terminal/options/op_menu.py | 8 ++-- gamestonk_terminal/options/volume.py | 54 ++++++++++++++++----------- 2 files changed, 36 insertions(+), 26 deletions(-) diff --git a/gamestonk_terminal/options/op_menu.py b/gamestonk_terminal/options/op_menu.py index 3ecda13fd2d..7b973c97c11 100644 --- a/gamestonk_terminal/options/op_menu.py +++ b/gamestonk_terminal/options/op_menu.py @@ -6,6 +6,7 @@ from gamestonk_terminal.helper_funcs import get_flair from gamestonk_terminal.options import volume as vol from gamestonk_terminal.menu import session + def opt_menu(df_stock, s_ticker, s_start, s_interval): # Add list of arguments that the options parser accepts @@ -33,7 +34,6 @@ def opt_menu(df_stock, s_ticker, s_start, s_interval): else: as_input = input(f"{get_flair()} (opt)> ") - # Parse fundamental analysis command of the list of possible commands try: (ns_known_args, l_args) = opt_parser.parse_known_args(as_input.split()) @@ -44,7 +44,7 @@ def opt_menu(df_stock, s_ticker, s_start, s_interval): if ns_known_args.cmd == "help": pass - #print_options(s_ticker) + # print_options(s_ticker) elif ns_known_args.cmd == "q": # Just leave the options menu @@ -55,11 +55,11 @@ def opt_menu(df_stock, s_ticker, s_start, s_interval): return True elif ns_known_args.cmd == "volume": - # call the volume graph + # call the volume graph vol.volume_graph(l_args, s_ticker) elif ns_known_args.cmd == "oi": - # call the volume graph + # call the volume graph vol.open_interest_graph(l_args, s_ticker) else: print("Command not recognized!") diff --git a/gamestonk_terminal/options/volume.py b/gamestonk_terminal/options/volume.py index 2456e199199..d9eba64305f 100644 --- a/gamestonk_terminal/options/volume.py +++ b/gamestonk_terminal/options/volume.py @@ -7,6 +7,7 @@ import argparse import datetime from gamestonk_terminal.helper_funcs import get_flair, parse_known_args_and_warn, valid_date + def open_interest_graph(l_args, s_ticker): """ Show traded options volume. [Source: Yahoo Finance] """ parser = argparse.ArgumentParser( @@ -28,13 +29,15 @@ def open_interest_graph(l_args, s_ticker): if not ns_parser: return - __get_open_interest_graph(s_ticker, ns_parser.s_expiry_date.strftime("%Y-%m-%d")) + __get_open_interest_graph( + s_ticker, ns_parser.s_expiry_date.strftime("%Y-%m-%d")) except SystemExit: - print("") + print("") except Exception as e: print(e) + def volume_graph(l_args, s_ticker): """ Show traded options volume. [Source: Yahoo Finance] """ parser = argparse.ArgumentParser( @@ -56,27 +59,31 @@ def volume_graph(l_args, s_ticker): if not ns_parser: return - __get_volume_graph(s_ticker, ns_parser.s_expiry_date.strftime("%Y-%m-%d")) + __get_volume_graph( + s_ticker, ns_parser.s_expiry_date.strftime("%Y-%m-%d")) except SystemExit: - print("") + print("") except Exception as e: print(e) + def __get_open_interest_graph(ticker_name, exp_date): df = __get_volume_data(ticker_name, exp_date) __generate_graph_sns(df, ticker_name, exp_date, True) #__generate_graph_plotly(df, ticker_name, exp_date) + def __get_volume_graph(ticker_name, exp_date): df = __get_volume_data(ticker_name, exp_date) __generate_graph_sns(df, ticker_name, exp_date) #__generate_graph_plotly(df, ticker_name, exp_date) + def __pull_call_put_data(call_put, flag): df = call_put.pivot_table( index='strike', - values = ['volume','openInterest'], + values=['volume', 'openInterest'], aggfunc='sum') df.reindex() @@ -86,6 +93,7 @@ def __pull_call_put_data(call_put, flag): return df + def __get_volume_data(ticker_name, exp_date): option_chain = yf.Ticker(ticker_name).option_chain(exp_date) @@ -93,25 +101,26 @@ def __get_volume_data(ticker_name, exp_date): calls = __pull_call_put_data( option_chain.calls, 'calls' - ) + ) puts = __pull_call_put_data( option_chain.puts, 'puts' - ) + ) volume_data = pd.concat( [ calls, puts ], - axis = 0 - ) - #dataframe + axis=0 + ) + # dataframe return volume_data -def __generate_graph_plotly(df, ticker_name, exp_date, for_open_interest = False): - #version with plotly express + +def __generate_graph_plotly(df, ticker_name, exp_date, for_open_interest=False): + # version with plotly express op_type = 'openInterest'if for_open_interest else 'volume' fig = px.line( @@ -119,22 +128,23 @@ def __generate_graph_plotly(df, ticker_name, exp_date, for_open_interest = False x="strike", y=op_type, title=f'{ticker_name} options {op_type} for {exp_date}', - color= 'type' - ) + color='type' + ) fig.show() return -def __generate_graph_sns(df, ticker_name, exp_date, for_open_interest = False): - #version with seaborn express + +def __generate_graph_sns(df, ticker_name, exp_date, for_open_interest=False): + # version with seaborn express op_type = 'openInterest'if for_open_interest else 'volume' - plt.figure(figsize=(12,6)) + plt.figure(figsize=(12, 6)) fig = sns.lineplot( - data = df, - x = 'strike', - y = 'op_type', - hue = 'type', - palette= ['limegreen', 'tomato']) + data=df, + x='strike', + y='op_type', + hue='type', + palette=['limegreen', 'tomato']) plt.title(f'{ticker_name} options {op_type} for {exp_date}') From 37588f438321d950de6659d836aa27305ac0a27c Mon Sep 17 00:00:00 2001 From: Laurent Andriamifidy Date: Wed, 17 Mar 2021 20:40:14 +0100 Subject: [PATCH 42/64] blacked files --- gamestonk_terminal/options/op_menu.py | 10 +---- gamestonk_terminal/options/volume.py | 64 +++++++++++---------------- 2 files changed, 28 insertions(+), 46 deletions(-) diff --git a/gamestonk_terminal/options/op_menu.py b/gamestonk_terminal/options/op_menu.py index 7b973c97c11..6180939dfe1 100644 --- a/gamestonk_terminal/options/op_menu.py +++ b/gamestonk_terminal/options/op_menu.py @@ -11,17 +11,11 @@ def opt_menu(df_stock, s_ticker, s_start, s_interval): # Add list of arguments that the options parser accepts opt_parser = argparse.ArgumentParser(prog="opt", add_help=False) - choices = [ - "help", - "q", - "quit", - "volume", - "oi" - ] + choices = ["help", "q", "quit", "volume", "oi"] opt_parser.add_argument("cmd", choices=choices) completer = NestedCompleter.from_nested_dict({c: None for c in choices}) - #print_options(s_ticker, s_start, s_interval) + # print_options(s_ticker, s_start, s_interval) # Loop forever and ever while True: diff --git a/gamestonk_terminal/options/volume.py b/gamestonk_terminal/options/volume.py index d9eba64305f..c91a6f96295 100644 --- a/gamestonk_terminal/options/volume.py +++ b/gamestonk_terminal/options/volume.py @@ -5,7 +5,11 @@ import matplotlib.pyplot as plt import plotly.express as px import argparse import datetime -from gamestonk_terminal.helper_funcs import get_flair, parse_known_args_and_warn, valid_date +from gamestonk_terminal.helper_funcs import ( + get_flair, + parse_known_args_and_warn, + valid_date, +) def open_interest_graph(l_args, s_ticker): @@ -21,7 +25,7 @@ def open_interest_graph(l_args, s_ticker): type=valid_date, dest="s_expiry_date", help="The expiry date (format YYYY-MM-DD) for the option chain", - required=True + required=True, ) l_similar = [] try: @@ -30,7 +34,8 @@ def open_interest_graph(l_args, s_ticker): return __get_open_interest_graph( - s_ticker, ns_parser.s_expiry_date.strftime("%Y-%m-%d")) + s_ticker, ns_parser.s_expiry_date.strftime("%Y-%m-%d") + ) except SystemExit: print("") @@ -51,7 +56,7 @@ def volume_graph(l_args, s_ticker): type=valid_date, dest="s_expiry_date", help="The expiry date (format YYYY-MM-DD) for the option chain", - required=True + required=True, ) l_similar = [] try: @@ -59,8 +64,7 @@ def volume_graph(l_args, s_ticker): if not ns_parser: return - __get_volume_graph( - s_ticker, ns_parser.s_expiry_date.strftime("%Y-%m-%d")) + __get_volume_graph(s_ticker, ns_parser.s_expiry_date.strftime("%Y-%m-%d")) except SystemExit: print("") @@ -71,25 +75,24 @@ def volume_graph(l_args, s_ticker): def __get_open_interest_graph(ticker_name, exp_date): df = __get_volume_data(ticker_name, exp_date) __generate_graph_sns(df, ticker_name, exp_date, True) - #__generate_graph_plotly(df, ticker_name, exp_date) + # __generate_graph_plotly(df, ticker_name, exp_date) def __get_volume_graph(ticker_name, exp_date): df = __get_volume_data(ticker_name, exp_date) __generate_graph_sns(df, ticker_name, exp_date) - #__generate_graph_plotly(df, ticker_name, exp_date) + # __generate_graph_plotly(df, ticker_name, exp_date) def __pull_call_put_data(call_put, flag): df = call_put.pivot_table( - index='strike', - values=['volume', 'openInterest'], - aggfunc='sum') + index="strike", values=["volume", "openInterest"], aggfunc="sum" + ) df.reindex() - df['strike'] = df.index - df['type'] = flag + df["strike"] = df.index + df["type"] = flag return df @@ -98,37 +101,25 @@ def __get_volume_data(ticker_name, exp_date): option_chain = yf.Ticker(ticker_name).option_chain(exp_date) - calls = __pull_call_put_data( - option_chain.calls, - 'calls' - ) + calls = __pull_call_put_data(option_chain.calls, "calls") - puts = __pull_call_put_data( - option_chain.puts, - 'puts' - ) + puts = __pull_call_put_data(option_chain.puts, "puts") - volume_data = pd.concat( - [ - calls, - puts - ], - axis=0 - ) + volume_data = pd.concat([calls, puts], axis=0) # dataframe return volume_data def __generate_graph_plotly(df, ticker_name, exp_date, for_open_interest=False): # version with plotly express - op_type = 'openInterest'if for_open_interest else 'volume' + op_type = "openInterest" if for_open_interest else "volume" fig = px.line( df, x="strike", y=op_type, - title=f'{ticker_name} options {op_type} for {exp_date}', - color='type' + title=f"{ticker_name} options {op_type} for {exp_date}", + color="type", ) fig.show() @@ -137,16 +128,13 @@ def __generate_graph_plotly(df, ticker_name, exp_date, for_open_interest=False): def __generate_graph_sns(df, ticker_name, exp_date, for_open_interest=False): # version with seaborn express - op_type = 'openInterest'if for_open_interest else 'volume' + op_type = "openInterest" if for_open_interest else "volume" plt.figure(figsize=(12, 6)) fig = sns.lineplot( - data=df, - x='strike', - y='op_type', - hue='type', - palette=['limegreen', 'tomato']) + data=df, x="strike", y="op_type", hue="type", palette=["limegreen", "tomato"] + ) - plt.title(f'{ticker_name} options {op_type} for {exp_date}') + plt.title(f"{ticker_name} options {op_type} for {exp_date}") plt.show() return From e953b8b200b2bb969cd256ba96a149aab04f1aaa Mon Sep 17 00:00:00 2001 From: Laurent Andriamifidy Date: Wed, 17 Mar 2021 20:44:21 +0100 Subject: [PATCH 43/64] blacked --- gamestonk_terminal/main_helper.py | 4 +--- terminal.py | 7 +++---- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/gamestonk_terminal/main_helper.py b/gamestonk_terminal/main_helper.py index f971ca0ad6f..4010e85a204 100644 --- a/gamestonk_terminal/main_helper.py +++ b/gamestonk_terminal/main_helper.py @@ -76,9 +76,7 @@ def print_help(s_ticker, s_start, s_interval, b_is_market_open): print( " pred prediction techniques, \t e.g.: regression, arima, rnn, lstm, prophet" ) - print( - " op options info, \t e.g.: volume" - ) + print(" op options info, \t e.g.: volume") print("") diff --git a/terminal.py b/terminal.py index 318ed535266..5b3c3f83727 100644 --- a/terminal.py +++ b/terminal.py @@ -1,4 +1,4 @@ - #!/usr/bin/env python +#!/usr/bin/env python import argparse @@ -73,8 +73,7 @@ def main(): "dd", "pred", "ca", - "op" - "fred", + "op" "fred", ] menu_parser.add_argument("opt", choices=choices) completer = NestedCompleter.from_nested_dict({c: None for c in choices}) @@ -184,7 +183,7 @@ def main(): elif ns_known_args.opt == "op": b_quit = opm.opt_menu(df_stock, s_ticker, s_start, s_interval) - + elif ns_known_args.opt == "fred": b_quit = fm.fred_menu() From add1b99f4abac48ea14047ee3d26bdbb3de22e75 Mon Sep 17 00:00:00 2001 From: Laurent Andriamifidy Date: Wed, 17 Mar 2021 20:51:36 +0100 Subject: [PATCH 44/64] made linter happy --- gamestonk_terminal/options/op_menu.py | 2 +- gamestonk_terminal/options/volume.py | 9 +++------ terminal.py | 5 +++-- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/gamestonk_terminal/options/op_menu.py b/gamestonk_terminal/options/op_menu.py index 6180939dfe1..a386a584ec2 100644 --- a/gamestonk_terminal/options/op_menu.py +++ b/gamestonk_terminal/options/op_menu.py @@ -7,7 +7,7 @@ from gamestonk_terminal.options import volume as vol from gamestonk_terminal.menu import session -def opt_menu(df_stock, s_ticker, s_start, s_interval): +def opt_menu(s_ticker): # Add list of arguments that the options parser accepts opt_parser = argparse.ArgumentParser(prog="opt", add_help=False) diff --git a/gamestonk_terminal/options/volume.py b/gamestonk_terminal/options/volume.py index c91a6f96295..8abf73bd8fa 100644 --- a/gamestonk_terminal/options/volume.py +++ b/gamestonk_terminal/options/volume.py @@ -1,12 +1,10 @@ +import argparse import yfinance as yf import seaborn as sns import pandas as pd import matplotlib.pyplot as plt import plotly.express as px -import argparse -import datetime from gamestonk_terminal.helper_funcs import ( - get_flair, parse_known_args_and_warn, valid_date, ) @@ -27,7 +25,6 @@ def open_interest_graph(l_args, s_ticker): help="The expiry date (format YYYY-MM-DD) for the option chain", required=True, ) - l_similar = [] try: ns_parser = parse_known_args_and_warn(parser, l_args) if not ns_parser: @@ -58,7 +55,7 @@ def volume_graph(l_args, s_ticker): help="The expiry date (format YYYY-MM-DD) for the option chain", required=True, ) - l_similar = [] + try: ns_parser = parse_known_args_and_warn(parser, l_args) if not ns_parser: @@ -137,4 +134,4 @@ def __generate_graph_sns(df, ticker_name, exp_date, for_open_interest=False): plt.title(f"{ticker_name} options {op_type} for {exp_date}") plt.show() - return + return fig diff --git a/terminal.py b/terminal.py index 5b3c3f83727..bb8f8ee39c5 100644 --- a/terminal.py +++ b/terminal.py @@ -73,7 +73,8 @@ def main(): "dd", "pred", "ca", - "op" "fred", + "op", + "fred", ] menu_parser.add_argument("opt", choices=choices) completer = NestedCompleter.from_nested_dict({c: None for c in choices}) @@ -182,7 +183,7 @@ def main(): b_quit = ddm.dd_menu(df_stock, s_ticker, s_start, s_interval) elif ns_known_args.opt == "op": - b_quit = opm.opt_menu(df_stock, s_ticker, s_start, s_interval) + b_quit = opm.opt_menu(s_ticker) elif ns_known_args.opt == "fred": b_quit = fm.fred_menu() From 3d3c3a06d50456063f59cab43e0b72ddab4499a0 Mon Sep 17 00:00:00 2001 From: Laurent Andriamifidy Date: Wed, 17 Mar 2021 21:06:36 +0100 Subject: [PATCH 45/64] fixed op_type flag value --- gamestonk_terminal/options/volume.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/gamestonk_terminal/options/volume.py b/gamestonk_terminal/options/volume.py index 8abf73bd8fa..14440ef101e 100644 --- a/gamestonk_terminal/options/volume.py +++ b/gamestonk_terminal/options/volume.py @@ -109,7 +109,10 @@ def __get_volume_data(ticker_name, exp_date): def __generate_graph_plotly(df, ticker_name, exp_date, for_open_interest=False): # version with plotly express - op_type = "openInterest" if for_open_interest else "volume" + if for_open_interest: + op_type = "openInterest" + else: + op_type = "volume" fig = px.line( df, @@ -125,10 +128,14 @@ def __generate_graph_plotly(df, ticker_name, exp_date, for_open_interest=False): def __generate_graph_sns(df, ticker_name, exp_date, for_open_interest=False): # version with seaborn express - op_type = "openInterest" if for_open_interest else "volume" + if for_open_interest: + op_type = "openInterest" + else: + op_type = "volume" + plt.figure(figsize=(12, 6)) fig = sns.lineplot( - data=df, x="strike", y="op_type", hue="type", palette=["limegreen", "tomato"] + data=df, x="strike", y=op_type, hue="type", palette=["limegreen", "tomato"] ) plt.title(f"{ticker_name} options {op_type} for {exp_date}") From b4c36b8df130f85166d4a0a4e6a680f962bb1791 Mon Sep 17 00:00:00 2001 From: didier Date: Wed, 17 Mar 2021 21:43:01 +0000 Subject: [PATCH 46/64] Refactor sentiment menu --- gamestonk_terminal/README.md | 4 +- .../README.md | 14 +- .../__init__.py | 0 .../behavioural_analysis/ba_controller.py | 235 ++++++++++++++++++ .../google_view.py} | 0 .../reddit_view.py} | 1 + .../sen_menu.py | 0 .../stocktwits_view.py} | 0 .../twitter_view.py} | 1 + gamestonk_terminal/main_helper.py | 6 +- terminal.py | 8 +- 11 files changed, 253 insertions(+), 16 deletions(-) rename gamestonk_terminal/{sentiment => behavioural_analysis}/README.md (95%) rename gamestonk_terminal/{sentiment => behavioural_analysis}/__init__.py (100%) create mode 100644 gamestonk_terminal/behavioural_analysis/ba_controller.py rename gamestonk_terminal/{sentiment/google_api.py => behavioural_analysis/google_view.py} (100%) rename gamestonk_terminal/{sentiment/reddit_api.py => behavioural_analysis/reddit_view.py} (99%) rename gamestonk_terminal/{sentiment => behavioural_analysis}/sen_menu.py (100%) rename gamestonk_terminal/{sentiment/stocktwits_api.py => behavioural_analysis/stocktwits_view.py} (100%) rename gamestonk_terminal/{sentiment/twitter_api.py => behavioural_analysis/twitter_view.py} (99%) diff --git a/gamestonk_terminal/README.md b/gamestonk_terminal/README.md index aaf9a94e512..208ff4bcd21 100644 --- a/gamestonk_terminal/README.md +++ b/gamestonk_terminal/README.md @@ -2,7 +2,7 @@ ### Table of contents * [Discover Stocks](#Discover-Stocks-) -* [Market Sentiment](#Market-Sentiment-) +* [Behavioural Analysis](#Behavioural-Analysis-) * [Research Web pages](#Research-Web-pages) * [Fundamental Analysis](#Fundamental-Analysis-) * [Technical Analysis](#Technical-Analysis-) @@ -66,7 +66,7 @@ Command|Description|Source   -## Market Sentiment [»](sentiment/README.md) +## Behavioural Analysis [»](behavioural_analysis/README.md) Command|Description ----|---- [Reddit](https://reddit.com)| diff --git a/gamestonk_terminal/sentiment/README.md b/gamestonk_terminal/behavioural_analysis/README.md similarity index 95% rename from gamestonk_terminal/sentiment/README.md rename to gamestonk_terminal/behavioural_analysis/README.md index 1a9e973c128..bdb8fed002f 100644 --- a/gamestonk_terminal/sentiment/README.md +++ b/gamestonk_terminal/behavioural_analysis/README.md @@ -1,6 +1,6 @@ -# MARKET SENTIMENT +# BEHAVIOURAL ANALYSIS -This menu aims to extrapolate market's sentiment regarding a pre-loaded ticker, and the usage of the following commands along with an example will be exploited below. +This menu aims to extrapolate behavioural analysis regarding a pre-loaded ticker, and the usage of the following commands along with an example will be exploited below. [REDDIT](#REDDIT) * [wsb](#wsb) @@ -12,11 +12,11 @@ This menu aims to extrapolate market's sentiment regarding a pre-loaded ticker, * [spac_c](#spac_c) - show other users spacs announcements from subreddit SPACs community * [spac](#spac) - - show other users spacs announcements from other subs + - show other users spacs announcements from other subs [STOCKTWITS](#STOCKTWITS) * [bullbear](#bullbear) - - estimate quick sentiment from last 30 messages on board + - estimate quick sentiment from last 30 messages on board * [messages](#messages) - output up to the 30 last messages on the board * [trending](#trending) @@ -29,7 +29,7 @@ This menu aims to extrapolate market's sentiment regarding a pre-loaded ticker, - infer about stock's sentiment from latest tweets * [sentiment](#sentiment) - in-depth sentiment prediction from tweets over time - + [GOOGLE](#GOOGLE) * [mentions](#mentions) - interest over time based on stock's mentions @@ -106,7 +106,7 @@ usage: bullbear [-t S_TICKER] ``` Print bullbear sentiment based on last 30 messages on the board. Also prints the watchlist_count. [Source: Stocktwits] * -t : ticker to gather sentiment from. - + sentiment @@ -156,7 +156,7 @@ Print quick sentiment inference from last tweets that contain the ticker. This m ``` usage: sentiment [-n N_NUM] [-d N_DAYS_PAST] ``` -Plot in-depth sentiment extracted from tweets from last days that contain pre-defined ticker. This model splits the text into character-level tokens and uses the DistilBERT model to make predictions. DistilBERT is a distilled version of the powerful BERT transformer model. Note that a big num of tweets extracted per hour in conjunction with a high number of days in the past, will make the algorithm take a long period of time to estimate sentiment. Inspired by https://towardsdatascience.com/sentiment-analysis-for-stock-price-prediction-in-python-bed40c65d178. [Source: Twitter] +Plot in-depth sentiment extracted from tweets from last days that contain pre-defined ticker. This model splits the text into character-level tokens and uses the DistilBERT model to make predictions. DistilBERT is a distilled version of the powerful BERT transformer model. Note that a big num of tweets extracted per hour in conjunction with a high number of days in the past, will make the algorithm take a long period of time to estimate sentiment. Inspired by https://towardsdatascience.com/sentiment-analysis-for-stock-price-prediction-in-python-bed40c65d178. [Source: Twitter] * -n : num of tweets to extract per hour. Default 100. * -d : num of days in the past to extract tweets. Default 7. diff --git a/gamestonk_terminal/sentiment/__init__.py b/gamestonk_terminal/behavioural_analysis/__init__.py similarity index 100% rename from gamestonk_terminal/sentiment/__init__.py rename to gamestonk_terminal/behavioural_analysis/__init__.py diff --git a/gamestonk_terminal/behavioural_analysis/ba_controller.py b/gamestonk_terminal/behavioural_analysis/ba_controller.py new file mode 100644 index 00000000000..6a204c82f26 --- /dev/null +++ b/gamestonk_terminal/behavioural_analysis/ba_controller.py @@ -0,0 +1,235 @@ +"""Behavioural Analysis Controller Module""" +__docformat__ = "numpy" + +import argparse +from typing import List +from datetime import datetime + +from gamestonk_terminal import feature_flags as gtff +from gamestonk_terminal.helper_funcs import get_flair +from gamestonk_terminal.menu import session +from gamestonk_terminal.behavioural_analysis import ( + google_view, + reddit_view, + stocktwits_view, +) +from prompt_toolkit.completion import NestedCompleter + + +class BehaviouralAnalysisController: + """Behavioural Analysis Controller class""" + + def __init__(self, ticker: str, start: datetime): + """Constructor""" + self.ticker = ticker + self.start = start + self.ba_parser = argparse.ArgumentParser(add_help=False, prog="ba") + self.ba_parser.add_argument( + "cmd", + choices=[ + "help", + "q", + "quit", + "watchlist", + "spac", + "spac_c", + "wsb", + "popular", + "bullbear", + "messages", + "trending", + "stalker", + "infer", + "sentiment", + "mentions", + "regions", + "queries", + "rise", + ], + ) + + @staticmethod + def print_help(): + """Print help""" + + print("\nBehavioural Analysis:") + print(" help show this behavioural analysis menu again") + print(" q quit this menu, and shows back to main menu") + print(" quit quit to abandon program") + print("") + print("Reddit:") + print( + " wsb show what WSB gang is up to in subreddit wallstreetbets" + ) + print(" watchlist show other users watchlist") + print(" popular show popular tickers") + print( + " spac_c show other users spacs announcements from subreddit SPACs community" + ) + print(" spac show other users spacs announcements from other subs") + print("") + print("Stocktwits:") + print( + " bullbear estimate quick sentiment from last 30 messages on board" + ) + print(" messages output up to the 30 last messages on the board") + print(" trending trending stocks") + print(" stalker stalk stocktwits user's last messages") + print("") + print("Twitter:") + print(" infer infer about stock's sentiment from latest tweets") + print(" sentiment in-depth sentiment prediction from tweets over time") + print("") + print("Google:") + print(" mentions interest over time based on stock's mentions") + print(" regions regions that show highest interest in stock") + print(" queries top related queries with this stock") + print(" rise top rising related queries with stock") + print("") + + def switch(self, an_input: str): + """Process and dispatch input + + Returns + ------- + True, False or None + False - quit the menu + True - quit the program + None - continue in the menu + """ + (known_args, other_args) = self.ba_parser.parse_known_args(an_input.split()) + + return getattr( + self, "call_" + known_args.cmd, lambda: "Command not recognized!" + )(other_args) + + def call_help(self, _): + """Process Help command""" + self.print_help() + + def call_q(self, _): + """Process Q command - quit the menu""" + return False + + def call_quit(self, _): + """Process Quit command - quit the program""" + return True + + def call_watchlist(self, other_args: List[str]): + """Process watchlist command""" + reddit_view.watchlist(other_args) + + def call_spac(self, other_args: List[str]): + """Process spac command""" + reddit_view.spac(other_args) + + def call_spac_c(self, other_args: List[str]): + """Process spac_c command""" + reddit_view.spac_community(other_args) + + def call_wsb(self, other_args: List[str]): + """Process wsb command""" + reddit_view.wsb_community(other_args) + + def call_popular(self, other_args: List[str]): + """Process popular command""" + reddit_view.popular_tickers(other_args) + + def call_bullbear(self, other_args: List[str]): + """Process bullbear command""" + stocktwits_view.bullbear(other_args, self.ticker) + + def call_messages(self, other_args: List[str]): + """Process messages command""" + stocktwits_view.messages(other_args, self.ticker) + + def call_trending(self, other_args: List[str]): + """Process trending command""" + stocktwits_view.trending(other_args) + + def call_stalker(self, other_args: List[str]): + """Process stalker command""" + stocktwits_view.stalker(other_args) + + def call_mentions(self, other_args: List[str]): + """Process mentions command""" + google_view.mentions(other_args, self.ticker, self.start) + + def call_regions(self, other_args: List[str]): + """Process regions command""" + google_view.regions(other_args, self.ticker) + + def call_queries(self, other_args: List[str]): + """Process queries command""" + google_view.queries(other_args, self.ticker) + + def call_rise(self, other_args: List[str]): + """Process rise command""" + google_view.rise(other_args, self.ticker) + + def call_infer(self, other_args: List[str]): + """Process infer command""" + if not gtff.ENABLE_PREDICT: + print("Predict is not enabled in feature_flags.py") + print("Twitter inference menu is disabled", "\n") + return + + try: + # pylint: disable=import-outside-toplevel + from gamestonk_terminal.behavioural_analysis import twitter_view + except ModuleNotFoundError as e: + print("Optional packages need to be installed") + print(e, "\n") + return + except Exception as e: + print(e, "\n") + return + + twitter_view.inference(other_args, self.ticker) + + def call_sentiment(self, other_args: List[str]): + """Process sentiment command""" + if not gtff.ENABLE_PREDICT: + print("Predict is not enabled in feature_flags.py") + print("Twitter inference menu is disabled", "\n") + return + + try: + # pylint: disable=import-outside-toplevel + from gamestonk_terminal.behavioural_analysis import twitter_view + except ModuleNotFoundError as e: + print("Optional packages need to be installed") + print(e, "\n") + return + except Exception as e: + print(e, "\n") + return + + twitter_view.sentiment(other_args, self.ticker) + + +def menu(ticker: str, start: datetime): + """Behavioural Analysis Menu""" + + ba_controller = BehaviouralAnalysisController(ticker, start) + ba_controller.call_help(None) + + while True: + # Get input command from user + if session and gtff.USE_PROMPT_TOOLKIT: + an_input = session.prompt( + f"{get_flair()} (ba)> ", + completer=completer, + ) + else: + an_input = input(f"{get_flair()} (ba)> ") + + try: + process_input = ba_controller.switch(an_input) + + if process_input is not None: + return process_input + + except SystemExit: + print("The command selected doesn't exist\n") + continue diff --git a/gamestonk_terminal/sentiment/google_api.py b/gamestonk_terminal/behavioural_analysis/google_view.py similarity index 100% rename from gamestonk_terminal/sentiment/google_api.py rename to gamestonk_terminal/behavioural_analysis/google_view.py diff --git a/gamestonk_terminal/sentiment/reddit_api.py b/gamestonk_terminal/behavioural_analysis/reddit_view.py similarity index 99% rename from gamestonk_terminal/sentiment/reddit_api.py rename to gamestonk_terminal/behavioural_analysis/reddit_view.py index ae51bde4e86..5093b2c4658 100644 --- a/gamestonk_terminal/sentiment/reddit_api.py +++ b/gamestonk_terminal/behavioural_analysis/reddit_view.py @@ -17,6 +17,7 @@ from gamestonk_terminal.reddit_helpers import ( def watchlist(l_args): + # other_args: List[str] parser = argparse.ArgumentParser( add_help=False, prog="watchlist", diff --git a/gamestonk_terminal/sentiment/sen_menu.py b/gamestonk_terminal/behavioural_analysis/sen_menu.py similarity index 100% rename from gamestonk_terminal/sentiment/sen_menu.py rename to gamestonk_terminal/behavioural_analysis/sen_menu.py diff --git a/gamestonk_terminal/sentiment/stocktwits_api.py b/gamestonk_terminal/behavioural_analysis/stocktwits_view.py similarity index 100% rename from gamestonk_terminal/sentiment/stocktwits_api.py rename to gamestonk_terminal/behavioural_analysis/stocktwits_view.py diff --git a/gamestonk_terminal/sentiment/twitter_api.py b/gamestonk_terminal/behavioural_analysis/twitter_view.py similarity index 99% rename from gamestonk_terminal/sentiment/twitter_api.py rename to gamestonk_terminal/behavioural_analysis/twitter_view.py index ca31ee737a7..dcad4037e8d 100644 --- a/gamestonk_terminal/sentiment/twitter_api.py +++ b/gamestonk_terminal/behavioural_analysis/twitter_view.py @@ -357,6 +357,7 @@ def sentiment(l_args, s_ticker): df_tweets["created_at"].values[0], "%Y-%m-%d %H:%M:%S" ).day n_idx = 0 + n_next_idx = 0 for n_next_idx, dt_created in enumerate(df_tweets["created_at"]): if datetime.strptime(dt_created, "%Y-%m-%d %H:%M:%S").day > n_day: l_xticks.append(n_next_idx) diff --git a/gamestonk_terminal/main_helper.py b/gamestonk_terminal/main_helper.py index 78272a3b092..ce0b0e4d46c 100644 --- a/gamestonk_terminal/main_helper.py +++ b/gamestonk_terminal/main_helper.py @@ -50,14 +50,14 @@ def print_help(s_ticker, s_start, s_interval, b_is_market_open): " disc discover trending stocks, \t e.g. map, sectors, high short interest" ) print(" mill papermill menu, \t\t\t menu to generate notebook reports") - print( - " sen sentiment of the market, \t from: reddit, stocktwits, twitter" - ) print( " fred economic data, \t\t\t from: Federal Reserve Bank of St. Louis " ) if s_ticker: + print( + " ba behavioural analysis, \t\t\t from: reddit, stocktwits, twitter" + ) print( " res research web page, \t e.g.: macroaxis, yahoo finance, fool" ) diff --git a/terminal.py b/terminal.py index 7fdcdf2f797..4e826a1e686 100644 --- a/terminal.py +++ b/terminal.py @@ -19,7 +19,7 @@ from gamestonk_terminal.helper_funcs import b_is_stock_market_open, get_flair from gamestonk_terminal.main_helper import clear, export, load, print_help, view, candle from gamestonk_terminal.menu import session from gamestonk_terminal.papermill import papermill_controller as mill -from gamestonk_terminal.sentiment import sen_menu as sm +from gamestonk_terminal.behavioural_analysis import ba_controller from gamestonk_terminal.technical_analysis import ta_menu as tam from gamestonk_terminal.comparison_analysis import ca_menu as cam from gamestonk_terminal.fred import fred_menu as fm @@ -65,7 +65,7 @@ def main(): "export", "disc", "mill", - "sen", + "ba", "res", "fa", "ta", @@ -162,8 +162,8 @@ def main(): elif ns_known_args.opt == "mill": b_quit = mill.papermill_menu() - elif ns_known_args.opt == "sen": - b_quit = sm.sen_menu(s_ticker, s_start) + elif ns_known_args.opt == "ba": + b_quit = ba_controller.menu(s_ticker, s_start) elif ns_known_args.opt == "res": b_quit = rm.res_menu(s_ticker, s_start, s_interval) From 810d81445ca94ae4723f0ff99570e1273adf82f4 Mon Sep 17 00:00:00 2001 From: DidierRLopes Date: Wed, 17 Mar 2021 21:45:39 +0000 Subject: [PATCH 47/64] Update test.yml for mypy to ignore 'ba' --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 860ceb1ca81..382a74b02e7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -23,7 +23,7 @@ jobs: - run: pip install bandit black codespell flake8 mypy pyupgrade safety pylint - run: bandit -r . || true - run: black --check . - - run: codespell --ignore-words-list="mape" --quiet-level=2 --skip=".git" + - run: codespell --ignore-words-list="mape","ba" --quiet-level=2 --skip=".git" - run: flake8 . --count --ignore=E203,W503 --max-line-length=122 --show-source --statistics - run: mypy --ignore-missing-imports . - run: shopt -s globstar && pyupgrade --py36-plus **/*.py From 1496602d02efb70253bde01663f72f4b1f2f651c Mon Sep 17 00:00:00 2001 From: didier Date: Wed, 17 Mar 2021 21:50:43 +0000 Subject: [PATCH 48/64] fix prompt-toolkit --- .../behavioural_analysis/ba_controller.py | 46 +++++++++++-------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/gamestonk_terminal/behavioural_analysis/ba_controller.py b/gamestonk_terminal/behavioural_analysis/ba_controller.py index 6a204c82f26..20bb0d1c1a4 100644 --- a/gamestonk_terminal/behavioural_analysis/ba_controller.py +++ b/gamestonk_terminal/behavioural_analysis/ba_controller.py @@ -19,6 +19,28 @@ from prompt_toolkit.completion import NestedCompleter class BehaviouralAnalysisController: """Behavioural Analysis Controller class""" + # Command choices + CHOICES = [ + "help", + "q", + "quit", + "watchlist", + "spac", + "spac_c", + "wsb", + "popular", + "bullbear", + "messages", + "trending", + "stalker", + "infer", + "sentiment", + "mentions", + "regions", + "queries", + "rise", + ] + def __init__(self, ticker: str, start: datetime): """Constructor""" self.ticker = ticker @@ -26,26 +48,7 @@ class BehaviouralAnalysisController: self.ba_parser = argparse.ArgumentParser(add_help=False, prog="ba") self.ba_parser.add_argument( "cmd", - choices=[ - "help", - "q", - "quit", - "watchlist", - "spac", - "spac_c", - "wsb", - "popular", - "bullbear", - "messages", - "trending", - "stalker", - "infer", - "sentiment", - "mentions", - "regions", - "queries", - "rise", - ], + choices=self.CHOICES, ) @staticmethod @@ -217,6 +220,9 @@ def menu(ticker: str, start: datetime): while True: # Get input command from user if session and gtff.USE_PROMPT_TOOLKIT: + completer = NestedCompleter.from_nested_dict( + {c: None for c in ba_controller.CHOICES} + ) an_input = session.prompt( f"{get_flair()} (ba)> ", completer=completer, From 470f8e6e7d4f48f2a2cce6c35ad2118007ccc944 Mon Sep 17 00:00:00 2001 From: DidierRLopes Date: Wed, 17 Mar 2021 21:57:13 +0000 Subject: [PATCH 49/64] Update test.yml --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 860ceb1ca81..382a74b02e7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -23,7 +23,7 @@ jobs: - run: pip install bandit black codespell flake8 mypy pyupgrade safety pylint - run: bandit -r . || true - run: black --check . - - run: codespell --ignore-words-list="mape" --quiet-level=2 --skip=".git" + - run: codespell --ignore-words-list="mape","ba" --quiet-level=2 --skip=".git" - run: flake8 . --count --ignore=E203,W503 --max-line-length=122 --show-source --statistics - run: mypy --ignore-missing-imports . - run: shopt -s globstar && pyupgrade --py36-plus **/*.py From 478406332f3c4f78fee91a3406fb215d42451928 Mon Sep 17 00:00:00 2001 From: didier Date: Wed, 17 Mar 2021 22:07:54 +0000 Subject: [PATCH 50/64] fix init --- gamestonk_terminal/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gamestonk_terminal/__init__.py b/gamestonk_terminal/__init__.py index f46858e0d22..bc157798fe5 100644 --- a/gamestonk_terminal/__init__.py +++ b/gamestonk_terminal/__init__.py @@ -4,5 +4,5 @@ from . import due_diligence from . import fundamental_analysis from . import papermill from . import prediction_techniques -from . import sentiment +from . import behavioural_analysis from . import technical_analysis From accd72514e15203cc192126c04e650ff19029dec Mon Sep 17 00:00:00 2001 From: didier Date: Wed, 17 Mar 2021 22:10:11 +0000 Subject: [PATCH 51/64] fix test import --- tests/test_sentiment_twitter_api.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_sentiment_twitter_api.py b/tests/test_sentiment_twitter_api.py index fa536db0754..ffe00494486 100644 --- a/tests/test_sentiment_twitter_api.py +++ b/tests/test_sentiment_twitter_api.py @@ -4,7 +4,10 @@ import unittest # pylint: disable=unused-import try: - from gamestonk_terminal.sentiment.twitter_api import inference, sentiment + from gamestonk_terminal.behavioural_analysis.twitter_view import ( + inference, + sentiment, + ) except ModuleNotFoundError as e: print("One of the optional packages seems to be missing") print(e) From 70fa023b9efe2a567ec3e6c61af2ea08cfe0a1d0 Mon Sep 17 00:00:00 2001 From: DidierRLopes Date: Wed, 17 Mar 2021 22:17:59 +0000 Subject: [PATCH 52/64] Update twitter_view to remove unused variable --- gamestonk_terminal/behavioural_analysis/twitter_view.py | 1 - 1 file changed, 1 deletion(-) diff --git a/gamestonk_terminal/behavioural_analysis/twitter_view.py b/gamestonk_terminal/behavioural_analysis/twitter_view.py index dcad4037e8d..ca31ee737a7 100644 --- a/gamestonk_terminal/behavioural_analysis/twitter_view.py +++ b/gamestonk_terminal/behavioural_analysis/twitter_view.py @@ -357,7 +357,6 @@ def sentiment(l_args, s_ticker): df_tweets["created_at"].values[0], "%Y-%m-%d %H:%M:%S" ).day n_idx = 0 - n_next_idx = 0 for n_next_idx, dt_created in enumerate(df_tweets["created_at"]): if datetime.strptime(dt_created, "%Y-%m-%d %H:%M:%S").day > n_day: l_xticks.append(n_next_idx) From 92f6204317bbba480febe96ac3c0eccc14287eb5 Mon Sep 17 00:00:00 2001 From: DidierRLopes Date: Wed, 17 Mar 2021 22:18:43 +0000 Subject: [PATCH 53/64] remove typo --- gamestonk_terminal/behavioural_analysis/reddit_view.py | 1 - 1 file changed, 1 deletion(-) diff --git a/gamestonk_terminal/behavioural_analysis/reddit_view.py b/gamestonk_terminal/behavioural_analysis/reddit_view.py index 5093b2c4658..ae51bde4e86 100644 --- a/gamestonk_terminal/behavioural_analysis/reddit_view.py +++ b/gamestonk_terminal/behavioural_analysis/reddit_view.py @@ -17,7 +17,6 @@ from gamestonk_terminal.reddit_helpers import ( def watchlist(l_args): - # other_args: List[str] parser = argparse.ArgumentParser( add_help=False, prog="watchlist", From 858967667efca2069486fe232968959398790abe Mon Sep 17 00:00:00 2001 From: didier Date: Wed, 17 Mar 2021 23:55:45 +0000 Subject: [PATCH 54/64] remove old sentiment menu --- .../behavioural_analysis/sen_menu.py | 192 ------------------ 1 file changed, 192 deletions(-) delete mode 100644 gamestonk_terminal/behavioural_analysis/sen_menu.py diff --git a/gamestonk_terminal/behavioural_analysis/sen_menu.py b/gamestonk_terminal/behavioural_analysis/sen_menu.py deleted file mode 100644 index c35cf5df95e..00000000000 --- a/gamestonk_terminal/behavioural_analysis/sen_menu.py +++ /dev/null @@ -1,192 +0,0 @@ -import argparse - -from gamestonk_terminal import feature_flags as gtff -from gamestonk_terminal.helper_funcs import get_flair -from gamestonk_terminal.menu import session -from gamestonk_terminal.sentiment import google_api, reddit_api, stocktwits_api -from prompt_toolkit.completion import NestedCompleter - - -def print_sentiment(): - """ Print help """ - - print("\nSentiment:") - print(" help show this sentiment menu again") - print(" q quit this menu, and shows back to main menu") - print(" quit quit to abandon program") - print("") - print("Reddit:") - print(" wsb show what WSB gang is up to in subreddit wallstreetbets") - print(" watchlist show other users watchlist") - print(" popular show popular tickers") - print( - " spac_c show other users spacs announcements from subreddit SPACs community" - ) - print(" spac show other users spacs announcements from other subs") - print("") - print("Stocktwits:") - print(" bullbear estimate quick sentiment from last 30 messages on board") - print(" messages output up to the 30 last messages on the board") - print(" trending trending stocks") - print(" stalker stalk stocktwits user's last messages") - print("") - print("Twitter:") - print(" infer infer about stock's sentiment from latest tweets") - print(" sentiment in-depth sentiment prediction from tweets over time") - print("") - print("Google:") - print(" mentions interest over time based on stock's mentions") - print(" regions regions that show highest interest in stock") - print(" queries top related queries with this stock") - print(" rise top rising related queries with stock") - print("") - - return - - -def sen_menu(s_ticker, s_start): - - # Add list of arguments that the discovery parser accepts - sen_parser = argparse.ArgumentParser(prog="sen", add_help=False) - choices = [ - "help", - "q", - "quit", - "watchlist", - "spac", - "spac_c", - "wsb", - "popular", - "bullbear", - "messages", - "trending", - "stalker", - "infer", - "sentiment", - "mentions", - "regions", - "queries", - "rise", - ] - sen_parser.add_argument("cmd", choices=choices) - completer = NestedCompleter.from_nested_dict({c: None for c in choices}) - - print_sentiment() - - # Loop forever and ever - while True: - # Get input command from user - if session and gtff.USE_PROMPT_TOOLKIT: - as_input = session.prompt( - f"{get_flair()} (sen)> ", - completer=completer, - ) - else: - as_input = input(f"{get_flair()} (sen)> ") - - # Parse sentiment command of the list of possible commands - try: - (ns_known_args, l_args) = sen_parser.parse_known_args(as_input.split()) - - except SystemExit: - print("The command selected doesn't exist\n") - continue - - if ns_known_args.cmd == "help": - print_sentiment() - - elif ns_known_args.cmd == "q": - # Just leave the DISC menu - return False - - elif ns_known_args.cmd == "quit": - # Abandon the program - return True - - elif ns_known_args.cmd == "watchlist": - reddit_api.watchlist(l_args) - - elif ns_known_args.cmd == "spac": - reddit_api.spac(l_args) - - elif ns_known_args.cmd == "spac_c": - reddit_api.spac_community(l_args) - - elif ns_known_args.cmd == "wsb": - reddit_api.wsb_community(l_args) - - elif ns_known_args.cmd == "popular": - reddit_api.popular_tickers(l_args) - - elif ns_known_args.cmd == "bullbear": - stocktwits_api.bullbear(l_args, s_ticker) - - elif ns_known_args.cmd == "messages": - stocktwits_api.messages(l_args, s_ticker) - - elif ns_known_args.cmd == "trending": - stocktwits_api.trending(l_args) - - elif ns_known_args.cmd == "stalker": - stocktwits_api.stalker(l_args) - - elif ns_known_args.cmd == "infer": - if not gtff.ENABLE_PREDICT: - print("Predict is not enabled in feature_flags.py") - print("Twitter inference menu is disabled") - print("") - continue - - try: - # pylint: disable=import-outside-toplevel - from gamestonk_terminal.sentiment import twitter_api - except ModuleNotFoundError as e: - print("One of the optional packages seems to be missing") - print("Optional packages need to be installed") - print(e) - print("") - continue - except Exception as e: - print(e) - print("") - continue - - twitter_api.inference(l_args, s_ticker) - - elif ns_known_args.cmd == "sentiment": - if not gtff.ENABLE_PREDICT: - print("Predict is not enabled in config_terminal.py") - print("Twitter sentiment menu is disabled") - print("") - continue - - try: - # pylint: disable=import-outside-toplevel - from gamestonk_terminal.sentiment import twitter_api - except ModuleNotFoundError as e: - print("One of the optional packages seems to be missing") - print("Optional packages need to be installed") - print(e) - print("") - continue - except Exception as e: - print(e) - print("") - continue - - twitter_api.sentiment(l_args, s_ticker) - - elif ns_known_args.cmd == "mentions": - google_api.mentions(l_args, s_ticker, s_start) - - elif ns_known_args.cmd == "regions": - google_api.regions(l_args, s_ticker) - - elif ns_known_args.cmd == "queries": - google_api.queries(l_args, s_ticker) - - elif ns_known_args.cmd == "rise": - google_api.rise(l_args, s_ticker) - - else: - print("Command not recognized!") From 4f0d1e3504b031c3c3698f7dcfc7fed02c33fc1d Mon Sep 17 00:00:00 2001 From: didier Date: Thu, 18 Mar 2021 01:15:47 +0000 Subject: [PATCH 55/64] fix identation --- gamestonk_terminal/main_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gamestonk_terminal/main_helper.py b/gamestonk_terminal/main_helper.py index ce0b0e4d46c..f3ecaa45072 100644 --- a/gamestonk_terminal/main_helper.py +++ b/gamestonk_terminal/main_helper.py @@ -56,7 +56,7 @@ def print_help(s_ticker, s_start, s_interval, b_is_market_open): if s_ticker: print( - " ba behavioural analysis, \t\t\t from: reddit, stocktwits, twitter" + " ba behavioural analysis, \t from: reddit, stocktwits, twitter" ) print( " res research web page, \t e.g.: macroaxis, yahoo finance, fool" From 0963cc9ce4ed218ed1e3807da8b96683b9afa91c Mon Sep 17 00:00:00 2001 From: didier Date: Thu, 18 Mar 2021 01:16:08 +0000 Subject: [PATCH 56/64] Add Sentiment Analysis from FinBrain --- .../behavioural_analysis/ba_controller.py | 8 + .../behavioural_analysis/finbrain_view.py | 159 ++++++++++++++++++ tests/test_ba_finbrain_api.py | 10 ++ 3 files changed, 177 insertions(+) create mode 100644 gamestonk_terminal/behavioural_analysis/finbrain_view.py create mode 100644 tests/test_ba_finbrain_api.py diff --git a/gamestonk_terminal/behavioural_analysis/ba_controller.py b/gamestonk_terminal/behavioural_analysis/ba_controller.py index 20bb0d1c1a4..6cfd52414a1 100644 --- a/gamestonk_terminal/behavioural_analysis/ba_controller.py +++ b/gamestonk_terminal/behavioural_analysis/ba_controller.py @@ -12,6 +12,7 @@ from gamestonk_terminal.behavioural_analysis import ( google_view, reddit_view, stocktwits_view, + finbrain_view, ) from prompt_toolkit.completion import NestedCompleter @@ -39,6 +40,7 @@ class BehaviouralAnalysisController: "regions", "queries", "rise", + "finbrain", ] def __init__(self, ticker: str, start: datetime): @@ -89,6 +91,8 @@ class BehaviouralAnalysisController: print(" queries top related queries with this stock") print(" rise top rising related queries with stock") print("") + print(" finbrain sentiment from 15+ major news headlines") + print("") def switch(self, an_input: str): """Process and dispatch input @@ -210,6 +214,10 @@ class BehaviouralAnalysisController: twitter_view.sentiment(other_args, self.ticker) + def call_finbrain(self, other_args: List[str]): + """Process finbrain command""" + finbrain_view.sentiment_analysis(other_args, self.ticker) + def menu(ticker: str, start: datetime): """Behavioural Analysis Menu""" diff --git a/gamestonk_terminal/behavioural_analysis/finbrain_view.py b/gamestonk_terminal/behavioural_analysis/finbrain_view.py new file mode 100644 index 00000000000..592b31afa00 --- /dev/null +++ b/gamestonk_terminal/behavioural_analysis/finbrain_view.py @@ -0,0 +1,159 @@ +"""Finbrain View Module""" +__docformat__ = "numpy" + +import argparse +import warnings +import requests +import pandas as pd +from matplotlib import pyplot as plt +from pandas.plotting import register_matplotlib_converters +from datetime import datetime +import numpy as np +from typing import List +from gamestonk_terminal.helper_funcs import ( + parse_known_args_and_warn, + plot_autoscale, + patch_pandas_text_adjustment, +) +from gamestonk_terminal.config_plot import PLOT_DPI +from gamestonk_terminal import feature_flags as gtff +from colorama import Fore, Style + + +register_matplotlib_converters() + + +def sentiment_coloring(val: float, last_val: float) -> str: + if float(val) > last_val: + color = Fore.GREEN + else: + color = Fore.RED + return f"{color}{val}{Style.RESET_ALL}" + + +def get_sentiment(ticker: str) -> pd.DataFrame: + """Gets Sentiment analysis provided by FinBrain's API + + Parameters + ---------- + ticker : str + Ticker to get the sentiment analysis from + + Returns + ------- + DataFrame() + Empty if there was an issue with data retrieval + """ + result = requests.get(f"https://api.finbrain.tech/v0/sentiments/{ticker}") + sentiment = pd.DataFrame() + if result.status_code == 200: + if "sentimentAnalysis" in result.json(): + sentiment = pd.DataFrame.from_dict( + result.json()["sentimentAnalysis"], orient="index" + ) + sentiment.index = pd.to_datetime(sentiment.index).to_pydatetime() + sentiment.index.name = "date" + sentiment.columns = ["Sentiment Analysis"] + else: + print("Unexpected data format from FinBrain API") + else: + print("Request error in retrieving sentiment from FinBrain API") + + return sentiment + + +def plot_sentiment(sentiment: pd.DataFrame, ticker: str): + """Plot Sentiment analysis provided by FinBrain's API + + Parameters + ---------- + sentiment : pd.DataFrame + Dataframe with sentiment data to plot + ticker : str + Ticker to get the sentiment analysis from + """ + plt.figure(figsize=plot_autoscale(), dpi=PLOT_DPI) + for index, row in sentiment.iterrows(): + if float(row["Sentiment Analysis"]) >= 0: + plt.scatter(index, float(row["Sentiment Analysis"]), s=100, c="green") + else: + plt.scatter(index, float(row["Sentiment Analysis"]), s=100, c="red") + plt.axhline(y=0, color="k", linestyle="--", lw=2) + plt.xlabel("Time") + plt.ylabel("Sentiment") + plt.grid(b=True, which="major", color="#666666", linestyle="-") + plt.minorticks_on() + plt.grid(b=True, which="minor", color="#999999", linestyle="-", alpha=0.2) + start_date = sentiment.index[0].strftime("%Y/%m/%d") + plt.title(f"FinBrain's Sentiment Analysis For {ticker} since {start_date}") + plt.ylim([-1.1, 1.1]) + senValues = np.array(pd.to_numeric(sentiment["Sentiment Analysis"].values)) + senNone = np.array(0 * len(sentiment)) + plt.fill_between( + sentiment.index, + pd.to_numeric(sentiment["Sentiment Analysis"].values), + 0, + where=(senValues < senNone), + alpha=0.30, + color="red", + interpolate=True, + ) + plt.fill_between( + sentiment.index, + pd.to_numeric(sentiment["Sentiment Analysis"].values), + 0, + where=(senValues >= senNone), + alpha=0.30, + color="green", + interpolate=True, + ) + if gtff.USE_ION: + plt.ion() + plt.show() + + +def sentiment_analysis(other_args: List[str], ticker: str): + """Sentiment analysis from FinBrain + + Parameters + ---------- + other_args : List[str] + Command line arguments to be processed with argparse + ticker : str + Ticker to get the sentiment analysis from + """ + parser = argparse.ArgumentParser( + add_help=False, + prog="finbrain", + description="""FinBrain collects the news headlines from 15+ major financial news + sources on a daily basis and analyzes them to generate sentiment scores + for more than 4500 US stocks. + + [Source: FinBrain Technologies develops deep learning algorithms for + financial analysis and prediction, which currently serves traders from + more than 150 countries all around the world.]""", + ) + + try: + ns_parser = parse_known_args_and_warn(parser, other_args) + if not ns_parser: + return + + df_sentiment = get_sentiment(ticker) + + if not df_sentiment.empty: + plot_sentiment(df_sentiment, ticker) + + if gtff.USE_COLOR: + print( + df_sentiment["Sentiment Analysis"] + .apply(sentiment_coloring, last_val=0) + .to_string() + ) + else: + print(df_sentiment.to_string()) + print("") + + except Exception as e: + print(e, "\n") + print("") diff --git a/tests/test_ba_finbrain_api.py b/tests/test_ba_finbrain_api.py new file mode 100644 index 00000000000..03893ac0166 --- /dev/null +++ b/tests/test_ba_finbrain_api.py @@ -0,0 +1,10 @@ +""" behavioural_analysis/finbrain_view.py tests """ +import unittest + +# from gamestonk_terminal.behavioural_analysis.finbrain_view import sentiment_analysis + + +class TestBaFinbrainApi(unittest.TestCase): + def test_sentiment_analysis(self): + # sentiment_analysis([], "TSLA") + print("") From c0b979cbb2bfd602000a2b1bf0e87f6326c1a626 Mon Sep 17 00:00:00 2001 From: didier Date: Thu, 18 Mar 2021 01:17:22 +0000 Subject: [PATCH 57/64] Add FinBrain hyperlink --- gamestonk_terminal/behavioural_analysis/finbrain_view.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gamestonk_terminal/behavioural_analysis/finbrain_view.py b/gamestonk_terminal/behavioural_analysis/finbrain_view.py index 592b31afa00..9cdf0f98272 100644 --- a/gamestonk_terminal/behavioural_analysis/finbrain_view.py +++ b/gamestonk_terminal/behavioural_analysis/finbrain_view.py @@ -131,7 +131,7 @@ def sentiment_analysis(other_args: List[str], ticker: str): [Source: FinBrain Technologies develops deep learning algorithms for financial analysis and prediction, which currently serves traders from - more than 150 countries all around the world.]""", + more than 150 countries all around the world. See https://finbrain.tech]""", ) try: From 936d9995386b62b0bdeb59ed40df8093874af044 Mon Sep 17 00:00:00 2001 From: didier Date: Thu, 18 Mar 2021 01:19:45 +0000 Subject: [PATCH 58/64] remove unused imports --- .../behavioural_analysis/finbrain_view.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/gamestonk_terminal/behavioural_analysis/finbrain_view.py b/gamestonk_terminal/behavioural_analysis/finbrain_view.py index 9cdf0f98272..e4459a7fb7b 100644 --- a/gamestonk_terminal/behavioural_analysis/finbrain_view.py +++ b/gamestonk_terminal/behavioural_analysis/finbrain_view.py @@ -2,18 +2,15 @@ __docformat__ = "numpy" import argparse -import warnings -import requests -import pandas as pd -from matplotlib import pyplot as plt -from pandas.plotting import register_matplotlib_converters -from datetime import datetime -import numpy as np from typing import List +import requests +from matplotlib import pyplot as plt +import pandas as pd +from pandas.plotting import register_matplotlib_converters +import numpy as np from gamestonk_terminal.helper_funcs import ( parse_known_args_and_warn, plot_autoscale, - patch_pandas_text_adjustment, ) from gamestonk_terminal.config_plot import PLOT_DPI from gamestonk_terminal import feature_flags as gtff From 366f53e14f417e52ed390542c4a26071d8e41ee9 Mon Sep 17 00:00:00 2001 From: didier Date: Thu, 18 Mar 2021 01:35:57 +0000 Subject: [PATCH 59/64] Fix dataframe order when printing into console --- .../behavioural_analysis/finbrain_view.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/gamestonk_terminal/behavioural_analysis/finbrain_view.py b/gamestonk_terminal/behavioural_analysis/finbrain_view.py index e4459a7fb7b..8b9f7f33199 100644 --- a/gamestonk_terminal/behavioural_analysis/finbrain_view.py +++ b/gamestonk_terminal/behavioural_analysis/finbrain_view.py @@ -82,7 +82,7 @@ def plot_sentiment(sentiment: pd.DataFrame, ticker: str): plt.minorticks_on() plt.grid(b=True, which="minor", color="#999999", linestyle="-", alpha=0.2) start_date = sentiment.index[0].strftime("%Y/%m/%d") - plt.title(f"FinBrain's Sentiment Analysis For {ticker} since {start_date}") + plt.title(f"FinBrain's Sentiment Analysis for {ticker.upper()} since {start_date}") plt.ylim([-1.1, 1.1]) senValues = np.array(pd.to_numeric(sentiment["Sentiment Analysis"].values)) senNone = np.array(0 * len(sentiment)) @@ -124,11 +124,10 @@ def sentiment_analysis(other_args: List[str], ticker: str): prog="finbrain", description="""FinBrain collects the news headlines from 15+ major financial news sources on a daily basis and analyzes them to generate sentiment scores - for more than 4500 US stocks. - - [Source: FinBrain Technologies develops deep learning algorithms for - financial analysis and prediction, which currently serves traders from - more than 150 countries all around the world. See https://finbrain.tech]""", + for more than 4500 US stocks.FinBrain Technologies develops deep learning + algorithms for financial analysis and prediction, which currently serves + traders from more than 150 countries all around the world. + [Source: https://finbrain.tech]""", ) try: @@ -141,6 +140,8 @@ def sentiment_analysis(other_args: List[str], ticker: str): if not df_sentiment.empty: plot_sentiment(df_sentiment, ticker) + df_sentiment.sort_index(ascending=True, inplace=True) + if gtff.USE_COLOR: print( df_sentiment["Sentiment Analysis"] From 82e401f4208b8c0825104368a76bbc20cf4d4ef9 Mon Sep 17 00:00:00 2001 From: didier Date: Thu, 18 Mar 2021 01:36:15 +0000 Subject: [PATCH 60/64] improve finbrain cosmetics and add to readme --- gamestonk_terminal/behavioural_analysis/README.md | 9 +++++++++ gamestonk_terminal/behavioural_analysis/ba_controller.py | 4 ++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/gamestonk_terminal/behavioural_analysis/README.md b/gamestonk_terminal/behavioural_analysis/README.md index bdb8fed002f..b52d85641e5 100644 --- a/gamestonk_terminal/behavioural_analysis/README.md +++ b/gamestonk_terminal/behavioural_analysis/README.md @@ -2,6 +2,8 @@ This menu aims to extrapolate behavioural analysis regarding a pre-loaded ticker, and the usage of the following commands along with an example will be exploited below. + * [finbrain](#finbrain) + [REDDIT](#REDDIT) * [wsb](#wsb) - show what WSB gang is up to in subreddit wallstreetbets @@ -41,6 +43,13 @@ This menu aims to extrapolate behavioural analysis regarding a pre-loaded ticker - top rising related queries with stock +## finbrain +``` +usage: finbrain +``` +FinBrain collects the news headlines from 15+ major financial news sources on a daily basis and analyzes them to generate sentiment scores for more than 4500 US stocks. FinBrain Technologies develops deep learning algorithms for financial analysis and prediction, which currently serves traders from more than 150 countries all around the world. [Source: See https://finbrain.tech] + + # REDDIT ## wsb diff --git a/gamestonk_terminal/behavioural_analysis/ba_controller.py b/gamestonk_terminal/behavioural_analysis/ba_controller.py index 6cfd52414a1..57e9a66cbe6 100644 --- a/gamestonk_terminal/behavioural_analysis/ba_controller.py +++ b/gamestonk_terminal/behavioural_analysis/ba_controller.py @@ -62,6 +62,8 @@ class BehaviouralAnalysisController: print(" q quit this menu, and shows back to main menu") print(" quit quit to abandon program") print("") + print(" finbrain sentiment from 15+ major news headlines") + print("") print("Reddit:") print( " wsb show what WSB gang is up to in subreddit wallstreetbets" @@ -91,8 +93,6 @@ class BehaviouralAnalysisController: print(" queries top related queries with this stock") print(" rise top rising related queries with stock") print("") - print(" finbrain sentiment from 15+ major news headlines") - print("") def switch(self, an_input: str): """Process and dispatch input From a04834ef3de559bbbcc50c89a3e90f076c03e2f6 Mon Sep 17 00:00:00 2001 From: DidierRLopes Date: Thu, 18 Mar 2021 01:37:38 +0000 Subject: [PATCH 61/64] Add Finbrain example to readme --- gamestonk_terminal/behavioural_analysis/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gamestonk_terminal/behavioural_analysis/README.md b/gamestonk_terminal/behavioural_analysis/README.md index b52d85641e5..e715d87ef74 100644 --- a/gamestonk_terminal/behavioural_analysis/README.md +++ b/gamestonk_terminal/behavioural_analysis/README.md @@ -49,6 +49,10 @@ usage: finbrain ``` FinBrain collects the news headlines from 15+ major financial news sources on a daily basis and analyzes them to generate sentiment scores for more than 4500 US stocks. FinBrain Technologies develops deep learning algorithms for financial analysis and prediction, which currently serves traders from more than 150 countries all around the world. [Source: See https://finbrain.tech] +![finbrain](https://user-images.githubusercontent.com/25267873/111560461-6f735c80-878a-11eb-8f3a-72ef0ddbf8e3.png) + +Captura de ecrã 2021-03-18, às 01 33 35 + # REDDIT From 3551ef68cff4dd3fc24455c1533dfcd085578295 Mon Sep 17 00:00:00 2001 From: didier Date: Thu, 18 Mar 2021 09:16:35 +0000 Subject: [PATCH 62/64] Improve finbrain plot --- gamestonk_terminal/behavioural_analysis/finbrain_view.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gamestonk_terminal/behavioural_analysis/finbrain_view.py b/gamestonk_terminal/behavioural_analysis/finbrain_view.py index 8b9f7f33199..0887877e9a8 100644 --- a/gamestonk_terminal/behavioural_analysis/finbrain_view.py +++ b/gamestonk_terminal/behavioural_analysis/finbrain_view.py @@ -5,6 +5,7 @@ import argparse from typing import List import requests from matplotlib import pyplot as plt +import matplotlib.dates as mdates import pandas as pd from pandas.plotting import register_matplotlib_converters import numpy as np @@ -81,8 +82,10 @@ def plot_sentiment(sentiment: pd.DataFrame, ticker: str): plt.grid(b=True, which="major", color="#666666", linestyle="-") plt.minorticks_on() plt.grid(b=True, which="minor", color="#999999", linestyle="-", alpha=0.2) - start_date = sentiment.index[0].strftime("%Y/%m/%d") + start_date = sentiment.index[-1].strftime("%Y/%m/%d") plt.title(f"FinBrain's Sentiment Analysis for {ticker.upper()} since {start_date}") + plt.gca().xaxis.set_major_formatter(mdates.DateFormatter("%Y/%m/%d")) + plt.gcf().autofmt_xdate() plt.ylim([-1.1, 1.1]) senValues = np.array(pd.to_numeric(sentiment["Sentiment Analysis"].values)) senNone = np.array(0 * len(sentiment)) From c1984d3a44c3cb3051d71ec1aa95d502206919b0 Mon Sep 17 00:00:00 2001 From: didier Date: Thu, 18 Mar 2021 12:54:28 +0000 Subject: [PATCH 63/64] Improve finbrain plot --- gamestonk_terminal/behavioural_analysis/finbrain_view.py | 1 + 1 file changed, 1 insertion(+) diff --git a/gamestonk_terminal/behavioural_analysis/finbrain_view.py b/gamestonk_terminal/behavioural_analysis/finbrain_view.py index 0887877e9a8..efc530aa77f 100644 --- a/gamestonk_terminal/behavioural_analysis/finbrain_view.py +++ b/gamestonk_terminal/behavioural_analysis/finbrain_view.py @@ -85,6 +85,7 @@ def plot_sentiment(sentiment: pd.DataFrame, ticker: str): start_date = sentiment.index[-1].strftime("%Y/%m/%d") plt.title(f"FinBrain's Sentiment Analysis for {ticker.upper()} since {start_date}") plt.gca().xaxis.set_major_formatter(mdates.DateFormatter("%Y/%m/%d")) + plt.gca().xaxis.set_major_locator(mdates.DayLocator(interval=1)) plt.gcf().autofmt_xdate() plt.ylim([-1.1, 1.1]) senValues = np.array(pd.to_numeric(sentiment["Sentiment Analysis"].values)) From 0b5a943683342a6eb73430621f422fee9b563019 Mon Sep 17 00:00:00 2001 From: DidierRLopes Date: Thu, 18 Mar 2021 12:56:48 +0000 Subject: [PATCH 64/64] Improve finbrain's cmd readme --- gamestonk_terminal/behavioural_analysis/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gamestonk_terminal/behavioural_analysis/README.md b/gamestonk_terminal/behavioural_analysis/README.md index e715d87ef74..4a926a0c1e7 100644 --- a/gamestonk_terminal/behavioural_analysis/README.md +++ b/gamestonk_terminal/behavioural_analysis/README.md @@ -49,9 +49,9 @@ usage: finbrain ``` FinBrain collects the news headlines from 15+ major financial news sources on a daily basis and analyzes them to generate sentiment scores for more than 4500 US stocks. FinBrain Technologies develops deep learning algorithms for financial analysis and prediction, which currently serves traders from more than 150 countries all around the world. [Source: See https://finbrain.tech] -![finbrain](https://user-images.githubusercontent.com/25267873/111560461-6f735c80-878a-11eb-8f3a-72ef0ddbf8e3.png) +![finbrain2](https://user-images.githubusercontent.com/25267873/111629515-49c97000-87e9-11eb-92a6-a9eebb4b4bd9.png) -Captura de ecrã 2021-03-18, às 01 33 35 +Captura de ecrã 2021-03-18, às 12 53 31 # REDDIT