From bfeaa2b9a0be7232fef8f35954af4fc0dacd3137 Mon Sep 17 00:00:00 2001 From: Hinko Kocevar Date: Sun, 18 Apr 2021 23:01:21 +0200 Subject: [PATCH 1/7] add ftd command to dd menu --- gamestonk_terminal/due_diligence/README.md | 14 +++ .../due_diligence/dd_controller.py | 7 ++ gamestonk_terminal/due_diligence/sec_view.py | 89 +++++++++++++++++++ 3 files changed, 110 insertions(+) create mode 100644 gamestonk_terminal/due_diligence/sec_view.py diff --git a/gamestonk_terminal/due_diligence/README.md b/gamestonk_terminal/due_diligence/README.md index 70bfb49340b..297d07428fa 100644 --- a/gamestonk_terminal/due_diligence/README.md +++ b/gamestonk_terminal/due_diligence/README.md @@ -24,6 +24,8 @@ This menu aims to help in due-diligence of a pre-loaded stock, and the usage of * short interest [Quandl] * [warnings](#warnings) * company warnings according to Sean Seah book [Market Watch] +* [ftd](#ftd) + * fails-to-deliver data [SEC] ## news @@ -157,3 +159,15 @@ Sean Seah warnings. Check: Consistent historical earnings per share; Consistentl * -d : print insights into warnings calculation. Default False. warnings + +## ftd + +```text +usage: ftd [-n] +``` + +The fails-to-deliver data collected by SEC. Fails to deliver on a given day are a cumulative number of all fails outstanding until that day, plus new fails that occur that day, less fails that settle that day. See . [Source: SEC] + +* -n : number of latest fails-to-deliver being printed. Default 10. + +ftd diff --git a/gamestonk_terminal/due_diligence/dd_controller.py b/gamestonk_terminal/due_diligence/dd_controller.py index 14ecab8fc40..808df2ef426 100644 --- a/gamestonk_terminal/due_diligence/dd_controller.py +++ b/gamestonk_terminal/due_diligence/dd_controller.py @@ -13,6 +13,7 @@ from gamestonk_terminal.due_diligence import market_watch_view as mw_view from gamestonk_terminal.due_diligence import quandl_view as q_view from gamestonk_terminal.due_diligence import reddit_view as r_view from gamestonk_terminal.due_diligence import news_view +from gamestonk_terminal.due_diligence import sec_view from gamestonk_terminal import feature_flags as gtff from gamestonk_terminal.helper_funcs import get_flair from gamestonk_terminal.menu import session @@ -37,6 +38,7 @@ class DueDiligenceController: "analyst", "warnings", "sec", + "ftd", ] def __init__(self, stock: DataFrame, ticker: str, start: str, interval: str): @@ -97,6 +99,7 @@ class DueDiligenceController: print( " warnings company warnings according to Sean Seah book [Market Watch]" ) + print(" ftd fails-to-deliver data [SEC]") print("") def switch(self, an_input: str): @@ -175,6 +178,10 @@ class DueDiligenceController: """ Process short command """ q_view.short_interest(other_args, self.ticker, self.start) + def call_ftd(self, other_args: List[str]): + """ Process ftd command """ + sec_view.fails_to_deliver(other_args, self.ticker) + def menu(stock: DataFrame, ticker: str, start: str, interval: str): """Due Diligence Menu diff --git a/gamestonk_terminal/due_diligence/sec_view.py b/gamestonk_terminal/due_diligence/sec_view.py new file mode 100644 index 00000000000..35f19df8b1c --- /dev/null +++ b/gamestonk_terminal/due_diligence/sec_view.py @@ -0,0 +1,89 @@ +""" SEC View """ +__docformat__ = "numpy" + +import argparse +from typing import List +import requests +import pandas as pd +from bs4 import BeautifulSoup +from gamestonk_terminal.helper_funcs import ( + get_user_agent, + check_positive, + parse_known_args_and_warn, +) + + +def fails_to_deliver(other_args: List[str], ticker: str): + """Display fails-to-deliver data for a given ticker + + Parameters + ---------- + other_args : List[str] + argparse other args - ["-n", "10"] + ticker : str + Stock ticker + """ + + parser = argparse.ArgumentParser( + add_help=False, + prog="ftd", + description="""Prints latest fails-to-deliver data. [Source: SEC]""", + ) + parser.add_argument( + "-n", + "--num", + action="store", + dest="n_num", + type=check_positive, + default=10, + help="number of latest fails-to-deliver being printed", + ) + + try: + ns_parser = parse_known_args_and_warn(parser, other_args) + if not ns_parser: + return + + url_ftds = "https://www.sec.gov/data/foiadocsfailsdatahtm" + text_soup_ftds = BeautifulSoup( + requests.get(url_ftds, headers={"User-Agent": get_user_agent()}).text, + "lxml", + ) + + table = text_soup_ftds.find("table", {"class": "list"}) + links = table.findAll("a") + link_idx = 0 + ftds_data = pd.DataFrame() + while len(ftds_data) < ns_parser.n_num: + if link_idx > len(links): + break + link = links[link_idx] + url = "https://www.sec.gov" + link["href"] + all_ftds = pd.read_csv( + url, + compression="zip", + sep="|", + engine="python", + skipfooter=2, + usecols=[0, 2, 3, 5], + dtype={"QUANTITY (FAILS)": "int"}, + ) + tmp_ftds = all_ftds[all_ftds["SYMBOL"] == ticker] + del tmp_ftds["PRICE"] + del tmp_ftds["SYMBOL"] + # merge the data from this archive + ftds_data = pd.concat([ftds_data, tmp_ftds], ignore_index=True) + link_idx += 1 + + # clip away extra rows + ftds_data = ftds_data.sort_values("SETTLEMENT DATE", ascending=False)[ + 0 : ns_parser.n_num + ] + print( + ftds_data.rename( + columns={"SETTLEMENT DATE": "Date", "QUANTITY (FAILS)": "Quantity"} + ).to_string(index=False) + ) + + except Exception as e: + print(e, "\n") From e9b5c2262cd53079251df5ac3e127cb71d67ad5c Mon Sep 17 00:00:00 2001 From: Hinko Kocevar Date: Sun, 18 Apr 2021 23:46:49 +0200 Subject: [PATCH 2/7] update dd controller and README. --- gamestonk_terminal/due_diligence/README.md | 12 ++++++++++++ gamestonk_terminal/due_diligence/dd_controller.py | 7 +++++++ 2 files changed, 19 insertions(+) diff --git a/gamestonk_terminal/due_diligence/README.md b/gamestonk_terminal/due_diligence/README.md index 472867b0f7e..2d0c9fb2d0c 100644 --- a/gamestonk_terminal/due_diligence/README.md +++ b/gamestonk_terminal/due_diligence/README.md @@ -26,6 +26,8 @@ This menu aims to help in due-diligence of a pre-loaded stock, and the usage of * company warnings according to Sean Seah book [Market Watch] * [dp](#dp) * dark pools (ATS) vs OTC data [FINRA] +* [ftd](#ftd) + * fails-to-deliver data [SEC] ## news @@ -170,4 +172,14 @@ Display barchart of dark pool (ATS) and OTC (Non ATS) data ![dp](https://user-images.githubusercontent.com/25267873/115130908-7987b580-9feb-11eb-8bca-1999174178d0.png) +## ftd +```text +usage: ftd [-n] +``` + +The fails-to-deliver data collected by SEC. Fails to deliver on a given day are a cumulative number of all fails outstanding until that day, plus new fails that occur that day, less fails that settle that day. See . [Source: SEC] + +* -n : number of latest fails-to-deliver being printed. Default 10. + +ftd diff --git a/gamestonk_terminal/due_diligence/dd_controller.py b/gamestonk_terminal/due_diligence/dd_controller.py index 378499f05cd..2dbe1379927 100644 --- a/gamestonk_terminal/due_diligence/dd_controller.py +++ b/gamestonk_terminal/due_diligence/dd_controller.py @@ -14,6 +14,7 @@ from gamestonk_terminal.due_diligence import quandl_view as q_view from gamestonk_terminal.due_diligence import reddit_view as r_view from gamestonk_terminal.due_diligence import news_view from gamestonk_terminal.due_diligence import finra_view +from gamestonk_terminal.due_diligence import sec_view from gamestonk_terminal import feature_flags as gtff from gamestonk_terminal.helper_funcs import get_flair from gamestonk_terminal.menu import session @@ -39,6 +40,7 @@ class DueDiligenceController: "warnings", "sec", "dp", + "ftd", ] def __init__(self, stock: DataFrame, ticker: str, start: str, interval: str): @@ -100,6 +102,7 @@ class DueDiligenceController: " warnings company warnings according to Sean Seah book [Market Watch]" ) print(" dp dark pools (ATS) vs OTC data [FINRA]") + print(" ftd fails-to-deliver data [SEC]") print("") def switch(self, an_input: str): @@ -182,6 +185,10 @@ class DueDiligenceController: """ Process dp command """ finra_view.dark_pool(other_args, self.ticker) + def call_ftd(self, other_args: List[str]): + """ Process ftd command """ + sec_view.fails_to_deliver(other_args, self.ticker) + def menu(stock: DataFrame, ticker: str, start: str, interval: str): """Due Diligence Menu From 84c80be012f2756828599b77a4eb65e892acf75e Mon Sep 17 00:00:00 2001 From: Hinko Kocevar Date: Mon, 19 Apr 2021 00:12:33 +0200 Subject: [PATCH 3/7] fix date formatting --- gamestonk_terminal/due_diligence/sec_view.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gamestonk_terminal/due_diligence/sec_view.py b/gamestonk_terminal/due_diligence/sec_view.py index 35f19df8b1c..01a512b552d 100644 --- a/gamestonk_terminal/due_diligence/sec_view.py +++ b/gamestonk_terminal/due_diligence/sec_view.py @@ -3,6 +3,7 @@ __docformat__ = "numpy" import argparse from typing import List +from datetime import datetime import requests import pandas as pd from bs4 import BeautifulSoup @@ -79,6 +80,9 @@ def fails_to_deliver(other_args: List[str], ticker: str): ftds_data = ftds_data.sort_values("SETTLEMENT DATE", ascending=False)[ 0 : ns_parser.n_num ] + ftds_data["SETTLEMENT DATE"] = ftds_data["SETTLEMENT DATE"].apply( + lambda x: datetime.strptime(str(x), "%Y%m%d").strftime("%Y-%m-%d") + ) print( ftds_data.rename( columns={"SETTLEMENT DATE": "Date", "QUANTITY (FAILS)": "Quantity"} From 58320dc09dd954fec9a7cbb394ae879c8d2a1576 Mon Sep 17 00:00:00 2001 From: Hinko Kocevar Date: Mon, 19 Apr 2021 00:36:20 +0200 Subject: [PATCH 4/7] update ROADMAP --- ROADMAP.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ROADMAP.md b/ROADMAP.md index d47c8d9c7d3..277662bd9a8 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -80,6 +80,7 @@ ## Due Diligence * [x] Add dark pools (ATS) vs Non-ATS data over time (@didier) - [PR #363](https://github.com/DidierRLopes/GamestonkTerminal/pull/363) +* [x] Add failure to deliver command (@hinxx) - [PR #366](#https://github.com/DidierRLopes/GamestonkTerminal/pull/366) **NEXT** @@ -148,7 +149,7 @@ * [ ] Refactoring * [ ] Summaries / tear sheets (@jmaslek) -___ +___ ## Portfolio Optimization * [x] Basic Optimization through PyPortFolioOpt(@jmaslek) - [PR #329](https://github.com/DidierRLopes/GamestonkTerminal/pull/329) From 48d48cae1c9175d803aed651be5040eccccda975 Mon Sep 17 00:00:00 2001 From: Hinko Kocevar Date: Mon, 19 Apr 2021 01:01:14 +0200 Subject: [PATCH 5/7] plot bars instead of printing to console --- gamestonk_terminal/due_diligence/sec_view.py | 24 ++++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/gamestonk_terminal/due_diligence/sec_view.py b/gamestonk_terminal/due_diligence/sec_view.py index 01a512b552d..7dc0ee7ae4e 100644 --- a/gamestonk_terminal/due_diligence/sec_view.py +++ b/gamestonk_terminal/due_diligence/sec_view.py @@ -7,6 +7,9 @@ from datetime import datetime import requests import pandas as pd from bs4 import BeautifulSoup +from matplotlib import pyplot as plt +from gamestonk_terminal.config_plot import PLOT_DPI +from gamestonk_terminal import feature_flags as gtff from gamestonk_terminal.helper_funcs import ( get_user_agent, check_positive, @@ -77,17 +80,24 @@ def fails_to_deliver(other_args: List[str], ticker: str): link_idx += 1 # clip away extra rows - ftds_data = ftds_data.sort_values("SETTLEMENT DATE", ascending=False)[ - 0 : ns_parser.n_num - ] + ftds_data = ftds_data.sort_values("SETTLEMENT DATE")[-ns_parser.n_num:] ftds_data["SETTLEMENT DATE"] = ftds_data["SETTLEMENT DATE"].apply( lambda x: datetime.strptime(str(x), "%Y%m%d").strftime("%Y-%m-%d") ) - print( - ftds_data.rename( - columns={"SETTLEMENT DATE": "Date", "QUANTITY (FAILS)": "Quantity"} - ).to_string(index=False) + + plt.bar( + ftds_data["SETTLEMENT DATE"], + ftds_data["QUANTITY (FAILS)"], ) + plt.ylabel("Number of shares") + plt.title(f"Fails-to-deliver Data for {ticker}") + plt.gcf().autofmt_xdate() + plt.xlabel("Days") + + if gtff.USE_ION: + plt.ion() + + plt.show() except Exception as e: print(e, "\n") From 79c3d0a40f71c3bb2e6b431bcc8844d44f0a007a Mon Sep 17 00:00:00 2001 From: Hinko Kocevar Date: Mon, 19 Apr 2021 01:03:51 +0200 Subject: [PATCH 6/7] fix checks --- gamestonk_terminal/due_diligence/sec_view.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gamestonk_terminal/due_diligence/sec_view.py b/gamestonk_terminal/due_diligence/sec_view.py index 7dc0ee7ae4e..94dabc09a57 100644 --- a/gamestonk_terminal/due_diligence/sec_view.py +++ b/gamestonk_terminal/due_diligence/sec_view.py @@ -80,7 +80,7 @@ def fails_to_deliver(other_args: List[str], ticker: str): link_idx += 1 # clip away extra rows - ftds_data = ftds_data.sort_values("SETTLEMENT DATE")[-ns_parser.n_num:] + ftds_data = ftds_data.sort_values("SETTLEMENT DATE")[-ns_parser.n_num :] ftds_data["SETTLEMENT DATE"] = ftds_data["SETTLEMENT DATE"].apply( lambda x: datetime.strptime(str(x), "%Y%m%d").strftime("%Y-%m-%d") ) From e0e9745cdd7199982179fc94315659f5445424d7 Mon Sep 17 00:00:00 2001 From: Hinko Kocevar Date: Mon, 19 Apr 2021 01:37:11 +0200 Subject: [PATCH 7/7] fix plot output --- gamestonk_terminal/due_diligence/sec_view.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/gamestonk_terminal/due_diligence/sec_view.py b/gamestonk_terminal/due_diligence/sec_view.py index 94dabc09a57..803cf8ecf3c 100644 --- a/gamestonk_terminal/due_diligence/sec_view.py +++ b/gamestonk_terminal/due_diligence/sec_view.py @@ -8,7 +8,7 @@ import requests import pandas as pd from bs4 import BeautifulSoup from matplotlib import pyplot as plt -from gamestonk_terminal.config_plot import PLOT_DPI +import matplotlib.dates as mdates from gamestonk_terminal import feature_flags as gtff from gamestonk_terminal.helper_funcs import ( get_user_agent, @@ -82,15 +82,18 @@ def fails_to_deliver(other_args: List[str], ticker: str): # clip away extra rows ftds_data = ftds_data.sort_values("SETTLEMENT DATE")[-ns_parser.n_num :] ftds_data["SETTLEMENT DATE"] = ftds_data["SETTLEMENT DATE"].apply( - lambda x: datetime.strptime(str(x), "%Y%m%d").strftime("%Y-%m-%d") + lambda x: datetime.strptime(str(x), "%Y%m%d") ) plt.bar( ftds_data["SETTLEMENT DATE"], - ftds_data["QUANTITY (FAILS)"], + ftds_data["QUANTITY (FAILS)"] / 1000, ) - plt.ylabel("Number of shares") + plt.ylabel("Shares [K]") plt.title(f"Fails-to-deliver Data for {ticker}") + plt.grid(b=True, which="major", color="#666666", linestyle="-", alpha=0.2) + plt.gca().xaxis.set_major_formatter(mdates.DateFormatter("%Y/%m/%d")) + plt.gca().xaxis.set_major_locator(mdates.DayLocator(interval=7)) plt.gcf().autofmt_xdate() plt.xlabel("Days")