Merge pull request #366 from hinxx/add-ftd-data2

Add fails-to-deliver data

ROADMAP.md

@@ -80,6 +80,7 @@
 
 ## Due Diligence
 * [x] Add dark pools (ATS) vs Non-ATS data over time (@didier) - [PR #363](https://github.com/DidierRLopes/GamestonkTerminal/pull/363)
+* [x] Add failure to deliver command (@hinxx) - [PR #366](#https://github.com/DidierRLopes/GamestonkTerminal/pull/366)
 
 **NEXT**
 
@@ -148,7 +149,7 @@
 * [ ] Refactoring
 * [ ] Summaries / tear sheets (@jmaslek)
 
-___  
+___
 
 ## Portfolio Optimization
 * [x] Basic Optimization through PyPortFolioOpt(@jmaslek) - [PR #329](https://github.com/DidierRLopes/GamestonkTerminal/pull/329)

gamestonk_terminal/due_diligence/README.md

@@ -26,6 +26,8 @@ This menu aims to help in due-diligence of a pre-loaded stock, and the usage of
   * company warnings according to Sean Seah book [Market Watch]
 * [dp](#dp)
   * dark pools (ATS) vs OTC data [FINRA]
+* [ftd](#ftd)
+  * fails-to-deliver data [SEC]
 
 ## news <a name="news"></a>
 
@@ -170,4 +172,14 @@ Display barchart of dark pool (ATS) and OTC (Non ATS) data
 
 ![dp](https://user-images.githubusercontent.com/25267873/115130908-7987b580-9feb-11eb-8bca-1999174178d0.png)
 
+## ftd <a name="ftd"></a>
 
+```text
+usage: ftd [-n]
+```
+
+The fails-to-deliver data collected by SEC. Fails to deliver on a given day are a cumulative number of all fails outstanding until that day, plus new fails that occur that day, less fails that settle that day. See <https://www.sec.gov/data/foiadocsfailsdatahtm>. [Source: SEC]
+
+* -n : number of latest fails-to-deliver being printed. Default 10.
+
+<img width="927" alt="ftd" src="TODO">

gamestonk_terminal/due_diligence/dd_controller.py

@@ -14,6 +14,7 @@ from gamestonk_terminal.due_diligence import quandl_view as q_view
 from gamestonk_terminal.due_diligence import reddit_view as r_view
 from gamestonk_terminal.due_diligence import news_view
 from gamestonk_terminal.due_diligence import finra_view
+from gamestonk_terminal.due_diligence import sec_view
 from gamestonk_terminal import feature_flags as gtff
 from gamestonk_terminal.helper_funcs import get_flair
 from gamestonk_terminal.menu import session
@@ -39,6 +40,7 @@ class DueDiligenceController:
         "warnings",
         "sec",
         "dp",
+        "ftd",
     ]
 
     def __init__(self, stock: DataFrame, ticker: str, start: str, interval: str):
@@ -100,6 +102,7 @@ class DueDiligenceController:
             "   warnings      company warnings according to Sean Seah book [Market Watch]"
         )
         print("   dp            dark pools (ATS) vs OTC data [FINRA]")
+        print("   ftd           fails-to-deliver data [SEC]")
         print("")
 
     def switch(self, an_input: str):
@@ -182,6 +185,10 @@ class DueDiligenceController:
         """ Process dp command """
         finra_view.dark_pool(other_args, self.ticker)
 
+    def call_ftd(self, other_args: List[str]):
+        """ Process ftd command """
+        sec_view.fails_to_deliver(other_args, self.ticker)
+
 
 def menu(stock: DataFrame, ticker: str, start: str, interval: str):
     """Due Diligence Menu

gamestonk_terminal/due_diligence/sec_view.py

@@ -0,0 +1,106 @@
+""" SEC View """
+__docformat__ = "numpy"
+
+import argparse
+from typing import List
+from datetime import datetime
+import requests
+import pandas as pd
+from bs4 import BeautifulSoup
+from matplotlib import pyplot as plt
+import matplotlib.dates as mdates
+from gamestonk_terminal import feature_flags as gtff
+from gamestonk_terminal.helper_funcs import (
+    get_user_agent,
+    check_positive,
+    parse_known_args_and_warn,
+)
+
+
+def fails_to_deliver(other_args: List[str], ticker: str):
+    """Display fails-to-deliver data for a given ticker
+
+    Parameters
+    ----------
+    other_args : List[str]
+        argparse other args - ["-n", "10"]
+    ticker : str
+        Stock ticker
+    """
+
+    parser = argparse.ArgumentParser(
+        add_help=False,
+        prog="ftd",
+        description="""Prints latest fails-to-deliver data. [Source: SEC]""",
+    )
+    parser.add_argument(
+        "-n",
+        "--num",
+        action="store",
+        dest="n_num",
+        type=check_positive,
+        default=10,
+        help="number of latest fails-to-deliver being printed",
+    )
+
+    try:
+        ns_parser = parse_known_args_and_warn(parser, other_args)
+        if not ns_parser:
+            return
+
+        url_ftds = "https://www.sec.gov/data/foiadocsfailsdatahtm"
+        text_soup_ftds = BeautifulSoup(
+            requests.get(url_ftds, headers={"User-Agent": get_user_agent()}).text,
+            "lxml",
+        )
+
+        table = text_soup_ftds.find("table", {"class": "list"})
+        links = table.findAll("a")
+        link_idx = 0
+        ftds_data = pd.DataFrame()
+        while len(ftds_data) < ns_parser.n_num:
+            if link_idx > len(links):
+                break
+            link = links[link_idx]
+            url = "https://www.sec.gov" + link["href"]
+            all_ftds = pd.read_csv(
+                url,
+                compression="zip",
+                sep="|",
+                engine="python",
+                skipfooter=2,
+                usecols=[0, 2, 3, 5],
+                dtype={"QUANTITY (FAILS)": "int"},
+            )
+            tmp_ftds = all_ftds[all_ftds["SYMBOL"] == ticker]
+            del tmp_ftds["PRICE"]
+            del tmp_ftds["SYMBOL"]
+            # merge the data from this archive
+            ftds_data = pd.concat([ftds_data, tmp_ftds], ignore_index=True)
+            link_idx += 1
+
+        # clip away extra rows
+        ftds_data = ftds_data.sort_values("SETTLEMENT DATE")[-ns_parser.n_num :]
+        ftds_data["SETTLEMENT DATE"] = ftds_data["SETTLEMENT DATE"].apply(
+            lambda x: datetime.strptime(str(x), "%Y%m%d")
+        )
+
+        plt.bar(
+            ftds_data["SETTLEMENT DATE"],
+            ftds_data["QUANTITY (FAILS)"] / 1000,
+        )
+        plt.ylabel("Shares [K]")
+        plt.title(f"Fails-to-deliver Data for {ticker}")
+        plt.grid(b=True, which="major", color="#666666", linestyle="-", alpha=0.2)
+        plt.gca().xaxis.set_major_formatter(mdates.DateFormatter("%Y/%m/%d"))
+        plt.gca().xaxis.set_major_locator(mdates.DayLocator(interval=7))
+        plt.gcf().autofmt_xdate()
+        plt.xlabel("Days")
+
+        if gtff.USE_ION:
+            plt.ion()
+
+        plt.show()
+
+    except Exception as e:
+        print(e, "\n")