From 57d290ccbb3fe78c4916ef31ee9e5efb71238c86 Mon Sep 17 00:00:00 2001 From: didier Date: Sun, 18 Apr 2021 02:06:19 +0100 Subject: [PATCH 1/8] Add dark pools (ATS) vs Non-ATS data over time --- .../due_diligence/dd_controller.py | 7 + .../due_diligence/finra_view.py | 272 ++++++++++++++++++ 2 files changed, 279 insertions(+) create mode 100644 gamestonk_terminal/due_diligence/finra_view.py diff --git a/gamestonk_terminal/due_diligence/dd_controller.py b/gamestonk_terminal/due_diligence/dd_controller.py index 14ecab8fc40..1324cf79cd1 100644 --- a/gamestonk_terminal/due_diligence/dd_controller.py +++ b/gamestonk_terminal/due_diligence/dd_controller.py @@ -13,6 +13,7 @@ from gamestonk_terminal.due_diligence import market_watch_view as mw_view from gamestonk_terminal.due_diligence import quandl_view as q_view from gamestonk_terminal.due_diligence import reddit_view as r_view from gamestonk_terminal.due_diligence import news_view +from gamestonk_terminal.due_diligence import finra_view from gamestonk_terminal import feature_flags as gtff from gamestonk_terminal.helper_funcs import get_flair from gamestonk_terminal.menu import session @@ -37,6 +38,7 @@ class DueDiligenceController: "analyst", "warnings", "sec", + "dp", ] def __init__(self, stock: DataFrame, ticker: str, start: str, interval: str): @@ -97,6 +99,7 @@ class DueDiligenceController: print( " warnings company warnings according to Sean Seah book [Market Watch]" ) + print(" dp dark pools vs OTC data [FINRA]") print("") def switch(self, an_input: str): @@ -175,6 +178,10 @@ class DueDiligenceController: """ Process short command """ q_view.short_interest(other_args, self.ticker, self.start) + def call_dp(self, other_args: List[str]): + """ Process dp command """ + finra_view.dark_pool(other_args, self.ticker) + def menu(stock: DataFrame, ticker: str, start: str, interval: str): """Due Diligence Menu diff --git a/gamestonk_terminal/due_diligence/finra_view.py b/gamestonk_terminal/due_diligence/finra_view.py new file mode 100644 index 00000000000..13767897840 --- /dev/null +++ b/gamestonk_terminal/due_diligence/finra_view.py @@ -0,0 +1,272 @@ +""" FINRA View """ +__docformat__ = "numpy" + +import argparse +from typing import List +import requests +import pandas as pd +from matplotlib import pyplot as plt +import matplotlib.dates as mdates +from gamestonk_terminal.config_plot import PLOT_DPI +from gamestonk_terminal import feature_flags as gtff +from gamestonk_terminal.helper_funcs import parse_known_args_and_warn, plot_autoscale + + +def getFINRAweeks(tier, is_ats) -> List: + """Get FINRA weeks + + Parameters + ---------- + tier : str + Stock tier between T1, T2, or OTCE + is_ats : bool + ATS data if true, NON-ATS otherwise + + Returns + ---------- + List + List of response data + """ + req_hdr = {"Accept": "application/json", "Content-Type": "application/json"} + + req_data = { + "compareFilters": [ + { + "compareType": "EQUAL", + "fieldName": "summaryTypeCode", + "fieldValue": "ATS_W_SMBL" if is_ats else "OTC_W_SMBL", + }, + { + "compareType": "EQUAL", + "fieldName": "tierIdentifier", + "fieldValue": tier, + }, + ], + "delimiter": "|", + "fields": ["weekStartDate"], + "limit": 27, + "quoteValues": False, + "sortFields": ["-weekStartDate"], + } + + response = requests.post( + "https://api.finra.org/data/group/otcMarket/name/weeklyDownloadDetails", + headers=req_hdr, + json=req_data, + ) + + return response.json() if response.status_code == 200 else list() + + +def getFINRAdata(weekStartDate, tier, ticker, is_ats) -> List: + """Get FINRA data + + Parameters + ---------- + weekStartDate : str + Weekly data to get FINRA data + tier : str + Stock tier between T1, T2, or OTCE + ticker : str + Stock ticker to get data from + is_ats : bool + ATS data if true, NON-ATS otherwise + + Returns + ---------- + str + Status code from request + List + List of response data + """ + req_hdr = {"Accept": "application/json", "Content-Type": "application/json"} + + l_cmp_filters = [ + { + "compareType": "EQUAL", + "fieldName": "weekStartDate", + "fieldValue": weekStartDate, + }, + {"compareType": "EQUAL", "fieldName": "tierIdentifier", "fieldValue": tier}, + { + "compareType": "EQUAL", + "description": "", + "fieldName": "summaryTypeCode", + "fieldValue": "ATS_W_SMBL" if is_ats else "OTC_W_SMBL", + }, + ] + + if ticker: + l_cmp_filters.append( + { + "compareType": "EQUAL", + "fieldName": "issueSymbolIdentifier", + "fieldValue": ticker, + } + ) + + req_data = { + "compareFilters": l_cmp_filters, + "delimiter": "|", + "fields": [ + "issueSymbolIdentifier", + "totalWeeklyShareQuantity", + "totalWeeklyTradeCount", + "lastUpdateDate", + ], + "limit": 5000, + "quoteValues": False, + "sortFields": ["totalWeeklyShareQuantity"], + } + + response = requests.post( + "https://api.finra.org/data/group/otcMarket/name/weeklySummary", + headers=req_hdr, + json=req_data, + ) + + return ( + response.status_code, + response.json() if response.status_code == 200 else list(), + ) + + +def getTickerFINRAdata(ticker): + """Get all FINRA data associated with a ticker + + Parameters + ---------- + ticker : str + Stock ticker to get data from + + Returns + ---------- + pd.DataFrame + Dark Pools (ATS) Data + pd.DataFrame + OTC (Non-ATS) Data + """ + tiers = ["T1", "T2", "OTCE"] + + l_data = list() + for tier in tiers: + for d_week in getFINRAweeks(tier, is_ats=True): + status_code, response = getFINRAdata( + d_week["weekStartDate"], tier, ticker, True + ) + if status_code == 200: + if response: + d_data = response[0] + d_data.update(d_week) + l_data.append(d_data) + else: + break + + df_ats = pd.DataFrame(l_data) + df_ats = df_ats.sort_values("weekStartDate") + df_ats = df_ats.set_index("weekStartDate") + + l_data = list() + for tier in tiers: + for d_week in getFINRAweeks(tier, is_ats=False): + status_code, response = getFINRAdata( + d_week["weekStartDate"], tier, ticker, False + ) + if status_code == 200: + if response: + d_data = response[0] + d_data.update(d_week) + l_data.append(d_data) + else: + break + + df_otc = pd.DataFrame(l_data) + df_otc = df_otc.sort_values("weekStartDate") + df_otc = df_otc.set_index("weekStartDate") + + return df_ats, df_otc + + +def plot_dark_pools(ticker: str, ats: pd.DataFrame, otc: pd.DataFrame): + """Plots ATS and NON-ATS data + + Parameters + ---------- + ticker : str + Stock ticker to get data from + ats : pd.DataFrame + Dark Pools (ATS) Data + otc : pd.DataFrame + OTC (Non-ATS) Data + """ + _, axData = plt.subplots(figsize=plot_autoscale(), dpi=PLOT_DPI) + + plt.subplot(3, 1, (1, 2)) + plt.bar( + ats.index, + (ats["totalWeeklyShareQuantity"] + otc["totalWeeklyShareQuantity"]) / 1_000_000, + color="tab:orange", + ) + plt.bar(otc.index, otc["totalWeeklyShareQuantity"] / 1_000_000, color="tab:blue") + plt.ylabel("Total Weekly Shares [Million]") + plt.legend(["ATS", "OTC"]) + plt.grid(b=True, which="major", color="#666666", linestyle="-", alpha=0.2) + plt.title(f"Dark Pools (ATS) vs OTC (Non-ATS) Data for {ticker}") + + # _ = axData.twinx() + plt.subplot(313) + plt.plot( + ats.index, + ats["totalWeeklyShareQuantity"] / ats["totalWeeklyTradeCount"], + color="tab:orange", + ) + plt.plot( + otc.index, + otc["totalWeeklyShareQuantity"] / otc["totalWeeklyTradeCount"], + color="tab:blue", + ) + plt.ylabel("Shares per Trade") + plt.legend(["ATS", "OTC"]) + plt.grid(b=True, which="major", color="#666666", linestyle="-", alpha=0.2) + plt.gcf().autofmt_xdate() + plt.xlabel("Weeks") + + if gtff.USE_ION: + plt.ion() + + plt.show() + + +def dark_pool(other_args: List[str], ticker: str): + """Display barchart of dark pool (ATS) and OTC (Non ATS) data + + Parameters + ---------- + other_args : List[str] + argparse other args + ticker : str + Stock ticker + """ + parser = argparse.ArgumentParser( + add_help=False, + prog="dp", + description="Display barchart of dark pool (ATS) and OTC (Non ATS) data [Source: FINRA]", + ) + + if 1: + ns_parser = parse_known_args_and_warn(parser, other_args) + if not ns_parser: + return + + df_ats, df_otc = getTickerFINRAdata(ticker) + + plot_dark_pools(ticker, df_ats, df_otc) + + print("") + + """ + except Exception as e: + print(e) + print("") + return + """ From 0032e12abc99486538ab9e8c8cdb2852eb203bad Mon Sep 17 00:00:00 2001 From: DidierRLopes Date: Sun, 18 Apr 2021 02:08:54 +0100 Subject: [PATCH 2/8] Update README.md --- gamestonk_terminal/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/gamestonk_terminal/README.md b/gamestonk_terminal/README.md index 192f21ca288..eb3dfae535b 100644 --- a/gamestonk_terminal/README.md +++ b/gamestonk_terminal/README.md @@ -213,6 +213,7 @@ Command|Description|Source `sec` |SEC filings |[MarketWatch](https://www.marketwatch.com/) `short` |short interest |[Quandl](https://www.quandl.com/) `warnings` |company warnings according to Sean Seah book |[MarketWatch](https://www.marketwatch.com/) +`dp` |dark pools (ATS) vs OTC data [FINRA](https://www.finra.org/#/)   From e04e4053c0ebdba9eff7e5065808d0874767bb6c Mon Sep 17 00:00:00 2001 From: didier Date: Sun, 18 Apr 2021 02:09:58 +0100 Subject: [PATCH 3/8] remove unused import --- gamestonk_terminal/due_diligence/finra_view.py | 1 - 1 file changed, 1 deletion(-) diff --git a/gamestonk_terminal/due_diligence/finra_view.py b/gamestonk_terminal/due_diligence/finra_view.py index 13767897840..a3358a71a5b 100644 --- a/gamestonk_terminal/due_diligence/finra_view.py +++ b/gamestonk_terminal/due_diligence/finra_view.py @@ -6,7 +6,6 @@ from typing import List import requests import pandas as pd from matplotlib import pyplot as plt -import matplotlib.dates as mdates from gamestonk_terminal.config_plot import PLOT_DPI from gamestonk_terminal import feature_flags as gtff from gamestonk_terminal.helper_funcs import parse_known_args_and_warn, plot_autoscale From 477751915c448841bac6c1849f8153bcf0676b7f Mon Sep 17 00:00:00 2001 From: DidierRLopes Date: Sun, 18 Apr 2021 02:12:19 +0100 Subject: [PATCH 4/8] Update README.md --- gamestonk_terminal/due_diligence/README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/gamestonk_terminal/due_diligence/README.md b/gamestonk_terminal/due_diligence/README.md index 70bfb49340b..472867b0f7e 100644 --- a/gamestonk_terminal/due_diligence/README.md +++ b/gamestonk_terminal/due_diligence/README.md @@ -24,6 +24,8 @@ This menu aims to help in due-diligence of a pre-loaded stock, and the usage of * short interest [Quandl] * [warnings](#warnings) * company warnings according to Sean Seah book [Market Watch] +* [dp](#dp) + * dark pools (ATS) vs OTC data [FINRA] ## news @@ -157,3 +159,15 @@ Sean Seah warnings. Check: Consistent historical earnings per share; Consistentl * -d : print insights into warnings calculation. Default False. warnings + +## dp + +```text +usage: dp +``` + +Display barchart of dark pool (ATS) and OTC (Non ATS) data + +![dp](https://user-images.githubusercontent.com/25267873/115130908-7987b580-9feb-11eb-8bca-1999174178d0.png) + + From 73681c2d75eff6dfb6fc88599849914eb9d6184f Mon Sep 17 00:00:00 2001 From: didier Date: Sun, 18 Apr 2021 02:35:30 +0100 Subject: [PATCH 5/8] Improve code of dd/dp --- .../due_diligence/dd_controller.py | 2 +- .../due_diligence/finra_view.py | 88 +++++++++++++------ 2 files changed, 63 insertions(+), 27 deletions(-) diff --git a/gamestonk_terminal/due_diligence/dd_controller.py b/gamestonk_terminal/due_diligence/dd_controller.py index 1324cf79cd1..378499f05cd 100644 --- a/gamestonk_terminal/due_diligence/dd_controller.py +++ b/gamestonk_terminal/due_diligence/dd_controller.py @@ -99,7 +99,7 @@ class DueDiligenceController: print( " warnings company warnings according to Sean Seah book [Market Watch]" ) - print(" dp dark pools vs OTC data [FINRA]") + print(" dp dark pools (ATS) vs OTC data [FINRA]") print("") def switch(self, an_input: str): diff --git a/gamestonk_terminal/due_diligence/finra_view.py b/gamestonk_terminal/due_diligence/finra_view.py index a3358a71a5b..ea3e80bfa71 100644 --- a/gamestonk_terminal/due_diligence/finra_view.py +++ b/gamestonk_terminal/due_diligence/finra_view.py @@ -2,7 +2,7 @@ __docformat__ = "numpy" import argparse -from typing import List +from typing import List, Any import requests import pandas as pd from matplotlib import pyplot as plt @@ -57,7 +57,7 @@ def getFINRAweeks(tier, is_ats) -> List: return response.json() if response.status_code == 200 else list() -def getFINRAdata(weekStartDate, tier, ticker, is_ats) -> List: +def getFINRAdata(weekStartDate, tier, ticker, is_ats) -> [int, List]: """Get FINRA data Parameters @@ -130,7 +130,7 @@ def getFINRAdata(weekStartDate, tier, ticker, is_ats) -> List: ) -def getTickerFINRAdata(ticker): +def getTickerFINRAdata(ticker) -> [pd.DataFrame, pd.DataFrame]: """Get all FINRA data associated with a ticker Parameters @@ -162,8 +162,9 @@ def getTickerFINRAdata(ticker): break df_ats = pd.DataFrame(l_data) - df_ats = df_ats.sort_values("weekStartDate") - df_ats = df_ats.set_index("weekStartDate") + if not df_ats.empty: + df_ats = df_ats.sort_values("weekStartDate") + df_ats = df_ats.set_index("weekStartDate") l_data = list() for tier in tiers: @@ -180,8 +181,9 @@ def getTickerFINRAdata(ticker): break df_otc = pd.DataFrame(l_data) - df_otc = df_otc.sort_values("weekStartDate") - df_otc = df_otc.set_index("weekStartDate") + if not df_otc.empty: + df_otc = df_otc.sort_values("weekStartDate") + df_otc = df_otc.set_index("weekStartDate") return df_ats, df_otc @@ -201,31 +203,62 @@ def plot_dark_pools(ticker: str, ats: pd.DataFrame, otc: pd.DataFrame): _, axData = plt.subplots(figsize=plot_autoscale(), dpi=PLOT_DPI) plt.subplot(3, 1, (1, 2)) - plt.bar( - ats.index, - (ats["totalWeeklyShareQuantity"] + otc["totalWeeklyShareQuantity"]) / 1_000_000, - color="tab:orange", - ) - plt.bar(otc.index, otc["totalWeeklyShareQuantity"] / 1_000_000, color="tab:blue") + if not ats.empty and not otc.empty: + plt.bar( + ats.index, + (ats["totalWeeklyShareQuantity"] + otc["totalWeeklyShareQuantity"]) + / 1_000_000, + color="tab:orange", + ) + plt.bar( + otc.index, otc["totalWeeklyShareQuantity"] / 1_000_000, color="tab:blue" + ) + plt.legend(["ATS", "OTC"]) + + elif not ats.empty: + plt.bar( + ats.index, + ats["totalWeeklyShareQuantity"] / 1_000_000, + color="tab:orange", + ) + plt.legend(["ATS"]) + + elif not otc.empty: + plt.bar( + otc.index, otc["totalWeeklyShareQuantity"] / 1_000_000, color="tab:blue" + ) + plt.legend(["OTC"]) + plt.ylabel("Total Weekly Shares [Million]") - plt.legend(["ATS", "OTC"]) plt.grid(b=True, which="major", color="#666666", linestyle="-", alpha=0.2) plt.title(f"Dark Pools (ATS) vs OTC (Non-ATS) Data for {ticker}") - # _ = axData.twinx() plt.subplot(313) - plt.plot( - ats.index, - ats["totalWeeklyShareQuantity"] / ats["totalWeeklyTradeCount"], - color="tab:orange", - ) - plt.plot( - otc.index, - otc["totalWeeklyShareQuantity"] / otc["totalWeeklyTradeCount"], - color="tab:blue", - ) + if not ats.empty: + plt.plot( + ats.index, + ats["totalWeeklyShareQuantity"] / ats["totalWeeklyTradeCount"], + color="tab:orange", + ) + plt.legend(["ATS"]) + + if not otc.empty: + plt.plot( + otc.index, + otc["totalWeeklyShareQuantity"] / otc["totalWeeklyTradeCount"], + color="tab:blue", + ) + plt.legend(["ATS", "OTC"]) + + else: + plt.plot( + otc.index, + otc["totalWeeklyShareQuantity"] / otc["totalWeeklyTradeCount"], + color="tab:blue", + ) + plt.legend(["OTC"]) + plt.ylabel("Shares per Trade") - plt.legend(["ATS", "OTC"]) plt.grid(b=True, which="major", color="#666666", linestyle="-", alpha=0.2) plt.gcf().autofmt_xdate() plt.xlabel("Weeks") @@ -259,6 +292,9 @@ def dark_pool(other_args: List[str], ticker: str): df_ats, df_otc = getTickerFINRAdata(ticker) + if df_ats.empty and df_otc.empty: + print("No ticker data found!") + plot_dark_pools(ticker, df_ats, df_otc) print("") From eae456cc3f9c1022957a77888c16964150c4e18d Mon Sep 17 00:00:00 2001 From: didier Date: Sun, 18 Apr 2021 02:37:29 +0100 Subject: [PATCH 6/8] remove unused import --- gamestonk_terminal/due_diligence/finra_view.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gamestonk_terminal/due_diligence/finra_view.py b/gamestonk_terminal/due_diligence/finra_view.py index ea3e80bfa71..47ce2f7c817 100644 --- a/gamestonk_terminal/due_diligence/finra_view.py +++ b/gamestonk_terminal/due_diligence/finra_view.py @@ -2,7 +2,7 @@ __docformat__ = "numpy" import argparse -from typing import List, Any +from typing import List import requests import pandas as pd from matplotlib import pyplot as plt From e02a4a8baaa0c7cbfedb679e501a03512ca5b559 Mon Sep 17 00:00:00 2001 From: didier Date: Sun, 18 Apr 2021 02:49:25 +0100 Subject: [PATCH 7/8] fix typing --- gamestonk_terminal/due_diligence/finra_view.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gamestonk_terminal/due_diligence/finra_view.py b/gamestonk_terminal/due_diligence/finra_view.py index 47ce2f7c817..ab4fc2465aa 100644 --- a/gamestonk_terminal/due_diligence/finra_view.py +++ b/gamestonk_terminal/due_diligence/finra_view.py @@ -2,7 +2,7 @@ __docformat__ = "numpy" import argparse -from typing import List +from typing import List, Tuple import requests import pandas as pd from matplotlib import pyplot as plt @@ -57,7 +57,7 @@ def getFINRAweeks(tier, is_ats) -> List: return response.json() if response.status_code == 200 else list() -def getFINRAdata(weekStartDate, tier, ticker, is_ats) -> [int, List]: +def getFINRAdata(weekStartDate, tier, ticker, is_ats) -> Tuple[int, List]: """Get FINRA data Parameters @@ -130,7 +130,7 @@ def getFINRAdata(weekStartDate, tier, ticker, is_ats) -> [int, List]: ) -def getTickerFINRAdata(ticker) -> [pd.DataFrame, pd.DataFrame]: +def getTickerFINRAdata(ticker) -> Tuple[pd.DataFrame, pd.DataFrame]: """Get all FINRA data associated with a ticker Parameters From 33ced3512928597a9754d89b53a9f42fd6f09418 Mon Sep 17 00:00:00 2001 From: DidierRLopes Date: Sun, 18 Apr 2021 20:14:28 +0100 Subject: [PATCH 8/8] Update ROADMAP.md --- ROADMAP.md | 1 + 1 file changed, 1 insertion(+) diff --git a/ROADMAP.md b/ROADMAP.md index 82d5b910ed9..d47c8d9c7d3 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -79,6 +79,7 @@ --- ## Due Diligence +* [x] Add dark pools (ATS) vs Non-ATS data over time (@didier) - [PR #363](https://github.com/DidierRLopes/GamestonkTerminal/pull/363) **NEXT**