Hotfix/databento + bugs (#4973)

* bump pywry

* fix rm command (#4890)

Co-authored-by: Henrique Joaquim <h.joaquim@campus.fct.unl.pt>

* Fix `crypto/defi/anchor` (#4892)

* handle defi anchor exceptions

* reason

* mypy

---------

Co-authored-by: Henrique Joaquim <h.joaquim@campus.fct.unl.pt>

* fix windows pyinstaller subprocess fail if space in path

* remove API_IEX_TOKEN

* remove API_SENTIMENTINVESTOR_TOKEN

* Fix data bento calls

* Fix options eod chain bug

* Fix finviz get

* Fix yf balance bug

* removing breakline

* Add error message

---------

Co-authored-by: teh_coderer <me@tehcoderer.com>
Co-authored-by: montezdesousa <79287829+montezdesousa@users.noreply.github.com>
Co-authored-by: Henrique Joaquim <h.joaquim@campus.fct.unl.pt>
This commit is contained in:
James Maslek
2023-05-09 01:16:32 -07:00
committed by GitHub
parent eaf27ab522
commit 425b740eda
9 changed files with 85 additions and 107 deletions

View File

@@ -61,11 +61,11 @@
"\n",
"from openbb_terminal import theme\n",
"\n",
"#cfg.theme = TerminalStyle(\n",
"# cfg.theme = TerminalStyle(\n",
"# user.preferences.MPL_STYLE,\n",
"# user.preferences.PMF_STYLE,\n",
"# user.preferences.RICH_STYLE,\n",
"#)"
"# )"
]
},
{
@@ -214,9 +214,7 @@
" if not isinstance(table, pd.DataFrame):\n",
" continue\n",
"\n",
" table[0].loc[\"URL\"] = (\n",
" table[0].loc[\"URL\"].apply(lambda x: f'<a href=\"{x}\">{x}</a>')\n",
" )"
" table[0].loc[\"URL\"] = table[0].loc[\"URL\"].apply(lambda x: f'<a href=\"{x}\">{x}</a>')"
]
},
{
@@ -249,7 +247,7 @@
"metadata": {},
"outputs": [],
"source": [
"#this command works 1/2 the time - really need to fix this - but this will catch the errors and stop it from breaking the report\n",
"# this command works 1/2 the time - really need to fix this - but this will catch the errors and stop it from breaking the report\n",
"try:\n",
" df_sec_filings = openbb.stocks.fa.sec(symbol)[\n",
" [\"Company Name\", \"Reporting Owner\", \"Form Type\", \"Period\", \"View\"]\n",
@@ -266,9 +264,9 @@
"metadata": {},
"outputs": [],
"source": [
"df_analyst = openbb.stocks.fa.pt(symbol=symbol).sort_index(ascending = False)\n",
"#print(df_analyst)\n",
"#if not df_analyst.empty:\n",
"df_analyst = openbb.stocks.fa.pt(symbol=symbol).sort_index(ascending=False)\n",
"# print(df_analyst)\n",
"# if not df_analyst.empty:\n",
"# if \"target\" in df_analyst.columns:\n",
"# if \"target_to\" in df_analyst.columns:\n",
"# df_analyst[\"target\"] = df_analyst[\"target_to\"].combine_first(\n",
@@ -826,7 +824,7 @@
"ticker_data_all = openbb.stocks.load(\n",
" symbol=symbol,\n",
" start_date=datetime.datetime.now() - datetime.timedelta(days=5 * 12 * 21),\n",
" verbose = False\n",
" verbose=False,\n",
")\n",
"ticker_data_all[\"Returns\"] = ticker_data_all[\"Adj Close\"].pct_change()"
]
@@ -876,8 +874,8 @@
"metadata": {},
"outputs": [],
"source": [
"income_df = openbb.stocks.fa.income(symbol, source = 'FinancialModelingPrep')\n",
"data_df = openbb.stocks.fa.overview(symbol, source = 'FinancialModelingPrep')\n",
"income_df = openbb.stocks.fa.income(symbol, source=\"FinancialModelingPrep\")\n",
"data_df = openbb.stocks.fa.overview(symbol, source=\"FinancialModelingPrep\")\n",
"mgmt_df = openbb.stocks.fa.mgmt(symbol)\n",
"try:\n",
" if not mgmt_df.empty:\n",
@@ -1149,11 +1147,11 @@
" htmlcode += widgets.row([widgets.h(3, \"Analyst Ratings\") + df_analyst.to_html()])\n",
"except:\n",
" pass\n",
"#try:\n",
"# try:\n",
"# htmlcode += widgets.row(\n",
"# [widgets.h(3, \"Analyst Recommendations\") + df_rating.to_html()]\n",
"# )\n",
"#except:\n",
"# except:\n",
"# pass\n",
"\n",
"body += widgets.add_tab(\"Analyst Opinions\", htmlcode)\n",

View File

@@ -292,7 +292,6 @@ class ComparisonAnalysisController(BaseController):
console.print(
f"[{self.user}] Similar Companies: {', '.join(self.similar)}",
"\n",
)
elif ns_parser.source == "Polygon":
self.similar = polygon_model.get_similar_companies(

View File

@@ -41,9 +41,9 @@ def get_similar_companies(
"""
try:
compare_list = ["Sector", "Industry"] if compare_list is None else compare_list
similar = (
Overview().compare(symbol, compare_list, verbose=0)["Ticker"].to_list()
)
similar = Overview().compare(symbol, compare_list, verbose=0)
similar.columns = [x.strip() for x in similar.columns]
return similar.Ticker.to_list()
except Exception as e:
logger.exception(str(e))
console.print(e)

View File

@@ -36,54 +36,23 @@ class DataBento(BaseModel):
Note that only nonadjusted data is available."""
self.exchange = "XNAS.ITCH"
self.stype = "native"
if not self.validate_symbol():
return pd.DataFrame()
return self.get_historical()
def validate_symbol(self):
"""Validates the symbol is supported by DataBento. There is no API for this, so we will use their
Symbology endpoint to see if something exists. Doing a long time range can make this request on the order of
one minute, so try to keep smaller requests."""
base_url = "https://hist.databento.com/v0/symbology.resolve"
params = {
"dataset": self.exchange,
"symbols": self.symbol,
"start_date": self.start,
"end_date": self.end,
"stype_in": "smart",
"stype_out": "product_id",
}
auth = requests.auth.HTTPBasicAuth(
get_current_user().credentials.API_DATABENTO_KEY, ""
)
# This seems to only work for futures? Assume the user is entering correct stock ticker
if self.exchange == "XNAS.ITCH":
return True
result = request(base_url, params=params, auth=auth, timeout=30)
if "message" not in result.json():
console.print(
"Issue validating symbol in the date range. Please check with DataBento for symbology."
)
return False
return result.json()["message"] != "Not found"
def get_historical_futures(self):
"""Gets historical EODfutures data from DataBento. Currently, just CME is supported.
Note this gets the highest volume contract each day"""
self.exchange = "GLBX.MDP3"
self.stype = "smart"
if not self.validate_symbol():
return pd.DataFrame()
self.stype = "continuous"
return self.get_historical()
@log_start_end(log=logger)
@check_api_key(["API_DATABENTO_KEY"])
def get_historical(self) -> pd.DataFrame:
"""Prepares the request to be made. I am using their https interface instead of their python client.
Their next update will change symbol.V.0 to symbol.C.0"""
This updated to .C.0"""
base_url = "https://hist.databento.com/v0/timeseries.get_range"
symbol = self.symbol + ".V.0" if self.exchange == "GLBX.MDP3" else self.symbol
symbol = self.symbol + ".C.0" if self.exchange == "GLBX.MDP3" else self.symbol
params = {
"dataset": self.exchange,
"symbols": symbol,
@@ -98,20 +67,24 @@ class DataBento(BaseModel):
)
data = self.process_request(base_url, params, auth)
if data.empty:
console.print(f"No data found for symbol `{self.symbol}`.")
return pd.DataFrame()
return data
def process_request(self, base_url, params, auth) -> pd.DataFrame:
"""Takes the request and returns the adjusted dataframe"""
r = request(base_url, params=params, auth=auth)
if r.status_code == 422 and "Unprocessable" in r.text:
console.print(r.json()["detail"])
return pd.DataFrame()
if r.status_code != 200:
raise Exception(f"Error: Status Code {r.status_code}")
console.print(f"Error: Status Code {r.status_code}")
return pd.DataFrame()
df = pd.read_csv(StringIO(r.text))
df["time"] = pd.to_datetime(df.ts_event, unit="ns")
df[["open", "high", "low", "close"]] /= 1_000_000_000
df = df.drop(columns=["ts_event", "publisher_id", "product_id"]).set_index(
"time"
)
df = df.drop(columns=["ts_event", "publisher_id"]).set_index("time")
df = df.rename(
columns={
"open": "Open",

View File

@@ -134,6 +134,7 @@ class OptionsController(BaseController):
self.full_chain: pd.DataFrame = pd.DataFrame()
self.expiry_dates: List[str] = []
self.current_price = 0.0
self.is_eod = False
# Keeps track of initial source of load so we can use correct commands later
self.source = get_ordered_list_sources(f"{self.PATH}load")[0]
if ticker:
@@ -654,6 +655,7 @@ class OptionsController(BaseController):
):
self.ticker = ns_parser.ticker.upper()
self.source = ns_parser.source
self.is_eod = False
self.set_option_chain()
self.set_current_price()
@@ -924,9 +926,11 @@ class OptionsController(BaseController):
df_chain = self.chain.copy()
needed = ["symbol", "code", "optionType", "expiration", "strike"]
needed = [col for col in needed if col in df_chain.columns]
if ns_parser.to_display:
to_display = ns_parser.to_display.split(",")
if self.is_eod:
for col in ["contractSymbol", "lastPrice", "bid", "ask"]:
to_display.remove(col)
display = [col for col in to_display if col not in needed]
df_chain = df_chain[needed + display]
@@ -1447,3 +1451,4 @@ class OptionsController(BaseController):
self.full_chain = intrinio_model.get_full_chain_eod(
self.ticker, datetime.strftime(ns_parser.date, "%Y-%m-%d")
)
self.is_eod = True

View File

@@ -321,6 +321,9 @@ BALANCE_PLOT = {
"accumulated_other_comprehensive_income": "accumulated_other_comprehensive_income",
"total_stockholders'_equity": "total_shareholder_equity",
"total_liabilities_and_stockholders'_equity": "total_liabilities_and_stockholders_equity",
"goodwill": "goodwill",
"intangible_assets": "intangible_assets_excluding_goodwill",
"taxes_payable": "taxes_payable",
},
"FinancialModelingPrep": {
"reported_currency": "reported_currency",

View File

@@ -11,10 +11,10 @@ interactions:
Connection:
- keep-alive
method: GET
uri: https://hist.databento.com/v0/timeseries.get_range?dataset=XNAS.ITCH&symbols=AAPL&schema=ohlcv-1d&start=2022-08-01&end=2022-12-01&stype_in=native&encoding=csv
uri: https://hist.databento.com/v0/timeseries.get_range?dataset=XNAS.ITCH&symbols=AAPL&schema=ohlcv-1d&start=2022-08-01&end=2022-10-01&stype_in=native&encoding=csv
response:
body:
string: 'ts_event,publisher_id,product_id,open,high,low,close,volume
string: 'ts_event,publisher_id,instrument_id,open,high,low,close,volume
1659312000000000000,2,27,162200000000,163590000000,160830000000,161400000000,10045637
@@ -107,7 +107,7 @@ interactions:
'
headers:
date:
- Mon, 17 Apr 2023 18:16:03 GMT
- Thu, 04 May 2023 19:33:41 GMT
server:
- uvicorn
transfer-encoding:

View File

@@ -1,45 +1,45 @@
time,Open,High,Low,Close,Volume
2022-08-01,162.2,163.59,160.83,161.4,10045637
2022-08-02,160.73,162.41,159.55,160.01,10065910
2022-08-03,160.06,166.595,159.85,165.84,13326012
2022-08-04,165.83,167.19,164.43,165.85,9578445
2022-08-05,166.0,166.23,162.79,164.92,9683719
2022-08-08,166.08,167.8,164.2,165.15,9135766
2022-08-09,164.99,165.81,163.46,164.97,9230005
2022-08-10,165.01,169.31,164.56,169.13,11638513
2022-08-11,169.24,170.99,168.19,168.8,9732568
2022-08-12,169.18,172.17,168.65,171.85,11916663
2022-08-15,171.14,173.39,170.74,173.15,8485815
2022-08-16,172.83,174.08,171.67,173.7,9979182
2022-08-17,172.67,176.15,172.14,173.32,13581411
2022-08-18,173.2,174.89,173.13,174.09,11175367
2022-08-19,173.6,173.73,170.7,170.79,11296793
2022-08-22,169.4,169.86,167.135,167.95,13070702
2022-08-23,167.61,168.7,166.65,167.13,10629743
2022-08-24,166.93,168.41,166.245,167.91,9482192
2022-08-25,169.27,170.34,168.23,170.2,9874644
2022-08-26,169.9,171.13,162.3,162.45,14062181
2022-08-29,161.79,162.91,159.83,161.78,12363097
2022-08-30,162.6,163.34,157.73,158.96,11853126
2022-08-31,159.65,160.57,155.77,156.0,13343001
2022-09-01,155.89,158.69,154.675,158.62,13016463
2022-09-02,157.71,160.36,154.965,155.65,12975036
2022-09-06,156.79,157.39,153.73,154.49,14840703
2022-09-07,154.8,156.66,153.61,155.96,13205605
2022-09-08,155.95,156.43,152.68,154.62,14072194
2022-09-09,155.26,157.805,154.76,157.52,10213517
2022-09-12,158.0,164.26,158.0,164.1,15959461
2022-09-13,164.15,164.9,153.385,154.25,19181953
2022-09-14,154.47,157.1,153.25,156.0,15250245
2022-09-15,155.96,156.04,151.1,151.27,13929824
2022-09-16,151.35,151.51,148.36,151.29,19608708
2022-09-19,149.82,155.0,148.7,155.0,13024172
2022-09-20,154.27,158.08,153.09,156.48,16126729
2022-09-21,156.58,158.74,152.24,152.4,16938650
2022-09-22,153.53,154.96,150.92,153.09,14902502
2022-09-23,152.25,152.55,148.56,150.57,16074945
2022-09-26,150.42,153.77,149.14,151.1,14918149
2022-09-27,152.84,154.69,149.95,151.87,14873042
2022-09-28,148.5,150.64,144.84,149.37,24532649
2022-09-29,148.0,148.18,140.68,142.41,21075773
2022-09-30,143.16,143.58,137.22,137.54,20801834
time,instrument_id,Open,High,Low,Close,Volume
2022-08-01,27,162.2,163.59,160.83,161.4,10045637
2022-08-02,27,160.73,162.41,159.55,160.01,10065910
2022-08-03,27,160.06,166.595,159.85,165.84,13326012
2022-08-04,27,165.83,167.19,164.43,165.85,9578445
2022-08-05,27,166.0,166.23,162.79,164.92,9683719
2022-08-08,27,166.08,167.8,164.2,165.15,9135766
2022-08-09,29,164.99,165.81,163.46,164.97,9230005
2022-08-10,29,165.01,169.31,164.56,169.13,11638513
2022-08-11,29,169.24,170.99,168.19,168.8,9732568
2022-08-12,29,169.18,172.17,168.65,171.85,11916663
2022-08-15,29,171.14,173.39,170.74,173.15,8485815
2022-08-16,29,172.83,174.08,171.67,173.7,9979182
2022-08-17,29,172.67,176.15,172.14,173.32,13581411
2022-08-18,29,173.2,174.89,173.13,174.09,11175367
2022-08-19,29,173.6,173.73,170.7,170.79,11296793
2022-08-22,29,169.4,169.86,167.135,167.95,13070702
2022-08-23,29,167.61,168.7,166.65,167.13,10629743
2022-08-24,29,166.93,168.41,166.245,167.91,9482192
2022-08-25,29,169.27,170.34,168.23,170.2,9874644
2022-08-26,29,169.9,171.13,162.3,162.45,14062181
2022-08-29,29,161.79,162.91,159.83,161.78,12363097
2022-08-30,29,162.6,163.34,157.73,158.96,11853126
2022-08-31,29,159.65,160.57,155.77,156.0,13343001
2022-09-01,29,155.89,158.69,154.675,158.62,13016463
2022-09-02,29,157.71,160.36,154.965,155.65,12975036
2022-09-06,29,156.79,157.39,153.73,154.49,14840703
2022-09-07,29,154.8,156.66,153.61,155.96,13205605
2022-09-08,29,155.95,156.43,152.68,154.62,14072194
2022-09-09,29,155.26,157.805,154.76,157.52,10213517
2022-09-12,29,158.0,164.26,158.0,164.1,15959461
2022-09-13,29,164.15,164.9,153.385,154.25,19181953
2022-09-14,29,154.47,157.1,153.25,156.0,15250245
2022-09-15,29,155.96,156.04,151.1,151.27,13929824
2022-09-16,29,151.35,151.51,148.36,151.29,19608708
2022-09-19,29,149.82,155.0,148.7,155.0,13024172
2022-09-20,29,154.27,158.08,153.09,156.48,16126729
2022-09-21,29,156.58,158.74,152.24,152.4,16938650
2022-09-22,29,153.53,154.96,150.92,153.09,14902502
2022-09-23,29,152.25,152.55,148.56,150.57,16074945
2022-09-26,29,150.42,153.77,149.14,151.1,14918149
2022-09-27,29,152.84,154.69,149.95,151.87,14873042
2022-09-28,29,148.5,150.64,144.84,149.37,24532649
2022-09-29,29,148.0,148.18,140.68,142.41,21075773
2022-09-30,29,143.16,143.58,137.22,137.54,20801834
1 time instrument_id Open High Low Close Volume
2 2022-08-01 27 162.2 163.59 160.83 161.4 10045637
3 2022-08-02 27 160.73 162.41 159.55 160.01 10065910
4 2022-08-03 27 160.06 166.595 159.85 165.84 13326012
5 2022-08-04 27 165.83 167.19 164.43 165.85 9578445
6 2022-08-05 27 166.0 166.23 162.79 164.92 9683719
7 2022-08-08 27 166.08 167.8 164.2 165.15 9135766
8 2022-08-09 29 164.99 165.81 163.46 164.97 9230005
9 2022-08-10 29 165.01 169.31 164.56 169.13 11638513
10 2022-08-11 29 169.24 170.99 168.19 168.8 9732568
11 2022-08-12 29 169.18 172.17 168.65 171.85 11916663
12 2022-08-15 29 171.14 173.39 170.74 173.15 8485815
13 2022-08-16 29 172.83 174.08 171.67 173.7 9979182
14 2022-08-17 29 172.67 176.15 172.14 173.32 13581411
15 2022-08-18 29 173.2 174.89 173.13 174.09 11175367
16 2022-08-19 29 173.6 173.73 170.7 170.79 11296793
17 2022-08-22 29 169.4 169.86 167.135 167.95 13070702
18 2022-08-23 29 167.61 168.7 166.65 167.13 10629743
19 2022-08-24 29 166.93 168.41 166.245 167.91 9482192
20 2022-08-25 29 169.27 170.34 168.23 170.2 9874644
21 2022-08-26 29 169.9 171.13 162.3 162.45 14062181
22 2022-08-29 29 161.79 162.91 159.83 161.78 12363097
23 2022-08-30 29 162.6 163.34 157.73 158.96 11853126
24 2022-08-31 29 159.65 160.57 155.77 156.0 13343001
25 2022-09-01 29 155.89 158.69 154.675 158.62 13016463
26 2022-09-02 29 157.71 160.36 154.965 155.65 12975036
27 2022-09-06 29 156.79 157.39 153.73 154.49 14840703
28 2022-09-07 29 154.8 156.66 153.61 155.96 13205605
29 2022-09-08 29 155.95 156.43 152.68 154.62 14072194
30 2022-09-09 29 155.26 157.805 154.76 157.52 10213517
31 2022-09-12 29 158.0 164.26 158.0 164.1 15959461
32 2022-09-13 29 164.15 164.9 153.385 154.25 19181953
33 2022-09-14 29 154.47 157.1 153.25 156.0 15250245
34 2022-09-15 29 155.96 156.04 151.1 151.27 13929824
35 2022-09-16 29 151.35 151.51 148.36 151.29 19608708
36 2022-09-19 29 149.82 155.0 148.7 155.0 13024172
37 2022-09-20 29 154.27 158.08 153.09 156.48 16126729
38 2022-09-21 29 156.58 158.74 152.24 152.4 16938650
39 2022-09-22 29 153.53 154.96 150.92 153.09 14902502
40 2022-09-23 29 152.25 152.55 148.56 150.57 16074945
41 2022-09-26 29 150.42 153.77 149.14 151.1 14918149
42 2022-09-27 29 152.84 154.69 149.95 151.87 14873042
43 2022-09-28 29 148.5 150.64 144.84 149.37 24532649
44 2022-09-29 29 148.0 148.18 140.68 142.41 21075773
45 2022-09-30 29 143.16 143.58 137.22 137.54 20801834

View File

@@ -29,7 +29,7 @@ def test_databento_model():
def test_stock_load(recorder):
"""Tests the stock load function."""
result = databento_model.get_historical_stock(
symbol="AAPL", start_date="2022-08-01", end_date="2022-12-01"
symbol="AAPL", start_date="2022-08-01", end_date="2022-10-01"
)
assert result.empty is False
recorder.capture(result)