diff --git a/README.md b/README.md index ad3132ec7f9..9059695775c 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ | OpenBB is committed to build the future of investment research by focusing on an open source infrastructure accessible to everyone, everywhere. | |:--:| -| ![openbb.jpg](/images/openbb_gradient.png) | +| [![openbb.jpg](/images/openbb_gradient.png)](https://openbb.co) | | Check our website at [openbb.co](https://openbb.co) |
diff --git a/openbb_terminal/config_terminal.py b/openbb_terminal/config_terminal.py index 53791280034..117ecec303b 100644 --- a/openbb_terminal/config_terminal.py +++ b/openbb_terminal/config_terminal.py @@ -200,3 +200,6 @@ OPENBB_PERSONAL_ACCESS_TOKEN = ( # https://intrinio.com/starter-plan API_INTRINIO_KEY = os.getenv("OPENBB_API_INTRINIO_KEY") or "REPLACE_ME" + +# https://databento.com/ +API_DATABENTO_KEY = os.getenv("OPENBB_API_DATABENTO_KEY") or "REPLACE_ME" diff --git a/openbb_terminal/core/integration_tests/integration_controller.py b/openbb_terminal/core/integration_tests/integration_controller.py index cf473ae1f6c..a68b997ca95 100644 --- a/openbb_terminal/core/integration_tests/integration_controller.py +++ b/openbb_terminal/core/integration_tests/integration_controller.py @@ -13,7 +13,7 @@ from multiprocessing import cpu_count from multiprocessing.pool import Pool from pathlib import Path from traceback import FrameSummary, extract_tb, format_list -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Callable, Dict, List, Optional, Tuple from openbb_terminal.core.config.paths import ( MISCELLANEOUS_DIRECTORY, @@ -353,6 +353,7 @@ def run_test_files( verbose: bool = False, special_arguments: Optional[Dict[str, str]] = None, subprocesses: Optional[int] = None, + ordered: bool = False, ) -> Tuple[int, int, Dict[str, Dict[str, Any]], float]: """Runs the test scripts and returns the fails dictionary @@ -366,6 +367,8 @@ def run_test_files( The special arguments to use in the scripts subprocesses: Optional[int] The number of subprocesses to use to run the tests + ordered: bool + Multiprocessing is not ordered by default. Use this flag to run the tests in order Returns ------- @@ -410,8 +413,10 @@ def run_test_files( if extra: chunksize += 1 + runner: Callable = pool.imap if ordered else pool.imap_unordered + for i, result in enumerate( - pool.imap( + runner( partial( run_test, verbose=verbose, @@ -531,6 +536,7 @@ def run_test_session( special_arguments: Optional[Dict[str, str]] = None, verbose: bool = False, subprocesses: Optional[int] = None, + ordered: bool = False, ): """Run the integration test session @@ -552,11 +558,13 @@ def run_test_session( Whether or not to print the output of the scripts subprocesses The number of subprocesses to use to run the tests + ordered: bool + Multiprocessing is not ordered by default. Use this flag to run the tests in order. """ console.print(to_section_title("integration test session starts"), style="bold") test_files = collect_test_files(path_list, skip_list) n_successes, n_failures, fails, seconds = run_test_files( - test_files, verbose, special_arguments, subprocesses + test_files, verbose, special_arguments, subprocesses, ordered ) display_failures(fails) display_summary(fails, n_successes, n_failures, seconds) @@ -645,6 +653,14 @@ def parse_args_and_run(): action="store_true", default=False, ) + parser.add_argument( + "-o", + "--ordered", + help="Multiprocessing is not ordered by default. Use this flag to run the tests in order.", + dest="ordered", + action="store_true", + default=False, + ) for arg in special_arguments_values: parser.add_argument( f"--{arg}", @@ -680,6 +696,7 @@ def parse_args_and_run(): special_arguments=special_args_dict, verbose=ns_parser.verbose, subprocesses=ns_parser.subprocesses, + ordered=ns_parser.ordered, ) diff --git a/openbb_terminal/cryptocurrency/cryptocurrency_helpers.py b/openbb_terminal/cryptocurrency/cryptocurrency_helpers.py index abfb6b01e11..7a1fdf04631 100644 --- a/openbb_terminal/cryptocurrency/cryptocurrency_helpers.py +++ b/openbb_terminal/cryptocurrency/cryptocurrency_helpers.py @@ -511,11 +511,19 @@ def load( start_date: Optional[Union[datetime, Union[str, None]]] = None, interval: Union[str, int] = "1440", exchange: str = "binance", - to_symbol: str = "usdt", + to_symbol: str = "usd", end_date: Optional[Union[datetime, Union[str, None]]] = None, - source: str = "CCXT", + source: str = "CoinGecko", ) -> pd.DataFrame: - """Load crypto currency to get data for + """ + Load crypto currency to get data for + Note: + take into consideration that the data might not be found if the data provider + (example) uses `usd` instead of `usdt` as the quote currency, so you might need to adjust + the `to_symbol` parameter. + For `CCXT` a valid pair would be: "BTC/USDT" + For `CoinGecko` a valid pair would be: "BTC-USD" + For `YahooFinance` a valid pair would be: "BTC-USD" Parameters ---------- @@ -549,7 +557,7 @@ def load( if isinstance(interval, int): interval = str(interval) if start_date is None: - start_date = (datetime.now() - timedelta(days=1100)).strftime("%Y-%m-%d") + start_date = (datetime.now() - timedelta(days=365)).strftime("%Y-%m-%d") if end_date is None: end_date = datetime.now().strftime("%Y-%m-%d") diff --git a/openbb_terminal/futures/databento_view.py b/openbb_terminal/futures/databento_view.py new file mode 100644 index 00000000000..e3d02ebe705 --- /dev/null +++ b/openbb_terminal/futures/databento_view.py @@ -0,0 +1,118 @@ +"""DataBento view""" +__docformat__ = "numpy" + +import os +from typing import List, Optional + +import mplfinance as mpf +from matplotlib import pyplot as plt + +from openbb_terminal.config_terminal import theme +from openbb_terminal.helper_funcs import ( + export_data, + is_valid_axes_count, + lambda_long_number_format_y_axis, + plot_autoscale, + print_rich_table, +) +from openbb_terminal.rich_config import console +from openbb_terminal.stocks import databento_model + + +def display_historical( + symbol: str, + start_date: Optional[str] = None, + end_date: Optional[str] = None, + raw: bool = False, + export: str = "", + sheet_name: Optional[str] = None, + external_axes: Optional[List[plt.Axes]] = None, +): + """Display historical futures [Source: DataBento] + + Parameters + ---------- + symbol: List[str] + Symbol to display + start_date: Optional[str] + Start date of the historical data with format YYYY-MM-DD + end_date: Optional[str] + End date of the historical data with format YYYY-MM-DD + raw: bool + Display futures timeseries in raw format + sheet_name: str + Optionally specify the name of the sheet the data is exported to. + export: str + Type of format to export data + external_axes : Optional[List[plt.Axes]], optional + External axes (1 axis is expected in the list), by default None + """ + data = databento_model.get_historical_futures( + symbol.upper(), start_date=start_date, end_date=end_date + ) + if data.empty: + console.print(f"No data found for {symbol}.") + return + # We check if there's Volume data to avoid errors and empty subplots + has_volume = False + if "Volume" in data.columns: + has_volume = bool(data["Volume"].sum() > 0) + + candle_chart_kwargs = { + "type": "candle", + "style": theme.mpf_style, + "volume": has_volume, + "addplot": [], + "xrotation": theme.xticks_rotation, + "scale_padding": {"left": 0.3, "right": 1, "top": 0.8, "bottom": 0.8}, + "update_width_config": { + "candle_linewidth": 0.6, + "candle_width": 0.8, + "volume_linewidth": 0.8, + "volume_width": 0.8, + }, + "warn_too_much_data": 10000, + } + if external_axes is None: + candle_chart_kwargs["returnfig"] = True + candle_chart_kwargs["figratio"] = (10, 7) + candle_chart_kwargs["figscale"] = 1.10 + candle_chart_kwargs["figsize"] = plot_autoscale() + candle_chart_kwargs["warn_too_much_data"] = 100_000 + + fig, ax = mpf.plot(data, **candle_chart_kwargs) + + if has_volume: + lambda_long_number_format_y_axis(data, "Volume", ax) + + fig.suptitle( + f"{symbol}", + x=0.055, + y=0.965, + horizontalalignment="left", + ) + theme.visualize_output(force_tight_layout=False) + elif (has_volume and is_valid_axes_count(external_axes, 2)) or ( + not has_volume and is_valid_axes_count(external_axes, 1) + ): + candle_chart_kwargs["ax"] = external_axes[0] + if has_volume: + candle_chart_kwargs["volume"] = external_axes[1] + mpf.plot(data, **candle_chart_kwargs) + + if raw: + print_rich_table( + data, + headers=list(data.columns), + show_index=True, + title="Futures timeseries", + ) + console.print() + + export_data( + export, + os.path.dirname(os.path.abspath(__file__)), + "historical_db", + data, + sheet_name, + ) diff --git a/openbb_terminal/futures/futures_controller.py b/openbb_terminal/futures/futures_controller.py index 26713534bea..1eea8f581a2 100644 --- a/openbb_terminal/futures/futures_controller.py +++ b/openbb_terminal/futures/futures_controller.py @@ -11,7 +11,7 @@ from typing import List, Optional from openbb_terminal import feature_flags as obbff from openbb_terminal.custom_prompt_toolkit import NestedCompleter from openbb_terminal.decorators import log_start_end -from openbb_terminal.futures import yfinance_model, yfinance_view +from openbb_terminal.futures import databento_view, yfinance_model, yfinance_view from openbb_terminal.helper_funcs import ( EXPORT_BOTH_RAW_DATA_AND_FIGURES, EXPORT_ONLY_RAW_DATA_ALLOWED, @@ -165,8 +165,8 @@ class FuturesController(BaseController): "--start", dest="start", type=valid_date, - help="Initial date. Default: 3 years ago", - default=(datetime.now() - timedelta(days=3 * 365)), + help="Initial date. Default: 2 years ago", + default=(datetime.now() - timedelta(days=2 * 365)), ) parser.add_argument( "-e", @@ -193,17 +193,29 @@ class FuturesController(BaseController): raw=True, ) if ns_parser: - yfinance_view.display_historical( - symbols=ns_parser.ticker.upper().split(","), - expiry=ns_parser.expiry, - start_date=ns_parser.start.strftime("%Y-%m-%d"), - end_date=ns_parser.end.strftime("%Y-%m-%d"), - raw=ns_parser.raw, - export=ns_parser.export, - sheet_name=" ".join(ns_parser.sheet_name) - if ns_parser.sheet_name - else None, - ) + if ns_parser.source == "YahooFinance": + yfinance_view.display_historical( + symbols=ns_parser.ticker.upper().split(","), + expiry=ns_parser.expiry, + start_date=ns_parser.start.strftime("%Y-%m-%d"), + end_date=ns_parser.end.strftime("%Y-%m-%d"), + raw=ns_parser.raw, + export=ns_parser.export, + sheet_name=" ".join(ns_parser.sheet_name) + if ns_parser.sheet_name + else None, + ) + if ns_parser.source == "DataBento": + databento_view.display_historical( + symbol=ns_parser.ticker, + start_date=ns_parser.start.strftime("%Y-%m-%d"), + end_date=ns_parser.end.strftime("%Y-%m-%d"), + raw=ns_parser.raw, + export=ns_parser.export, + sheet_name=" ".join(ns_parser.sheet_name) + if ns_parser.sheet_name + else None, + ) @log_start_end(log=logger) def call_curve(self, other_args: List[str]): diff --git a/openbb_terminal/futures/sdk_helper.py b/openbb_terminal/futures/sdk_helper.py new file mode 100644 index 00000000000..7115b782be1 --- /dev/null +++ b/openbb_terminal/futures/sdk_helper.py @@ -0,0 +1,31 @@ +"""SDK Helper Functions""" +__docformat__ = "numpy" + +from typing import List, Optional, Union + +import pandas as pd + +from openbb_terminal.futures import yfinance_model +from openbb_terminal.stocks import databento_model + + +def get_historical( + symbols: Union[str, List[str]], + start_date: str, + end_date: str, + source: Optional[str] = "YahooFinance", + expiry: Optional[str] = "", +) -> pd.DataFrame: + """Get historical futures data""" + if source == "YahooFinance": + if isinstance(symbols, str): + symbols = [symbols] + return yfinance_model.get_historical_futures( + symbols, expiry, start_date, end_date + ) + if source == "DataBento": + if isinstance(symbols, list): + print("DataBento only supports one symbol at a time. Using first symbol.") + symbols = symbols[0] + return databento_model.get_historical_futures(symbols, start_date, end_date) + return pd.DataFrame() diff --git a/openbb_terminal/futures/yfinance_model.py b/openbb_terminal/futures/yfinance_model.py index d9c76af35fb..d8989b99f8b 100644 --- a/openbb_terminal/futures/yfinance_model.py +++ b/openbb_terminal/futures/yfinance_model.py @@ -99,7 +99,7 @@ def get_historical_futures( Returns ------- pd.DataFrame - Dictionary with sector weightings allocation + Historical futures data """ if start_date is None: diff --git a/openbb_terminal/helper_funcs.py b/openbb_terminal/helper_funcs.py index 92ecb4ce37a..a74a444ad1a 100644 --- a/openbb_terminal/helper_funcs.py +++ b/openbb_terminal/helper_funcs.py @@ -44,10 +44,7 @@ from openbb_terminal import ( config_terminal as cfg, feature_flags as obbff, ) -from openbb_terminal.core.config.paths import ( - HOME_DIRECTORY, - USER_EXPORTS_DIRECTORY, -) +from openbb_terminal.core.config.paths import HOME_DIRECTORY, USER_EXPORTS_DIRECTORY from openbb_terminal.rich_config import console try: @@ -1967,7 +1964,9 @@ def check_start_less_than_end(start_date: str, end_date: str) -> bool: # Write an abstract helper to make requests from a url with potential headers and params -def request(url: str, method="GET", **kwargs) -> requests.Response: +def request( + url: str, method: str = "GET", timeout: int = 0, **kwargs +) -> requests.Response: """Abstract helper to make requests from a url with potential headers and params. Parameters @@ -1990,15 +1989,15 @@ def request(url: str, method="GET", **kwargs) -> requests.Response: # We want to add a user agent to the request, so check if there are any headers # If there are headers, check if there is a user agent, if not add one. # Some requests seem to work only with a specific user agent, so we want to be able to override it. - headers = kwargs.pop("headers") if "headers" in kwargs else {} + headers = kwargs.pop("headers", {}) + timeout = timeout or cfg.REQUEST_TIMEOUT + if "User-Agent" not in headers: headers["User-Agent"] = get_user_agent() if method.upper() == "GET": - return requests.get(url, headers=headers, timeout=cfg.REQUEST_TIMEOUT, **kwargs) + return requests.get(url, headers=headers, timeout=timeout, **kwargs) if method.upper() == "POST": - return requests.post( - url, headers=headers, timeout=cfg.REQUEST_TIMEOUT, **kwargs - ) + return requests.post(url, headers=headers, timeout=timeout, **kwargs) raise ValueError("Method must be GET or POST") diff --git a/openbb_terminal/keys_controller.py b/openbb_terminal/keys_controller.py index 164fab8a62c..e9dd9a61ff9 100644 --- a/openbb_terminal/keys_controller.py +++ b/openbb_terminal/keys_controller.py @@ -1140,6 +1140,34 @@ class KeysController(BaseController): # pylint: disable=too-many-public-methods key=ns_parser.key, persist=True, show_output=True ) + @log_start_end(log=logger) + def call_databento(self, other_args: List[str]): + """Process databento command""" + parser = argparse.ArgumentParser( + add_help=False, + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + prog="databento", + description="Set DataBento API key.", + ) + parser.add_argument( + "-k", + "--key", + type=str, + dest="key", + help="key", + ) + if not other_args: + console.print("For your API Key, https://databento.com") + return + + if other_args and "-" not in other_args[0][0]: + other_args.insert(0, "-k") + ns_parser = self.parse_simple_args(parser, other_args) + if ns_parser: + self.status_dict["databento"] = keys_model.set_databento_key( + key=ns_parser.key, persist=True, show_output=True + ) + @log_start_end(log=logger) def call_openbb(self, other_args: List[str]): """Process openbb command""" diff --git a/openbb_terminal/keys_model.py b/openbb_terminal/keys_model.py index 1e4eb961cb8..c127c44dd93 100644 --- a/openbb_terminal/keys_model.py +++ b/openbb_terminal/keys_model.py @@ -58,6 +58,7 @@ API_DICT: Dict = { "quandl": "QUANDL", "polygon": "POLYGON", "intrinio": "INTRINIO", + "databento": "DATABENTO", "fred": "FRED", "news": "NEWSAPI", "tradier": "TRADIER", @@ -2592,7 +2593,7 @@ def set_intrinio_key(key: str, persist: bool = False, show_output: bool = False) def check_intrinio_key(show_output: bool = False) -> str: - """Check Polygon key + """Check Intrinio key Parameters ---------- @@ -2626,3 +2627,72 @@ def check_intrinio_key(show_output: bool = False) -> str: console.print(status.colorize()) return str(status) + + +def set_databento_key( + key: str, persist: bool = False, show_output: bool = False +) -> str: + """Set DataBento key + + Parameters + ---------- + key: str + API key + persist: bool, optional + If False, api key change will be contained to where it was changed. For example, a Jupyter notebook session. + If True, api key change will be global, i.e. it will affect terminal environment variables. + By default, False. + show_output: bool, optional + Display status string or not. By default, False. + + Returns + ------- + str + Status of key set + + Examples + -------- + >>> from openbb_terminal.sdk import openbb + >>> openbb.keys.databento(key="example_key") + """ + + set_key("OPENBB_API_DATABENTO_KEY", key, persist) + return check_databento_key(show_output) + + +def check_databento_key(show_output: bool = False) -> str: + """Check DataBento key + + Parameters + ---------- + show_output: bool + Display status string or not. By default, False. + + Returns + ------- + str + Status of key set + """ + + if cfg.API_DATABENTO_KEY == "REPLACE_ME": + logger.info("DataBento key not defined") + status = KeyStatus.NOT_DEFINED + else: + r = request( + "https://hist.databento.com/v0/metadata.list_datasets", + auth=(f"{cfg.API_DATABENTO_KEY}", ""), + ) + if r.status_code in [403, 401, 429]: + logger.warning("DataBento key defined, test failed") + status = KeyStatus.DEFINED_TEST_FAILED + elif r.status_code == 200: + logger.info("DataBento key defined, test passed") + status = KeyStatus.DEFINED_TEST_PASSED + else: + logger.warning("DataBento key defined, test inconclusive") + status = KeyStatus.DEFINED_TEST_INCONCLUSIVE + + if show_output: + console.print(status.colorize()) + + return str(status) diff --git a/openbb_terminal/miscellaneous/data_sources_default.json b/openbb_terminal/miscellaneous/data_sources_default.json index ce627503055..f67ed36fc39 100644 --- a/openbb_terminal/miscellaneous/data_sources_default.json +++ b/openbb_terminal/miscellaneous/data_sources_default.json @@ -6,7 +6,7 @@ "candle": [], "codes": ["Polygon"], "news": ["Feedparser", "NewsApi"], - "load": ["YahooFinance", "AlphaVantage", "Polygon", "EODHD", "Intrinio"], + "load": ["YahooFinance", "AlphaVantage", "Polygon", "EODHD", "Intrinio", "DataBento"], "th": { "symbol": ["FinanceDatabase"], "open": ["Bursa"], @@ -488,7 +488,7 @@ }, "futures": { "search": ["YahooFinance"], - "historical": ["YahooFinance"], + "historical": ["YahooFinance", "DataBento"], "curve": ["YahooFinance"] }, "fixedincome": { diff --git a/openbb_terminal/reports/templates/equity.ipynb b/openbb_terminal/reports/templates/equity.ipynb index a4e7d1cf8c0..2757330bc7f 100644 --- a/openbb_terminal/reports/templates/equity.ipynb +++ b/openbb_terminal/reports/templates/equity.ipynb @@ -134,10 +134,15 @@ "source": [ "info = openbb.stocks.fa.info(symbol=symbol).transpose()\n", "\n", - "if info[\"Long business summary\"][0] != \"NA\":\n", - " overview = info[\"Long business summary\"][0]\n", + "long_summary = \"Long business summary\"\n", + "long_name = \"Long name\"\n", + "\n", + "if long_summary in info and info[long_summary][0] != \"NA\":\n", + " overview = info[long_summary][0]\n", + "elif long_name in info:\n", + " overview = info[long_name][0]\n", "else:\n", - " overview = info[\"Long name\"][0]" + " overview = \"\"" ] }, { @@ -181,7 +186,7 @@ "metadata": {}, "outputs": [], "source": [ - "tables = openbb.etf.news(info[\"Short name\"][0], 5)\n", + "tables = openbb.etf.news(info[\"Short name\"][0], 5) if \"Short name\" in info else []\n", "for table in tables:\n", " table[0].loc[\"link\"] = (\n", " table[0].loc[\"link\"].apply(lambda x: f'{x}')\n", @@ -195,7 +200,7 @@ "metadata": {}, "outputs": [], "source": [ - "quote_data = openbb.stocks.quote_fmp(symbol)\n", + "quote_data = openbb.stocks.quote(symbol)\n", "quote_data" ] }, @@ -240,9 +245,7 @@ "metadata": {}, "outputs": [], "source": [ - "df_sec_filings = openbb.stocks.fa.sec(symbol=symbol)[[\"Type\", \"Category\", \"Link\"]].head(\n", - " 5\n", - ")\n", + "df_sec_filings = openbb.stocks.fa.sec(symbol)[[\"Type\", \"Category\", \"Link\"]].head(5)\n", "df_sec_filings[\"Link\"] = df_sec_filings[\"Link\"].apply(\n", " lambda x: f'{x}'\n", ")\n", @@ -1195,7 +1198,7 @@ " \"Buffet Score is neutral\",\n", " \"Buffet Score is favourable\",\n", " ],\n", - " float(score[\"Total Score\"], 2),\n", + " float(round(score[\"Total Score\"], 2)),\n", " )\n", "if predictions:\n", " htmlcode += widgets.kpi(\n", @@ -1208,7 +1211,9 @@ " )\n", "body += widgets.add_tab(\"SUMMARY\", htmlcode)\n", "\n", - "htmlcode = widgets.row([widgets.h(3, \"Description\") + widgets.p(overview)])\n", + "htmlcode=\"\"\n", + "if overview:\n", + " htmlcode += widgets.row([widgets.h(3, \"Description\") + widgets.p(overview)])\n", "htmlcode += widgets.row([widgets.h(3, \"Price Chart\") + price_chart])\n", "htmlcode += widgets.row([widgets.h(3, \"Quote\") + quote_data.to_html()])\n", "htmlcode += widgets.row([widgets.h(3, \"Latest News for \" + symbol)])\n", @@ -1466,7 +1471,7 @@ "metadata": { "celltoolbar": "Tags", "kernelspec": { - "display_name": "obb", + "display_name": "Python 3", "language": "python", "name": "python3" }, @@ -1480,11 +1485,11 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.6" + "version": "3.9.13" }, "vscode": { "interpreter": { - "hash": "100174a9203096c0c10fb537684ff280825ee9e252451beb8786068677204f06" + "hash": "af870c0f9615f5bb08692efcc9bf540319fe60e6793056508a4dab09c9fa8dda" } } }, diff --git a/openbb_terminal/sdk.py b/openbb_terminal/sdk.py index 8e8dbf80976..bfd2fd12df1 100644 --- a/openbb_terminal/sdk.py +++ b/openbb_terminal/sdk.py @@ -297,7 +297,7 @@ class OpenBBSDK: Attributes: `curve`: Get curve futures [Source: Yahoo Finance]\n `curve_chart`: Display curve futures [Source: Yahoo Finance]\n - `historical`: Get historical futures [Source: Yahoo Finance]\n + `historical`: Get historical futures data\n `historical_chart`: Display historical futures [Source: Yahoo Finance]\n `search`: Get search futures [Source: Yahoo Finance]\n """ @@ -316,6 +316,7 @@ class OpenBBSDK: `coinbase`: Set Coinbase key\n `coinglass`: Set Coinglass key.\n `cpanic`: Set Cpanic key.\n + `databento`: Set DataBento key\n `degiro`: Set Degiro key\n `eodhd`: Set Eodhd key.\n `ethplorer`: Set Ethplorer key.\n diff --git a/openbb_terminal/sdk_core/models/futures_sdk_model.py b/openbb_terminal/sdk_core/models/futures_sdk_model.py index 87f0f8771e1..fece0f27561 100644 --- a/openbb_terminal/sdk_core/models/futures_sdk_model.py +++ b/openbb_terminal/sdk_core/models/futures_sdk_model.py @@ -10,7 +10,7 @@ class FuturesRoot(Category): Attributes: `curve`: Get curve futures [Source: Yahoo Finance]\n `curve_chart`: Display curve futures [Source: Yahoo Finance]\n - `historical`: Get historical futures [Source: Yahoo Finance]\n + `historical`: Get historical futures data\n `historical_chart`: Display historical futures [Source: Yahoo Finance]\n `search`: Get search futures [Source: Yahoo Finance]\n """ @@ -21,6 +21,6 @@ class FuturesRoot(Category): super().__init__() self.curve = lib.futures_yfinance_model.get_curve_futures self.curve_chart = lib.futures_yfinance_view.display_curve - self.historical = lib.futures_yfinance_model.get_historical_futures + self.historical = lib.futures_sdk_model.get_historical self.historical_chart = lib.futures_yfinance_view.display_historical self.search = lib.futures_yfinance_model.get_search_futures diff --git a/openbb_terminal/sdk_core/models/keys_sdk_model.py b/openbb_terminal/sdk_core/models/keys_sdk_model.py index 27539200faf..a32c97e45c0 100644 --- a/openbb_terminal/sdk_core/models/keys_sdk_model.py +++ b/openbb_terminal/sdk_core/models/keys_sdk_model.py @@ -15,6 +15,7 @@ class KeysRoot(Category): `coinbase`: Set Coinbase key\n `coinglass`: Set Coinglass key.\n `cpanic`: Set Cpanic key.\n + `databento`: Set DataBento key\n `degiro`: Set Degiro key\n `eodhd`: Set Eodhd key.\n `ethplorer`: Set Ethplorer key.\n @@ -55,6 +56,7 @@ class KeysRoot(Category): self.coinbase = lib.keys_model.set_coinbase_key self.coinglass = lib.keys_model.set_coinglass_key self.cpanic = lib.keys_model.set_cpanic_key + self.databento = lib.keys_model.set_databento_key self.degiro = lib.keys_model.set_degiro_key self.eodhd = lib.keys_model.set_eodhd_key self.ethplorer = lib.keys_model.set_ethplorer_key diff --git a/openbb_terminal/sdk_core/sdk_init.py b/openbb_terminal/sdk_core/sdk_init.py index 221c3854e76..7054136d8e4 100644 --- a/openbb_terminal/sdk_core/sdk_init.py +++ b/openbb_terminal/sdk_core/sdk_init.py @@ -260,6 +260,7 @@ from openbb_terminal.forex import ( from openbb_terminal.futures import ( yfinance_model as futures_yfinance_model, yfinance_view as futures_yfinance_view, + sdk_helper as futures_sdk_model, ) # Stocks Helpers diff --git a/openbb_terminal/sdk_core/trail_map.csv b/openbb_terminal/sdk_core/trail_map.csv index 2b8edf697a3..2ddddedc647 100644 --- a/openbb_terminal/sdk_core/trail_map.csv +++ b/openbb_terminal/sdk_core/trail_map.csv @@ -260,7 +260,7 @@ forex.oanda.price,forex_oanda_model.fx_price_request,forex_oanda_view.get_fx_pri forex.oanda.summary,forex_oanda_model.account_summary_request,forex_oanda_view.get_account_summary forex.quote,forex_sdk_helpers.quote, futures.curve,futures_yfinance_model.get_curve_futures,futures_yfinance_view.display_curve -futures.historical,futures_yfinance_model.get_historical_futures,futures_yfinance_view.display_historical +futures.historical,futures_sdk_model.get_historical,futures_yfinance_view.display_historical futures.search,futures_yfinance_model.get_search_futures, keys.av,keys_model.set_av_key, keys.binance,keys_model.set_binance_key, @@ -269,6 +269,7 @@ keys.cmc,keys_model.set_cmc_key, keys.coinbase,keys_model.set_coinbase_key, keys.coinglass,keys_model.set_coinglass_key, keys.cpanic,keys_model.set_cpanic_key, +keys.databento,keys_model.set_databento_key, keys.degiro,keys_model.set_degiro_key, keys.eodhd,keys_model.set_eodhd_key, keys.ethplorer,keys_model.set_ethplorer_key, diff --git a/openbb_terminal/stocks/databento_model.py b/openbb_terminal/stocks/databento_model.py new file mode 100644 index 00000000000..392b7ca4afa --- /dev/null +++ b/openbb_terminal/stocks/databento_model.py @@ -0,0 +1,154 @@ +__docformat__ = "numpy" +import logging +from datetime import datetime, timedelta +from io import StringIO + +import pandas as pd +import requests +from pydantic import BaseModel + +from openbb_terminal import config_terminal as cfg +from openbb_terminal.decorators import check_api_key, log_start_end +from openbb_terminal.helper_funcs import request +from openbb_terminal.rich_config import console + +logger = logging.getLogger(__name__) + +# pylint:disable=too-few-public-methods + + +class DataBento(BaseModel): + symbol: str + exchange: str = "XNAS.ITCH" + stype: str = "native" + start: str = (datetime.now() - timedelta(days=365)).strftime("%Y-%m-%d") + end: str = datetime.now().strftime("%Y-%m-%d") + + class Config: + arbitrary_types_allowed = True + + def get_historical_stock(self): + """Gets historical EOD stock data from DataBento. Currnetly, just NASDAQ is supported. + Note that only nonadjusted data is available.""" + self.exchange = "XNAS.ITCH" + self.stype = "native" + if not self.validate_symbol(): + return pd.DataFrame() + return self.get_historical() + + def validate_symbol(self): + """Validates the symbol is supported by DataBento. There is no API for this, so we will use their + Symbology endpoint to see if something exists. Doing a long time range can make this request on the order of + one minute, so try to keep smaller requests.""" + base_url = "https://hist.databento.com/v0/symbology.resolve" + params = { + "dataset": self.exchange, + "symbols": self.symbol, + "start_date": self.start, + "end_date": self.end, + "stype_in": "smart", + "stype_out": "product_id", + } + auth = requests.auth.HTTPBasicAuth(cfg.API_DATABENTO_KEY, "") + # This seems to only work for futures? Assume the user is entering correct stock ticker + if self.exchange == "XNAS.ITCH": + return True + result = request(base_url, params=params, auth=auth, timeout=30) + if "message" not in result.json(): + console.print( + "Issue validating symbol in the date range. Please check with DataBento for symbology." + ) + return False + return result.json()["message"] != "Not found" + + def get_historical_futures(self): + """Gets historical EODfutures data from DataBento. Currently, just CME is supported. + Note this gets the highest volume contract each day""" + self.exchange = "GLBX.MDP3" + self.stype = "smart" + if not self.validate_symbol(): + return pd.DataFrame() + return self.get_historical() + + @log_start_end(log=logger) + @check_api_key(["API_DATABENTO_KEY"]) + def get_historical(self) -> pd.DataFrame: + """Prepares the request to be made. I am using their https interface instead of their python client. + Their next update will change symbol.V.0 to symbol.C.0""" + + base_url = "https://hist.databento.com/v0/timeseries.stream" + symbol = self.symbol + ".V.0" if self.exchange == "GLBX.MDP3" else self.symbol + params = { + "dataset": self.exchange, + "symbols": symbol, + "schema": "ohlcv-1d", + "start": self.start, + "end": self.end, + "stype_in": self.stype, + "encoding": "csv", + } + auth = requests.auth.HTTPBasicAuth(cfg.API_DATABENTO_KEY, "") + data = self.process_request(base_url, params, auth) + if data.empty: + console.print(f"No data found for symbol `{self.symbol}`.") + return data + + def process_request(self, base_url, params, auth) -> pd.DataFrame: + """Takes the request and returns the adjusted dataframe""" + r = request(base_url, params=params, auth=auth) + if r.status_code != 200: + raise Exception(f"Error: Status Code {r.status_code}") + df = pd.read_csv(StringIO(r.text)) + df["time"] = pd.to_datetime(df.ts_event, unit="ns") + df[["open", "high", "low", "close"]] /= 1_000_000_000 + df = df.drop(columns=["ts_event", "publisher_id", "product_id"]).set_index( + "time" + ) + df = df.rename( + columns={ + "open": "Open", + "high": "High", + "low": "Low", + "close": "Close", + "volume": "Volume", + } + ) + return df + + +@log_start_end(log=logger) +@check_api_key(["API_DATABENTO_KEY"]) +def get_historical_stock(symbol: str, start_date: str, end_date: str) -> pd.DataFrame: + """Gets historical EOD data from DataBento. Currently, just NASDAQ is supported. + + Parameters + ---------- + symbol : str + Symbol to get data for + start_date : str + Start date of data + end_date : str + End date to get data for + + """ + db = DataBento(symbol=symbol, start=start_date, end=end_date) + return db.get_historical_stock() + + +@log_start_end(log=logger) +@check_api_key(["API_DATABENTO_KEY"]) +def get_historical_futures(symbol: str, start_date: str, end_date: str) -> pd.DataFrame: + """Gets historical EODfutures data from DataBento. Currently, just CME is supported. + + Parameters + ---------- + symbol : str + Symbol to get data for + start_date : str + Start date of data + end_date : str + End date to get data for + + """ + db = DataBento(symbol=symbol, start=start_date, end=end_date) + return db.get_historical_futures() diff --git a/openbb_terminal/stocks/options/nasdaq_model.py b/openbb_terminal/stocks/options/nasdaq_model.py index f703a8b3ee3..5c508308bd6 100644 --- a/openbb_terminal/stocks/options/nasdaq_model.py +++ b/openbb_terminal/stocks/options/nasdaq_model.py @@ -176,12 +176,12 @@ def option_expirations(symbol: str) -> List[str]: List of expiration dates """ df = get_full_option_chain(symbol) + if df.empty: return [] + # get everything that is not an empty string - exps = [exp for exp in list(df.expirygroup.unique()) if exp] - # Convert 'January 11, 1993' into '1993-01-11' - return [datetime.strptime(exp, "%B %d, %Y").strftime("%Y-%m-%d") for exp in exps] + return [exp for exp in list(df.expiration.unique()) if exp] @log_start_end(log=logger) diff --git a/openbb_terminal/stocks/stocks_helper.py b/openbb_terminal/stocks/stocks_helper.py index 5c39d7e8cba..8fb27613041 100644 --- a/openbb_terminal/stocks/stocks_helper.py +++ b/openbb_terminal/stocks/stocks_helper.py @@ -56,6 +56,8 @@ from openbb_terminal.stocks.stocks_model import ( load_stock_yf, ) +from . import databento_model + logger = logging.getLogger(__name__) @@ -335,6 +337,12 @@ def load( elif source == "Intrinio": df_stock_candidate = load_stock_intrinio(symbol, start_date, end_date) + + elif source == "DataBento": + df_stock_candidate = databento_model.get_historical_stock( + symbol, start_date.strftime("%Y-%m-%d"), end_date.strftime("%Y-%m-%d") + ) + else: console.print("[red]Invalid source for stock[/red]\n") return diff --git a/tests/openbb_terminal/futures/txt/test_futures_controller/test_print_help.txt b/tests/openbb_terminal/futures/txt/test_futures_controller/test_print_help.txt index a816e32836d..3d687932c2c 100644 --- a/tests/openbb_terminal/futures/txt/test_futures_controller/test_print_help.txt +++ b/tests/openbb_terminal/futures/txt/test_futures_controller/test_print_help.txt @@ -1,5 +1,5 @@ search search for a future based on category or exchange [YahooFinance] - historical get historical timeseries [YahooFinance] + historical get historical timeseries [YahooFinance, DataBento] curve get futures curves [YahooFinance] diff --git a/tests/openbb_terminal/stocks/cassettes/test_databento_model/test_stock_load.yaml b/tests/openbb_terminal/stocks/cassettes/test_databento_model/test_stock_load.yaml new file mode 100644 index 00000000000..2b1f3cb4636 --- /dev/null +++ b/tests/openbb_terminal/stocks/cassettes/test_databento_model/test_stock_load.yaml @@ -0,0 +1,118 @@ +interactions: +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Authorization: + - MOCK_AUTHORIZATION + Connection: + - keep-alive + method: GET + uri: https://hist.databento.com/v0/timeseries.stream?dataset=XNAS.ITCH&symbols=AAPL&schema=ohlcv-1d&start=2022-08-01&end=2022-12-01&stype_in=native&encoding=csv + response: + body: + string: 'ts_event,publisher_id,product_id,open,high,low,close,volume + + 1659312000000000000,2,27,162200000000,163590000000,160830000000,161400000000,10045637 + + 1659398400000000000,2,27,160730000000,162410000000,159550000000,160010000000,10065910 + + 1659484800000000000,2,27,160060000000,166595000000,159850000000,165840000000,13326012 + + 1659571200000000000,2,27,165830000000,167190000000,164430000000,165850000000,9578445 + + 1659657600000000000,2,27,166000000000,166230000000,162790000000,164920000000,9683719 + + 1659916800000000000,2,27,166080000000,167800000000,164200000000,165150000000,9135766 + + 1660003200000000000,2,29,164990000000,165810000000,163460000000,164970000000,9230005 + + 1660089600000000000,2,29,165010000000,169310000000,164560000000,169130000000,11638513 + + 1660176000000000000,2,29,169240000000,170990000000,168190000000,168800000000,9732568 + + 1660262400000000000,2,29,169180000000,172170000000,168650000000,171850000000,11916663 + + 1660521600000000000,2,29,171140000000,173390000000,170740000000,173150000000,8485815 + + 1660608000000000000,2,29,172830000000,174080000000,171670000000,173700000000,9979182 + + 1660694400000000000,2,29,172670000000,176150000000,172140000000,173320000000,13581411 + + 1660780800000000000,2,29,173200000000,174890000000,173130000000,174090000000,11175367 + + 1660867200000000000,2,29,173600000000,173730000000,170700000000,170790000000,11296793 + + 1661126400000000000,2,29,169400000000,169860000000,167135000000,167950000000,13070702 + + 1661212800000000000,2,29,167610000000,168700000000,166650000000,167130000000,10629743 + + 1661299200000000000,2,29,166930000000,168410000000,166245000000,167910000000,9482192 + + 1661385600000000000,2,29,169270000000,170340000000,168230000000,170200000000,9874644 + + 1661472000000000000,2,29,169900000000,171130000000,162300000000,162450000000,14062181 + + 1661731200000000000,2,29,161790000000,162910000000,159830000000,161780000000,12363097 + + 1661817600000000000,2,29,162600000000,163340000000,157730000000,158960000000,11853126 + + 1661904000000000000,2,29,159650000000,160570000000,155770000000,156000000000,13343001 + + 1661990400000000000,2,29,155890000000,158690000000,154675000000,158620000000,13016463 + + 1662076800000000000,2,29,157710000000,160360000000,154965000000,155650000000,12975036 + + 1662422400000000000,2,29,156790000000,157390000000,153730000000,154490000000,14840703 + + 1662508800000000000,2,29,154800000000,156660000000,153610000000,155960000000,13205605 + + 1662595200000000000,2,29,155950000000,156430000000,152680000000,154620000000,14072194 + + 1662681600000000000,2,29,155260000000,157805000000,154760000000,157520000000,10213517 + + 1662940800000000000,2,29,158000000000,164260000000,158000000000,164100000000,15959461 + + 1663027200000000000,2,29,164150000000,164900000000,153385000000,154250000000,19181953 + + 1663113600000000000,2,29,154470000000,157100000000,153250000000,156000000000,15250245 + + 1663200000000000000,2,29,155960000000,156040000000,151100000000,151270000000,13929824 + + 1663286400000000000,2,29,151350000000,151510000000,148360000000,151290000000,19608708 + + 1663545600000000000,2,29,149820000000,155000000000,148700000000,155000000000,13024172 + + 1663632000000000000,2,29,154270000000,158080000000,153090000000,156480000000,16126729 + + 1663718400000000000,2,29,156580000000,158740000000,152240000000,152400000000,16938650 + + 1663804800000000000,2,29,153530000000,154960000000,150920000000,153090000000,14902502 + + 1663891200000000000,2,29,152250000000,152550000000,148560000000,150570000000,16074945 + + 1664150400000000000,2,29,150420000000,153770000000,149140000000,151100000000,14918149 + + 1664236800000000000,2,29,152840000000,154690000000,149950000000,151870000000,14873042 + + 1664323200000000000,2,29,148500000000,150640000000,144840000000,149370000000,24532649 + + 1664409600000000000,2,29,148000000000,148180000000,140680000000,142410000000,21075773 + + 1664496000000000000,2,29,143160000000,143580000000,137220000000,137540000000,20801834 + + ' + headers: + date: + - Tue, 21 Feb 2023 14:37:29 GMT + server: + - uvicorn + transfer-encoding: + - chunked + status: + code: 200 + message: OK +version: 1 diff --git a/tests/openbb_terminal/stocks/csv/test_databento_model/test_stock_load.csv b/tests/openbb_terminal/stocks/csv/test_databento_model/test_stock_load.csv new file mode 100644 index 00000000000..e90e6321b7d --- /dev/null +++ b/tests/openbb_terminal/stocks/csv/test_databento_model/test_stock_load.csv @@ -0,0 +1,45 @@ +time,Open,High,Low,Close,Volume +2022-08-01,162.2,163.59,160.83,161.4,10045637 +2022-08-02,160.73,162.41,159.55,160.01,10065910 +2022-08-03,160.06,166.595,159.85,165.84,13326012 +2022-08-04,165.83,167.19,164.43,165.85,9578445 +2022-08-05,166.0,166.23,162.79,164.92,9683719 +2022-08-08,166.08,167.8,164.2,165.15,9135766 +2022-08-09,164.99,165.81,163.46,164.97,9230005 +2022-08-10,165.01,169.31,164.56,169.13,11638513 +2022-08-11,169.24,170.99,168.19,168.8,9732568 +2022-08-12,169.18,172.17,168.65,171.85,11916663 +2022-08-15,171.14,173.39,170.74,173.15,8485815 +2022-08-16,172.83,174.08,171.67,173.7,9979182 +2022-08-17,172.67,176.15,172.14,173.32,13581411 +2022-08-18,173.2,174.89,173.13,174.09,11175367 +2022-08-19,173.6,173.73,170.7,170.79,11296793 +2022-08-22,169.4,169.86,167.135,167.95,13070702 +2022-08-23,167.61,168.7,166.65,167.13,10629743 +2022-08-24,166.93,168.41,166.245,167.91,9482192 +2022-08-25,169.27,170.34,168.23,170.2,9874644 +2022-08-26,169.9,171.13,162.3,162.45,14062181 +2022-08-29,161.79,162.91,159.83,161.78,12363097 +2022-08-30,162.6,163.34,157.73,158.96,11853126 +2022-08-31,159.65,160.57,155.77,156.0,13343001 +2022-09-01,155.89,158.69,154.675,158.62,13016463 +2022-09-02,157.71,160.36,154.965,155.65,12975036 +2022-09-06,156.79,157.39,153.73,154.49,14840703 +2022-09-07,154.8,156.66,153.61,155.96,13205605 +2022-09-08,155.95,156.43,152.68,154.62,14072194 +2022-09-09,155.26,157.805,154.76,157.52,10213517 +2022-09-12,158.0,164.26,158.0,164.1,15959461 +2022-09-13,164.15,164.9,153.385,154.25,19181953 +2022-09-14,154.47,157.1,153.25,156.0,15250245 +2022-09-15,155.96,156.04,151.1,151.27,13929824 +2022-09-16,151.35,151.51,148.36,151.29,19608708 +2022-09-19,149.82,155.0,148.7,155.0,13024172 +2022-09-20,154.27,158.08,153.09,156.48,16126729 +2022-09-21,156.58,158.74,152.24,152.4,16938650 +2022-09-22,153.53,154.96,150.92,153.09,14902502 +2022-09-23,152.25,152.55,148.56,150.57,16074945 +2022-09-26,150.42,153.77,149.14,151.1,14918149 +2022-09-27,152.84,154.69,149.95,151.87,14873042 +2022-09-28,148.5,150.64,144.84,149.37,24532649 +2022-09-29,148.0,148.18,140.68,142.41,21075773 +2022-09-30,143.16,143.58,137.22,137.54,20801834 diff --git a/tests/openbb_terminal/stocks/test_databento_model.py b/tests/openbb_terminal/stocks/test_databento_model.py new file mode 100644 index 00000000000..b0f0ca94529 --- /dev/null +++ b/tests/openbb_terminal/stocks/test_databento_model.py @@ -0,0 +1,35 @@ +import pytest + +from openbb_terminal.stocks import databento_model + + +@pytest.fixture(scope="module") +def vcr_config(): + return { + "filter_headers": [ + ("User-Agent", None), + ("Authorization", "MOCK_AUTHORIZATION"), + ], + } + + +def test_databento_model(): + """Tests the DataBento model.""" + model = databento_model.DataBento( + symbol="AAPL", start="2021-08-10", end="2021-08-11" + ) + assert model.symbol == "AAPL" + assert model.exchange == "XNAS.ITCH" + assert model.stype == "native" + assert model.start == "2021-08-10" + assert model.end == "2021-08-11" + + +@pytest.mark.vcr +def test_stock_load(recorder): + """Tests the stock load function.""" + result = databento_model.get_historical_stock( + symbol="AAPL", start_date="2022-08-01", end_date="2022-12-01" + ) + assert result.empty is False + recorder.capture(result) diff --git a/tests/openbb_terminal/stocks/txt/test_stocks_controller/test_print_help.txt b/tests/openbb_terminal/stocks/txt/test_stocks_controller/test_print_help.txt index 42006f18e71..549e4cf5203 100644 --- a/tests/openbb_terminal/stocks/txt/test_stocks_controller/test_print_help.txt +++ b/tests/openbb_terminal/stocks/txt/test_stocks_controller/test_print_help.txt @@ -1,5 +1,5 @@ search search a specific stock ticker for analysis [FinanceDatabase] - load load a specific stock ticker and additional info for analysis [YahooFinance, AlphaVantage, Polygon, EODHD, Intrinio] + load load a specific stock ticker and additional info for analysis [YahooFinance, AlphaVantage, Polygon, EODHD, Intrinio, DataBento] Stock: