From 264feb04a42ebd24d39588654b04c5a0b35d66cb Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Wed, 20 Sep 2023 04:44:54 -0700 Subject: [PATCH] Hotfix/fix-cboe-regressions (#5447) * fix cboe regressions * Lint * fixes intraday dates --------- Co-authored-by: Igor Radovanovic <74266147+IgorWounds@users.noreply.github.com> Co-authored-by: Diogo Sousa --- openbb_sdk/openbb/package/economy.py | 40 +-- openbb_sdk/openbb/package/extension_map.json | 12 +- openbb_sdk/openbb/package/news.py | 16 +- openbb_sdk/openbb/package/stocks.py | 18 +- openbb_sdk/openbb/package/stocks_fa.py | 8 +- openbb_sdk/openbb/package/stocks_options.py | 2 +- .../models/european_index_historical.py | 5 +- .../models/major_indices_historical.py | 28 +- .../openbb_cboe/models/stock_historical.py | 298 ++++++++++-------- .../cboe/openbb_cboe/models/stock_info.py | 54 ++-- .../openbb_polygon/models/stock_historical.py | 19 +- .../models/stock_historical.py | 22 +- .../standard_models/stock_historical.py | 10 +- 13 files changed, 289 insertions(+), 243 deletions(-) diff --git a/openbb_sdk/openbb/package/economy.py b/openbb_sdk/openbb/package/economy.py index a61dfb438c9..58e0e9ea5cf 100644 --- a/openbb_sdk/openbb/package/economy.py +++ b/openbb_sdk/openbb/package/economy.py @@ -186,7 +186,7 @@ class CLASS_economy(Container): Parameters ---------- - provider : Union[Literal['quandl'], NoneType] + provider : Union[Literal['quandl'], None] The provider to use for the query, by default None. If None, the provider specified in defaults is selected or 'quandl' if there is no default. @@ -197,7 +197,7 @@ class CLASS_economy(Container): Certain symbols, such as "ES=F", or exact names are also valid. Default report is: S&P 500 Consolidated (CME)) (provider: quandl) - data_type : Union[Literal['F', 'FO', 'CITS'], NoneType] + data_type : Union[Literal['F', 'FO', 'CITS'], None] The type of data to reuturn. Default is "FO". @@ -207,9 +207,9 @@ class CLASS_economy(Container): CITS = Commodity Index Trader Supplemental. Only valid for commodities. (provider: quandl) - legacy_format : Union[bool, NoneType] + legacy_format : Union[bool, None] Returns the legacy format of report. Default is False. (provider: quandl) - report_type : Union[Literal['ALL', 'CHG', 'OLD', 'OTR'], NoneType] + report_type : Union[Literal['ALL', 'CHG', 'OLD', 'OTR'], None] The type of report to return. Default is "ALL". @@ -221,7 +221,7 @@ class CLASS_economy(Container): OTR = Other Crop Years (provider: quandl) - measure : Union[Literal['CR', 'NT', 'OI', 'CHG'], NoneType] + measure : Union[Literal['CR', 'NT', 'OI', 'CHG'], None] The measure to return. Default is None. @@ -233,11 +233,11 @@ class CLASS_economy(Container): CHG = Change in Positions. Only valid when data_type is "CITS". (provider: quandl) - start_date : Union[datetime.date, NoneType] + start_date : Union[datetime.date, None] The start date of the time series. Defaults to all. (provider: quandl) - end_date : Union[datetime.date, NoneType] + end_date : Union[datetime.date, None] The end date of the time series. Defaults to the most recent data. (provider: quandl) - transform : Union[Literal['diff', 'rdiff', 'cumul', 'normalize'], NoneType] + transform : Union[Literal['diff', 'rdiff', 'cumul', 'normalize'], None] Transform the data as w/w difference, percent change, cumulative, or normalize. (provider: quandl) Returns @@ -285,7 +285,7 @@ class CLASS_economy(Container): ---------- query : str Search query. - provider : Union[Literal['quandl'], NoneType] + provider : Union[Literal['quandl'], None] The provider to use for the query, by default None. If None, the provider specified in defaults is selected or 'quandl' if there is no default. @@ -688,9 +688,9 @@ class CLASS_economy(Container): The provider to use for the query, by default None. If None, the provider specified in defaults is selected or 'cboe' if there is no default. - interval : Union[Literal['1d', '1m'], NoneType, Literal['1min', '5min', '15min', '30min', '1hour', '4hour', '1day'], Literal['1m', '2m', '5m', '15m', '30m', '60m', '90m', '1h', '1d', '5d', '1wk', '1mo', '3mo']] + interval : Union[Literal['1d', '1m'], None, Literal['1min', '5min', '15min', '30min', '1hour', '4hour', '1day'], Literal['1m', '2m', '5m', '15m', '30m', '60m', '90m', '1h', '1d', '5d', '1wk', '1mo', '3mo']] Use interval, 1m, for intraday prices during the most recent trading period. (provider: cboe); Data granularity. (provider: fmp); Data granularity. (provider: yfinance) - timeseries : Union[pydantic.types.NonNegativeInt, NoneType] + timeseries : Union[pydantic.types.NonNegativeInt, None] Number of days to look back. (provider: fmp) timespan : Literal['minute', 'hour', 'day', 'week', 'month', 'quarter', 'year'] Timespan of the data. (provider: polygon) @@ -795,7 +795,7 @@ class CLASS_economy(Container): Search query. symbol : Union[bool, List[str]] Whether to search by ticker symbol. - provider : Union[Literal['cboe'], NoneType] + provider : Union[Literal['cboe'], None] The provider to use for the query, by default None. If None, the provider specified in defaults is selected or 'cboe' if there is no default. @@ -877,9 +877,9 @@ class CLASS_economy(Container): Parameters ---------- - region : Union[Literal['US', 'EU'], NoneType] + region : Union[Literal['US', 'EU'], None] The region to return. Currently supports US and EU. - provider : Union[Literal['cboe'], NoneType] + provider : Union[Literal['cboe'], None] The provider to use for the query, by default None. If None, the provider specified in defaults is selected or 'cboe' if there is no default. @@ -1068,17 +1068,17 @@ class CLASS_economy(Container): Parameters ---------- - series_name : Literal['Shiller PE Ratio by Month', 'Shiller PE Ratio by Year', 'PE Ratio by Year', 'PE Ratio by Month', 'Dividend by Year', 'Dividend by Month', 'Dividend Growth by Quarter', 'Dividend Growth by Year', 'Dividend Yield by Year', 'Dividend Yield by Month', 'Earnings by Year', 'Earnings by Month', 'Earnings Growth by Year', 'Earnings Growth by Quarter', 'Real Earnings Growth by Year', 'Real Earnings Growth by Quarter', 'Earnings Yield by Year', 'Earnings Yield by Month', 'Real Price by Year', 'Real Price by Month', 'Inflation Adjusted Price by Year', 'Inflation Adjusted Price by Month', 'Sales by Year', 'Sales by Quarter', 'Sales Growth by Year', 'Sales Growth by Quarter', 'Real Sales by Year', 'Real Sales by Quarter', 'Real Sales Growth by Year', 'Real Sales Growth by Quarter', 'Price to Sales Ratio by Year', 'Price to Sales Ratio by Quarter', 'Price to Book Value Ratio by Year', 'Price to Book Value Ratio by Quarter', 'Book Value per Share by Year', 'Book Value per Share by Quarter'] + series_name : Literal['Shiller PE Ratio by Month', 'Shiller PE Ratio by Year', 'PE Rat... The name of the series. Defaults to 'PE Ratio by Month'. - start_date : Union[str, NoneType] + start_date : Union[str, None] The start date of the time series. Format: YYYY-MM-DD - end_date : Union[str, NoneType] + end_date : Union[str, None] The end date of the time series. Format: YYYY-MM-DD - collapse : Union[Literal['daily', 'weekly', 'monthly', 'quarterly', 'annual'], NoneType] + collapse : Union[Literal['daily', 'weekly', 'monthly', 'quarterly', 'annual'... Collapse the frequency of the time series. - transform : Union[Literal['diff', 'rdiff', 'cumul', 'normalize'], NoneType] + transform : Union[Literal['diff', 'rdiff', 'cumul', 'normalize'], None] The transformation of the time series. - provider : Union[Literal['quandl'], NoneType] + provider : Union[Literal['quandl'], None] The provider to use for the query, by default None. If None, the provider specified in defaults is selected or 'quandl' if there is no default. diff --git a/openbb_sdk/openbb/package/extension_map.json b/openbb_sdk/openbb/package/extension_map.json index 1f363b68fee..b11b4e0560d 100644 --- a/openbb_sdk/openbb/package/extension_map.json +++ b/openbb_sdk/openbb/package/extension_map.json @@ -9,18 +9,18 @@ "news@0.1.0a0", "openbb_charting@0.1.0a1", "qa@0.1.0a0", - "stocks@0.1.0a1", + "stocks@0.1.0a0", "ta@0.1.0a0" ], "openbb_provider_extension": [ "alpha_vantage@0.1.0a0", - "benzinga@0.1.0a1", + "benzinga@0.1.0a0", "cboe@0.1.0a0", - "fmp@0.1.0a1", + "fmp@0.1.0a0", "fred@0.1.0a0", - "intrinio@0.1.0a1", - "polygon@0.1.0a1", + "intrinio@0.1.0a0", + "polygon@0.1.0a0", "quandl@0.1.0a0", - "yfinance@0.1.0a1" + "yfinance@0.1.0a0" ] } \ No newline at end of file diff --git a/openbb_sdk/openbb/package/news.py b/openbb_sdk/openbb/package/news.py index b9875d40637..26ea2ca55e4 100644 --- a/openbb_sdk/openbb/package/news.py +++ b/openbb_sdk/openbb/package/news.py @@ -35,31 +35,31 @@ class CLASS_news(Container): ---------- limit : NonNegativeInt Number of articles to return. - provider : Union[Literal['benzinga', 'fmp', 'intrinio'], NoneType] + provider : Union[Literal['benzinga', 'fmp', 'intrinio'], None] The provider to use for the query, by default None. If None, the provider specified in defaults is selected or 'benzinga' if there is no default. display : Literal['headline', 'abstract', 'full'] Specify headline only (headline), headline + teaser (abstract), or headline + full body (full). (provider: benzinga) - date : Union[str, NoneType] + date : Union[str, None] Date of the news to retrieve. (provider: benzinga) - start_date : Union[str, NoneType] + start_date : Union[str, None] Start date of the news to retrieve. (provider: benzinga) - end_date : Union[str, NoneType] + end_date : Union[str, None] End date of the news to retrieve. (provider: benzinga) updated_since : Union[int, None] Number of seconds since the news was updated. (provider: benzinga) published_since : Union[int, None] Number of seconds since the news was published. (provider: benzinga) - sort : Union[Literal['id', 'created', 'updated'], NoneType] + sort : Union[Literal['id', 'created', 'updated'], None] Key to sort the news by. (provider: benzinga) - order : Union[Literal['asc', 'desc'], NoneType] + order : Union[Literal['asc', 'desc'], None] Order to sort the news by. (provider: benzinga) - isin : Union[str, NoneType] + isin : Union[str, None] The ISIN of the news to retrieve. (provider: benzinga) cusip : Union[str, None] The CUSIP of the news to retrieve. (provider: benzinga) - channels : Union[str, NoneType] + channels : Union[str, None] Channels of the news to retrieve. (provider: benzinga) topics : Union[str, None] Topics of the news to retrieve. (provider: benzinga) diff --git a/openbb_sdk/openbb/package/stocks.py b/openbb_sdk/openbb/package/stocks.py index d1c835e845b..ae2914f47f2 100644 --- a/openbb_sdk/openbb/package/stocks.py +++ b/openbb_sdk/openbb/package/stocks.py @@ -539,7 +539,7 @@ class CLASS_stocks(Container): ---------- symbols : str Comma separated list of symbols. - limit : Union[pydantic.types.NonNegativeInt, NoneType] + limit : Union[pydantic.types.NonNegativeInt, None] Number of results to return per page. chart : bool Whether to create a chart or not, by default False. @@ -549,21 +549,21 @@ class CLASS_stocks(Container): no default. display : Literal['headline', 'abstract', 'full'] Specify headline only (headline), headline + teaser (abstract), or headline + full body (full). (provider: benzinga) - date : Union[str, NoneType] + date : Union[str, None] Date of the news to retrieve. (provider: benzinga) - start_date : Union[str, NoneType] + start_date : Union[str, None] Start date of the news to retrieve. (provider: benzinga) - end_date : Union[str, NoneType] + end_date : Union[str, None] End date of the news to retrieve. (provider: benzinga) updated_since : Union[int, None] Number of seconds since the news was updated. (provider: benzinga) published_since : Union[int, None] Number of seconds since the news was published. (provider: benzinga) - sort : Union[Literal['id', 'created', 'updated'], NoneType] + sort : Union[Literal['id', 'created', 'updated'], None] Key to sort the news by. (provider: benzinga) - order : Union[Literal['asc', 'desc'], NoneType] + order : Union[Literal['asc', 'desc'], None] Order to sort the news by. (provider: benzinga); Sort order of the articles. (provider: polygon) - isin : Union[str, NoneType] + isin : Union[str, None] The ISIN of the news to retrieve. (provider: benzinga) cusip : Union[str, None] The CUSIP of the news to retrieve. (provider: benzinga) @@ -575,7 +575,7 @@ class CLASS_stocks(Container): Authors of the news to retrieve. (provider: benzinga) content_types : Union[str, None] Content types of the news to retrieve. (provider: benzinga) - published_utc : Union[str, NoneType] + published_utc : Union[str, None] Date query to fetch articles. Supports operators <, <=, >, >= (provider: polygon) Returns @@ -680,7 +680,7 @@ class CLASS_stocks(Container): ---------- symbol : Union[str, List[str]] Comma separated list of symbols. - provider : Union[Literal['fmp', 'intrinio'], NoneType] + provider : Union[Literal['fmp', 'intrinio'], None] The provider to use for the query, by default None. If None, the provider specified in defaults is selected or 'fmp' if there is no default. diff --git a/openbb_sdk/openbb/package/stocks_fa.py b/openbb_sdk/openbb/package/stocks_fa.py index f63c15ca84a..c78df4ca13c 100644 --- a/openbb_sdk/openbb/package/stocks_fa.py +++ b/openbb_sdk/openbb/package/stocks_fa.py @@ -3007,13 +3007,13 @@ class CLASS_stocks_fa(Container): ---------- symbol : Union[str, List[str]] Symbol to get data for. - type : Union[Literal['1', '1-A', '1-E', '1-K', '1-N', '1-SA', '1-U', '1-Z', '10', '10-D', '10-K', '10-M', '10-Q', '11-K', '12b-25', '13F', '13H', '144', '15', '15F', '17-H', '18', '18-K', '19b-4', '19b-4(e)', '19b-7', '2-E', '20-F', '24F-2', '25', '3', '4', '40-F', '5', '6-K', '7-M', '8-A', '8-K', '8-M', '9-M', 'ABS-15G', 'ABS-EE', 'ABS DD-15E', 'ADV', 'ADV-E', 'ADV-H', 'ADV-NR', 'ADV-W', 'ATS', 'ATS-N', 'ATS-R', 'BD', 'BD-N', 'BDW', 'C', 'CA-1', 'CB', 'CFPORTAL', 'CRS', 'CUSTODY', 'D', 'F-1', 'F-10', 'F-3', 'F-4', 'F-6', 'F-7', 'F-8', 'F-80', 'F-N', 'F-X', 'ID', 'MA', 'MA-I', 'MA-NR', 'MA-W', 'MSD', 'MSDW', 'N-14', 'N-17D-1', 'N-17f-1', 'N-17f-2', 'N-18f-1', 'N-1A', 'N-2', 'N-23c-3', 'N-27D-1', 'N-3', 'N-4', 'N-5', 'N-54A', 'N-54C', 'N-6', 'N-6EI-1', 'N-6F', 'N-8A', 'N-8B-2', 'N-8B-4', 'N-8F', 'N-CEN'], NoneType] + type : Union[Literal['1', '1-A', '1-E', '1-K', '1-N', '1-SA', '1-U', '1-... Type of the SEC filing form. - page : Union[int, NoneType] + page : Union[int, None] Page number of the results. - limit : Union[int, NoneType] + limit : Union[int, None] The number of data entries to return. - provider : Union[Literal['fmp'], NoneType] + provider : Union[Literal['fmp'], None] The provider to use for the query, by default None. If None, the provider specified in defaults is selected or 'fmp' if there is no default. diff --git a/openbb_sdk/openbb/package/stocks_options.py b/openbb_sdk/openbb/package/stocks_options.py index baeb430ca02..b2dae87cfde 100644 --- a/openbb_sdk/openbb/package/stocks_options.py +++ b/openbb_sdk/openbb/package/stocks_options.py @@ -38,7 +38,7 @@ class CLASS_stocks_options(Container): The provider to use for the query, by default None. If None, the provider specified in defaults is selected or 'cboe' if there is no default. - date : Union[str, NoneType] + date : Union[str, None] Date for which the options chains are returned. (provider: intrinio) Returns diff --git a/openbb_sdk/providers/cboe/openbb_cboe/models/european_index_historical.py b/openbb_sdk/providers/cboe/openbb_cboe/models/european_index_historical.py index e437c2f9151..da8949ef70f 100644 --- a/openbb_sdk/providers/cboe/openbb_cboe/models/european_index_historical.py +++ b/openbb_sdk/providers/cboe/openbb_cboe/models/european_index_historical.py @@ -46,7 +46,10 @@ class CboeEuropeanIndexHistoricalData(EuropeanIndexHistoricalData): @validator("date", pre=True, check_fields=False) def date_validate(cls, v): # pylint: disable=E0213 """Return datetime object from string.""" - return datetime.strptime(v, "%Y-%m-%d") + try: + return datetime.strptime(v, "%Y-%m-%d") + except Exception: + return datetime.strptime(v, "%Y-%m-%dT%H:%M:%S") class CboeEuropeanIndexHistoricalFetcher( diff --git a/openbb_sdk/providers/cboe/openbb_cboe/models/major_indices_historical.py b/openbb_sdk/providers/cboe/openbb_cboe/models/major_indices_historical.py index e825b086f5a..fbb84c385b6 100644 --- a/openbb_sdk/providers/cboe/openbb_cboe/models/major_indices_historical.py +++ b/openbb_sdk/providers/cboe/openbb_cboe/models/major_indices_historical.py @@ -45,11 +45,15 @@ class CboeMajorIndicesHistoricalData(MajorIndicesHistoricalData): @validator("date", pre=True, check_fields=False) def date_validate(cls, v): # pylint: disable=E0213 - """Return datetime object from string.""" - try: - return datetime.strptime(v, "%Y-%m-%d %H:%M:%S") - except Exception: - return datetime.strptime(v, "%Y-%m-%d") + """Return formatted datetime string.""" + if isinstance(v, str): + try: + dt = datetime.strptime(v, "%Y-%m-%d %H:%M:%S") + return dt.strftime("%Y-%m-%d %H:%M:%S") + except ValueError: + dt = datetime.strptime(v, "%Y-%m-%d") + return dt.strftime("%Y-%m-%d") + return v class CboeMajorIndicesHistoricalFetcher( @@ -73,18 +77,16 @@ class CboeMajorIndicesHistoricalFetcher( credentials: Optional[Dict[str, str]], **kwargs: Any, ) -> List[Dict]: - """Return the raw data from the CBOE endpoint""" - + """Return the raw data from the CBOE endpoint.""" # Synbol directories are cached for seven days and are used for error handling and URL generation. INDEXES = get_cboe_index_directory().index.to_list() query.symbol = query.symbol.upper() data = pd.DataFrame() if "^" in query.symbol: query.symbol = query.symbol.replace("^", "") - if query.symbol == "NDX" and query.interval == "1d": - raise RuntimeError( - "NDX time series data is only supported when `interval='1m'`." - ) + query.interval = ( + "1m" if query.symbol == "NDX" and query.interval == "1d" else query.interval + ) now = datetime.now() query.start_date = ( @@ -198,6 +200,8 @@ class CboeMajorIndicesHistoricalFetcher( data_list[i]["volume"]["total_options_volume"] ) data = pd.DataFrame() + date = [d.replace("T", " ") for d in date] + date = [datetime.strptime(d, "%Y-%m-%d %H:%M:%S") for d in date] data["date"] = pd.to_datetime(date) data["open"] = open_ data["high"] = high @@ -215,5 +219,5 @@ class CboeMajorIndicesHistoricalFetcher( @staticmethod def transform_data(data: List[Dict]) -> List[CboeMajorIndicesHistoricalData]: - """Transform the data to the standard format""" + """Transform the data to the standard format.""" return [CboeMajorIndicesHistoricalData.parse_obj(d) for d in data] diff --git a/openbb_sdk/providers/cboe/openbb_cboe/models/stock_historical.py b/openbb_sdk/providers/cboe/openbb_cboe/models/stock_historical.py index 82ea36c226d..e5ba52bac9b 100644 --- a/openbb_sdk/providers/cboe/openbb_cboe/models/stock_historical.py +++ b/openbb_sdk/providers/cboe/openbb_cboe/models/stock_historical.py @@ -1,4 +1,5 @@ -"""CBOE Major Indices End of Day fetcher.""" +"""CBOE Stock Historical fetcher.""" + from datetime import datetime, timedelta from typing import Any, Dict, List, Literal, Optional @@ -16,7 +17,7 @@ from openbb_provider.standard_models.stock_historical import ( StockHistoricalQueryParams, ) from openbb_provider.utils.helpers import make_request -from pydantic import Field, validator +from pydantic import Field class CboeStockHistoricalQueryParams(StockHistoricalQueryParams): @@ -44,14 +45,6 @@ class CboeStockHistoricalData(StockHistoricalData): description="Total number of options traded during the most recent trading period. Only valid if interval is 1m." ) - @validator("date", pre=True, check_fields=False) - def date_validate(cls, v): # pylint: disable=E0213 - """Return datetime object from string.""" - try: - return datetime.strptime(v, "%Y-%m-%d %H:%M:%S") - except Exception: - return datetime.strptime(v, "%Y-%m-%d") - class CboeStockHistoricalFetcher( Fetcher[ @@ -72,146 +65,171 @@ class CboeStockHistoricalFetcher( credentials: Optional[Dict[str, str]], **kwargs: Any, ) -> List[Dict]: - """Return the raw data from the CBOE endpoint.""" + """Return the raw data from the CBOE endpoint""" + # Synbol directories are cached for seven days and are used for error handling and URL generation. SYMBOLS = get_cboe_directory() INDEXES = get_cboe_index_directory().index.to_list() - query.symbol = query.symbol.upper() - data = pd.DataFrame() - if "^" in query.symbol: - query.symbol = query.symbol.replace("^", "") - if query.symbol == "NDX" and query.interval == "1d": - raise RuntimeError( - "NDX time series data is only supported when `interval='1m'`." - ) - + multi = pd.DataFrame() now = datetime.now() - query.start_date = ( - query.start_date if query.start_date else now - timedelta(days=50000) + tickers = ( + query.symbol.split(",") if "," in query.symbol else [query.symbol.upper()] ) + if len(tickers) > 1: + query.start_date = ( + query.start_date if query.start_date else now - timedelta(days=720) + ) + if len(tickers) == 1: + query.start_date = ( + query.start_date if query.start_date else now - timedelta(days=50000) + ) query.end_date = query.end_date if query.end_date else now - if query.symbol not in SYMBOLS.index and query.symbol not in INDEXES: - raise RuntimeError( - f"The symbol, {query.symbol}, was not found in the CBOE directory. Use `search()`." - ) - - def __generate_historical_prices_url( - symbol, - data_type: Optional[Literal["intraday", "historical"]] = "historical", - ) -> str: - """Generate the final URL for historical prices data.""" - url: str = ( - f"https://cdn.cboe.com/api/global/delayed_quotes/charts/{data_type}" - ) - url = ( - url + f"/_{symbol}.json" - if symbol in TICKER_EXCEPTIONS or symbol in INDEXES - else url + f"/{symbol}.json" - ) - return url - - url = ( - __generate_historical_prices_url(query.symbol, "intraday") - if query.interval == "1m" - else __generate_historical_prices_url(query.symbol) - ) - r = make_request(url) - - if r.status_code != 200: - raise RuntimeError(r.status_code) - - if query.interval == "1d": - data = ( - pd.DataFrame(r.json()["data"])[ - ["date", "open", "high", "low", "close", "volume"] - ] - ).set_index("date") - - # Fill in missing data from current or most recent trading session. - - today = pd.to_datetime(datetime.now().date()) - if today.weekday() > 4: - day_minus = today.weekday() - 4 - today = pd.to_datetime(today - timedelta(days=day_minus)) - if today != data.index[-1]: - _today = pd.Series(get_ticker_info(query.symbol)) - today_df = pd.Series(dtype="object") - today_df["open"] = round(_today["open"], 2) - today_df["high"] = round(_today["high"], 2) - today_df["low"] = round(_today["low"], 2) - today_df["close"] = round(_today["close"], 2) - if ( - query.symbol not in INDEXES - and query.symbol not in TICKER_EXCEPTIONS - ): - data = data[data["volume"] > 0] - today_df["volume"] = _today["volume"] - today_df["date"] = today.date() - today_df = pd.DataFrame(today_df).transpose().set_index("date") - - data = pd.concat([data, today_df], axis=0) - - # If ticker is an index there is no volume data and the types must be set. - - if query.symbol in INDEXES or query.symbol in TICKER_EXCEPTIONS: - data = data[["open", "high", "low", "close", "volume"]] - data["open"] = round(data.open.astype(float), 2) - data["high"] = round(data.high.astype(float), 2) - data["low"] = round(data.low.astype(float), 2) - data["close"] = round(data.close.astype(float), 2) - data["volume"] = 0 - - data.index = pd.to_datetime(data.index, format="%Y-%m-%d") - data = data[data["open"] > 0] - data = data[ - (data.index >= pd.to_datetime(query.start_date, format="%Y-%m-%d")) - & (data.index <= pd.to_datetime(query.end_date, format="%Y-%m-%d")) - ] - data.index = data.index.astype(str) - if query.interval == "1m": - data_list = r.json()["data"] - date: List[datetime] = [] - open: List[float] = [] - high: List[float] = [] - low: List[float] = [] - close: List[float] = [] - volume: List[float] = [] - calls_volume: List[float] = [] - puts_volume: List[float] = [] - total_options_volume: List[float] = [] - - for i in enumerate(data_list): - date.append(data_list[i[0]]["datetime"]) - open.append(data_list[i[0]]["price"]["open"]) - high.append(data_list[i[0]]["price"]["high"]) - low.append(data_list[i[0]]["price"]["low"]) - close.append(data_list[i[0]]["price"]["close"]) - volume.append(data_list[i[0]]["volume"]["stock_volume"]) - calls_volume.append(data_list[i[0]]["volume"]["calls_volume"]) - puts_volume.append(data_list[i[0]]["volume"]["puts_volume"]) - total_options_volume.append( - data_list[i[0]]["volume"]["total_options_volume"] - ) + def get_one(symbol, start_date, end_date, interval): + symbol = symbol.upper() + if "^" in symbol: + symbol = symbol.replace("^", "") + if len(tickers) == 1 and symbol == "NDX": + raise RuntimeError("NDX is not supported by this function.") data = pd.DataFrame() - data["date"] = pd.to_datetime(date) - data["open"] = open - data["high"] = high - data["low"] = low - data["close"] = close - data["volume"] = volume - data["calls_volume"] = calls_volume - data["puts_volume"] = puts_volume - data["total_options_volume"] = total_options_volume - data = data.set_index("date").sort_index() - data.index = data.index.astype(str) - data = data[data["open"] > 0] - data["volume"] = round(data["volume"].astype("int64")) - return data.drop_duplicates().reset_index().to_dict("records") + if symbol not in SYMBOLS.index and symbol not in INDEXES: + raise RuntimeError( + f"The symbol, {symbol}, was not found in the CBOE directory. Use `search()`." + ) + + def __generate_historical_prices_url( + symbol, + data_type: Optional[Literal["intraday", "historical"]] = "historical", + ) -> str: + """Generate the final URL for historical prices data.""" + url: str = ( + f"https://cdn.cboe.com/api/global/delayed_quotes/charts/{data_type}" + ) + url = ( + url + f"/_{symbol}.json" + if symbol in TICKER_EXCEPTIONS or symbol in INDEXES + else url + f"/{symbol}.json" + ) + return url + + url = ( + __generate_historical_prices_url(symbol, "intraday") + if interval == "1m" + else __generate_historical_prices_url(symbol) + ) + r = make_request(url) + + if r.status_code != 200: + raise RuntimeError(r.status_code) + + if interval == "1d": + data = ( + pd.DataFrame(r.json()["data"])[ + ["date", "open", "high", "low", "close", "volume"] + ] + ).set_index("date") + + # Fill in missing data from current or most recent trading session. + + today = pd.to_datetime(datetime.now().date()) + if today.weekday() > 4: + day_minus = today.weekday() - 4 + today = pd.to_datetime(today - timedelta(days=day_minus)) + if today != data.index[-1]: + _today = pd.Series(get_ticker_info(symbol)) + today_df = pd.Series(dtype="object") + today_df["open"] = round(_today["open"], 2) + today_df["high"] = round(_today["high"], 2) + today_df["low"] = round(_today["low"], 2) + today_df["close"] = round(_today["close"], 2) + if symbol not in INDEXES and symbol not in TICKER_EXCEPTIONS: + data = data[data["volume"] > 0] + today_df["volume"] = _today["volume"] + today_df["date"] = today.date() + today_df = pd.DataFrame(today_df).transpose().set_index("date") + + data = pd.concat([data, today_df], axis=0) + + # If ticker is an index there is no volume data and the types must be set. + + if symbol in INDEXES or symbol in TICKER_EXCEPTIONS: + data = data[["open", "high", "low", "close", "volume"]] + data["open"] = round(data.open.astype(float), 2) + data["high"] = round(data.high.astype(float), 2) + data["low"] = round(data.low.astype(float), 2) + data["close"] = round(data.close.astype(float), 2) + data["volume"] = 0 + + data.index = pd.to_datetime(data.index, format="%Y-%m-%d") + data = data[data["open"] > 0] + data = data[ + (data.index >= pd.to_datetime(start_date, format="%Y-%m-%d")) + & (data.index <= pd.to_datetime(end_date, format="%Y-%m-%d")) + ] + data.index = data.index.strftime("%Y-%m-%d") + if query.interval == "1m": + data_list = r.json()["data"] + date: list[datetime] = [] + open: list[float] = [] + high: list[float] = [] + low: list[float] = [] + close: list[float] = [] + volume: list[float] = [] + calls_volume: list[float] = [] + puts_volume: list[float] = [] + total_options_volume: list[float] = [] + + for i in range(0, len(data_list)): + date.append(data_list[i]["datetime"]) + open.append(data_list[i]["price"]["open"]) + high.append(data_list[i]["price"]["high"]) + low.append(data_list[i]["price"]["low"]) + close.append(data_list[i]["price"]["close"]) + volume.append(data_list[i]["volume"]["stock_volume"]) + calls_volume.append(data_list[i]["volume"]["calls_volume"]) + puts_volume.append(data_list[i]["volume"]["puts_volume"]) + total_options_volume.append( + data_list[i]["volume"]["total_options_volume"] + ) + data = pd.DataFrame() + date = [d.replace("T", " ") for d in date] + date = [datetime.strptime(d, "%Y-%m-%d %H:%M:%S") for d in date] + data["date"] = date + data["open"] = open + data["high"] = high + data["low"] = low + data["close"] = close + data["volume"] = volume + data["calls_volume"] = calls_volume + data["puts_volume"] = puts_volume + data["total_options_volume"] = total_options_volume + data = data.set_index("date").sort_index() + data.index = data.index.strftime("%Y-%m-%d %H:%M:%S") + data = data[data["open"] > 0] + + data["volume"] = round(data["volume"].astype("int64")) + data["symbol"] = symbol + return data.drop_duplicates().reset_index() + + for ticker in tickers: + if ticker != "NDX": + data = get_one( + ticker, + start_date=query.start_date, + end_date=query.end_date, + interval=query.interval, + ).set_index(["date", "symbol"]) + multi = pd.concat([multi, data], axis=0) + + multi = multi.sort_index().reset_index() + if len(tickers) == 1: + multi = multi.drop(columns=["symbol"]) + + return multi.to_dict("records") @staticmethod def transform_data(data: List[Dict]) -> List[CboeStockHistoricalData]: - """Transform the data to the standard format""" - return [CboeStockHistoricalData.parse_obj(d) for d in data] + """Transform the data to the standard format.""" + return [CboeStockHistoricalData(**d) for d in data] diff --git a/openbb_sdk/providers/cboe/openbb_cboe/models/stock_info.py b/openbb_sdk/providers/cboe/openbb_cboe/models/stock_info.py index 35f22814935..5ed24bca252 100644 --- a/openbb_sdk/providers/cboe/openbb_cboe/models/stock_info.py +++ b/openbb_sdk/providers/cboe/openbb_cboe/models/stock_info.py @@ -1,6 +1,6 @@ """CBOE Stock Info fetcher.""" - +import concurrent.futures from datetime import datetime from typing import Any, Dict, List, Optional @@ -92,7 +92,7 @@ class CboeStockInfoFetcher( @staticmethod def transform_query(params: Dict[str, Any]) -> CboeStockInfoQueryParams: - """Transform the query""" + """Transform the query.""" return CboeStockInfoQueryParams(**params) @staticmethod @@ -100,30 +100,40 @@ class CboeStockInfoFetcher( query: CboeStockInfoQueryParams, credentials: Optional[Dict[str, str]], **kwargs: Any, - ) -> dict: - """Return the raw data from the CBOE endpoint""" - - query.symbol = query.symbol.upper().replace("^", "") + ) -> List[Dict]: + """Return the raw data from the CBOE endpoint.""" + results = [] + query.symbol = query.symbol.upper() + symbols = ( + query.symbol.split(",") if "," in query.symbol else [query.symbol.upper()] + ) INDEXES = get_cboe_index_directory().index.to_list() SYMBOLS = get_cboe_directory() - if query.symbol not in SYMBOLS.index and query.symbol not in INDEXES: - raise RuntimeError( - f"The symbol, {query.symbol}, was not found in the CBOE directory." - ) + def get_one(symbol): + data = pd.Series(dtype="object") + if symbol in SYMBOLS.index or symbol in INDEXES: + _info = pd.Series(get_ticker_info(symbol)) + _iv = pd.Series(get_ticker_iv(symbol)) + data = ( + pd.DataFrame(pd.concat([_info, _iv])) + .transpose() + .drop(columns="seqno") + .iloc[0] + ) - _info = pd.Series(get_ticker_info(query.symbol)) - _iv = pd.Series(get_ticker_iv(query.symbol)) - data = ( - pd.DataFrame(pd.concat([_info, _iv])) - .transpose() - .drop(columns="seqno") - .iloc[0] + results.append(data.to_dict()) + + with concurrent.futures.ThreadPoolExecutor() as executor: + executor.map(get_one, symbols) + + return ( + pd.DataFrame.from_records(results) + .sort_values(by="symbol") + .to_dict("records") ) - return data.to_dict() - @staticmethod - def transform_data(data: dict) -> List[CboeStockInfoData]: - """Transform the data to the standard format""" - return [CboeStockInfoData.parse_obj(data)] + def transform_data(data: List[Dict]) -> List[CboeStockInfoData]: + """Transform the data to the standard format.""" + return [CboeStockInfoData.parse_obj(d) for d in data] diff --git a/openbb_sdk/providers/polygon/openbb_polygon/models/stock_historical.py b/openbb_sdk/providers/polygon/openbb_polygon/models/stock_historical.py index a7ebf9118da..cd762fc66c2 100644 --- a/openbb_sdk/providers/polygon/openbb_polygon/models/stock_historical.py +++ b/openbb_sdk/providers/polygon/openbb_polygon/models/stock_historical.py @@ -14,7 +14,7 @@ from openbb_provider.standard_models.stock_historical import ( StockHistoricalQueryParams, ) from openbb_provider.utils.descriptions import QUERY_DESCRIPTIONS -from pydantic import Field, PositiveInt, validator +from pydantic import Field, PositiveInt class PolygonStockHistoricalQueryParams(StockHistoricalQueryParams): @@ -60,10 +60,6 @@ class PolygonStockHistoricalData(StockHistoricalData): alias="n", ) - @validator("t", pre=True, check_fields=False) - def time_validate(cls, v): # pylint: disable=E0213 - return datetime.fromtimestamp(v / 1000) - class PolygonStockHistoricalFetcher( Fetcher[ @@ -118,10 +114,11 @@ class PolygonStockHistoricalFetcher( if "," in query.symbol: results = [dict(symbol=symbol, **d) for d in results] - # return data.extend( - # [PolygonStockHistoricalData.parse_obj(d) for d in results] - # ) - # extend the data without using the Pydantic model + for r in results: + r["t"] = datetime.fromtimestamp(r["t"] / 1000) + if query.timespan not in ["minute", "hour"]: + r["t"] = r["t"].date() + data.extend(results) with ThreadPoolExecutor() as executor: @@ -131,8 +128,6 @@ class PolygonStockHistoricalFetcher( @staticmethod def transform_data(data: List[dict]) -> List[PolygonStockHistoricalData]: - transformed_data: List[PolygonStockHistoricalData] = [ - PolygonStockHistoricalData.parse_obj(d) for d in data - ] + transformed_data = [PolygonStockHistoricalData.parse_obj(d) for d in data] transformed_data.sort(key=lambda x: x.date) return transformed_data diff --git a/openbb_sdk/providers/yfinance/openbb_yfinance/models/stock_historical.py b/openbb_sdk/providers/yfinance/openbb_yfinance/models/stock_historical.py index c6fd663089f..9cdce3fd10b 100644 --- a/openbb_sdk/providers/yfinance/openbb_yfinance/models/stock_historical.py +++ b/openbb_sdk/providers/yfinance/openbb_yfinance/models/stock_historical.py @@ -13,8 +13,8 @@ from openbb_provider.standard_models.stock_historical import ( from openbb_provider.utils.descriptions import QUERY_DESCRIPTIONS from openbb_yfinance.utils.helpers import yf_download from openbb_yfinance.utils.references import INTERVALS, PERIODS -from pandas import to_datetime -from pydantic import Field +from pandas import Timestamp, to_datetime +from pydantic import Field, validator class YFinanceStockHistoricalQueryParams(StockHistoricalQueryParams): @@ -56,6 +56,13 @@ class YFinanceStockHistoricalQueryParams(StockHistoricalQueryParams): class YFinanceStockHistoricalData(StockHistoricalData): """YFinance Stock End of Day Data.""" + @validator("date", pre=True, check_fields=False) + def date_validate(cls, v): # pylint: disable=E0213 + """Return formatted datetime.""" + if isinstance(v, Timestamp): + return v.to_pydatetime() + return v + class YFinanceStockHistoricalFetcher( Fetcher[ @@ -126,10 +133,14 @@ class YFinanceStockHistoricalFetcher( if query.start_date: data.set_index("date", inplace=True) - data.index = to_datetime(data.index).date + data.index = to_datetime(data.index) + + start_date_dt = datetime.combine(query.start_date, datetime.min.time()) + end_date_dt = datetime.combine(query.end_date, datetime.min.time()) + data = data[ - (data.index >= query.start_date - timedelta(days=days)) - & (data.index <= query.end_date) + (data.index >= start_date_dt - timedelta(days=days)) + & (data.index <= end_date_dt) ] data.reset_index(inplace=True) @@ -142,4 +153,5 @@ class YFinanceStockHistoricalFetcher( data: List[Dict], ) -> List[YFinanceStockHistoricalData]: """Transform the data to the standard format.""" + return [YFinanceStockHistoricalData.parse_obj(d) for d in data] diff --git a/openbb_sdk/sdk/provider/openbb_provider/standard_models/stock_historical.py b/openbb_sdk/sdk/provider/openbb_provider/standard_models/stock_historical.py index 9d018a6525a..24e4f9952a2 100644 --- a/openbb_sdk/sdk/provider/openbb_provider/standard_models/stock_historical.py +++ b/openbb_sdk/sdk/provider/openbb_provider/standard_models/stock_historical.py @@ -7,6 +7,7 @@ from datetime import ( ) from typing import List, Optional, Set, Union +from dateutil import parser from pydantic import Field, NonNegativeInt, PositiveFloat, validator from openbb_provider.abstract.data import Data @@ -36,12 +37,15 @@ class StockHistoricalQueryParams(QueryParams): class StockHistoricalData(Data): """Stock end of day price Data.""" - date: Union[dateType, datetime] = Field( - description=DATA_DESCRIPTIONS.get("date", "") - ) + date: datetime = Field(description=DATA_DESCRIPTIONS.get("date", "")) open: PositiveFloat = Field(description=DATA_DESCRIPTIONS.get("open", "")) high: PositiveFloat = Field(description=DATA_DESCRIPTIONS.get("high", "")) low: PositiveFloat = Field(description=DATA_DESCRIPTIONS.get("low", "")) close: PositiveFloat = Field(description=DATA_DESCRIPTIONS.get("close", "")) volume: NonNegativeInt = Field(description=DATA_DESCRIPTIONS.get("volume", "")) vwap: Optional[PositiveFloat] = Field(description=DATA_DESCRIPTIONS.get("vwap", "")) + + @validator("date", pre=True, check_fields=False) + def date_validate(cls, v): # pylint: disable=E0213 + """Return formatted datetime.""" + return parser.isoparse(str(v))