From a905799b94cf42f17ae85b2f6a5d8237a095c48e Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Thu, 21 Nov 2024 13:40:22 -0800 Subject: [PATCH] [Enhancement] Add Multi-Country Support & ECB Yield Curves To `fixedincome.goverment.yield_curve(provider='econdb')` (#6946) * add multi-country suppport and ECB yield curves to EconDB * lint * do igor's suggestion * mypy --- .../provider/standard_models/yield_curve.py | 28 +- .../openbb_fixedincome/fixedincome_views.py | 64 ++-- openbb_platform/openbb/assets/reference.json | 33 +- .../openbb/package/fixedincome_government.py | 39 ++- .../openbb_econdb/models/yield_curve.py | 286 +++++++++++------- .../openbb_econdb/utils/yield_curves.py | 72 +++-- 6 files changed, 339 insertions(+), 183 deletions(-) diff --git a/openbb_platform/core/openbb_core/provider/standard_models/yield_curve.py b/openbb_platform/core/openbb_core/provider/standard_models/yield_curve.py index 9d7169f80b7..382d6c2ce07 100644 --- a/openbb_platform/core/openbb_core/provider/standard_models/yield_curve.py +++ b/openbb_platform/core/openbb_core/provider/standard_models/yield_curve.py @@ -3,20 +3,19 @@ from datetime import date as dateType from typing import Optional, Union -from pydantic import Field, field_validator - from openbb_core.provider.abstract.data import Data from openbb_core.provider.abstract.query_params import QueryParams from openbb_core.provider.utils.descriptions import ( DATA_DESCRIPTIONS, QUERY_DESCRIPTIONS, ) +from pydantic import Field, computed_field, field_validator class YieldCurveQueryParams(QueryParams): """Yield Curve Query.""" - date: Union[None, dateType, str] = Field( + date: Optional[Union[dateType, str]] = Field( default=None, description=QUERY_DESCRIPTIONS.get("date", "") + " By default is the current data.", @@ -34,9 +33,10 @@ class YieldCurveQueryParams(QueryParams): if isinstance(v, dateType): return v.strftime("%Y-%m-%d") new_dates: list = [] + dates: list = [] if isinstance(v, str): dates = v.split(",") - if isinstance(v, list): + elif isinstance(v, list): dates = v for date in dates: new_dates.append(to_datetime(date).date().strftime("%Y-%m-%d")) @@ -52,7 +52,21 @@ class YieldCurveData(Data): description=DATA_DESCRIPTIONS.get("date", ""), ) maturity: str = Field(description="Maturity length of the security.") - rate: float = Field( - description="The yield as a normalized percent (0.05 is 5%)", - json_schema_extra={"x-unit_measurement": "percent", "x-frontend_multiply": 100}, + + @computed_field( # type: ignore + description="Maturity length, in years, as a decimal.", + return_type=Optional[float], ) + @property + def maturity_years(self) -> Optional[float]: + """Get the maturity in years as a decimal.""" + if "_" not in self.maturity: # pylint: disable=E1135 + return None + + parts = self.maturity.split("_") # pylint: disable=E1101 + months = sum( + int(parts[i + 1]) * (12 if parts[i] == "year" else 1) + for i in range(0, len(parts), 2) + ) + + return months / 12 diff --git a/openbb_platform/extensions/fixedincome/openbb_fixedincome/fixedincome_views.py b/openbb_platform/extensions/fixedincome/openbb_fixedincome/fixedincome_views.py index 5a56923e1b3..a8b6cae73f2 100644 --- a/openbb_platform/extensions/fixedincome/openbb_fixedincome/fixedincome_views.py +++ b/openbb_platform/extensions/fixedincome/openbb_fixedincome/fixedincome_views.py @@ -28,7 +28,7 @@ class FixedIncomeViews: from openbb_core.app.utils import basemodel_to_df from pandas import DataFrame - data = kwargs.get("data", None) + data = kwargs.get("data") df: DataFrame = DataFrame() if data: if isinstance(data, DataFrame) and not data.empty: # noqa: SIM108 @@ -55,6 +55,9 @@ class FixedIncomeViews: provider = kwargs.get("provider") df["date"] = df["date"].astype(str) maturities = duration_sorter(df["maturity"].unique().tolist()) + countries: list = ( + df["country"].unique().tolist() if "country" in df.columns else [] + ) # Use the supplied colors, if any. colors = kwargs.get("colors", []) @@ -66,11 +69,13 @@ class FixedIncomeViews: figure.update_layout(ChartStyle().plotly_template.get("layout", {})) text_color = "white" if ChartStyle().plt_style == "dark" else "black" - def create_fig(figure, df, dates, color_count, country: Optional[str] = None): + def create_fig( + figure, dataframe, dates, color_count, country: Optional[str] = None + ): """Create a scatter for each date in the data.""" for date in dates: color = colors[color_count % len(colors)] - plot_df = df[df["date"] == date].copy() + plot_df = dataframe[dataframe["date"] == date].copy() plot_df["rate"] = plot_df["rate"].apply(lambda x: x * 100) plot_df = plot_df.rename(columns={"rate": "Yield"}) plot_df = ( @@ -81,31 +86,41 @@ class FixedIncomeViews: ) plot_df = plot_df.rename(columns={"index": "Maturity"}) plot_df["Maturity"] = [ - ( - d.split("_")[1] + " " + d.split("_")[0].title() - if d != "long_term" - else "Long Term" - ) + (d.split("_")[1] + " " + d.split("_")[0].title()) for d in plot_df["Maturity"] ] figure.add_scatter( x=plot_df["Maturity"], y=plot_df["Yield"], mode="lines+markers", - name=f"{country} - {date}" if country else date, + name=( + f"{country.replace('_', ' ').title().replace('Ecb', 'ECB')} {date}" + if country + else date + ), line=dict(width=3, color=color), marker=dict(size=10, color=color), hovertemplate=( "Maturity: %{x}
Yield: %{y}%" - if len(dates) == 1 + if len(dates) == 1 and not countries else "%{fullData.name}
Maturity: %{x}
Yield: %{y}%" ), ) color_count += 1 return figure, color_count - dates = df.date.unique().tolist() - figure, color_count = create_fig(figure, df, dates, color_count) + if countries: + for _country in countries: + _df = df[df["country"] == _country] + dates = _df.date.unique().tolist() + figure, color_count = create_fig( + figure, _df, dates, color_count, _country + ) + + else: + dates = df.date.unique().tolist() + figure, color_count = create_fig(figure, df, dates, color_count) + extra_params = kwargs.get("extra_params", {}) extra_params = ( extra_params if isinstance(extra_params, dict) else extra_params.__dict__ @@ -130,14 +145,26 @@ class FixedIncomeViews: ) country = f"United States {curve_type}" elif provider == "econdb": - country = extra_params.get("country", "") - country = country.replace("_", " ").title() if country else "United States" + country = ( + "" + if countries + else ( + extra_params.get("country", "") + .replace("_", " ") + .title() + .replace("Ecb", "ECB") + or "United States" + ) + ) + country = country + " " if country else "" title = kwargs.get("title", "") if not title: title = f"{country}Yield Curve" - if len(dates) == 1: + if len(dates) == 1 and len(countries) == 1: title = f"{country} Yield Curve - {dates[0]}" + elif countries: + title = f"Yield Curve - {', '.join(countries).replace('_', ' ').title().replace('Ecb', 'ECB')}" # Update the layout of the figure. figure.update_layout( @@ -156,11 +183,7 @@ class FixedIncomeViews: categoryorder="array", categoryarray=( [ - ( - d.split("_")[1] + " " + d.split("_")[0].title() - if d != "long_term" - else "Long Term" - ) + (d.split("_")[1] + " " + d.split("_")[0].title()) for d in maturities ] ), @@ -186,6 +209,7 @@ class FixedIncomeViews: margin=dict( b=25, t=10, + l=20, ), ) diff --git a/openbb_platform/openbb/assets/reference.json b/openbb_platform/openbb/assets/reference.json index 590301883f0..73c4d32ef27 100644 --- a/openbb_platform/openbb/assets/reference.json +++ b/openbb_platform/openbb/assets/reference.json @@ -3020,7 +3020,7 @@ }, { "name": "date", - "type": "Union[Union[str, date], List[Union[str, date]]]", + "type": "Union[Union[date, str], List[Union[date, str]]]", "description": "A specific date to get data for. Multiple items allowed for provider(s): yfinance.", "default": null, "optional": true, @@ -4671,7 +4671,7 @@ "standard": [ { "name": "date", - "type": "Union[Union[str, date], List[Union[str, date]]]", + "type": "Union[Union[date, str], List[Union[date, str]]]", "description": "A specific date to get data for. Default is the latest report. Multiple items allowed for provider(s): fred.", "default": null, "optional": true, @@ -9657,7 +9657,7 @@ "standard": [ { "name": "date", - "type": "Union[Union[str, date], List[Union[str, date]]]", + "type": "Union[Union[date, str], List[Union[date, str]]]", "description": "A specific date to get data for. Default is the latest report. Multiple items allowed for provider(s): fred.", "default": null, "optional": true, @@ -33795,7 +33795,7 @@ "fmp": [ { "name": "date", - "type": "Union[Union[str, date], str]", + "type": "Union[Union[date, str], str]", "description": "A specific date to get data for. Entering a date will attempt to return the NPORT-P filing for the entered date. This needs to be _exactly_ the date of the filing. Use the holdings_date command/endpoint to find available filing dates for the ETF.", "default": null, "optional": true, @@ -33823,7 +33823,7 @@ "sec": [ { "name": "date", - "type": "Union[Union[str, date], str]", + "type": "Union[Union[date, str], str]", "description": "A specific date to get data for. The date represents the period ending. The date entered will return the closest filing.", "default": null, "optional": true, @@ -36674,7 +36674,7 @@ "standard": [ { "name": "date", - "type": "Union[Union[None, date, str], List[Union[None, date, str]]]", + "type": "Union[Union[date, str], List[Union[date, str]]]", "description": "A specific date to get data for. By default is the current data. Multiple items allowed for provider(s): econdb, federal_reserve, fmp, fred.", "default": null, "optional": true, @@ -36684,14 +36684,17 @@ "econdb": [ { "name": "country", - "type": "Literal['australia', 'canada', 'china', 'hong_kong', 'india', 'japan', 'mexico', 'new_zealand', 'russia', 'saudi_arabia', 'singapore', 'south_africa', 'south_korea', 'taiwan', 'thailand', 'united_kingdom', 'united_states']", - "description": "The country to get data. New Zealand, Mexico, Singapore, and Thailand have only monthly data. The nearest date to the requested one will be used.", + "type": "Union[str, List[str]]", + "description": "The country to get data. New Zealand, Mexico, Singapore, and Thailand have only monthly data. The nearest date to the requested one will be used. Multiple items allowed for provider(s): econdb.", "default": "united_states", "optional": true, "choices": [ "australia", "canada", "china", + "ecb_instantaneous_forward", + "ecb_par_yield", + "ecb_spot_rate", "hong_kong", "india", "japan", @@ -36783,14 +36786,6 @@ "default": "", "optional": false, "choices": null - }, - { - "name": "rate", - "type": "float", - "description": "The yield as a normalized percent (0.05 is 5%)", - "default": "", - "optional": false, - "choices": null } ], "econdb": [], @@ -37456,7 +37451,7 @@ "standard": [ { "name": "date", - "type": "Union[Union[str, date], List[Union[str, date]]]", + "type": "Union[Union[date, str], List[Union[date, str]]]", "description": "A specific date to get data for. Multiple items allowed for provider(s): fred.", "default": null, "optional": true, @@ -38767,7 +38762,7 @@ }, { "name": "date_first_added", - "type": "Union[str, date]", + "type": "Union[date, str]", "description": "Date the constituent company was added to the index.", "default": null, "optional": true, @@ -38783,7 +38778,7 @@ }, { "name": "founded", - "type": "Union[str, date]", + "type": "Union[date, str]", "description": "Founding year of the constituent company in the index.", "default": null, "optional": true, diff --git a/openbb_platform/openbb/package/fixedincome_government.py b/openbb_platform/openbb/package/fixedincome_government.py index e9c5204b8cf..023694a40ea 100644 --- a/openbb_platform/openbb/package/fixedincome_government.py +++ b/openbb_platform/openbb/package/fixedincome_government.py @@ -338,7 +338,7 @@ class ROUTER_fixedincome_government(Container): def yield_curve( self, date: Annotated[ - Union[str, datetime.date, None, List[Union[str, datetime.date, None]]], + Union[datetime.date, str, None, List[Union[datetime.date, str, None]]], OpenBBField( description="A specific date to get data for. By default is the current data. Multiple comma separated items allowed for provider(s): econdb, federal_reserve, fmp, fred." ), @@ -355,12 +355,12 @@ class ROUTER_fixedincome_government(Container): Parameters ---------- - date : Union[str, date, None, List[Union[str, date, None]]] + date : Union[date, str, None, List[Union[date, str, None]]] A specific date to get data for. By default is the current data. Multiple comma separated items allowed for provider(s): econdb, federal_reserve, fmp, fred. provider : Optional[Literal['econdb', 'federal_reserve', 'fmp', 'fred']] The provider to use, by default None. If None, the priority list configured in the settings is used. Default priority: econdb, federal_reserve, fmp, fred. - country : Literal['australia', 'canada', 'china', 'hong_kong', 'india', 'japan', 'mexico', 'new_zealand', 'russia', 'saudi_arabia', 'singapore', 'south_africa', 'south_korea', 'taiwan', 'thailand', 'united_kingdom', 'united_states'] - The country to get data. New Zealand, Mexico, Singapore, and Thailand have only monthly data. The nearest date to the requested one will be used. (provider: econdb) + country : str + The country to get data. New Zealand, Mexico, Singapore, and Thailand have only monthly data. The nearest date to the requested one will be used. Multiple comma separated items allowed. (provider: econdb) use_cache : bool If true, cache the request for four hours. (provider: econdb) yield_curve_type : Literal['nominal', 'real', 'breakeven', 'treasury_minus_fed_funds', 'corporate_spot', 'corporate_par'] @@ -386,8 +386,6 @@ class ROUTER_fixedincome_government(Container): The date of the data. maturity : str Maturity length of the security. - rate : float - The yield as a normalized percent (0.05 is 5%) Examples -------- @@ -421,7 +419,34 @@ class ROUTER_fixedincome_government(Container): }, "fmp": {"multiple_items_allowed": True, "choices": None}, "fred": {"multiple_items_allowed": True, "choices": None}, - } + }, + "country": { + "econdb": { + "multiple_items_allowed": True, + "choices": [ + "australia", + "canada", + "china", + "ecb_instantaneous_forward", + "ecb_par_yield", + "ecb_spot_rate", + "hong_kong", + "india", + "japan", + "mexico", + "new_zealand", + "russia", + "saudi_arabia", + "singapore", + "south_africa", + "south_korea", + "taiwan", + "thailand", + "united_kingdom", + "united_states", + ], + } + }, }, ) ) diff --git a/openbb_platform/providers/econdb/openbb_econdb/models/yield_curve.py b/openbb_platform/providers/econdb/openbb_econdb/models/yield_curve.py index 8b84371a08d..748b9a2c9a5 100644 --- a/openbb_platform/providers/econdb/openbb_econdb/models/yield_curve.py +++ b/openbb_platform/providers/econdb/openbb_econdb/models/yield_curve.py @@ -14,7 +14,7 @@ from openbb_core.provider.standard_models.yield_curve import ( ) from openbb_core.provider.utils.descriptions import QUERY_DESCRIPTIONS from openbb_core.provider.utils.errors import EmptyDataError -from openbb_econdb.utils.yield_curves import COUNTRIES, COUNTRIES_DICT, COUNTRIES_LIST +from openbb_econdb.utils.yield_curves import COUNTRIES_DICT, COUNTRIES_LIST from pydantic import Field, field_validator @@ -22,14 +22,14 @@ class EconDbYieldCurveQueryParams(YieldCurveQueryParams): """EconDB Economic Indicators Query.""" __json_schema_extra__ = { - "date": ["multiple_items_allowed"], + "date": {"multiple_items_allowed": True}, + "country": {"multiple_items_allowed": True, "choices": COUNTRIES_LIST}, } - country: COUNTRIES = Field( + country: str = Field( default="united_states", description=QUERY_DESCRIPTIONS.get("country", "") + " New Zealand, Mexico, Singapore, and Thailand have only monthly data." + " The nearest date to the requested one will be used.", - json_schema_extra={"choices": COUNTRIES_LIST}, ) use_cache: bool = Field( default=True, @@ -42,9 +42,18 @@ class EconDbYieldCurveQueryParams(YieldCurveQueryParams): """Validate the country.""" if v is None: return "united_states" - if v not in COUNTRIES_DICT: - raise OpenBBError(f"Country must be one of {COUNTRIES_DICT}") - return v + + countries = v.split(",") + new_countries: list = [] + + for country in countries: + if country not in COUNTRIES_DICT: + raise ValueError( + f"Invalid country, {country}, must be one of {list(COUNTRIES_DICT)}" + ) + new_countries.append(country) + + return ",".join(new_countries) class EconDbYieldCurveData(YieldCurveData): @@ -59,7 +68,7 @@ class EconDbYieldCurveFetcher( require_credentials = False @staticmethod - def transform_query(params: Dict[str, Any]) -> EconDbYieldCurveQueryParams: + def transform_query(params: dict[str, Any]) -> EconDbYieldCurveQueryParams: """Transform the query parameters.""" if not params.get("date"): params["date"] = datetime.now().strftime("%Y-%m-%d") @@ -68,133 +77,194 @@ class EconDbYieldCurveFetcher( @staticmethod async def aextract_data( # pylint: disable=R0914.R0912,R0915 query: EconDbYieldCurveQueryParams, - credentials: Optional[Dict[str, str]], + credentials: Optional[dict[str, str]], **kwargs: Any, - ) -> List[Dict]: + ) -> dict: """Extract the data.""" # pylint: disable=import-outside-toplevel + import asyncio # noqa from openbb_econdb.utils import helpers + from warnings import warn token = credentials.get("econdb_api_key") if credentials else "" # type: ignore # Attempt to create a temporary token if one is not supplied. if not token: token = await helpers.create_token(use_cache=query.use_cache) credentials.update({"econdb_api_key": token}) # type: ignore - base_url = "https://www.econdb.com/api/series/?ticker=" - symbols = list(COUNTRIES_DICT[query.country].keys()) - url = ( - base_url - + f"%5B{','.join(symbols)}%5D&page_size=50&format=json&token={token}" - ) - data: List[Dict] = [] - response: Union[Dict, List[Dict]] = {} - if query.use_cache is True: - cache_dir = f"{helpers.get_user_cache_directory()}/http/econdb_yield_curve" - async with helpers.CachedSession( - cache=helpers.SQLiteBackend( - cache_dir, expire_after=3600 * 4, ignored_params=["token"] - ) - ) as session: - try: - response = await helpers.amake_request( # type: ignore - url, - session=session, - timeout=20, - **kwargs, - ) - finally: - await session.close() - else: - response = await helpers.amake_request( # type: ignore - url, timeout=20, **kwargs + + results: dict = {} + messages: list = [] + + async def get_one_country(country): + """Get the data for one country.""" + base_url = "https://www.econdb.com/api/series/?ticker=" + symbols = list(COUNTRIES_DICT[country].keys()) + url = ( + base_url + + f"%5B{','.join(symbols)}%5D&page_size=50&format=json&token={token}" ) - if not response: - raise OpenBBError("No data was returned.") - data = response.get("results") # type: ignore - if not data: - raise EmptyDataError("The response was returned empty.") - return data + data: list = [] + response: Union[dict, list[dict]] = {} + if query.use_cache is True: + cache_dir = ( + f"{helpers.get_user_cache_directory()}/http/econdb_yield_curve" + ) + async with helpers.CachedSession( + cache=helpers.SQLiteBackend( + cache_dir, expire_after=3600 * 4, ignored_params=["token"] + ) + ) as session: + await session.delete_expired_responses() + try: + response = await helpers.amake_request( # type: ignore + url, + session=session, + timeout=20, + **kwargs, + ) + finally: + await session.close() + else: + response = await helpers.amake_request( # type: ignore + url, timeout=20, **kwargs + ) + if not response: + messages.append(f"No data was returned for, {country}") + return + data = response.get("results") # type: ignore + if not data: + messages.append(f"The response for, {country}, was returned empty.") + return + results[country] = data + + return + + _countries = query.country.split(",") + + tasks = [asyncio.create_task(get_one_country(c)) for c in _countries] + await asyncio.gather(*tasks) + + if not results and messages: + msg_str = "\n".join(messages) + raise OpenBBError(msg_str) + + if not results and not messages: + raise OpenBBError("Unexpected outcome -> All requests were returned empty.") + + if results and messages: + for message in messages: + warn(message) + + return results @staticmethod def transform_data( query: EconDbYieldCurveQueryParams, - data: List[Dict], + data: dict, **kwargs: Any, - ) -> AnnotatedResult[List[EconDbYieldCurveData]]: + ) -> AnnotatedResult[list[EconDbYieldCurveData]]: """Transform the data.""" # pylint: disable=import-outside-toplevel + from numpy import nan from pandas import Categorical, DataFrame, DatetimeIndex - maturity_order = list(COUNTRIES_DICT[query.country].values()) - dates = query.date.split(",") # type: ignore - dates_list = DatetimeIndex(dates) - new_data: Dict = {} - metadata: Dict = {} - # Unpack the data for each maturity. - for item in data: - ticker = item.get("ticker") - maturity = COUNTRIES_DICT[query.country].get(ticker) - dataset = item.get("dataset") - additional_metadata = item.get("additional_metadata") - units = "percent" - description = item.get("description") - dates = item["data"]["dates"] - rates = item["data"]["values"] - meta = { - "symbol": ticker, - "description": description, - "dataset": dataset, - "units": units, - "additional_metadata": additional_metadata, - } - metadata[maturity] = meta - for d, r in zip(dates, rates): - if maturity not in new_data: - new_data[maturity] = {} - new_data[maturity][d] = r + results_metadata: dict = {} + results_data: list = [] - # Create a DataFrame from the data - df = DataFrame(new_data) + def process_country_data(country, country_data): + """Process the data for one country.""" + maturity_order = list(COUNTRIES_DICT[country].values()) + dates = query.date.split(",") # type: ignore + dates_list = DatetimeIndex(dates) + new_data: Dict = {} + metadata: Dict = {} + # Unpack the data for each maturity. + for item in country_data: + ticker = item.get("ticker") + maturity = COUNTRIES_DICT[country].get(ticker) + dataset = item.get("dataset") + additional_metadata = item.get("additional_metadata") + units = "percent" + description = item.get("description") + dates = item["data"]["dates"] + rates = item["data"]["values"] + meta = { + "symbol": ticker, + "description": description, + "dataset": dataset, + "units": units, + "additional_metadata": additional_metadata, + } + metadata[maturity] = meta + for d, r in zip(dates, rates): + if maturity not in new_data: + new_data[maturity] = {} + new_data[maturity][d] = r - # Convert the index to a DatetimeIndex - df.index = DatetimeIndex(df.index) + # Create a DataFrame from the data + df = DataFrame(new_data) - # Find the nearest date in the DataFrame to each date in dates_list - def get_nearest_date(df, target_date): - differences = (df.index - target_date).to_series().abs() - nearest_date_index = differences.argmin() - nearest_date = df.index[nearest_date_index] - return nearest_date + # Convert the index to a DatetimeIndex + df.index = DatetimeIndex(df.index) - nearest_dates = [get_nearest_date(df, date) for date in dates_list] + # Find the nearest date in the DataFrame to each date in dates_list + def get_nearest_date(df, target_date): + differences = (df.index - target_date).to_series().abs() + nearest_date_index = differences.argmin() + nearest_date = df.index[nearest_date_index] + return nearest_date - # Filter for only the nearest dates - df = df[df.index.isin(nearest_dates)] + nearest_dates = [get_nearest_date(df, date) for date in dates_list] - # Flatten the DataFrame - flattened_data = df.reset_index().melt( - id_vars="index", var_name="maturity", value_name="rate" - ) - flattened_data = flattened_data.rename(columns={"index": "date"}).sort_values( - "date" - ) - flattened_data["maturity"] = Categorical( - flattened_data["maturity"], categories=maturity_order, ordered=True - ) - flattened_data["rate"] = flattened_data["rate"].astype(float) / 100 - flattened_data = flattened_data.sort_values( - by=["date", "maturity"] - ).reset_index(drop=True) - flattened_data.loc[:, "date"] = flattened_data["date"].dt.strftime("%Y-%m-%d") - records = flattened_data.copy() - records["rate"] = records["rate"].fillna("N/A").replace("N/A", None) - records = records.to_dict("records") - results: List[EconDbYieldCurveData] = [ - EconDbYieldCurveData.model_validate(r) for r in records if r.get("rate") - ] - if not results: + # Filter for only the nearest dates + df = df[df.index.isin(nearest_dates)] + + # Flatten the DataFrame + flattened_data = df.reset_index().melt( + id_vars="index", var_name="maturity", value_name="rate" + ) + flattened_data = flattened_data.rename( + columns={"index": "date"} + ).sort_values("date") + flattened_data["maturity"] = Categorical( + flattened_data["maturity"], categories=maturity_order, ordered=True + ) + flattened_data["rate"] = flattened_data["rate"].astype(float) / 100 + flattened_data = flattened_data.sort_values( + by=["date", "maturity"] + ).reset_index(drop=True) + flattened_data.loc[:, "date"] = flattened_data["date"].dt.strftime( + "%Y-%m-%d" + ) + new_df = flattened_data.copy() + new_df.loc[:, "country"] = country + + def convert_duration(x): + """Convert the duration to a decimal representation of years.""" + period, unit = x.split("_") + if period == "long" and unit == "term": + return 30 + if period == "year": + return int(unit) + return int(unit) / 12 + + new_df.loc[:, "maturity_years"] = new_df.maturity.apply(convert_duration) + + new_df = new_df.replace({nan: None}) + records = [ + EconDbYieldCurveData.model_validate(r) + for r in new_df.to_dict("records") + if r.get("rate") + ] + results_data.extend(records) + results_metadata[country] = metadata + + for country, country_data in data.items(): + process_country_data(country, country_data) + + if not results_data: raise EmptyDataError( f"No data was found for the country, {query.country}," f" and dates, {query.date}" ) - return AnnotatedResult(result=results, metadata=metadata) + return AnnotatedResult(result=results_data, metadata=results_metadata) diff --git a/openbb_platform/providers/econdb/openbb_econdb/utils/yield_curves.py b/openbb_platform/providers/econdb/openbb_econdb/utils/yield_curves.py index d8d546eca8e..ed31e15d4f2 100644 --- a/openbb_platform/providers/econdb/openbb_econdb/utils/yield_curves.py +++ b/openbb_platform/providers/econdb/openbb_econdb/utils/yield_curves.py @@ -1,7 +1,5 @@ """EconDB Yield Curve Utilities""" -from typing import Literal - DAILY = { "australia": { "RBA_F02D.FCMYGBAG2D.D.AU": "year_2", @@ -20,7 +18,7 @@ DAILY = { "BOC_BOND_YIELDS_ALL.BD_CDN_5YR_DQ_YLD.D.CA": "year_5", "BOC_BOND_YIELDS_ALL.BD_CDN_7YR_DQ_YLD.D.CA": "year_7", "BOC_BOND_YIELDS_ALL.BD_CDN_10YR_DQ_YLD.D.CA": "year_10", - "BOC_BOND_YIELDS_ALL.BD_CDN_LONG_DQ_YLD.D.CA": "long_term", + "BOC_BOND_YIELDS_ALL.BD_CDN_LONG_DQ_YLD.D.CA": "year_30", }, "china": { "CCDC_CGB.CHB_3M.D.CN": "month_3", @@ -134,7 +132,56 @@ DAILY = { "FRB_H15.C279E.D.US": "year_20", "FRB_H15.F5280.D.US": "year_30", }, + "ecb_spot_rate": { + "ECBYC.035E9053E2.B.EA": "month_3", + "ECBYC.03610053E2.B.EA": "month_6", + "ECBYC.0354C053E2.B.EA": "year_1", + "ECBYC.035D1053E2.B.EA": "year_2", + "ECBYC.035EA053E2.B.EA": "year_3", + "ECBYC.035F7053E2.B.EA": "year_4", + "ECBYC.03604053E2.B.EA": "year_5", + "ECBYC.03611053E2.B.EA": "year_6", + "ECBYC.0361E053E2.B.EA": "year_7", + "ECBYC.0362B053E2.B.EA": "year_8", + "ECBYC.03638053E2.B.EA": "year_9", + "ECBYC.03558053E2.B.EA": "year_10", + "ECBYC.034D2053E2.B.EA": "year_20", + "ECBYC.035DD053E2.B.EA": "year_30", + }, + "ecb_par_yield": { + "ECBYC.0317C053E2.B.EA": "month_3", + "ECBYC.031A3053E2.B.EA": "month_6", + "ECBYC.030DF053E2.B.EA": "year_1", + "ECBYC.03164053E2.B.EA": "year_2", + "ECBYC.0317D053E2.B.EA": "year_3", + "ECBYC.0318A053E2.B.EA": "year_4", + "ECBYC.03197053E2.B.EA": "year_5", + "ECBYC.031A4053E2.B.EA": "year_6", + "ECBYC.031B1053E2.B.EA": "year_7", + "ECBYC.031BE053E2.B.EA": "year_8", + "ECBYC.031CB053E2.B.EA": "year_9", + "ECBYC.03065053E2.B.EA": "year_10", + "ECBYC.030EB053E2.B.EA": "year_20", + "ECBYC.03170053E2.B.EA": "year_30", + }, + "ecb_instantaneous_forward": { + "ECBYC.02C71053E2.B.EA": "month_3", + "ECBYC.02C98053E2.B.EA": "month_6", + "ECBYC.02BD4053E2.B.EA": "year_1", + "ECBYC.02C59053E2.B.EA": "year_2", + "ECBYC.02C72053E2.B.EA": "year_3", + "ECBYC.02C7F053E2.B.EA": "year_4", + "ECBYC.02C8C053E2.B.EA": "year_5", + "ECBYC.02C99053E2.B.EA": "year_6", + "ECBYC.02CA6053E2.B.EA": "year_7", + "ECBYC.02CB3053E2.B.EA": "year_8", + "ECBYC.02CC0053E2.B.EA": "year_9", + "ECBYC.02B5A053E2.B.EA": "year_10", + "ECBYC.02BE0053E2.B.EA": "year_20", + "ECBYC.02C65053E2.B.EA": "year_30", + }, } + MONTHLY = { "thailand": { "BOTFM_RT_001_S2.36.M.TH": "month_1", @@ -198,22 +245,3 @@ COUNTRIES_DICT = { **DAILY, **MONTHLY, } -COUNTRIES = Literal[ - "australia", - "canada", - "china", - "hong_kong", - "india", - "japan", - "mexico", - "new_zealand", - "russia", - "saudi_arabia", - "singapore", - "south_africa", - "south_korea", - "taiwan", - "thailand", - "united_kingdom", - "united_states", -]