mirror of
https://github.com/OpenBB-finance/OpenBB.git
synced 2026-10-27 04:31:13 +00:00
[Feature] Add Fred Release Tables (#6595)
* add fred release tables * docstring * pylint * description * improve EmptyDataError message * even better message
This commit is contained in:
@@ -0,0 +1,73 @@
|
||||
"""FRED Release Table Standard Model."""
|
||||
|
||||
from datetime import date as dateType
|
||||
from typing import Optional, Union
|
||||
|
||||
from pydantic import Field
|
||||
|
||||
from openbb_core.provider.abstract.data import Data
|
||||
from openbb_core.provider.abstract.query_params import QueryParams
|
||||
from openbb_core.provider.utils.descriptions import (
|
||||
DATA_DESCRIPTIONS,
|
||||
QUERY_DESCRIPTIONS,
|
||||
)
|
||||
|
||||
|
||||
class ReleaseTableQueryParams(QueryParams):
|
||||
"""FRED Release Table Query."""
|
||||
|
||||
release_id: str = Field(
|
||||
description="The ID of the release." + " Use `fred_search` to find releases.",
|
||||
)
|
||||
element_id: Optional[str] = Field(
|
||||
default=None,
|
||||
description="The element ID of a specific table in the release.",
|
||||
)
|
||||
date: Optional[Union[dateType, str]] = Field(
|
||||
default=None,
|
||||
description=QUERY_DESCRIPTIONS.get("date", ""),
|
||||
)
|
||||
|
||||
|
||||
class ReleaseTableData(Data):
|
||||
"""FRED Release Table Data."""
|
||||
|
||||
date: Optional[dateType] = Field(
|
||||
default=None, description=DATA_DESCRIPTIONS.get("date", "")
|
||||
)
|
||||
level: Optional[int] = Field(
|
||||
default=None,
|
||||
description="The indentation level of the element.",
|
||||
)
|
||||
element_type: Optional[str] = Field(
|
||||
default=None,
|
||||
description="The type of the element.",
|
||||
)
|
||||
line: Optional[int] = Field(
|
||||
default=None,
|
||||
description="The line number of the element.",
|
||||
)
|
||||
element_id: Optional[str] = Field(
|
||||
default=None,
|
||||
description="The element id in the parent/child relationship.",
|
||||
)
|
||||
parent_id: Optional[str] = Field(
|
||||
default=None,
|
||||
description="The parent id in the parent/child relationship.",
|
||||
)
|
||||
children: Optional[str] = Field(
|
||||
default=None,
|
||||
description="The element_id of each child, as a comma-separated string.",
|
||||
)
|
||||
symbol: Optional[str] = Field(
|
||||
default=None,
|
||||
description=DATA_DESCRIPTIONS.get("symbol", ""),
|
||||
)
|
||||
name: Optional[str] = Field(
|
||||
default=None,
|
||||
description="The name of the series.",
|
||||
)
|
||||
value: Optional[float] = Field(
|
||||
default=None,
|
||||
description="The reported value of the series.",
|
||||
)
|
||||
@@ -993,3 +993,36 @@ def test_economy_pce(params, headers):
|
||||
result = requests.get(url, headers=headers, timeout=10)
|
||||
assert isinstance(result, requests.Response)
|
||||
assert result.status_code == 200
|
||||
|
||||
|
||||
@parametrize(
|
||||
"params",
|
||||
[
|
||||
(
|
||||
{
|
||||
"provider": "fred",
|
||||
"date": None,
|
||||
"release_id": "14",
|
||||
"element_id": "7930",
|
||||
}
|
||||
),
|
||||
(
|
||||
{
|
||||
"provider": "fred",
|
||||
"date": None,
|
||||
"release_id": "14",
|
||||
"element_id": None,
|
||||
}
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.integration
|
||||
def test_economy_fred_release_table(params, headers):
|
||||
"""Test the economy fred release table"""
|
||||
params = {p: v for p, v in params.items() if v}
|
||||
|
||||
query_str = get_querystring(params, [])
|
||||
url = f"http://0.0.0.0:8000/api/v1/economy/fred_release_table?{query_str}"
|
||||
result = requests.get(url, headers=headers, timeout=10)
|
||||
assert isinstance(result, requests.Response)
|
||||
assert result.status_code == 200
|
||||
|
||||
@@ -955,3 +955,35 @@ def test_economy_pce(params, obb):
|
||||
assert result
|
||||
assert isinstance(result, OBBject)
|
||||
assert len(result.results) > 0
|
||||
|
||||
|
||||
@parametrize(
|
||||
"params",
|
||||
[
|
||||
(
|
||||
{
|
||||
"provider": "fred",
|
||||
"date": None,
|
||||
"release_id": "14",
|
||||
"element_id": "7930",
|
||||
}
|
||||
),
|
||||
(
|
||||
{
|
||||
"provider": "fred",
|
||||
"date": None,
|
||||
"release_id": "14",
|
||||
"element_id": None,
|
||||
}
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.integration
|
||||
def test_economy_fred_release_table(params, obb):
|
||||
"""Test the economy fred release table endpoint"""
|
||||
params = {p: v for p, v in params.items() if v}
|
||||
|
||||
result = obb.economy.fred_release_table(**params)
|
||||
assert result
|
||||
assert isinstance(result, OBBject)
|
||||
assert len(result.results) > 0
|
||||
|
||||
@@ -155,6 +155,33 @@ async def fred_series(
|
||||
return await OBBject.from_query(Query(**locals()))
|
||||
|
||||
|
||||
@router.command(
|
||||
model="FredReleaseTable",
|
||||
examples=[
|
||||
APIEx(
|
||||
description="Get the top-level elements of a release by not supplying an element ID.",
|
||||
parameters={"release_id": "50", "provider": "fred"},
|
||||
),
|
||||
APIEx(
|
||||
description="Drill down on a specific section of the release.",
|
||||
parameters={"release_id": "50", "element_id": "4880", "provider": "fred"},
|
||||
),
|
||||
APIEx(
|
||||
description="Drill down on a specific table of the release.",
|
||||
parameters={"release_id": "50", "element_id": "4881", "provider": "fred"},
|
||||
),
|
||||
],
|
||||
)
|
||||
async def fred_release_table(
|
||||
cc: CommandContext,
|
||||
provider_choices: ProviderChoices,
|
||||
standard_params: StandardParams,
|
||||
extra_params: ExtraParams,
|
||||
) -> OBBject:
|
||||
"""Get economic release data by ID and/or element from FRED."""
|
||||
return await OBBject.from_query(Query(**locals()))
|
||||
|
||||
|
||||
@router.command(
|
||||
model="MoneyMeasures",
|
||||
examples=[
|
||||
|
||||
159
openbb_platform/openbb/assets/reference.json
vendored
159
openbb_platform/openbb/assets/reference.json
vendored
@@ -5738,6 +5738,165 @@
|
||||
},
|
||||
"model": "FredSeries"
|
||||
},
|
||||
"/economy/fred_release_table": {
|
||||
"deprecated": {
|
||||
"flag": null,
|
||||
"message": null
|
||||
},
|
||||
"description": "Get economic release data by ID and/or element from FRED.",
|
||||
"examples": "\nExamples\n--------\n\n```python\nfrom openbb import obb\n# Get the top-level elements of a release by not supplying an element ID.\nobb.economy.fred_release_table(release_id='50', provider='fred')\n# Drill down on a specific section of the release.\nobb.economy.fred_release_table(release_id='50', element_id='4880', provider='fred')\n# Drill down on a specific table of the release.\nobb.economy.fred_release_table(release_id='50', element_id='4881', provider='fred')\n```\n\n",
|
||||
"parameters": {
|
||||
"standard": [
|
||||
{
|
||||
"name": "release_id",
|
||||
"type": "str",
|
||||
"description": "The ID of the release. Use `fred_search` to find releases.",
|
||||
"default": "",
|
||||
"optional": false,
|
||||
"choices": null
|
||||
},
|
||||
{
|
||||
"name": "element_id",
|
||||
"type": "str",
|
||||
"description": "The element ID of a specific table in the release.",
|
||||
"default": null,
|
||||
"optional": true,
|
||||
"choices": null
|
||||
},
|
||||
{
|
||||
"name": "date",
|
||||
"type": "Union[Union[Union[str, date], str], List[Union[Union[str, date], str]]]",
|
||||
"description": "A specific date to get data for. Multiple items allowed for provider(s): fred.",
|
||||
"default": null,
|
||||
"optional": true,
|
||||
"choices": null
|
||||
},
|
||||
{
|
||||
"name": "provider",
|
||||
"type": "Literal['fred']",
|
||||
"description": "The provider to use, by default None. If None, the priority list configured in the settings is used. Default priority: fred.",
|
||||
"default": null,
|
||||
"optional": true
|
||||
}
|
||||
],
|
||||
"fred": []
|
||||
},
|
||||
"returns": {
|
||||
"OBBject": [
|
||||
{
|
||||
"name": "results",
|
||||
"type": "List[FredReleaseTable]",
|
||||
"description": "Serializable results."
|
||||
},
|
||||
{
|
||||
"name": "provider",
|
||||
"type": "Optional[Literal['fred']]",
|
||||
"description": "Provider name."
|
||||
},
|
||||
{
|
||||
"name": "warnings",
|
||||
"type": "Optional[List[Warning_]]",
|
||||
"description": "List of warnings."
|
||||
},
|
||||
{
|
||||
"name": "chart",
|
||||
"type": "Optional[Chart]",
|
||||
"description": "Chart object."
|
||||
},
|
||||
{
|
||||
"name": "extra",
|
||||
"type": "Dict[str, Any]",
|
||||
"description": "Extra info."
|
||||
}
|
||||
]
|
||||
},
|
||||
"data": {
|
||||
"standard": [
|
||||
{
|
||||
"name": "date",
|
||||
"type": "date",
|
||||
"description": "The date of the data.",
|
||||
"default": null,
|
||||
"optional": true,
|
||||
"choices": null
|
||||
},
|
||||
{
|
||||
"name": "level",
|
||||
"type": "int",
|
||||
"description": "The indentation level of the element.",
|
||||
"default": null,
|
||||
"optional": true,
|
||||
"choices": null
|
||||
},
|
||||
{
|
||||
"name": "element_type",
|
||||
"type": "str",
|
||||
"description": "The type of the element.",
|
||||
"default": null,
|
||||
"optional": true,
|
||||
"choices": null
|
||||
},
|
||||
{
|
||||
"name": "line",
|
||||
"type": "int",
|
||||
"description": "The line number of the element.",
|
||||
"default": null,
|
||||
"optional": true,
|
||||
"choices": null
|
||||
},
|
||||
{
|
||||
"name": "element_id",
|
||||
"type": "str",
|
||||
"description": "The element id in the parent/child relationship.",
|
||||
"default": null,
|
||||
"optional": true,
|
||||
"choices": null
|
||||
},
|
||||
{
|
||||
"name": "parent_id",
|
||||
"type": "str",
|
||||
"description": "The parent id in the parent/child relationship.",
|
||||
"default": null,
|
||||
"optional": true,
|
||||
"choices": null
|
||||
},
|
||||
{
|
||||
"name": "children",
|
||||
"type": "str",
|
||||
"description": "The element_id of each child, as a comma-separated string.",
|
||||
"default": null,
|
||||
"optional": true,
|
||||
"choices": null
|
||||
},
|
||||
{
|
||||
"name": "symbol",
|
||||
"type": "str",
|
||||
"description": "The date of the data.",
|
||||
"default": null,
|
||||
"optional": true,
|
||||
"choices": null
|
||||
},
|
||||
{
|
||||
"name": "name",
|
||||
"type": "str",
|
||||
"description": "The name of the series.",
|
||||
"default": null,
|
||||
"optional": true,
|
||||
"choices": null
|
||||
},
|
||||
{
|
||||
"name": "value",
|
||||
"type": "float",
|
||||
"description": "The reported value of the series.",
|
||||
"default": null,
|
||||
"optional": true,
|
||||
"choices": null
|
||||
}
|
||||
],
|
||||
"fred": []
|
||||
},
|
||||
"model": "FredReleaseTable"
|
||||
},
|
||||
"/economy/money_measures": {
|
||||
"deprecated": {
|
||||
"flag": null,
|
||||
|
||||
@@ -21,6 +21,7 @@ class ROUTER_economy(Container):
|
||||
country_profile
|
||||
cpi
|
||||
fred_regional
|
||||
fred_release_table
|
||||
fred_search
|
||||
fred_series
|
||||
/gdp
|
||||
@@ -997,6 +998,117 @@ class ROUTER_economy(Container):
|
||||
)
|
||||
)
|
||||
|
||||
@exception_handler
|
||||
@validate
|
||||
def fred_release_table(
|
||||
self,
|
||||
release_id: Annotated[
|
||||
str,
|
||||
OpenBBField(
|
||||
description="The ID of the release. Use `fred_search` to find releases."
|
||||
),
|
||||
],
|
||||
element_id: Annotated[
|
||||
Optional[str],
|
||||
OpenBBField(
|
||||
description="The element ID of a specific table in the release."
|
||||
),
|
||||
] = None,
|
||||
date: Annotated[
|
||||
Union[str, datetime.date, None, List[Union[str, datetime.date, None]]],
|
||||
OpenBBField(
|
||||
description="A specific date to get data for. Multiple comma separated items allowed for provider(s): fred."
|
||||
),
|
||||
] = None,
|
||||
provider: Annotated[
|
||||
Optional[Literal["fred"]],
|
||||
OpenBBField(
|
||||
description="The provider to use, by default None. If None, the priority list configured in the settings is used. Default priority: fred."
|
||||
),
|
||||
] = None,
|
||||
**kwargs
|
||||
) -> OBBject:
|
||||
"""Get economic release data by ID and/or element from FRED.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
release_id : str
|
||||
The ID of the release. Use `fred_search` to find releases.
|
||||
element_id : Optional[str]
|
||||
The element ID of a specific table in the release.
|
||||
date : Union[str, datetime.date, None, List[Union[str, datetime.d...
|
||||
A specific date to get data for. Multiple comma separated items allowed for provider(s): fred.
|
||||
provider : Optional[Literal['fred']]
|
||||
The provider to use, by default None. If None, the priority list configured in the settings is used. Default priority: fred.
|
||||
|
||||
Returns
|
||||
-------
|
||||
OBBject
|
||||
results : List[FredReleaseTable]
|
||||
Serializable results.
|
||||
provider : Optional[Literal['fred']]
|
||||
Provider name.
|
||||
warnings : Optional[List[Warning_]]
|
||||
List of warnings.
|
||||
chart : Optional[Chart]
|
||||
Chart object.
|
||||
extra : Dict[str, Any]
|
||||
Extra info.
|
||||
|
||||
FredReleaseTable
|
||||
----------------
|
||||
date : Optional[date]
|
||||
The date of the data.
|
||||
level : Optional[int]
|
||||
The indentation level of the element.
|
||||
element_type : Optional[str]
|
||||
The type of the element.
|
||||
line : Optional[int]
|
||||
The line number of the element.
|
||||
element_id : Optional[str]
|
||||
The element id in the parent/child relationship.
|
||||
parent_id : Optional[str]
|
||||
The parent id in the parent/child relationship.
|
||||
children : Optional[str]
|
||||
The element_id of each child, as a comma-separated string.
|
||||
symbol : Optional[str]
|
||||
The date of the data.
|
||||
name : Optional[str]
|
||||
The name of the series.
|
||||
value : Optional[float]
|
||||
The reported value of the series.
|
||||
|
||||
Examples
|
||||
--------
|
||||
>>> from openbb import obb
|
||||
>>> # Get the top-level elements of a release by not supplying an element ID.
|
||||
>>> obb.economy.fred_release_table(release_id='50', provider='fred')
|
||||
>>> # Drill down on a specific section of the release.
|
||||
>>> obb.economy.fred_release_table(release_id='50', element_id='4880', provider='fred')
|
||||
>>> # Drill down on a specific table of the release.
|
||||
>>> obb.economy.fred_release_table(release_id='50', element_id='4881', provider='fred')
|
||||
""" # noqa: E501
|
||||
|
||||
return self._run(
|
||||
"/economy/fred_release_table",
|
||||
**filter_inputs(
|
||||
provider_choices={
|
||||
"provider": self._get_provider(
|
||||
provider,
|
||||
"economy.fred_release_table",
|
||||
("fred",),
|
||||
)
|
||||
},
|
||||
standard_params={
|
||||
"release_id": release_id,
|
||||
"element_id": element_id,
|
||||
"date": date,
|
||||
},
|
||||
extra_params=kwargs,
|
||||
info={"date": {"fred": {"multiple_items_allowed": True}}},
|
||||
)
|
||||
)
|
||||
|
||||
@exception_handler
|
||||
@validate
|
||||
def fred_search(
|
||||
|
||||
@@ -32,6 +32,7 @@ from openbb_fred.models.personal_consumption_expenditures import (
|
||||
FredPersonalConsumptionExpendituresFetcher,
|
||||
)
|
||||
from openbb_fred.models.regional import FredRegionalDataFetcher
|
||||
from openbb_fred.models.release_table import FredReleaseTableFetcher
|
||||
from openbb_fred.models.retail_prices import FredRetailPricesFetcher
|
||||
from openbb_fred.models.search import (
|
||||
FredSearchFetcher,
|
||||
@@ -83,6 +84,7 @@ Research division of the Federal Reserve Bank of St. Louis that has more than
|
||||
"OvernightBankFundingRate": FredOvernightBankFundingRateFetcher,
|
||||
"PersonalConsumptionExpenditures": FredPersonalConsumptionExpendituresFetcher,
|
||||
"CommercialPaper": FREDCommercialPaperFetcher,
|
||||
"FredReleaseTable": FredReleaseTableFetcher,
|
||||
"FredSearch": FredSearchFetcher,
|
||||
"FredSeries": FredSeriesFetcher,
|
||||
"FredRegional": FredRegionalDataFetcher,
|
||||
|
||||
@@ -0,0 +1,263 @@
|
||||
"""FRED Release Table Table Model."""
|
||||
|
||||
# pylint: disable=unused-argument
|
||||
|
||||
from datetime import (
|
||||
date as dateType,
|
||||
datetime,
|
||||
)
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from openbb_core.app.model.abstract.error import OpenBBError
|
||||
from openbb_core.provider.abstract.fetcher import Fetcher
|
||||
from openbb_core.provider.standard_models.fred_release_table import (
|
||||
ReleaseTableData,
|
||||
ReleaseTableQueryParams,
|
||||
)
|
||||
from openbb_core.provider.utils.errors import EmptyDataError
|
||||
from pydantic import field_validator
|
||||
|
||||
|
||||
class FredReleaseTableQueryParams(ReleaseTableQueryParams):
|
||||
"""FRED Release Table Query Params."""
|
||||
|
||||
__json_schema_extra__ = {"date": {"multiple_items_allowed": True}}
|
||||
|
||||
@field_validator("date", mode="before", check_fields=False)
|
||||
@classmethod
|
||||
def validate_date(cls, v):
|
||||
"""Validate the dates entered."""
|
||||
if v is None:
|
||||
return None
|
||||
if isinstance(v, (list, dateType)):
|
||||
return v
|
||||
new_dates: List = []
|
||||
date_param = v
|
||||
if isinstance(date_param, str):
|
||||
new_dates = date_param.split(",")
|
||||
elif isinstance(date_param, dateType):
|
||||
new_dates.append(date_param.strftime("%Y-%m-%d"))
|
||||
elif isinstance(date_param, list) and isinstance(date_param[0], dateType):
|
||||
new_dates = [d.strftime("%Y-%m-%d") for d in new_dates]
|
||||
else:
|
||||
new_dates = date_param
|
||||
return ",".join(new_dates) if len(new_dates) > 1 else new_dates[0]
|
||||
|
||||
|
||||
class FredReleaseTableData(ReleaseTableData):
|
||||
"""FRED Release Table Data."""
|
||||
|
||||
__alias_dict__ = {
|
||||
"date": "observation_date",
|
||||
"value": "observation_value",
|
||||
"symbol": "series_id",
|
||||
"element_type": "type",
|
||||
}
|
||||
|
||||
|
||||
class FredReleaseTableFetcher(
|
||||
Fetcher[
|
||||
FredReleaseTableQueryParams,
|
||||
List[FredReleaseTableData],
|
||||
]
|
||||
):
|
||||
"""FRED Release Table Fetcher."""
|
||||
|
||||
@staticmethod
|
||||
def transform_query(params: Dict[str, Any]) -> FredReleaseTableQueryParams:
|
||||
"""Transform query."""
|
||||
return FredReleaseTableQueryParams(**params)
|
||||
|
||||
@staticmethod
|
||||
async def aextract_data(
|
||||
query: FredReleaseTableQueryParams,
|
||||
credentials: Optional[Dict[str, str]],
|
||||
**kwargs: Any,
|
||||
) -> List[Dict]:
|
||||
"""Extract data."""
|
||||
# pylint: disable=import-outside-toplevel
|
||||
import asyncio # noqa
|
||||
from openbb_core.provider.utils.helpers import amake_request
|
||||
from openbb_fred.models.search import FredSearchFetcher
|
||||
from numpy import nan
|
||||
from pandas import DataFrame, to_datetime
|
||||
|
||||
api_key = credentials.get("fred_api_key") if credentials else ""
|
||||
dates: List = [""]
|
||||
|
||||
# We'll verify that the release_id is valid and check the frequency for monthly/quarterly data.
|
||||
release_info = await FredSearchFetcher.fetch_data(
|
||||
{"release_id": query.release_id}, credentials
|
||||
)
|
||||
|
||||
if not release_info:
|
||||
raise OpenBBError(f"No release information found for, {query.release_id}.")
|
||||
|
||||
release_freq = list(set([d.model_dump().get("frequency_short") for d in release_info]))[0] # type: ignore # pylint: disable=R1718
|
||||
|
||||
if query.date is not None:
|
||||
if isinstance(query.date, dateType):
|
||||
query.date = query.date.strftime("%Y-%m-%d") # type: ignore
|
||||
dates = query.date.split(",") # type: ignore
|
||||
# Set the date to the first of the observation frequency for each date.
|
||||
# Daily observations should automatically fetch the closest date.
|
||||
if release_freq == "M":
|
||||
dates = [d.replace(d[-2:], "01") if len(d) == 10 else d for d in dates]
|
||||
elif release_freq == "Q":
|
||||
|
||||
def to_quarter_start(d):
|
||||
date_obj = datetime.strptime(d, "%Y-%m-%d")
|
||||
quarter = (date_obj.month - 1) // 3 + 1
|
||||
start_month = (quarter - 1) * 3 + 1
|
||||
return date_obj.replace(month=start_month, day=1).strftime(
|
||||
"%Y-%m-%d"
|
||||
)
|
||||
|
||||
dates = [to_quarter_start(d) for d in dates if len(d) == 10]
|
||||
|
||||
dates = list(set(dates))
|
||||
dates = (
|
||||
[f"&observation_date={date}" for date in dates if date] if dates else "" # type: ignore
|
||||
)
|
||||
|
||||
element_id = (
|
||||
f"&element_id={query.element_id}" if query.element_id is not None else ""
|
||||
)
|
||||
|
||||
URLS = [
|
||||
f"https://api.stlouisfed.org/fred/release/tables?release_id={query.release_id}"
|
||||
+ f"{element_id}{date}&include_observation_values=true&api_key={api_key}"
|
||||
+ "&file_type=json"
|
||||
for date in dates
|
||||
]
|
||||
results: List = []
|
||||
|
||||
async def get_one(URL):
|
||||
"""Get the observations for a single date."""
|
||||
response = await amake_request(URL)
|
||||
|
||||
# If the response has no elements we return empty and try the next URL.
|
||||
# If all URLs return empty, it will raise in `transform_data`.
|
||||
if "elements" not in response:
|
||||
return
|
||||
|
||||
res: List = []
|
||||
data: List = []
|
||||
# We use `res` to store the table and section elements
|
||||
# and to identify if observation values are returned.
|
||||
# We use `data` to store the observation values.
|
||||
# Only one scenario should unfold.
|
||||
for v in response.get("elements", {}).values(): # type: ignore
|
||||
if v and (v.get("type") == "section" or v.get("type") == "table"):
|
||||
v["element_id"] = str(v["element_id"])
|
||||
v["parent_id"] = str(v["parent_id"]) if v.get("parent_id") else None
|
||||
v.pop("children", None)
|
||||
v.pop("release_id", None)
|
||||
res.append(v)
|
||||
elif (
|
||||
"observation_value" in v
|
||||
and v.get("observation_value") != "."
|
||||
and v.get("type") != "header"
|
||||
):
|
||||
v["element_id"] = str(v["element_id"])
|
||||
data.append(v)
|
||||
# When observation values are returned, we parse and collect the parent elements while flattening the data.
|
||||
if data:
|
||||
index_cols = ["line", "element_id", "parent_id"]
|
||||
df = DataFrame(data).dropna(how="all", axis=1)
|
||||
for index_col in index_cols.copy():
|
||||
if index_col not in df.columns:
|
||||
index_cols.remove(index_col)
|
||||
df = (
|
||||
df.set_index(index_cols)
|
||||
.sort_index()[
|
||||
[
|
||||
"level",
|
||||
"series_id",
|
||||
"name",
|
||||
"observation_date",
|
||||
"observation_value",
|
||||
]
|
||||
]
|
||||
.reset_index()
|
||||
)
|
||||
df["parent_id"] = df.parent_id.astype(str)
|
||||
df["element_id"] = df.element_id.astype(str)
|
||||
df["observation_value"] = df.observation_value.str.replace(
|
||||
",", ""
|
||||
).astype(float)
|
||||
|
||||
if "line" in df.columns:
|
||||
df["line"] = df.line.astype(int)
|
||||
# Some dates are in the format 'Jan 2021' and others are '2021-01-01'.
|
||||
try:
|
||||
df["observation_date"] = to_datetime(
|
||||
df["observation_date"], format="%b %Y"
|
||||
).dt.date
|
||||
except ValueError:
|
||||
df["observation_date"] = to_datetime(
|
||||
df["observation_date"], format="%Y-%m-%d"
|
||||
).dt.date
|
||||
children = (
|
||||
df.groupby("parent_id")["element_id"]
|
||||
.apply(lambda x: x.sort_values().unique().tolist())
|
||||
.to_dict()
|
||||
)
|
||||
children = {k: ",".join(v) for k, v in children.items()}
|
||||
df["children"] = df.element_id.map(children)
|
||||
new_index_cols = [
|
||||
"line",
|
||||
"element_id",
|
||||
"children",
|
||||
"parent_id",
|
||||
"level",
|
||||
]
|
||||
for index_col in new_index_cols.copy():
|
||||
if index_col not in df.columns:
|
||||
new_index_cols.remove(index_col)
|
||||
df = (
|
||||
df.set_index(new_index_cols)
|
||||
.sort_index()
|
||||
.reset_index()
|
||||
.replace({nan: None})
|
||||
)
|
||||
results.extend(df.to_dict("records"))
|
||||
# If no observation values are returned, we collect the unique element IDs for the user.
|
||||
elif res:
|
||||
for item in res:
|
||||
if not any(r["element_id"] == item["element_id"] for r in results):
|
||||
results.append(item)
|
||||
|
||||
await asyncio.gather(*[get_one(URL) for URL in URLS])
|
||||
|
||||
return results
|
||||
|
||||
@staticmethod
|
||||
def transform_data(
|
||||
query: FredReleaseTableQueryParams,
|
||||
data: List[Dict],
|
||||
**kwargs: Any,
|
||||
) -> List[FredReleaseTableData]:
|
||||
"""Transform data."""
|
||||
if not data:
|
||||
raise EmptyDataError(
|
||||
f"No tables were found for the release, {query.release_id}."
|
||||
+ " Use `fred_search()` to list all release IDs."
|
||||
+ "\n\nReleases without tables will not return data,"
|
||||
+ " nor will this endpoint return individual line items."
|
||||
+ "\n\nTry a different 'element_id' and/or 'date'."
|
||||
+ "\n\nExclude 'date' for the most recent observations."
|
||||
+ "\n\nExclude 'element_id' to reveal the top-level element IDs."
|
||||
+ "\n\nUse `fred_series` for single series data."
|
||||
)
|
||||
|
||||
return [
|
||||
FredReleaseTableData.model_validate(d)
|
||||
for d in sorted(
|
||||
data,
|
||||
key=lambda x: (
|
||||
x.get("observation_date", float("inf")),
|
||||
x.get("line", float("inf")),
|
||||
),
|
||||
)
|
||||
]
|
||||
@@ -0,0 +1,110 @@
|
||||
interactions:
|
||||
- request:
|
||||
body: null
|
||||
headers:
|
||||
Accept:
|
||||
- application/json
|
||||
Accept-Encoding:
|
||||
- gzip, deflate
|
||||
Connection:
|
||||
- keep-alive
|
||||
method: GET
|
||||
uri: https://api.stlouisfed.org/fred/release/series?api_key=MOCK_API_KEY&file_type=json&offset=0&release_id=483
|
||||
response:
|
||||
body:
|
||||
string: !!binary |
|
||||
H4sIAAAAAAAEA6pWKkpNzCnJzE2NLy5JLCpRslIyMjAy0TUw1zW0VNJByKbmpaDL5RelpBbFJ1Uq
|
||||
WSkVpxZlphbHZ6Yo6SgV5xeVxIPllKyUEouTlXSUkvNL80qUrEx0lPLT0opTS5SsDHSUcjJzM0uU
|
||||
rAwNDAx0oAYUK1lFVytlgmwK9ncLMrQwcHGMdAxzR3YIyc4sySzJSVWyUjK0MNB1SaxUcCxLLUpM
|
||||
T1UA2aCko5SfVJxaVJZYkpmfhxQEhha6hga6BoZoCrCEQlpRamFpal4yKBhcEjNzKpV0lOBi8cUZ
|
||||
+eBAdVHSUSrNyywpVrJSCkgtSk7NK4GJwNWoggIvNbE4Py8xJz4xJau0uCQXpM5KyS+/RCEYKpNT
|
||||
qeAIlksFBzZUFEk93Dy/YEclHaWcxOKS+NKClMSSVFC4ImJXwcDcysDQysBc18BUSUepIL+gNCex
|
||||
KLOkUsnK1ExHKb0ov7QgHl00L78kFeQHx2KFxDyF1IqS1LzizPw8hfw0hZKMVIXElJRMUEgm5iik
|
||||
5CeXgpwPDlmFxKT80hKwkpCi1MTi0qJKhaDUgnyFoNS01KLUvORUhaDEktRiPQWNjJKSgmKrGP0Y
|
||||
/fLycr281PLK/KLstNQUvfyi9Bj93MSi7NSS4hj9EqgxukWpBfm6RTBjdItAxuhm5qXlF+WCrdZU
|
||||
qtVBSlPG1ExSxqSlKANTXQOjEZCiLDBTlDnWFAUSHfIpypKaKcqSxBRlPmJTlBnWFAUSHfIpytPP
|
||||
xTWCKnUeqI5T8MxLSa1AK3YQtaihha6BCY1SEdhmBceCIgUjHQUjA0MLBVsFUJUKrgnh9RQuVcWY
|
||||
dZsSjetCS8ySy8wIW10IEh2S6Qyw2FoAKuX8fPMJAAA=
|
||||
headers:
|
||||
Cache-Control:
|
||||
- max-age=0, no-cache
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Encoding:
|
||||
- gzip
|
||||
Content-Length:
|
||||
- '659'
|
||||
Content-Type:
|
||||
- application/json; charset=UTF-8
|
||||
Date:
|
||||
- Sat, 20 Jul 2024 03:15:42 GMT
|
||||
Expires:
|
||||
- Sat, 20 Jul 2024 03:15:42 GMT
|
||||
Last-Modified:
|
||||
- Sat, 20 Jul 2024 03:14:16 GMT
|
||||
Pragma:
|
||||
- no-cache
|
||||
Server:
|
||||
- Apache
|
||||
Set-Cookie:
|
||||
- ak_bmsc=62672FD53EF83E813D9FAA69FDAFA432~000000000000000000000000000000~YAAQhnfZF3k5VM2QAQAANV0jzhiPnbAHsGoxXM5FRiF4eKap/SLQy8pcnmNwh0BkLtW+VXoSwoBqiRh2cFCro9WgxIpxIj9V/1tp9cVaARJ7ZULqQvFeAel9vMqIG/T4PoOC0J5aQum8jHG/+tpim/NdVG5Zd2kkqg7DpEAPUU7s6TYf4Dwn4wAvcXX1Z0nxcaDens0D9Rp8UUHQ1bb3EXCHO8r0mvGMREUdov23boE/IVygLxspB9J52wJZICmQqP52XNPyqdyzOn1Z0vbmgrjdIFg6/NvhmdjBzs68Un59PcjprDJ4t8e8azqOlqgM8qhFU8tnetw0UpguSc/Iq4aDWGCdnSsqraJWpVIpsOwgygsPXqkekzNPMXhQzgU=;
|
||||
Domain=.stlouisfed.org; Path=/; Expires=Sat, 20 Jul 2024 05:15:42 GMT; Max-Age=7200
|
||||
Strict-Transport-Security:
|
||||
- max-age=86400
|
||||
Vary:
|
||||
- Accept-Encoding
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
- request:
|
||||
body: null
|
||||
headers:
|
||||
Accept:
|
||||
- application/json
|
||||
Accept-Encoding:
|
||||
- gzip, deflate
|
||||
Connection:
|
||||
- keep-alive
|
||||
method: GET
|
||||
uri: https://api.stlouisfed.org/fred/release/tables?api_key=MOCK_API_KEY&element_id=1217633&file_type=json&include_observation_values=true&observation_date=2024-07-14&release_id=483
|
||||
response:
|
||||
body:
|
||||
string: !!binary |
|
||||
H4sIAAAAAAAEA6pWykvMTVWyUgr2dwtScCxLLUpMTy1W0lFKzUnNTc0ric9MUbIyNDI0NzM21lEq
|
||||
Ss1JTSxOBQsqmVgYI9QVK1lVK0HUmYKYmNpNUbWbWBjrKBWnFmWmFkOMAznA2MDFMdIxzF1JR6kg
|
||||
sQjD+pzMvFQlq7zSnBwdpZLKApCzISYo6cC8YWyg65JYCfOIAshQJR2lnNSy1BwlKyVDJR2l/KTi
|
||||
1KKyxJLM/Lz4ssScUpAppnrGxhYGFmiyKYklIEkjAyMTXQNzXUMjJR2l5IzMnJSi1Dwlq+jYWh2o
|
||||
j82w+9iMCB9bUuxjS3J9bGpqDIpA5PAg2sfm2H1sToSPDS0o9rKhBbl+tjA3RU8DBP0MWG0tAAqS
|
||||
49clAwAA
|
||||
headers:
|
||||
Cache-Control:
|
||||
- max-age=0, no-cache
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Encoding:
|
||||
- gzip
|
||||
Content-Length:
|
||||
- '291'
|
||||
Content-Type:
|
||||
- application/json; charset=UTF-8
|
||||
Date:
|
||||
- Sat, 20 Jul 2024 03:15:42 GMT
|
||||
Expires:
|
||||
- Sat, 20 Jul 2024 03:15:42 GMT
|
||||
Last-Modified:
|
||||
- Sat, 20 Jul 2024 03:14:16 GMT
|
||||
Pragma:
|
||||
- no-cache
|
||||
Server:
|
||||
- Apache
|
||||
Set-Cookie:
|
||||
- ak_bmsc=C2489C1E25D28A4A78AD99E0B47F8C1F~000000000000000000000000000000~YAAQhnfZF6Q5VM2QAQAAkl4jzhgRSEZTmErv1vo+EACjd+/0q1++YenEVUn0nMXlsQ8XbloHZ1kiqd5l0r7p9georSB0m6V68/tW1irvAFJNH/Jw9LpnrhpeEOZl5Qjs+mt1vq6sddV15SdVZcfPvZySBm91tW1HBavoU/5wWrEh65Fxipa08lK5bJ+cheQs7EOPF68jsrGBZdqnhz5IKW02SLZMzHyOi9yYylYoDMu9uMqjV685L2tNd53cilM8yzIjQ761rBoBX3kNk3fyKUvD5KeT0fWUONlwwnjT21Sp/tRnvn22lL235OXLdv8IRiaJXTgK/V6hixYjrfWHLY/N2twCROMC0Njak/tMk8F0AcD7t2isu8RfwRfcHg3l;
|
||||
Domain=.stlouisfed.org; Path=/; Expires=Sat, 20 Jul 2024 05:15:42 GMT; Max-Age=7200
|
||||
Strict-Transport-Security:
|
||||
- max-age=86400
|
||||
Vary:
|
||||
- Accept-Encoding
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
version: 1
|
||||
@@ -0,0 +1,107 @@
|
||||
interactions:
|
||||
- request:
|
||||
body: null
|
||||
headers:
|
||||
Accept:
|
||||
- application/json
|
||||
Accept-Encoding:
|
||||
- gzip, deflate
|
||||
Connection:
|
||||
- keep-alive
|
||||
method: GET
|
||||
uri: https://api.stlouisfed.org/fred/release/series?api_key=MOCK_API_KEY&file_type=json&offset=0&release_id=483
|
||||
response:
|
||||
body:
|
||||
string: !!binary |
|
||||
H4sIAAAAAAAEA6pWKkpNzCnJzE2NLy5JLCpRslIyMjAy0TUw1zW0VNJByKbmpaDL5RelpBbFJ1Uq
|
||||
WSkVpxZlphbHZ6Yo6SgV5xeVxIPllKyUEouTlXSUkvNL80qUrEx0lPLT0opTS5SsDHSUcjJzM0uU
|
||||
rAwNDAx0oAYUK1lFVytlgmwK9ncLMrQwcHGMdAxzR3YIyc4sySzJSVWyUjK0MNB1SaxUcCxLLUpM
|
||||
T1UA2aCko5SfVJxaVJZYkpmfhxQEhha6hga6BoZoCrCEQlpRamFpal4yKBhcEjNzKpV0lOBi8cUZ
|
||||
+eBAdVHSUSrNyywpVrJSCkgtSk7NK4GJwNWoggIvNbE4Py8xJz4xJau0uCQXpM5KyS+/RCEYKpNT
|
||||
qeAIlksFBzZUFEk93Dy/YEclHaWcxOKS+NKClMSSVFC4ImJXwcDcysDQysBc18BUSUepIL+gNCex
|
||||
KLOkUsnK1ExHKb0ov7QgHl00L78kFeQHx2KFxDyF1IqS1LzizPw8hfw0hZKMVIXElJRMUEgm5iik
|
||||
5CeXgpwPDlmFxKT80hKwkpCi1MTi0qJKhaDUgnyFoNS01KLUvORUhaDEktRiPQWNjJKSgmKrGP0Y
|
||||
/fLycr281PLK/KLstNQUvfyi9Bj93MSi7NSS4hj9EqgxukWpBfm6RTBjdItAxuhm5qXlF+WCrdZU
|
||||
qtVBSlPG1ExSxqSlKANTXQOjEZCiLDBTlDnWFAUSHfIpypKaKcqSxBRlPmJTlBnWFAUSHfIpytPP
|
||||
xTWCKnUeqI5T8MxLSa1AK3YQtaihha6BCY1SEdhmBceCIgUjHQUjA0MLBVsFUJUKrgnh9RQuVcWY
|
||||
dZsSjetCS8ySy8wIW10IEh2S6Qyw2FoAKuX8fPMJAAA=
|
||||
headers:
|
||||
Cache-Control:
|
||||
- max-age=0, no-cache
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Encoding:
|
||||
- gzip
|
||||
Content-Length:
|
||||
- '659'
|
||||
Content-Type:
|
||||
- application/json; charset=UTF-8
|
||||
Date:
|
||||
- Sat, 20 Jul 2024 03:14:16 GMT
|
||||
Expires:
|
||||
- Sat, 20 Jul 2024 03:14:16 GMT
|
||||
Last-Modified:
|
||||
- Sat, 20 Jul 2024 03:14:16 GMT
|
||||
Pragma:
|
||||
- no-cache
|
||||
Server:
|
||||
- Apache
|
||||
Set-Cookie:
|
||||
- ak_bmsc=ECA8DBF6CBB000B1069E9D75FD977737~000000000000000000000000000000~YAAQhnfZFzz5U82QAQAAGA4izhicvj04D6saom/DwbFXWvSyg0d9i2waekXRdbVjVHR2RWf9NO4eHKf5m3CJ5qNX2q/W1jBJLCmWxWMBGx0VvUmrnnKMDEGHPJZYn0KDVp+vzGJV+ZHk4qlR3G7fUWbaIMXNjHlkTD1YcKmnk7bZ1R3pjuKAC7dTQ5oQFZ6ebKpYbJQpNGJQZnBn3DBgYiEHGlfV0hCbR+h+hJ0ichyNHj91Tfdebcwke6G44I6ptGe6H7W82kW/2sBdKYCMjBqVIgDGhUuMX8NBWOhjDDs2u+JMRNvzf2BwJmMK6B89o1etzpEQ5tnDVoptKaTvkJvcO5ZRLscszS18uFJXkOKxsfqGLMegSwSAZcTEP2Cf;
|
||||
Domain=.stlouisfed.org; Path=/; Expires=Sat, 20 Jul 2024 05:14:16 GMT; Max-Age=7200
|
||||
Strict-Transport-Security:
|
||||
- max-age=86400
|
||||
Vary:
|
||||
- Accept-Encoding
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
- request:
|
||||
body: null
|
||||
headers:
|
||||
Accept:
|
||||
- application/json
|
||||
Accept-Encoding:
|
||||
- gzip, deflate
|
||||
Connection:
|
||||
- keep-alive
|
||||
method: GET
|
||||
uri: https://api.stlouisfed.org/fred/release/tables?api_key=MOCK_API_KEY&element_id=1217633&file_type=json&include_observation_values=true&observation_date=2024-07-14&release_id=483
|
||||
response:
|
||||
body:
|
||||
string: !!binary |
|
||||
H4sIAAAAAAAEA6pWykvMTVWyUgr2dwtScCxLLUpMTy1W0lFKzUnNTc0ric9MUbIyNDI0NzM21lEq
|
||||
Ss1JTSxOBQsqmVgYI9QVK1lVK0HUmYKYmNpNUbWbWBjrKBWnFmWmFkOMAznA2MDFMdIxzF1JR6kg
|
||||
sQjD+pzMvFQlq7zSnBwdpZLKApCzISYo6cC8YWyg65JYCfOIAshQJR2lnNSy1BwlKyVDJR2l/KTi
|
||||
1KKyxJLM/Lz4ssScUpAppnrGxhYGFmiyKYklIEkjAyMTXQNzXUMjJR2l5IzMnJSi1Dwlq+jYWh2o
|
||||
j82w+9iMCB9bUuxjS3J9bGpqDIpA5PAg2sfm2H1sToSPDS0o9rKhBbl+tjA3RU8DBP0MWG0tAAqS
|
||||
49clAwAA
|
||||
headers:
|
||||
Cache-Control:
|
||||
- max-age=0, no-cache
|
||||
Connection:
|
||||
- keep-alive
|
||||
Content-Encoding:
|
||||
- gzip
|
||||
Content-Length:
|
||||
- '291'
|
||||
Content-Type:
|
||||
- application/json; charset=UTF-8
|
||||
Date:
|
||||
- Sat, 20 Jul 2024 03:14:17 GMT
|
||||
Expires:
|
||||
- Sat, 20 Jul 2024 03:14:17 GMT
|
||||
Last-Modified:
|
||||
- Sat, 20 Jul 2024 03:14:16 GMT
|
||||
Pragma:
|
||||
- no-cache
|
||||
Server:
|
||||
- Apache
|
||||
Strict-Transport-Security:
|
||||
- max-age=86400
|
||||
Vary:
|
||||
- Accept-Encoding
|
||||
status:
|
||||
code: 200
|
||||
message: OK
|
||||
version: 1
|
||||
@@ -35,6 +35,7 @@ from openbb_fred.models.personal_consumption_expenditures import (
|
||||
FredPersonalConsumptionExpendituresFetcher,
|
||||
)
|
||||
from openbb_fred.models.regional import FredRegionalDataFetcher
|
||||
from openbb_fred.models.release_table import FredReleaseTableFetcher
|
||||
from openbb_fred.models.retail_prices import FredRetailPricesFetcher
|
||||
from openbb_fred.models.search import (
|
||||
FredSearchFetcher,
|
||||
@@ -503,3 +504,17 @@ def test_fred_personal_consumption_expenditures_fetcher(credentials=test_credent
|
||||
fetcher = FredPersonalConsumptionExpendituresFetcher()
|
||||
result = fetcher.test(params, credentials)
|
||||
assert result is None
|
||||
|
||||
|
||||
@pytest.mark.record_http
|
||||
def test_fred_release_table_fetcher(credentials=test_credentials):
|
||||
"""Test FredReleaseTableFetcher."""
|
||||
params = {
|
||||
"date": "2024-07-14",
|
||||
"release_id": "483",
|
||||
"element_id": "1217633",
|
||||
}
|
||||
|
||||
fetcher = FredReleaseTableFetcher()
|
||||
result = fetcher.test(params, credentials)
|
||||
assert result is None
|
||||
|
||||
Reference in New Issue
Block a user