fix warnings raised by federal_reserve on some xlsx workbook parsing

This commit is contained in:
Danglewood
2026-09-25 14:32:30 -07:00
parent 5c31fd48bb
commit 32b6d4a280
6 changed files with 74 additions and 20 deletions

View File

@@ -125,7 +125,9 @@ class FederalReserveAtlantaTaylorRuleFetcher(
"""Overlay each rule's prescription with the actual rate, keyed by date."""
from io import BytesIO
from pandas import isna, read_excel, to_datetime
from pandas import isna, read_excel
from openbb_federal_reserve.utils.workbook import cell_dates
frames = read_excel(
BytesIO(data[0]["_raw"]),
@@ -145,12 +147,11 @@ class FederalReserveAtlantaTaylorRuleFetcher(
(c for c in frame.columns if str(c).strip() == "Actual Fed Funds Rate"),
None,
)
dates = to_datetime(frame[frame.columns[0]], errors="coerce")
dates = cell_dates(frame[frame.columns[0]])
for index in range(len(frame)):
observation = dates.iloc[index]
if isna(observation):
observation = dates[index]
if observation is None:
continue
observation = observation.date()
record = merged.setdefault(observation, {"date": observation})
cell = frame[prescribed].iloc[index] if prescribed else None
record[field] = None if cell is None or isna(cell) else float(cell)

View File

@@ -27,7 +27,9 @@ def parse_dgei(
"""Melt a DGEI sheet's side-by-side measure blocks to long records."""
from io import BytesIO
from pandas import isna, notna, read_excel, to_datetime, to_numeric
from pandas import isna, notna, read_excel, to_numeric
from openbb_federal_reserve.utils.workbook import cell_dates
raw = read_excel(BytesIO(content), sheet_name=sheet, header=None)
label_row = next(
@@ -60,8 +62,7 @@ def parse_dgei(
if notna(candidate) and str(candidate).strip():
measure = re.split(r"[,=]", str(candidate).strip())[0].strip()
break
stamps = to_datetime(body.iloc[:, start], errors="coerce")
dates = [None if isna(stamp) else stamp.date() for stamp in stamps]
dates = cell_dates(body.iloc[:, start])
for index in range(start + 1, stop):
label = labels.iloc[index]
if isna(label):
@@ -143,6 +144,8 @@ def melt_two_level(
from pandas import isna, notna, read_excel, to_datetime, to_numeric
from openbb_federal_reserve.utils.workbook import cell_dates
raw = read_excel(BytesIO(content), sheet_name=sheet, header=None)
if raw.shape[0] <= label_row + 1:
return []
@@ -176,8 +179,7 @@ def melt_two_level(
)
dates = [None if isna(stamp) else stamp.date() for stamp in stamps]
else:
stamps = to_datetime(first, errors="coerce")
dates = [None if isna(stamp) else stamp.date() for stamp in stamps]
dates = cell_dates(first)
values = {
index: to_numeric(body.iloc[:, index], errors="coerce")

View File

@@ -146,7 +146,7 @@ def fetch_guide_concepts(
"""
from concurrent.futures import ThreadPoolExecutor
resolved = lines or ubpr_concept_lines()
resolved = ubpr_concept_lines() if lines is None else lines
report_date = _report_date()
pairs = sorted({(c, resolved[c]) for c in concepts if c in resolved})
if not pairs or not report_date:

View File

@@ -75,6 +75,31 @@ def _first_data_row(column: list[Any]) -> int | None:
return None
def cell_dates(values: Any) -> list[dateType | None]:
"""Return each Excel date cell as a date, and None for every other cell.
Parameters
----------
values : Any
The column's cells, as read from the workbook.
Returns
-------
list[date | None]
One entry per cell: the date for a date or datetime cell, else None.
"""
from pandas import notna
return [
value.date()
if isinstance(value, datetime) and notna(value)
else value
if isinstance(value, dateType) and not isinstance(value, datetime)
else None
for value in values
]
def _text(value: Any) -> str:
"""Return a trimmed, newline-collapsed string for a header cell, else empty."""
from pandas import isna

View File

@@ -214,14 +214,9 @@ class TestExecutiveSummaryCassette:
"""Replay the captured ESR transports through the full real fetcher."""
@pytest.fixture(scope="class")
def parsed(self):
"""Run transform_query -> extract_data -> transform_data over the cassette.
Serves the router responses, parsed guide concepts and filtered concept
index entirely from the recording, so the parse runs with no live network
and no pre-warmed cache. The Call Report filer set is served as the bank's
own RSSD so the lead-bank resolution is a no-op and never hits the network.
"""
@classmethod
def parsed(cls):
"""Run the full fetcher over the cassette with no live network."""
with pytest.MonkeyPatch.context() as patch, replay(_ESR_CASSETTE):
patch.setattr(
"openbb_federal_reserve.utils.ticker.call_report_filers",

View File

@@ -6,9 +6,10 @@ from zipfile import ZIP_DEFLATED, ZipFile
import pytest
from openpyxl import Workbook, load_workbook
from pandas import DataFrame
from pandas import DataFrame, NaT, Series, Timestamp
from openbb_federal_reserve.utils.workbook import (
cell_dates,
melt_sheet,
pivot_wide,
round_value,
@@ -238,3 +239,33 @@ class TestMeltSheet:
end_date=date(2024, 12, 31),
)
assert [r["date"] for r in records] == [date(2024, 1, 31)]
class TestCellDates:
"""Tests for ``cell_dates``."""
def test_keeps_only_date_cells(self):
"""Datetime and date cells become dates; text, numbers and NaT become None."""
cells = Series(
[
datetime(2026, 1, 1, 12, 30),
Timestamp("2026-02-01"),
date(2026, 3, 1),
NaT,
None,
"not a date",
"3",
202604,
],
dtype=object,
)
assert cell_dates(cells) == [
date(2026, 1, 1),
date(2026, 2, 1),
date(2026, 3, 1),
None,
None,
None,
None,
None,
]