mirror of
https://github.com/OpenBB-finance/OpenBB.git
synced 2026-11-11 20:01:16 +00:00
fix warnings raised by federal_reserve on some xlsx workbook parsing
This commit is contained in:
@@ -125,7 +125,9 @@ class FederalReserveAtlantaTaylorRuleFetcher(
|
||||
"""Overlay each rule's prescription with the actual rate, keyed by date."""
|
||||
from io import BytesIO
|
||||
|
||||
from pandas import isna, read_excel, to_datetime
|
||||
from pandas import isna, read_excel
|
||||
|
||||
from openbb_federal_reserve.utils.workbook import cell_dates
|
||||
|
||||
frames = read_excel(
|
||||
BytesIO(data[0]["_raw"]),
|
||||
@@ -145,12 +147,11 @@ class FederalReserveAtlantaTaylorRuleFetcher(
|
||||
(c for c in frame.columns if str(c).strip() == "Actual Fed Funds Rate"),
|
||||
None,
|
||||
)
|
||||
dates = to_datetime(frame[frame.columns[0]], errors="coerce")
|
||||
dates = cell_dates(frame[frame.columns[0]])
|
||||
for index in range(len(frame)):
|
||||
observation = dates.iloc[index]
|
||||
if isna(observation):
|
||||
observation = dates[index]
|
||||
if observation is None:
|
||||
continue
|
||||
observation = observation.date()
|
||||
record = merged.setdefault(observation, {"date": observation})
|
||||
cell = frame[prescribed].iloc[index] if prescribed else None
|
||||
record[field] = None if cell is None or isna(cell) else float(cell)
|
||||
|
||||
@@ -27,7 +27,9 @@ def parse_dgei(
|
||||
"""Melt a DGEI sheet's side-by-side measure blocks to long records."""
|
||||
from io import BytesIO
|
||||
|
||||
from pandas import isna, notna, read_excel, to_datetime, to_numeric
|
||||
from pandas import isna, notna, read_excel, to_numeric
|
||||
|
||||
from openbb_federal_reserve.utils.workbook import cell_dates
|
||||
|
||||
raw = read_excel(BytesIO(content), sheet_name=sheet, header=None)
|
||||
label_row = next(
|
||||
@@ -60,8 +62,7 @@ def parse_dgei(
|
||||
if notna(candidate) and str(candidate).strip():
|
||||
measure = re.split(r"[,=]", str(candidate).strip())[0].strip()
|
||||
break
|
||||
stamps = to_datetime(body.iloc[:, start], errors="coerce")
|
||||
dates = [None if isna(stamp) else stamp.date() for stamp in stamps]
|
||||
dates = cell_dates(body.iloc[:, start])
|
||||
for index in range(start + 1, stop):
|
||||
label = labels.iloc[index]
|
||||
if isna(label):
|
||||
@@ -143,6 +144,8 @@ def melt_two_level(
|
||||
|
||||
from pandas import isna, notna, read_excel, to_datetime, to_numeric
|
||||
|
||||
from openbb_federal_reserve.utils.workbook import cell_dates
|
||||
|
||||
raw = read_excel(BytesIO(content), sheet_name=sheet, header=None)
|
||||
if raw.shape[0] <= label_row + 1:
|
||||
return []
|
||||
@@ -176,8 +179,7 @@ def melt_two_level(
|
||||
)
|
||||
dates = [None if isna(stamp) else stamp.date() for stamp in stamps]
|
||||
else:
|
||||
stamps = to_datetime(first, errors="coerce")
|
||||
dates = [None if isna(stamp) else stamp.date() for stamp in stamps]
|
||||
dates = cell_dates(first)
|
||||
|
||||
values = {
|
||||
index: to_numeric(body.iloc[:, index], errors="coerce")
|
||||
|
||||
@@ -146,7 +146,7 @@ def fetch_guide_concepts(
|
||||
"""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
resolved = lines or ubpr_concept_lines()
|
||||
resolved = ubpr_concept_lines() if lines is None else lines
|
||||
report_date = _report_date()
|
||||
pairs = sorted({(c, resolved[c]) for c in concepts if c in resolved})
|
||||
if not pairs or not report_date:
|
||||
|
||||
@@ -75,6 +75,31 @@ def _first_data_row(column: list[Any]) -> int | None:
|
||||
return None
|
||||
|
||||
|
||||
def cell_dates(values: Any) -> list[dateType | None]:
|
||||
"""Return each Excel date cell as a date, and None for every other cell.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
values : Any
|
||||
The column's cells, as read from the workbook.
|
||||
|
||||
Returns
|
||||
-------
|
||||
list[date | None]
|
||||
One entry per cell: the date for a date or datetime cell, else None.
|
||||
"""
|
||||
from pandas import notna
|
||||
|
||||
return [
|
||||
value.date()
|
||||
if isinstance(value, datetime) and notna(value)
|
||||
else value
|
||||
if isinstance(value, dateType) and not isinstance(value, datetime)
|
||||
else None
|
||||
for value in values
|
||||
]
|
||||
|
||||
|
||||
def _text(value: Any) -> str:
|
||||
"""Return a trimmed, newline-collapsed string for a header cell, else empty."""
|
||||
from pandas import isna
|
||||
|
||||
@@ -214,14 +214,9 @@ class TestExecutiveSummaryCassette:
|
||||
"""Replay the captured ESR transports through the full real fetcher."""
|
||||
|
||||
@pytest.fixture(scope="class")
|
||||
def parsed(self):
|
||||
"""Run transform_query -> extract_data -> transform_data over the cassette.
|
||||
|
||||
Serves the router responses, parsed guide concepts and filtered concept
|
||||
index entirely from the recording, so the parse runs with no live network
|
||||
and no pre-warmed cache. The Call Report filer set is served as the bank's
|
||||
own RSSD so the lead-bank resolution is a no-op and never hits the network.
|
||||
"""
|
||||
@classmethod
|
||||
def parsed(cls):
|
||||
"""Run the full fetcher over the cassette with no live network."""
|
||||
with pytest.MonkeyPatch.context() as patch, replay(_ESR_CASSETTE):
|
||||
patch.setattr(
|
||||
"openbb_federal_reserve.utils.ticker.call_report_filers",
|
||||
|
||||
@@ -6,9 +6,10 @@ from zipfile import ZIP_DEFLATED, ZipFile
|
||||
|
||||
import pytest
|
||||
from openpyxl import Workbook, load_workbook
|
||||
from pandas import DataFrame
|
||||
from pandas import DataFrame, NaT, Series, Timestamp
|
||||
|
||||
from openbb_federal_reserve.utils.workbook import (
|
||||
cell_dates,
|
||||
melt_sheet,
|
||||
pivot_wide,
|
||||
round_value,
|
||||
@@ -238,3 +239,33 @@ class TestMeltSheet:
|
||||
end_date=date(2024, 12, 31),
|
||||
)
|
||||
assert [r["date"] for r in records] == [date(2024, 1, 31)]
|
||||
|
||||
|
||||
class TestCellDates:
|
||||
"""Tests for ``cell_dates``."""
|
||||
|
||||
def test_keeps_only_date_cells(self):
|
||||
"""Datetime and date cells become dates; text, numbers and NaT become None."""
|
||||
cells = Series(
|
||||
[
|
||||
datetime(2026, 1, 1, 12, 30),
|
||||
Timestamp("2026-02-01"),
|
||||
date(2026, 3, 1),
|
||||
NaT,
|
||||
None,
|
||||
"not a date",
|
||||
"3",
|
||||
202604,
|
||||
],
|
||||
dtype=object,
|
||||
)
|
||||
assert cell_dates(cells) == [
|
||||
date(2026, 1, 1),
|
||||
date(2026, 2, 1),
|
||||
date(2026, 3, 1),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user