From b0c092ece5eefd22983fe3380c0fd3fbc7ddd4fa Mon Sep 17 00:00:00 2001 From: Dex Hunter Date: Fri, 25 Sep 2026 03:12:59 +0100 Subject: [PATCH] perf(core): vectorize the date conversion in `basemodel_to_df` (#7622) * perf(core): vectorize the date conversion in basemodel_to_df basemodel_to_df ran three per-element Python passes over the date column of every payload it converted. Series.apply(to_datetime) calls pandas.to_datetime once per row, and because each call receives a lone scalar with no format hint, pandas also re-runs its datetime format guesser on every row. Parse the column in one call instead, and use vectorized comparisons for the all-midnight test and the reduction to datetime.date. The element-wise parse is kept as a fallback for one case that the vectorized path cannot represent: a column mixing UTC offsets, such as a series crossing a daylight-savings boundary. Parsing per element lets each value keep its own offset and yields an object column of tz-aware Timestamps, whereas a single to_datetime call over that column raises "Mixed timezones detected". That behaviour is covered by test_to_df_daylight_savings and is preserved exactly. Measured on a payload built through EquityHistoricalData, median of 9 under an exclusive CPU lock: 250 daily bars 8.92 ms -> 1.46 ms 1,000 daily bars 32.93 ms -> 3.13 ms 5,000 daily bars 162.25 ms -> 12.23 ms * fix a broken test - unrelated to this PR... --------- Co-authored-by: Danglewood <85772166+deeleeramone@users.noreply.github.com> Co-authored-by: deeleeramone <> Co-authored-by: Claude Opus 5.5 --- .../core/openbb_core/app/utils/__init__.py | 19 ++++++-- openbb_platform/core/tests/app/test_utils.py | 48 +++++++++++++++++++ 2 files changed, 64 insertions(+), 3 deletions(-) diff --git a/openbb_platform/core/openbb_core/app/utils/__init__.py b/openbb_platform/core/openbb_core/app/utils/__init__.py index 819ede0b940..62486a8b8c2 100644 --- a/openbb_platform/core/openbb_core/app/utils/__init__.py +++ b/openbb_platform/core/openbb_core/app/utils/__init__.py @@ -46,9 +46,22 @@ def basemodel_to_df( # If the date column contains dates only, convert them to a date to avoid encoding time data. if "date" in df.columns: - df["date"] = df["date"].apply(to_datetime) - if all(t.time() == time(0, 0) for t in df["date"]): - df["date"] = df["date"].apply(lambda x: x.date()) + try: + # Parse the column in a single call instead of once per row. + parsed = to_datetime(df["date"]) + except ValueError: + # A column that mixes UTC offsets - a series crossing a daylight + # savings boundary, for instance - has no single datetime64 + # representation, so fall back to parsing each value on its own. + parsed = df["date"].apply(to_datetime) + + df["date"] = parsed + + if parsed.dtype == object: + if all(t.time() == time(0, 0) for t in parsed): + df["date"] = parsed.apply(lambda x: x.date()) + elif (parsed == parsed.dt.normalize()).all(): + df["date"] = parsed.dt.date if index and index in df.columns: if index == "date": diff --git a/openbb_platform/core/tests/app/test_utils.py b/openbb_platform/core/tests/app/test_utils.py index dda31cd9a32..9607e7c75b2 100644 --- a/openbb_platform/core/tests/app/test_utils.py +++ b/openbb_platform/core/tests/app/test_utils.py @@ -196,6 +196,54 @@ def test_basemodel_to_df_with_date_index(): assert df.index.name == "date" +def test_basemodel_to_df_mixed_offsets_all_midnight_reduces_to_date(): + """A mixed-offset column that is entirely midnight still reduces to dates. + + The payload crosses a daylight-savings boundary, so it has no single + datetime64 representation and stays object dtype. Every value is midnight + local time, so it must still be reduced to ``datetime.date``. + """ + from datetime import date + + data = [ + Data(date="2023-11-03 00:00:00-04:00", value=1), + Data(date="2023-11-06 00:00:00-05:00", value=2), + Data(date="2023-11-08 00:00:00-05:00", value=3), + ] + result = basemodel_to_df(data) + assert list(result["date"]) == [ + date(2023, 11, 3), + date(2023, 11, 6), + date(2023, 11, 8), + ] + + +def test_basemodel_to_df_falls_back_to_per_element_parse(monkeypatch): + """A column the vectorized parse rejects is parsed one value at a time. + + Whether a mixed-offset column raises depends on the pandas version - 3.x + raises, 2.x returns object dtype - so the fallback is forced here rather + than left to the installed version. + """ + from datetime import date + + real_to_datetime = pd.to_datetime + + def fake_to_datetime(arg, *args, **kwargs): + if isinstance(arg, pd.Series): + raise ValueError("Mixed timezones detected. Pass utc=True") + return real_to_datetime(arg, *args, **kwargs) + + monkeypatch.setattr(pd, "to_datetime", fake_to_datetime) + + data = [ + Data(date="2023-11-03 00:00:00-04:00", value=1), + Data(date="2023-11-06 00:00:00-05:00", value=2), + ] + result = basemodel_to_df(data) + assert list(result["date"]) == [date(2023, 11, 3), date(2023, 11, 6)] + + def test_df_to_basemodel_with_date_only_column_keeps_date_format(): df = pd.DataFrame( {