mirror of
https://github.com/OpenBB-finance/OpenBB.git
synced 2026-11-09 10:51:57 +00:00
merge branch develop
This commit is contained in:
Danglewood
@@ -8,7 +8,7 @@ readme = "README.md"
|
||||
requires-python = ">=3.10,<4"
|
||||
dependencies = [
|
||||
"ruff>=0.15,<0.16",
|
||||
"ty>=0.0.1a14",
|
||||
"ty>=0.0.40",
|
||||
"pydocstyle>=6.3.0,<7.0.0",
|
||||
"codespell>=2.2.5,<3.0.0",
|
||||
"pre-commit>=3.5.0,<4.0.0",
|
||||
@@ -17,7 +17,7 @@ dependencies = [
|
||||
"pytest-subtests>=0.11.0,<1.0.0",
|
||||
"pytest-recorder>=0.6.5",
|
||||
# vcrpy (via pytest-recorder) references aiohttp.streams.AsyncStreamReaderMixin,
|
||||
# removed in aiohttp 3.14.0; cap until vcrpy ships 3.14 support.
|
||||
# removed in aiohttp 3.14.0; cap until vcrpy ships 3.14 support. https://github.com/kevin1024/vcrpy/pull/996
|
||||
"aiohttp<3.14",
|
||||
"pytest-asyncio>=1.3.0,<2.0.0",
|
||||
"pytest-order>=1.3.0,<2.0.0",
|
||||
|
||||
39
openbb_platform/extensions/devtools/uv.lock
generated
vendored
39
openbb_platform/extensions/devtools/uv.lock
generated
vendored
@@ -1326,7 +1326,7 @@ requires-dist = [
|
||||
{ name = "pytest-subtests", specifier = ">=0.11.0,<1.0.0" },
|
||||
{ name = "ruff", specifier = ">=0.15,<0.16" },
|
||||
{ name = "tox", specifier = ">=4.11.3,<5.0.0" },
|
||||
{ name = "ty", specifier = ">=0.0.1a14" },
|
||||
{ name = "ty", specifier = ">=0.0.40" },
|
||||
{ name = "types-python-dateutil", specifier = ">=2.8.19.14,<3.0.0" },
|
||||
{ name = "types-requests", specifier = ">=2.32,<3.0.0" },
|
||||
{ name = "types-toml", specifier = ">=0.10.8.7,<1.0.0" },
|
||||
@@ -2237,26 +2237,27 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "ty"
|
||||
version = "0.0.33"
|
||||
version = "0.0.42"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/84/44/9478c50c266826c1bf30d1692e589755bffa8f1c0a3eb7af8a346c255991/ty-0.0.33.tar.gz", hash = "sha256:46d63bda07403322cb6c28ccfdd5536be916e13df725c29f7ccd0a21f06bd9e8", size = 5559373, upload-time = "2026-04-28T10:45:13.18Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/d8/91/5b5ec4ed8721c18be8d9611778d7c07723cd755676f03b41bf0ea0caa5d3/ty-0.0.42.tar.gz", hash = "sha256:70f5553ac678fc63558d4d77b08a18a68a228f44be2a2fe1afc3f5988db662e7", size = 5769116, upload-time = "2026-06-01T19:40:32.869Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/e9/24/e287388c63a19191be26b32ff4dbd06029834068150ebe2532939bc4c851/ty-0.0.33-py3-none-linux_armv6l.whl", hash = "sha256:94d0a9d2234261a8911396d59e506b5923fe0971dbda43b9dcea287936887fcc", size = 11021308, upload-time = "2026-04-28T10:45:43.34Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/00/ca/ba1eed819895bd239fba8ee35dfcd5fcb266c203b0914a17a59579096bb5/ty-0.0.33-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:e4a2b5ba078f90de342f56b5f7979bb77c9b9b1d8625a041352ffc6ee93c4073", size = 10777272, upload-time = "2026-04-28T10:45:32.905Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/25/a8/c3131d37b44b3fea1d6654a1c929a0cd0873822f77a90482b8ec28f6fbbd/ty-0.0.33-py3-none-macosx_11_0_arm64.whl", hash = "sha256:84ff5707825e9af9668d2bcf66975f93e520a63b524ab494e3a8265735be2563", size = 10201078, upload-time = "2026-04-28T10:45:23.374Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7b/db/d8e37ff0045810cc65e1ff36aa0da0a2253c05659787ac987df8a16c7897/ty-0.0.33-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e375285736f57886868e7af0b11c7b0ec5b6543fa15e7ad2a714fed9f077d4e0", size = 10732347, upload-time = "2026-04-28T10:45:21.444Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e0/1a/20e83a412506a918e4684fc67b567cf7cc13b105470b3428cb23c3d5aa13/ty-0.0.33-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5680f6350c3b4e46b8bff6d7bb132366ea239463d6cad4892725d06046e65464", size = 10808238, upload-time = "2026-04-28T10:45:38.565Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5d/4b/d0a39f4464dc6cb4cc2c159473ce216bd1846bfb684c0323a3cb36dce5c6/ty-0.0.33-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c5535538bad8d0f7e62bcdff02197cdb30e41451d80b35d27e17d128f2e1dc5d", size = 11288348, upload-time = "2026-04-28T10:45:08.419Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/35/7e/f1745e0f9583363d7a83d9a4990fc244f76ecc30840ddad83dc16a33c52d/ty-0.0.33-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:da196c42bbbc069e1e21e3e52107c061aa9660352dae57a41930690b56e2c02d", size = 11789907, upload-time = "2026-04-28T10:45:19.064Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a5/71/25f39f46a12d662859d45bc648555d0661044eb43db6b5648c9947487da9/ty-0.0.33-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9281672921ef6d4460e03146b5e6c18cb1a3e3a3b8a1a88f6f33226d05a469b7", size = 11500774, upload-time = "2026-04-28T10:45:48.012Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/94/ec/136959ecbb7c71cb90537f5aea441c73f4ab24612868a6ecdc9d7444d32d/ty-0.0.33-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82c1b8f303f82da64e878108e764be3ecbcd7c9903ac0a7f7031614ed00b97ab", size = 11360314, upload-time = "2026-04-28T10:45:05.402Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/cf/95/32809575c222f00beed498cb728e9290a0f5009f930025381bb7253b2206/ty-0.0.33-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:efe3af412c9ff67bce5fa37d0a2b0d8555c24072b145a5bac6c79637f1c83abe", size = 10707785, upload-time = "2026-04-28T10:45:10.836Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/13/89/c8e9531f7aa4a093359e15fa32c8e1277fbbe90d16894d7c6032d29f4b34/ty-0.0.33-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:aeec29c91ea768601747da546c3efc20b72c2fb1bd52bcc786a5c6eeff51d27b", size = 10834987, upload-time = "2026-04-28T10:45:40.738Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/31/16/9835fbcf5338af1a1917bd28fdb8a7193c210b83f243aa286fa9f79cb3ad/ty-0.0.33-py3-none-musllinux_1_2_i686.whl", hash = "sha256:a535977c52bbb5f7e96b8b70a6ad375ad077f4a9ff2492508ea3816a2b403819", size = 10968968, upload-time = "2026-04-28T10:45:30.26Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/36/69/64c76aabc1bc70c7f24b686cd93c3407f8ea430905e395f59bf9603ef571/ty-0.0.33-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:1d732facf39fcb221ba279d469c5040d37883e964f123b1563888efd34818180", size = 11458077, upload-time = "2026-04-28T10:45:45.971Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/91/84/fae27b0c4718776a298690d31ca4cc1995f2e3e1c63a7b59e84c41498e9a/ty-0.0.33-py3-none-win32.whl", hash = "sha256:d90960b574428dc252f85e8598ec5fcb7f619794196b2fc95a90da075ed4681c", size = 10345364, upload-time = "2026-04-28T10:45:16.836Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3c/a0/a2938b23ae3e1a09a2d7c189e2ac5f7113676bae4e0e23948b568e18e5f8/ty-0.0.33-py3-none-win_amd64.whl", hash = "sha256:c1c3aec62c44de610c6e95f0a4e97ac3dbc07934bfdbf1fd90d758c9ff72f48e", size = 11342470, upload-time = "2026-04-28T10:45:26.455Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ab/62/7fb948aace38d2f6329261bb33c035a8484549c74f1db28649c7a4c6fed9/ty-0.0.33-py3-none-win_arm64.whl", hash = "sha256:0d44f99ba1b441e55e2aa301b2ac0a21112784931b46a5f66f4ea9efe5620d97", size = 10742673, upload-time = "2026-04-28T10:45:35.555Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d4/7c/2df5136ad7c0db69a3973b0b19da8f52bfacdc453c7dffc832d1bf7d23ff/ty-0.0.42-py3-none-linux_armv6l.whl", hash = "sha256:c08a0066610c13627b7d7ad758adb96ca99685791e641eb26837e20803851c53", size = 11544141, upload-time = "2026-06-01T19:40:41.065Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0a/82/96cf406d39d8976e825361a27e332224445812793060ac9506d8a5d32b39/ty-0.0.42-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:3e944ee4e3d5cdaf70e4ea87f9dd474cc3db612837b50a3ce57afa8da400ecc2", size = 11283538, upload-time = "2026-06-01T19:40:26.758Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/dc/fe/813b60b9332df835c16c05859ff5aac1896593d01b638ba0e461ede415ac/ty-0.0.42-py3-none-macosx_11_0_arm64.whl", hash = "sha256:603085306e4aac2ce592b39119a4b49ebf8b780cd394e2cfc7dbf3fd8228f954", size = 10711874, upload-time = "2026-06-01T19:40:28.77Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/07/64/1f609265be0302ce0f51aa03a27636d018947a76100ff1405258d8445e6c/ty-0.0.42-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a58f17834d7f078c49326a01111a5aac16c979a774b98cdfd8e2350068316676", size = 11213021, upload-time = "2026-06-01T19:40:45.01Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/88/c0/f147b2fde7cd01b5f77682937e85a5cdfe35f48b3f1d0f41021024cbd927/ty-0.0.42-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e6ed1027f313202c5c74e376007d1eb5d214494299beb0ea047078b8ad307d40", size = 11321604, upload-time = "2026-06-01T19:40:55.164Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a9/72/74a5e68a9bd194681f15c4aac7a0dfab378e76e7a107e8ecd93971a22377/ty-0.0.42-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:063838d2360c1d2c065b45ca76a56ecd6df07fff6570813e74183236559e16d9", size = 11802178, upload-time = "2026-06-01T19:40:19.558Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3c/9f/06a31dc9cc91faa2cb8a4bf2f0ba5f7a9a96a4828fb8434338682954ca86/ty-0.0.42-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c3f9ed508dfae4cbc943d7766324dd9c57ac8302c8543505fc29cae8ed425fe9", size = 12358436, upload-time = "2026-06-01T19:40:48.968Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/06/33/a5bb1afcb671e0b9197f007264682b22f1063bf0e83c151eeaf9958f9047/ty-0.0.42-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0fe1eb7d98472ac56ac19fec51c6ed8fe56d86ea0d232a11a127e8c62c882a66", size = 11997849, upload-time = "2026-06-01T19:40:42.951Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ee/8d/560e4ec4c2f69e68fa094bb93cd19b5eb92c9732d2e0f5e7cc3accde84c4/ty-0.0.42-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de75f9e78bfa81209f2f297528758977cfd4518ba35ef45a0acb516c892a27a5", size = 11869087, upload-time = "2026-06-01T19:40:24.38Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b2/dd/3794db15199c03eda60961690046a56c4fbf5d8ef073c82fe2402c851b8c/ty-0.0.42-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:c63281f2f1d4df339117fcd4a6dfe17cb999f84eafe707b30e9ebbe26f0bb54a", size = 12059000, upload-time = "2026-06-01T19:40:34.982Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/98/9a/02b61cc65ecbd90f18bd02178845c5b756c78fb92643571e77df52c6eed8/ty-0.0.42-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:19e3856477f25255f772851fa7f16f5356c4e1927324d074d49c7bc9a9b211e1", size = 11195698, upload-time = "2026-06-01T19:40:37.109Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7b/d6/3927335c956b06a806269dcd2e5b46bc4284b0a95ecc6b0246094a1de28c/ty-0.0.42-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:2f0f4acac9028264cee5ea0b88229df0b9b2586fc917dbadb6ee35a0e99e8b06", size = 11353487, upload-time = "2026-06-01T19:40:47.035Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d6/87/79e7ae4f5f9fb3bea5f3cfac2b4f8c60e905f13962e3c0d97f8b51a5bff6/ty-0.0.42-py3-none-musllinux_1_2_i686.whl", hash = "sha256:ae244c84e30fdf2bb1a3cbf2b973da8aa535e57c701f12db44b2939604586c04", size = 11463474, upload-time = "2026-06-01T19:40:30.812Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/49/f9/73305ead1b3ccc3d79c04258877db2cd7908c6a6c2060d4e598deca384d2/ty-0.0.42-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:2430434f4a52bec0da552ff6a061dcc1c5d11973259248679a1146d776c12f37", size = 11961710, upload-time = "2026-06-01T19:40:39.056Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e1/04/52b7325dad8d1a86653f90240208f3e3657bed5e3c8144eed367a0f736b0/ty-0.0.42-py3-none-win32.whl", hash = "sha256:984a55c2fe63b40dac03f5a144b99033c7ed720eb7611787e3f0bd49af8dcf12", size = 10783897, upload-time = "2026-06-01T19:40:22.189Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/94/d2/3d2d61255c76c0843766f00b39290115c23cc1cd4fcb0471d84a48f482f1/ty-0.0.42-py3-none-win_amd64.whl", hash = "sha256:f7afd81b10b377d9d4ce6aad355a4f47fd37d47f443118c01ca6e79d46fe6608", size = 11878640, upload-time = "2026-06-01T19:40:50.903Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b4/91/1eb0c1e3d558707ead7424f8bfd89b58f42e576714cbed7ad46dcceef34a/ty-0.0.42-py3-none-win_arm64.whl", hash = "sha256:4068c24b0b264fc9f1901e06b97988a041fcaa36c90f18d7747f05124701c7b3", size = 11202335, upload-time = "2026-06-01T19:40:53.155Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -56,7 +56,7 @@ class FredEconomicCalendarFetcher(
|
||||
from io import StringIO
|
||||
from math import ceil
|
||||
|
||||
from openbb_core.provider.utils.helpers import amake_request
|
||||
from openbb_fred.utils.rate_limiter import fred_get
|
||||
from pandas import (
|
||||
DataFrame,
|
||||
concat,
|
||||
@@ -137,13 +137,8 @@ class FredEconomicCalendarFetcher(
|
||||
) -> DataFrame:
|
||||
"""Fetch all pages of the FRED releases calendar for a single date or date range."""
|
||||
end_date = end_date or start_date
|
||||
base_url = (
|
||||
"https://fred.stlouisfed.org/releases/calendar"
|
||||
f"?po=1&ptic=0&vs={start_date}&ve={end_date}&rid={rid}"
|
||||
)
|
||||
first_page = await amake_request(
|
||||
base_url, response_callback=response_callback
|
||||
)
|
||||
base_url = f"https://fred.stlouisfed.org/releases/calendar?po=1&ptic=0&vs={start_date}&ve={end_date}&rid={rid}"
|
||||
first_page = await fred_get(base_url, response_callback=response_callback)
|
||||
n_results = first_page.get("ptic", 0) # type: ignore
|
||||
|
||||
if n_results == 0:
|
||||
@@ -161,7 +156,7 @@ class FredEconomicCalendarFetcher(
|
||||
]
|
||||
|
||||
async def _fetch_page(url: str):
|
||||
return await amake_request(url, response_callback=response_callback)
|
||||
return await fred_get(url, response_callback=response_callback)
|
||||
|
||||
responses = await asyncio.gather(*[_fetch_page(u) for u in remaining_urls])
|
||||
|
||||
@@ -222,7 +217,8 @@ class FredEconomicCalendarFetcher(
|
||||
try:
|
||||
all_results = await fetch_all_pages(start_date, end_date, rid)
|
||||
except Exception as e:
|
||||
raise OpenBBError(e) from e
|
||||
message = str(e) or f"FRED request failed ({type(e).__name__})."
|
||||
raise OpenBBError(message) from e
|
||||
|
||||
if all_results.empty:
|
||||
raise OpenBBError(
|
||||
|
||||
@@ -76,7 +76,7 @@ class FredHighQualityMarketCorporateBondFetcher(
|
||||
# pylint: disable=import-outside-toplevel
|
||||
import asyncio # noqa
|
||||
from dateutil import parser # noqa
|
||||
from openbb_core.provider.utils.helpers import amake_request # noqa
|
||||
from openbb_fred.utils.rate_limiter import fred_get # noqa
|
||||
|
||||
api_key = credentials.get("fred_api_key") if credentials else ""
|
||||
|
||||
@@ -99,7 +99,7 @@ class FredHighQualityMarketCorporateBondFetcher(
|
||||
|
||||
async def get_one(URL):
|
||||
"""Get the observations for a single date."""
|
||||
data = await amake_request(URL)
|
||||
data = await fred_get(URL)
|
||||
if data:
|
||||
elements = dict(data.get("elements", {}).items()) # type: ignore
|
||||
for k, v in elements.items(): # pylint: disable=W0612
|
||||
|
||||
@@ -489,7 +489,8 @@ class FredManufacturingOutlookNYFetcher(
|
||||
credentials,
|
||||
)
|
||||
except Exception as e:
|
||||
raise OpenBBError(e) from e
|
||||
message = str(e) or f"FRED request failed ({type(e).__name__})."
|
||||
raise OpenBBError(message) from e
|
||||
|
||||
return {
|
||||
"metadata": response.metadata,
|
||||
|
||||
@@ -278,7 +278,7 @@ class FredNonFarmPayrollsFetcher(
|
||||
"""Extract data."""
|
||||
# pylint: disable=import-outside-toplevel
|
||||
import asyncio # noqa
|
||||
from openbb_core.provider.utils.helpers import amake_request
|
||||
from openbb_fred.utils.rate_limiter import fred_get
|
||||
from numpy import nan
|
||||
from pandas import DataFrame, to_datetime
|
||||
|
||||
@@ -304,7 +304,7 @@ class FredNonFarmPayrollsFetcher(
|
||||
|
||||
async def get_one(URL):
|
||||
"""Get the observations for a single date."""
|
||||
response = await amake_request(URL)
|
||||
response = await fred_get(URL)
|
||||
data = [v for v in response.get("elements", {}).values() if v.get("observation_value") != "."] # type: ignore
|
||||
if data:
|
||||
df = (
|
||||
|
||||
@@ -122,7 +122,7 @@ class FredPersonalConsumptionExpendituresFetcher(
|
||||
"""Extract data."""
|
||||
# pylint: disable=import-outside-toplevel
|
||||
import asyncio # noqa
|
||||
from openbb_core.provider.utils.helpers import amake_request
|
||||
from openbb_fred.utils.rate_limiter import fred_get
|
||||
from numpy import nan
|
||||
from pandas import DataFrame, to_datetime
|
||||
|
||||
@@ -148,7 +148,7 @@ class FredPersonalConsumptionExpendituresFetcher(
|
||||
|
||||
async def get_one(URL):
|
||||
"""Get the observations for a single date."""
|
||||
response = await amake_request(URL)
|
||||
response = await fred_get(URL)
|
||||
data = [
|
||||
v
|
||||
for v in response.get("elements", {}).values() # type: ignore
|
||||
|
||||
@@ -219,10 +219,8 @@ class FredRegionalDataFetcher(
|
||||
) -> dict:
|
||||
"""Extract the raw data."""
|
||||
# pylint: disable=import-outside-toplevel
|
||||
from openbb_core.provider.utils.helpers import (
|
||||
amake_request,
|
||||
get_querystring,
|
||||
)
|
||||
from openbb_core.provider.utils.helpers import get_querystring
|
||||
from openbb_fred.utils.rate_limiter import fred_get
|
||||
|
||||
api_key = credentials.get("fred_api_key") if credentials else ""
|
||||
season = query.season.upper()
|
||||
@@ -254,7 +252,7 @@ class FredRegionalDataFetcher(
|
||||
)
|
||||
+ f"&file_type=json&api_key={api_key}"
|
||||
)
|
||||
return await amake_request(url) # type: ignore
|
||||
return await fred_get(url) # type: ignore
|
||||
|
||||
@staticmethod
|
||||
def transform_data(
|
||||
|
||||
@@ -77,7 +77,7 @@ class FredReleaseTableFetcher(
|
||||
"""Extract data."""
|
||||
# pylint: disable=import-outside-toplevel
|
||||
import asyncio # noqa
|
||||
from openbb_core.provider.utils.helpers import amake_request
|
||||
from openbb_fred.utils.rate_limiter import fred_get
|
||||
from openbb_fred.models.search import FredSearchFetcher
|
||||
from numpy import nan
|
||||
from pandas import DataFrame, to_datetime
|
||||
@@ -132,7 +132,7 @@ class FredReleaseTableFetcher(
|
||||
|
||||
async def get_one(URL):
|
||||
"""Get the observations for a single date."""
|
||||
response = await amake_request(URL)
|
||||
response = await fred_get(URL)
|
||||
|
||||
# If the response has no elements we return empty and try the next URL.
|
||||
# If all URLs return empty, it will raise in `transform_data`.
|
||||
|
||||
@@ -187,10 +187,8 @@ class FredSearchFetcher(
|
||||
"""Extract the raw data."""
|
||||
# pylint: disable=import-outside-toplevel
|
||||
import asyncio # noqa
|
||||
from openbb_core.provider.utils.helpers import (
|
||||
amake_request,
|
||||
get_querystring,
|
||||
)
|
||||
from openbb_core.provider.utils.helpers import get_querystring
|
||||
from openbb_fred.utils.rate_limiter import fred_get
|
||||
|
||||
api_key = credentials.get("fred_api_key") if credentials else ""
|
||||
|
||||
@@ -201,7 +199,7 @@ class FredSearchFetcher(
|
||||
"""Get data for one series."""
|
||||
data: dict = {}
|
||||
url = f"https://api.stlouisfed.org/geofred/series/group?series_id={_id}&api_key={api_key}&file_type=json"
|
||||
response = await amake_request(url)
|
||||
response = await fred_get(url)
|
||||
data = response.get("series_group") # type: ignore
|
||||
if data:
|
||||
data.update({"series_id": _id})
|
||||
@@ -215,7 +213,7 @@ class FredSearchFetcher(
|
||||
|
||||
if query.search_type == "release" and query.release_id is None:
|
||||
url = f"https://api.stlouisfed.org/fred/releases?api_key={api_key}&file_type=json"
|
||||
response = await amake_request(url)
|
||||
response = await fred_get(url)
|
||||
results = response.get("releases") # type: ignore
|
||||
if results:
|
||||
return results
|
||||
@@ -238,7 +236,7 @@ class FredSearchFetcher(
|
||||
|
||||
querystring = get_querystring(query.model_dump(), exclude).replace(" ", "%20")
|
||||
url = url + querystring + f"&file_type=json&api_key={api_key}"
|
||||
response = await amake_request(url)
|
||||
response = await fred_get(url)
|
||||
|
||||
if isinstance(response, dict) and "error_code" in response:
|
||||
raise OpenBBError(
|
||||
|
||||
@@ -117,12 +117,10 @@ class FredSeriesFetcher(
|
||||
) -> list[dict]:
|
||||
"""Extract data."""
|
||||
# pylint: disable=import-outside-toplevel
|
||||
from openbb_core.provider.utils.helpers import (
|
||||
ClientResponse,
|
||||
ClientSession,
|
||||
amake_requests,
|
||||
get_querystring,
|
||||
)
|
||||
import asyncio
|
||||
|
||||
from openbb_core.provider.utils.helpers import get_querystring
|
||||
from openbb_fred.utils.rate_limiter import fred_get
|
||||
from pandas import DataFrame
|
||||
|
||||
api_key = credentials.get("fred_api_key") if credentials else ""
|
||||
@@ -133,21 +131,15 @@ class FredSeriesFetcher(
|
||||
querystring = get_querystring(query.model_dump(), ["series_id"])
|
||||
series_ids = query.symbol.split(",") if "," in query.symbol else [query.symbol]
|
||||
|
||||
urls = [
|
||||
f"{base_url}?series_id={series_id}&{querystring}&file_type=json&api_key={api_key}"
|
||||
for series_id in series_ids
|
||||
]
|
||||
|
||||
async def callback(response: ClientResponse, session: ClientSession) -> dict:
|
||||
observations_response = await response.json()
|
||||
series_id = response.url.query.get("series_id")
|
||||
|
||||
metadata_response = await session.get_json(
|
||||
f"{metadata_url}?series_id={series_id}&file_type=json&api_key={api_key}",
|
||||
timeout=5,
|
||||
async def fetch_one(series_id: str) -> dict:
|
||||
obs_url = f"{base_url}?series_id={series_id}&{querystring}&file_type=json&api_key={api_key}"
|
||||
meta_url = (
|
||||
f"{metadata_url}?series_id={series_id}&file_type=json&api_key={api_key}"
|
||||
)
|
||||
|
||||
# seriess is not a typo, it's the actual key in the response
|
||||
observations_response = await fred_get(obs_url, timeout=5, **kwargs)
|
||||
metadata_response = await fred_get(meta_url, timeout=5, **kwargs)
|
||||
|
||||
_metadata = (
|
||||
metadata_response.get("seriess", [{}])[0]
|
||||
if isinstance(metadata_response, dict)
|
||||
@@ -158,6 +150,7 @@ class FredSeriesFetcher(
|
||||
if isinstance(observations_response, dict)
|
||||
else []
|
||||
) or []
|
||||
|
||||
try:
|
||||
for d in observations:
|
||||
d.pop("realtime_start")
|
||||
@@ -186,12 +179,20 @@ class FredSeriesFetcher(
|
||||
}
|
||||
|
||||
try:
|
||||
results = await amake_requests(
|
||||
urls, response_callback=callback, timeout=5, **kwargs
|
||||
)
|
||||
results: list[dict] = []
|
||||
for result in await asyncio.gather(
|
||||
*[fetch_one(sid) for sid in series_ids], return_exceptions=True
|
||||
):
|
||||
if isinstance(result, Exception):
|
||||
raise result
|
||||
if result:
|
||||
results.append(result) # type: ignore
|
||||
return results
|
||||
except OpenBBError:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise OpenBBError(e) from e
|
||||
message = str(e) or f"FRED request failed ({type(e).__name__})."
|
||||
raise OpenBBError(message) from e
|
||||
|
||||
@staticmethod
|
||||
def transform_data(
|
||||
|
||||
@@ -158,7 +158,8 @@ class FredTipsYieldsFetcher(
|
||||
ids_df = await get_tips_series()
|
||||
ids = ids_df.series_id.to_list()
|
||||
except Exception as e:
|
||||
raise OpenBBError(e) from e
|
||||
message = str(e) or f"FRED request failed ({type(e).__name__})."
|
||||
raise OpenBBError(message) from e
|
||||
|
||||
# If we are looking for a specific tenor, the request will be smaller.
|
||||
if query.maturity:
|
||||
@@ -197,7 +198,8 @@ class FredTipsYieldsFetcher(
|
||||
df = DataFrame([d.model_dump() for d in res.result]) # type: ignore
|
||||
meta: dict = res.metadata or {} # type: ignore
|
||||
except Exception as e:
|
||||
raise OpenBBError(e) from e
|
||||
message = str(e) or f"FRED request failed ({type(e).__name__})."
|
||||
raise OpenBBError(message) from e
|
||||
|
||||
for k, v in title_map.items():
|
||||
if k in meta:
|
||||
|
||||
315
openbb_platform/providers/fred/openbb_fred/utils/rate_limiter.py
Normal file
315
openbb_platform/providers/fred/openbb_fred/utils/rate_limiter.py
Normal file
@@ -0,0 +1,315 @@
|
||||
"""Process-wide FRED API rate limiter, in-memory cache, and request helper.
|
||||
|
||||
Every FRED API request in this provider should be routed through `fred_get`.
|
||||
The helper provides:
|
||||
|
||||
* A global minimum-interval throttle (default 600 ms ≈ 100 req/min), well
|
||||
below FRED's 120 req/min ceiling.
|
||||
* Auto-retry on HTTP 429 with `Retry-After` / exponential backoff, so
|
||||
transient throttling does not surface as an error.
|
||||
* An in-memory TTL LRU cache so that repeated identical requests (typical
|
||||
when a user refreshes or re-opens widgets) reuse the prior response.
|
||||
* Single-flight de-duplication: concurrent callers asking for the same URL
|
||||
share a single in-flight request instead of fanning out duplicates.
|
||||
"""
|
||||
|
||||
# pylint: disable=import-outside-toplevel
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import copy
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Awaitable, Callable
|
||||
from typing import Any
|
||||
|
||||
from openbb_core.app.model.abstract.error import OpenBBError
|
||||
|
||||
|
||||
def _float_env(name: str, default: float) -> float:
|
||||
raw = os.environ.get(name)
|
||||
if not raw:
|
||||
return default
|
||||
try:
|
||||
return float(raw)
|
||||
except ValueError:
|
||||
return default
|
||||
|
||||
|
||||
def _int_env(name: str, default: int) -> int:
|
||||
raw = os.environ.get(name)
|
||||
if not raw:
|
||||
return default
|
||||
try:
|
||||
return int(raw)
|
||||
except ValueError:
|
||||
return default
|
||||
|
||||
|
||||
MIN_INTERVAL_SECONDS = _float_env("OPENBB_FRED_MIN_INTERVAL", 0.6)
|
||||
MAX_RETRIES = _int_env("OPENBB_FRED_MAX_RETRIES", 4)
|
||||
DEFAULT_BACKOFF_SECONDS = _float_env("OPENBB_FRED_BACKOFF", 2.0)
|
||||
MAX_BACKOFF_SECONDS = _float_env("OPENBB_FRED_MAX_BACKOFF", 30.0)
|
||||
CACHE_TTL_SECONDS = _float_env("OPENBB_FRED_CACHE_TTL", 300.0)
|
||||
CACHE_MAX_ENTRIES = _int_env("OPENBB_FRED_CACHE_SIZE", 512)
|
||||
|
||||
_RATE_LIMIT_MESSAGE = (
|
||||
"FRED API rate limit exceeded (HTTP 429). Please wait a moment and try again."
|
||||
)
|
||||
|
||||
_throttle_lock = threading.Lock()
|
||||
_last_request_at: float = 0.0
|
||||
|
||||
_cache_lock = threading.Lock()
|
||||
_cache: OrderedDict[str, tuple[float, Any]] = OrderedDict()
|
||||
|
||||
_inflight_lock = threading.Lock()
|
||||
_inflight: dict[tuple[int, str], asyncio.Future] = {}
|
||||
|
||||
_API_KEY_RE = re.compile(r"([?&])api_key=[^&]*")
|
||||
|
||||
|
||||
class _RateLimitedResponse(Exception):
|
||||
"""Internal sentinel raised when a 429 is observed on a FRED response."""
|
||||
|
||||
def __init__(self, retry_after: float | None = None) -> None:
|
||||
super().__init__(_RATE_LIMIT_MESSAGE)
|
||||
self.retry_after = retry_after
|
||||
|
||||
|
||||
def _cache_key(url: str) -> str:
|
||||
"""Build a cache key from a URL, stripping the API key."""
|
||||
return _API_KEY_RE.sub(r"\1api_key=__redacted__", url)
|
||||
|
||||
|
||||
def _cache_get(key: str) -> Any:
|
||||
"""Return a cached value if present and not expired, else None.
|
||||
|
||||
Returns a deep copy so downstream mutation of dicts / lists does not
|
||||
corrupt subsequent cache hits.
|
||||
"""
|
||||
if CACHE_TTL_SECONDS <= 0 or CACHE_MAX_ENTRIES <= 0:
|
||||
return None
|
||||
with _cache_lock:
|
||||
entry = _cache.get(key)
|
||||
if entry is None:
|
||||
return None
|
||||
stored_at, value = entry
|
||||
if time.monotonic() - stored_at > CACHE_TTL_SECONDS:
|
||||
_cache.pop(key, None)
|
||||
return None
|
||||
_cache.move_to_end(key)
|
||||
cached_value = value
|
||||
return copy.deepcopy(cached_value)
|
||||
|
||||
|
||||
def _cache_set(key: str, value: Any) -> None:
|
||||
"""Insert a deep-copied value into the cache and evict oldest entries when full."""
|
||||
if CACHE_TTL_SECONDS <= 0 or CACHE_MAX_ENTRIES <= 0:
|
||||
return
|
||||
snapshot = copy.deepcopy(value)
|
||||
with _cache_lock:
|
||||
_cache[key] = (time.monotonic(), snapshot)
|
||||
_cache.move_to_end(key)
|
||||
while len(_cache) > CACHE_MAX_ENTRIES:
|
||||
_cache.popitem(last=False)
|
||||
|
||||
|
||||
def cache_clear() -> None:
|
||||
"""Clear the in-memory FRED response cache (useful for tests)."""
|
||||
with _cache_lock:
|
||||
_cache.clear()
|
||||
|
||||
|
||||
async def acquire() -> None:
|
||||
"""Enforce a minimum gap between successive FRED API requests."""
|
||||
global _last_request_at # noqa: PLW0603 # pylint: disable=global-statement
|
||||
while True:
|
||||
with _throttle_lock:
|
||||
now = time.monotonic()
|
||||
wait_for = _last_request_at + MIN_INTERVAL_SECONDS - now
|
||||
if wait_for <= 0:
|
||||
_last_request_at = now
|
||||
return
|
||||
await asyncio.sleep(wait_for)
|
||||
|
||||
|
||||
def _parse_retry_after(value: str | None) -> float | None:
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
return max(float(value), 0.0)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _check_payload_rate_limit(payload: Any) -> None:
|
||||
if not isinstance(payload, dict):
|
||||
return
|
||||
code = payload.get("error_code") or payload.get("status_code")
|
||||
message = payload.get("error_message") or payload.get("message") or ""
|
||||
if str(code) == "429" or "too many requests" in str(message).lower():
|
||||
raise _RateLimitedResponse()
|
||||
|
||||
|
||||
async def _fetch(
|
||||
url: str,
|
||||
response_callback: Callable[[Any, Any], Awaitable[dict | list[dict]]] | None,
|
||||
retries: int,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
"""Perform a single throttled fetch with retry-on-429."""
|
||||
from openbb_core.provider.utils.helpers import amake_request
|
||||
|
||||
async def _wrapped_callback(response, session):
|
||||
status = getattr(response, "status", None)
|
||||
if status == 429:
|
||||
retry_after = _parse_retry_after(
|
||||
response.headers.get("Retry-After")
|
||||
if hasattr(response, "headers")
|
||||
else None
|
||||
)
|
||||
raise _RateLimitedResponse(retry_after)
|
||||
if response_callback is not None:
|
||||
return await response_callback(response, session)
|
||||
payload = await response.json()
|
||||
_check_payload_rate_limit(payload)
|
||||
return payload
|
||||
|
||||
attempt = 0
|
||||
while True:
|
||||
await acquire()
|
||||
try:
|
||||
return await amake_request(
|
||||
url, response_callback=_wrapped_callback, **kwargs
|
||||
)
|
||||
except _RateLimitedResponse as rl:
|
||||
attempt += 1
|
||||
if attempt > retries:
|
||||
raise OpenBBError(ValueError(_RATE_LIMIT_MESSAGE)) from rl
|
||||
backoff = (
|
||||
rl.retry_after
|
||||
if rl.retry_after is not None
|
||||
else min(
|
||||
DEFAULT_BACKOFF_SECONDS * (2 ** (attempt - 1)),
|
||||
MAX_BACKOFF_SECONDS,
|
||||
)
|
||||
)
|
||||
await asyncio.sleep(max(backoff, MIN_INTERVAL_SECONDS))
|
||||
|
||||
|
||||
def _is_cacheable(value: Any) -> bool:
|
||||
"""Don't cache empty / falsy payloads — they're usually transient errors."""
|
||||
if value is None:
|
||||
return False
|
||||
return not (isinstance(value, (list, dict, str, bytes)) and len(value) == 0)
|
||||
|
||||
|
||||
async def fred_get(
|
||||
url: str,
|
||||
*,
|
||||
response_callback: Callable[[Any, Any], Awaitable[dict | list[dict]]] | None = None,
|
||||
max_retries: int | None = None,
|
||||
use_cache: bool = True,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
"""Throttled, retrying, cached GET wrapper around `amake_request` for FRED.
|
||||
|
||||
Concurrent calls for the same URL share a single in-flight request so
|
||||
duplicates collapse into one upstream API call.
|
||||
"""
|
||||
retries = MAX_RETRIES if max_retries is None else max_retries
|
||||
cache_key = _cache_key(url) if use_cache else None
|
||||
|
||||
if cache_key is not None:
|
||||
cached = _cache_get(cache_key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
inflight_key: tuple[int, str] | None = None
|
||||
future: asyncio.Future | None = None
|
||||
owner = False
|
||||
if cache_key is not None:
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
loop = None
|
||||
if loop is not None:
|
||||
inflight_key = (id(loop), cache_key)
|
||||
with _inflight_lock:
|
||||
existing = _inflight.get(inflight_key)
|
||||
if existing is not None:
|
||||
future = existing
|
||||
else:
|
||||
future = loop.create_future()
|
||||
_inflight[inflight_key] = future
|
||||
owner = True
|
||||
|
||||
if future is not None and not owner:
|
||||
shared = await future
|
||||
return copy.deepcopy(shared)
|
||||
|
||||
try:
|
||||
result = await _fetch(
|
||||
url,
|
||||
response_callback=response_callback,
|
||||
retries=retries,
|
||||
**kwargs,
|
||||
)
|
||||
except BaseException as exc:
|
||||
if future is not None and owner and not future.done():
|
||||
future.set_exception(exc)
|
||||
raise
|
||||
finally:
|
||||
if inflight_key is not None and owner:
|
||||
with _inflight_lock:
|
||||
_inflight.pop(inflight_key, None)
|
||||
|
||||
if cache_key is not None and _is_cacheable(result):
|
||||
_cache_set(cache_key, result)
|
||||
|
||||
if future is not None and owner and not future.done():
|
||||
future.set_result(result)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
async def fred_get_many(
|
||||
urls: list[str],
|
||||
*,
|
||||
response_callback: Callable[[Any, Any], Awaitable[dict | list[dict]]] | None = None,
|
||||
return_exceptions: bool = False,
|
||||
use_cache: bool = True,
|
||||
**kwargs: Any,
|
||||
) -> list[Any]:
|
||||
"""Issue many FRED requests under the global rate limit and cache."""
|
||||
tasks = [
|
||||
fred_get(
|
||||
url,
|
||||
response_callback=response_callback,
|
||||
use_cache=use_cache,
|
||||
**kwargs,
|
||||
)
|
||||
for url in urls
|
||||
]
|
||||
results = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
cleaned: list[Any] = []
|
||||
first_error: Exception | None = None
|
||||
for result in results:
|
||||
if isinstance(result, Exception):
|
||||
if return_exceptions:
|
||||
cleaned.append(result)
|
||||
elif first_error is None:
|
||||
first_error = result
|
||||
continue
|
||||
cleaned.append(result)
|
||||
|
||||
if first_error is not None and not cleaned:
|
||||
raise first_error
|
||||
|
||||
return cleaned
|
||||
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "openbb-fred"
|
||||
version = "1.6.1"
|
||||
version = "1.6.2"
|
||||
description = "FRED extension for OpenBB"
|
||||
authors = ["OpenBB Team <hello@openbb.co>"]
|
||||
license = "AGPL-3.0-only"
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
2
openbb_platform/pyproject.toml
vendored
2
openbb_platform/pyproject.toml
vendored
@@ -19,7 +19,7 @@ openbb-congress-gov = "^1.2.3"
|
||||
openbb-econdb = "^1.5.1"
|
||||
openbb-federal-reserve = "^1.6.2"
|
||||
openbb-fmp = "^1.6.1"
|
||||
openbb-fred = "^1.6.1"
|
||||
openbb-fred = "^1.6.2"
|
||||
openbb-government-us = "^1.6.1"
|
||||
openbb-imf = "^2.1.3"
|
||||
openbb-intrinio = "^1.6.1"
|
||||
|
||||
Reference in New Issue
Block a user