fix(finra): use persistent session to prevent TooManyRedirects (#7443)

* fix(finra): use persistent session to prevent TooManyRedirects

Resolves #6972.

The FINRA OTC data fetching functions (get_finra_weeks, get_finra_data,
get_full_data) previously created new HTTP sessions for each request.
When making dozens of sequential API calls to aggregate weekly data,
this caused Cloudflare's bot protection on api.finra.org to trigger
redirect loops, resulting in a TooManyRedirects error.

Changes:
- Refactored get_finra_weeks and get_finra_data to accept an optional
  session parameter via kwargs.
- Updated get_full_data to create a single requests.Session and perform
  a pre-flight GET to finra.org to establish Cloudflare cookies before
  making API calls.
- The session is reused across all requests within a single data fetch
  operation, preventing the redirect loop.
- Removed hardcoded User-Agent from get_finra_data in favor of the
  platform's standard user agent set by get_requests_session.
- Increased timeout for weeklyDownloadDetails from 3s to 20s for
  consistency and reliability.

* Apply Black Formatting

* perf(finra): use async requests for concurrent weekly data fetching

Addresses reviewer feedback on PR #7443.

The get_full_data function previously made up to 52 sequential POST
requests to fetch weekly OTC data, resulting in poor performance.

This commit adds async equivalents (aget_finra_weeks, aget_finra_data,
aget_full_data) that use the platform's amake_request utility and
asyncio.gather to fetch all weeks concurrently through a shared
aiohttp session.

The FinraOTCAggregateFetcher now uses aextract_data, which is natively
supported by the platform's Fetcher base class.

The pre-flight GET to finra.org for Cloudflare cookie establishment
is preserved in the async path.

* fix(finra): return all rows from data fetching instead of just the first one

* run black, add symbol field to data model, and recapture test cassette

* mypy

---------

Co-authored-by: Danglewood <85772166+deeleeramone@users.noreply.github.com>

openbb_platform/core/openbb_core/provider/standard_models/otc_aggregate.py

@@ -4,7 +4,10 @@ from datetime import date as dateType
 
 from openbb_core.provider.abstract.data import Data
 from openbb_core.provider.abstract.query_params import QueryParams
-from openbb_core.provider.utils.descriptions import QUERY_DESCRIPTIONS
+from openbb_core.provider.utils.descriptions import (
+    DATA_DESCRIPTIONS,
+    QUERY_DESCRIPTIONS,
+)
 from pydantic import Field
 
 
@@ -23,6 +26,7 @@ class OTCAggregateData(Data):
     update_date: dateType = Field(
         description="Most recent date on which total trades is updated based on data received from each ATS/OTC."
     )
+    symbol: str = Field(description=DATA_DESCRIPTIONS.get("symbol", ""))
     share_quantity: float = Field(
         description="Aggregate weekly total number of shares reported by each ATS for the Symbol."
     )

openbb_platform/providers/finra/openbb_finra/models/otc_aggregate.py

@@ -30,6 +30,7 @@ class FinraOTCAggregateData(OTCAggregateData):
         "share_quantity": "totalWeeklyShareQuantity",
         "trade_quantity": "totalWeeklyTradeCount",
         "update_date": "lastUpdateDate",
+        "symbol": "issueSymbolIdentifier",
     }
 
 
@@ -45,16 +46,16 @@ class FinraOTCAggregateFetcher(
 
     # pylint: disable=unused-argument
     @staticmethod
-    def extract_data(
+    async def aextract_data(
         query: FinraOTCAggregateQueryParams,
         credentials: dict[str, str] | None,
         **kwargs: Any,
     ) -> list[dict]:
         """Extract the data from the FINRA endpoint."""
         # pylint: disable=import-outside-toplevel
-        from openbb_finra.utils.helpers import get_full_data
+        from openbb_finra.utils.helpers import aget_full_data
 
-        return get_full_data(query.symbol, query.tier, query.is_ats)
+        return await aget_full_data(query.symbol, query.tier, query.is_ats)
 
     @staticmethod
     def transform_data(

openbb_platform/providers/finra/openbb_finra/utils/helpers.py

@@ -5,12 +5,17 @@ import datetime
 # pylint: disable=W0621
 
 
-def get_finra_weeks(tier: str = "T1", is_ats: bool = True):
+def get_finra_weeks(tier: str = "T1", is_ats: bool = True, **kwargs):
     """Fetch the available weeks from FINRA that can be used."""
     # pylint: disable=import-outside-toplevel
-    from openbb_core.provider.utils.helpers import make_request
+    from openbb_core.provider.utils.helpers import get_requests_session, make_request
 
-    request_header = {"Accept": "application/json", "Content-Type": "application/json"}
+    session = kwargs.get("session", get_requests_session())
+
+    request_header = {
+        "Accept": "application/json",
+        "Content-Type": "application/json",
+    }
 
     request_data = {
         "compareFilters": [
@@ -37,21 +42,23 @@ def get_finra_weeks(tier: str = "T1", is_ats: bool = True):
         url="https://api.finra.org/data/group/otcMarket/name/weeklyDownloadDetails",
         headers=request_header,
         json=request_data,
-        timeout=3,
+        timeout=20,
+        session=session,
     )
 
     return response.json() if response.status_code == 200 else []
 
 
-def get_finra_data(symbol, week_start, tier: str = "T1", is_ats: bool = True):
+def get_finra_data(symbol, week_start, tier: str = "T1", is_ats: bool = True, **kwargs):
     """Get the data for a symbol from FINRA."""
     # pylint: disable=import-outside-toplevel
-    from openbb_core.provider.utils.helpers import make_request
+    from openbb_core.provider.utils.helpers import get_requests_session, make_request
+
+    session = kwargs.get("session", get_requests_session())
 
     req_hdr = {
         "Accept": "application/json",
         "Content-Type": "application/json",
-        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) ",
     }
 
     filters = [
@@ -97,24 +104,188 @@ def get_finra_data(symbol, week_start, tier: str = "T1", is_ats: bool = True):
         headers=req_hdr,
         json=req_data,
         timeout=20,
+        session=session,
     )
     return response
 
 
 def get_full_data(symbol, tier: str = "T1", is_ats: bool = True):
     """Get the full data for a symbol from FINRA."""
-    weeks = [week["weekStartDate"] for week in get_finra_weeks(tier, is_ats)]
+    # pylint: disable=import-outside-toplevel
+    from openbb_core.provider.utils.helpers import get_requests_session
+
+    session = get_requests_session()
+
+    # We make a pre-flight request to the FINRA website to establish a session.
+    # This is to avoid the TooManyRedirects error that occurs when the FINRA
+    # API redirects to the FINRA website to establish a session.
+    session.get("https://www.finra.org/finra-data", timeout=10)
+
+    weeks = [
+        week["weekStartDate"] for week in get_finra_weeks(tier, is_ats, session=session)
+    ]
 
     data = []
     for week in weeks:
-        response = get_finra_data(symbol, week, tier, is_ats)
+        response = get_finra_data(symbol, week, tier, is_ats, session=session)
         r_json = response.json()
         if response.status_code == 200 and r_json:
-            data.append(response.json()[0])
+            data.extend(r_json)
 
     return data
 
 
+async def aget_finra_weeks(tier: str = "T1", is_ats: bool = True, **kwargs):
+    """Fetch the available weeks from FINRA asynchronously."""
+    # pylint: disable=import-outside-toplevel
+    from openbb_core.provider.utils.helpers import amake_request
+
+    session = kwargs.pop("session", None)
+
+    request_header = {
+        "Accept": "application/json",
+        "Content-Type": "application/json",
+    }
+
+    request_data = {
+        "compareFilters": [
+            {
+                "compareType": "EQUAL",
+                "fieldName": "summaryTypeCode",
+                "fieldValue": "ATS_W_SMBL" if is_ats else "OTC_W_SMBL",
+            },
+            {
+                "compareType": "EQUAL",
+                "fieldName": "tierIdentifier",
+                "fieldValue": tier,
+            },
+        ],
+        "delimiter": "|",
+        "fields": ["weekStartDate"],
+        "limit": 52,
+        "quoteValues": False,
+        "sortFields": ["-weekStartDate"],
+    }
+
+    kwargs_for_request = {
+        "headers": request_header,
+        "json": request_data,
+        "timeout": 20,
+    }
+    if session is not None:
+        kwargs_for_request["session"] = session
+
+    result = await amake_request(
+        url="https://api.finra.org/data/group/otcMarket/name/weeklyDownloadDetails",
+        method="POST",
+        **kwargs_for_request,  # type: ignore
+    )
+
+    return result if isinstance(result, list) else []
+
+
+async def aget_finra_data(
+    symbol, week_start, tier: str = "T1", is_ats: bool = True, **kwargs
+):
+    """Get the data for a symbol from FINRA asynchronously."""
+    # pylint: disable=import-outside-toplevel
+    from openbb_core.provider.utils.helpers import amake_request
+
+    session = kwargs.pop("session", None)
+
+    filters = [
+        {
+            "compareType": "EQUAL",
+            "fieldName": "weekStartDate",
+            "fieldValue": week_start,
+        },
+        {"compareType": "EQUAL", "fieldName": "tierIdentifier", "fieldValue": tier},
+        {
+            "compareType": "EQUAL",
+            "description": "",
+            "fieldName": "summaryTypeCode",
+            "fieldValue": "ATS_W_SMBL" if is_ats else "OTC_W_SMBL",
+        },
+    ]
+
+    if symbol:
+        filters.append(
+            {
+                "compareType": "EQUAL",
+                "fieldName": "issueSymbolIdentifier",
+                "fieldValue": symbol,
+            }
+        )
+
+    req_data = {
+        "compareFilters": filters,
+        "delimiter": "|",
+        "fields": [
+            "issueSymbolIdentifier",
+            "totalWeeklyShareQuantity",
+            "totalWeeklyTradeCount",
+            "lastUpdateDate",
+        ],
+        "limit": 5000,
+        "quoteValues": False,
+        "sortFields": ["totalWeeklyShareQuantity"],
+    }
+
+    req_hdr = {
+        "Accept": "application/json",
+        "Content-Type": "application/json",
+    }
+
+    kwargs_for_request = {"headers": req_hdr, "json": req_data, "timeout": 20}
+    if session is not None:
+        kwargs_for_request["session"] = session
+
+    return await amake_request(
+        url="https://api.finra.org/data/group/otcMarket/name/weeklySummary",
+        method="POST",
+        **kwargs_for_request,  # type: ignore
+    )
+
+
+async def aget_full_data(symbol, tier: str = "T1", is_ats: bool = True):
+    """Get the full data for a symbol from FINRA asynchronously."""
+    # pylint: disable=import-outside-toplevel
+    import asyncio
+
+    from openbb_core.provider.utils.helpers import get_async_requests_session
+
+    session = await get_async_requests_session()
+
+    try:
+        await session.request("GET", "https://www.finra.org/finra-data", timeout=10)
+
+        weeks_data = await aget_finra_weeks(tier, is_ats, session=session)
+        weeks = [week["weekStartDate"] for week in weeks_data]
+
+        async def fetch_week(week_start):
+            result = await aget_finra_data(
+                symbol, week_start, tier, is_ats, session=session
+            )
+            if isinstance(result, list) and result:
+                return result
+            if isinstance(result, dict) and result:
+                return [result]
+            return []
+
+        results = await asyncio.gather(
+            *[fetch_week(w) for w in weeks], return_exceptions=True
+        )
+
+        flat_results = []
+        for r in results:
+            if isinstance(r, list):
+                flat_results.extend(r)
+
+        return flat_results
+    finally:
+        await session.close()
+
+
 def get_adjusted_date(year, month, day):
     """Find the closest date if the date falls on a weekend."""
     # Get the date

openbb_platform/providers/finra/tests/test_finra_fetchers.py

@@ -1,5 +1,6 @@
 """Tests for the Finra fetchers."""
 
+import pandas  # noqa: F401
 import pytest
 from openbb_core.app.service.user_service import UserService
 from openbb_finra.models.equity_short_interest import FinraShortInterestFetcher
@@ -21,6 +22,7 @@ def vcr_config():
     }
 
 
+@pytest.mark.freeze_time("2026-04-06")
 @pytest.mark.record_http
 def test_finra_otc_aggregate_fetcher(credentials=test_credentials):
     """Test the Finra OTC Aggregate fetcher."""