Files
OpenBB/openbb_platform/providers/polygon/openbb_polygon/utils/helpers.py
teh_coderer da7886d0a7 Make Pro Fetchers async by replacing ThreadPoolExecutor with asyncio.gather (#5822)
* init

* Update helpers.py

* filter URL on raise

* polygon, fmp done, update cassettes

* updates

* cleanup

* revert tests for now

* Update discovery_filings.py

* Update helpers.py

* add `async` overload to `Fetcher.extract_data`

* update merged fred models

* tests, improve fred series

* Update insider_trading.py

* insider

* tests

* intrinio

* Update equity_historical.py

* implement abstract async extract on Fetcher, with simple switch

* cleanup

* Update helpers.py

* Update helpers.py

* fix fmp historical deprecated urls

* Revert "fix fmp historical deprecated urls"

This reverts commit 1a760a2473.

* fix fmp `index.market` 1day url missing query params

* Update test_fmp_market_indices_fetcher.yaml

* field fixes

* `extract_data_async` -> `aextract_data`, `async_request` -> `amake_request`

* `async_requests` -> `amake_requests`

* Update company_news.py

* add sort to index `integration` params

* add `client.py` tests

* Update equity_search.py

* Add lost v in platform api version string

* Resolve linter errors here and there

---------

Co-authored-by: Theodore Aptekarev <aptekarev@gmail.com>
2023-12-08 14:41:37 +00:00

142 lines
3.6 KiB
Python

"""Polygon Helpers Module."""
import json
from io import StringIO
from typing import Any, List, Optional, Tuple, TypeVar, Union
from openbb_core.provider.utils.errors import EmptyDataError
from openbb_core.provider.utils.helpers import (
ClientResponse,
ClientSession,
amake_request,
)
from pydantic import BaseModel
T = TypeVar("T", bound=BaseModel)
class BasicResponse:
"""Basic Response class."""
def __init__(self, response: StringIO):
"""Initialize the BasicResponse class."""
# Find a way to get the status code
self.status_code = 200
response.seek(0)
self.text = response.read()
def json(self) -> dict:
"""Return the response as a dictionary."""
return json.loads(self.text)
def request(url: str) -> BasicResponse:
"""
Request function for PyScript. Pass in Method and make sure to await.
Parameters:
-----------
url: str
URL to make request to
Return:
-------
response: BasicRequest
BasicRequest object with status_code and text attributes
"""
# pylint: disable=import-outside-toplevel
from pyodide.http import open_url # type: ignore
response = open_url(url)
return BasicResponse(response)
async def response_callback(
response: ClientResponse, _: ClientSession
) -> Union[dict, List[dict]]:
"""Callback for make_request."""
data: dict = await response.json()
if response.status != 200:
message = data.get("error", None) or data.get("message", None)
raise RuntimeError(f"Error in Polygon request -> {message}")
keys_in_data = "results" in data or "tickers" in data
if not keys_in_data or len(data) == 0:
raise EmptyDataError()
return data
async def get_data(url: str, **kwargs: Any) -> Union[list, dict]:
"""Get data from Polygon endpoint."""
return await amake_request(url, response_callback=response_callback, **kwargs)
async def get_data_many(
url: str, sub_dict: Optional[str] = None, **kwargs: Any
) -> List[dict]:
"""Get data from Polygon endpoint and convert to list of schemas.
Parameters:
-----------
url: str
The URL to get the data from.
sub_dict: Optional[str]
The sub-dictionary to use.
Returns:
--------
List[dict]
Dictionary of data.
"""
data = await get_data(url, **kwargs)
if sub_dict and isinstance(data, dict):
data = data.get(sub_dict, [])
if isinstance(data, dict):
raise ValueError("Expected list of dicts, got dict")
return data
async def get_data_one(url: str, **kwargs: Any) -> dict:
"""Get data from Polygon endpoint and convert to schema."""
data = await get_data(url, **kwargs)
if isinstance(data, list):
if len(data) == 0:
raise ValueError("Expected dict, got empty list")
try:
data = {i: data[i] for i in range(len(data))} if len(data) > 1 else data[0]
except TypeError as e:
raise ValueError("Expected dict, got list of dicts") from e
return data
def get_date_condition(date: str) -> Tuple:
"""Get the date condition for the querystring."""
date_conditions = {
"<": "lt",
"<=": "lte",
">": "gt",
">=": "gte",
}
for key, value in date_conditions.items():
if key in date:
return date.split(key)[1], value
return date, "eq"
def map_tape(tape: int) -> str:
"""Map the tape to a string."""
STOCK_TAPE_MAP = {
1: "Tape A: NYSE",
2: "Tape B: NYSE ARCA",
3: "Tape C: NASDAQ",
}
return STOCK_TAPE_MAP.get(tape, "") if tape else ""