Skip checking creds (#5737)

* skip checking creds

* fix bug

* ok mypy

* add comment

* rename required_credentials

* fix

* fix top_retail bug

* don't require creds in some nasdaq fetchers

* black

* this was not here before

* fix tests

* update fetcher test

* update contributing guidelines
This commit is contained in:
montezdesousa
2023-11-17 16:15:34 +00:00
committed by GitHub
parent 86aa18b7d0
commit 565cafba53
34 changed files with 136 additions and 74 deletions

View File

@@ -349,7 +349,7 @@ class <ProviderName>EquityHistoricalData(EquityHistoricalData):
The `Fetcher` class is responsible for making the request to the API endpoint and providing the output.
It will receive the Query Parameters, and it will return the output while leveraging the pydantic model schemas.
It will receive the query parameters, and it will return the output while leveraging the pydantic model schemas.
For the `EquityHistorical` example, this would look like the following:
@@ -382,7 +382,9 @@ class <ProviderName>EquityHistoricalFetcher(
@staticmethod
def transform_data(
query: <ProviderName>EquityHistoricalQueryParams,
data: dict,
**kwargs: Any,
) -> List[<ProviderName>EquityHistoricalData]:
"""Transform the data to the standard format."""
@@ -391,6 +393,22 @@ class <ProviderName>EquityHistoricalFetcher(
> Make sure that you're following the TET pattern when building a `Fetcher` - **Transform, Extract, Transform**. See more on this [here](#the-tet-pattern).
By default the credentials declared on each `Provider` are required. This means that before a query is executed, we check that all the credentials are present and if not an exception is raised. If you want to make credentials optional on a given fetcher, even though they are declared on the `Provider`, you can add `require_credentials=False` to the `Fetcher` class. See the following example:
```python
class <ProviderName>EquityHistoricalFetcher(
Fetcher[
<ProviderName>EquityHistoricalQueryParams,
List[<ProviderName>EquityHistoricalData],
]
):
"""Transform the query, extract and transform the data."""
require_credentials = False
...
```
#### Make the provider visible
In order to make the new provider visible to the OpenBB Platform, you'll need to add it to the `__init__.py` file of the `providers/<provider_name>/openbb_<provider_name>/` folder.
@@ -405,14 +423,14 @@ from openbb_<provider_name>.models.equity_historical import <ProviderName>Equity
name="<provider_name>",
website="<URL to the provider website>",
description="Provider description goes here",
required_credentials=["api_key"],
credentials=["api_key"],
fetcher_dict={
"EquityHistorical": <ProviderName>EquityHistoricalFetcher,
},
)
```
If the provider does not require any credentials, you can remove that parameter. On the other hand, if it requires more than 2 items to authenticate, you can add a list of all the required items to the `required_credentials` list.
If the provider does not require any credentials, you can remove that parameter. On the other hand, if it requires more than 2 items to authenticate, you can add a list of all the required items to the `credentials` list.
After running `pip install .` on `openbb_platform/providers/<provider_name>` your provider should be ready for usage, both from the Python interface and the API.

View File

@@ -243,7 +243,7 @@ Steps to create an `OBBject` extension:
```python
from openbb_core.app.model.extension import Extension
ext = Extension(name="example", required_credentials=["some_api_key"])
ext = Extension(name="example", credentials=["some_api_key"])
```
3. Optionally declare an `OBBject` accessor, it will use the extension name:

View File

@@ -36,11 +36,11 @@ class CredentialsLoader:
@staticmethod
def prepare(
required_credentials: Dict[str, Set[str]],
credentials: Dict[str, Set[str]],
) -> Dict[str, Tuple[object, None]]:
"""Prepare credentials map to be used in the Credentials model"""
formatted: Dict[str, Tuple[object, None]] = {}
for origin, creds in required_credentials.items():
for origin, creds in credentials.items():
for c in creds:
# Not sure we should do this, if you require the same credential it breaks
# if c in formatted:
@@ -61,7 +61,7 @@ class CredentialsLoader:
try:
entry = entry_point.load()
if isinstance(entry, Extension):
for c in entry.required_credentials:
for c in entry.credentials:
self.credentials["obbject"].add(c)
except Exception as e:
traceback.print_exception(type(e), e, e.__traceback__)
@@ -70,7 +70,7 @@ class CredentialsLoader:
def from_providers(self) -> None:
"""Load credentials from providers"""
self.credentials["providers"] = set()
for c in ProviderInterface().required_credentials:
for c in ProviderInterface().credentials:
self.credentials["providers"].add(c)
def load(self) -> BaseModel:

View File

@@ -11,7 +11,7 @@ class Extension:
def __init__(
self,
name: str,
required_credentials: Optional[List[str]] = None,
credentials: Optional[List[str]] = None,
) -> None:
"""Initialize the extension.
@@ -19,11 +19,11 @@ class Extension:
----------
name : str
Name of the extension.
required_credentials : Optional[List[str]], optional
credentials : Optional[List[str]], optional
List of required credentials, by default None
"""
self.name = name
self.required_credentials = required_credentials or []
self.credentials = credentials or []
@property
def obbject_accessor(self) -> Callable:

View File

@@ -64,8 +64,8 @@ class ProviderInterface(metaclass=SingletonMeta):
----------
map : MapType
Dictionary of provider information.
required_credentials: List[str]
List of required_credentials.
credentials: List[str]
List of credentials.
model_providers : Dict[str, ProviderChoices]
Dictionary of provider choices by model.
params : Dict[str, Dict[str, Union[StandardParams, ExtraParams]]]
@@ -110,9 +110,9 @@ class ProviderInterface(metaclass=SingletonMeta):
return self._map
@property
def required_credentials(self) -> List[str]:
def credentials(self) -> List[str]:
"""Dictionary of required credentials by provider."""
return self._registry_map.required_credentials
return self._registry_map.credentials
@property
def model_providers(self) -> Dict[str, ProviderChoices]:

View File

@@ -6,7 +6,7 @@ def test_credentials():
with patch(
target="openbb_core.app.model.credentials.ProviderInterface"
) as mock_provider_interface:
mock_provider_interface.required_credentials = {
mock_provider_interface.credentials = {
"benzinga_api_key": (typing.Optional[str], None),
"polygon_api_key": (typing.Optional[str], None),
}

View File

@@ -27,11 +27,11 @@ def test_map(provider_interface):
assert "EquityHistorical" in provider_interface_map
def test_required_credentials(provider_interface):
def test_credentials(provider_interface):
"""Test required credentials."""
required_credentials = provider_interface.required_credentials
assert isinstance(required_credentials, list)
assert len(required_credentials) > 0
credentials = provider_interface.credentials
assert isinstance(credentials, list)
assert len(credentials) > 0
def test_model_providers(provider_interface):

View File

@@ -29,6 +29,9 @@ class classproperty:
class Fetcher(Generic[Q, R]):
"""Abstract class for the fetcher."""
# Tell query executor if credentials are required. Can be overridden by subclasses.
require_credentials = True
@staticmethod
def transform_query(params: Dict[str, Any]) -> Q:
"""Transform the params to the provider-specific query."""
@@ -108,6 +111,9 @@ class Fetcher(Generic[Q, R]):
data = cls.extract_data(query=query, credentials=credentials, **kwargs)
transformed_data = cls.transform_data(query=query, data=data, **kwargs)
# Class Assertions
assert isinstance(cls.require_credentials, bool)
# Query Assertions
assert query
assert issubclass(type(query), cls.query_params_type)

View File

@@ -13,7 +13,7 @@ class Provider:
name: str,
description: str,
website: Optional[str] = None,
required_credentials: Optional[List[str]] = None,
credentials: Optional[List[str]] = None,
fetcher_dict: Optional[Dict[str, Type[Fetcher]]] = None,
) -> None:
"""Initialize the provider.
@@ -26,7 +26,7 @@ class Provider:
Description of the provider.
website : Optional[str]
Website of the provider, by default None.
required_credentials : Optional[List[str]], optional
credentials : Optional[List[str]], optional
List of required credentials, by default None
fetcher_dict : Optional[Dict[str, Type[Fetcher]]]
Dictionary of fetchers, by default None.
@@ -35,9 +35,9 @@ class Provider:
self.description = description
self.website = website
self.fetcher_dict = fetcher_dict or {}
if required_credentials is None:
self.required_credentials: List = []
if credentials is None:
self.credentials: List = []
else:
self.required_credentials = []
for rq in required_credentials:
self.required_credentials.append(f"{self.name.lower()}_{rq}")
self.credentials = []
for c in credentials:
self.credentials.append(f"{self.name.lower()}_{c}")

View File

@@ -36,19 +36,24 @@ class QueryExecutor:
@staticmethod
def filter_credentials(
provider: Provider, credentials: Optional[Dict[str, SecretStr]]
credentials: Optional[Dict[str, SecretStr]],
provider: Provider,
require_credentials: bool,
) -> Dict[str, str]:
"""Filter credentials and check if they match provider requirements."""
if provider.required_credentials is not None:
filtered_credentials = {}
if provider.credentials:
if credentials is None:
credentials = {}
filtered_credentials = {}
for c in provider.required_credentials:
for c in provider.credentials:
credential_value = credentials.get(c)
if c not in credentials or credential_value is None:
raise ProviderError(f"Missing credential '{c}'.")
filtered_credentials[c] = credential_value.get_secret_value()
if require_credentials:
raise ProviderError(f"Missing credential '{c}'.")
else:
filtered_credentials[c] = credential_value.get_secret_value()
return filtered_credentials
@@ -80,8 +85,10 @@ class QueryExecutor:
Query result.
"""
provider = self.get_provider(provider_name)
filtered_credentials = self.filter_credentials(provider, credentials)
fetcher = self.get_fetcher(provider, model_name)
filtered_credentials = self.filter_credentials(
credentials, provider, fetcher.require_credentials
)
try:
return fetcher.fetch_data(params, filtered_credentials, **kwargs)

View File

@@ -20,7 +20,7 @@ class RegistryMap:
def __init__(self, registry: Optional[Registry] = None) -> None:
"""Initialize Registry Map."""
self._registry = registry or RegistryLoader.from_extensions()
self._required_credentials = self._get_required_credentials(self._registry)
self._credentials = self._get_credentials(self._registry)
self._available_providers = self._get_available_providers(self._registry)
self._map, self._return_map = self._get_map(self._registry)
self._models = self._get_models(self._map)
@@ -36,9 +36,9 @@ class RegistryMap:
return self._available_providers
@property
def required_credentials(self) -> List[str]:
def credentials(self) -> List[str]:
"""Get list of required credentials."""
return self._required_credentials
return self._credentials
@property
def map(self) -> MapType:
@@ -55,11 +55,11 @@ class RegistryMap:
"""Get available models."""
return self._models
def _get_required_credentials(self, registry: Registry) -> List[str]:
def _get_credentials(self, registry: Registry) -> List[str]:
"""Get list of required credentials."""
cred_list = []
for provider in registry.providers.values():
for c in provider.required_credentials:
for c in provider.credentials:
cred_list.append(c)
return cred_list

View File

@@ -10,7 +10,7 @@ def test_provider_initialization():
assert provider.name == "TestProvider"
assert provider.description == "A simple test provider."
assert provider.website is None
assert provider.required_credentials == []
assert provider.credentials == []
assert provider.fetcher_dict == {}
@@ -20,25 +20,25 @@ def test_provider_with_optional_parameters():
name="TestProvider",
description="A simple test provider.",
website="https://testprovider.example.com",
required_credentials=["api_key"],
credentials=["api_key"],
fetcher_dict={"fetcher1": None},
)
assert provider.name == "TestProvider"
assert provider.description == "A simple test provider."
assert provider.website == "https://testprovider.example.com"
assert provider.required_credentials == ["testprovider_api_key"]
assert provider.credentials == ["testprovider_api_key"]
assert provider.fetcher_dict == {"fetcher1": None}
def test_provider_required_credentials_formatting():
def test_provider_credentials_formatting():
"""Test the formatting of required credentials."""
required_credentials = ["key1", "key2"]
credentials = ["key1", "key2"]
provider = Provider(
name="TestProvider",
description="A simple test provider.",
required_credentials=required_credentials,
credentials=credentials,
)
expected_credentials = ["testprovider_key1", "testprovider_key2"]
assert provider.required_credentials == expected_credentials
assert provider.credentials == expected_credentials

View File

@@ -55,21 +55,40 @@ def test_get_fetcher_failure(mock_query_executor):
def test_filter_credentials_success(mock_query_executor):
"""Test if credentials are properly filtered."""
provider = mock_query_executor.get_provider("test_provider")
provider.required_credentials = ["api_key"]
credentials = {"api_key": SecretStr("12345")}
provider.credentials = ["test_provider_api_key"]
credentials = {
"test_provider_api_key": SecretStr("12345"),
"other_api_key": SecretStr("12345"),
}
filtered_credentials = mock_query_executor.filter_credentials(provider, credentials)
filtered_credentials = mock_query_executor.filter_credentials(
credentials, provider, True
)
assert filtered_credentials == {"api_key": "12345"}
assert filtered_credentials == {"test_provider_api_key": "12345"}
def test_filter_credentials_missing(mock_query_executor):
def test_filter_credentials_missing_require(mock_query_executor):
"""Test if the proper error is raised when a credential is missing."""
provider = mock_query_executor.get_provider("test_provider")
provider.required_credentials = ["api_key"]
provider.credentials = ["test_provider_api_key"]
credentials = {"other_api_key": SecretStr("12345")}
with pytest.raises(ProviderError, match="Missing credential"):
mock_query_executor.filter_credentials(provider, {})
mock_query_executor.filter_credentials(credentials, provider, True)
def test_filter_credentials_missing_dont_require(mock_query_executor):
"""Test if the proper error is raised when a credential is missing."""
provider = mock_query_executor.get_provider("test_provider")
provider.credentials = ["test_provider_api_key"]
credentials = {"other_api_key": SecretStr("12345")}
filtered_credentials = mock_query_executor.filter_credentials(
credentials, provider, False
)
assert filtered_credentials == {}
def test_execute_success(mock_query_executor):

View File

@@ -12,9 +12,9 @@ def load_registry_map():
return registry_map
def test_get_required_credentials(load_registry_map):
"""Test if the _get_required_credentials method behaves as expected."""
required_creds = load_registry_map.required_credentials
def test_get_credentials(load_registry_map):
"""Test if the _get_credentials method behaves as expected."""
required_creds = load_registry_map.credentials
assert "fmp_api_key" in required_creds

View File

@@ -12,7 +12,7 @@ alpha_vantage_provider = Provider(
from fundamental data to technical indicators, Alpha Vantage
is your one-stop-shop for enterprise-grade global market data delivered through
cloud-based APIs, Excel, and Google Sheets. """,
required_credentials=["api_key"],
credentials=["api_key"],
fetcher_dict={
"EquityHistorical": AVEquityHistoricalFetcher,
},

View File

@@ -8,7 +8,7 @@ benzinga_provider = Provider(
website="https://www.benzinga.com/",
description="""Benzinga is a financial data provider that offers an API
focused on information that moves the market.""",
required_credentials=["api_key"],
credentials=["api_key"],
fetcher_dict={
"CompanyNews": BenzingaCompanyNewsFetcher,
"GlobalNews": BenzingaGlobalNewsFetcher,

View File

@@ -15,7 +15,7 @@ biztoc_provider = Provider(
If you're not a developer but would still like to use Biztoc outside of the main website,
we've partnered with OpenBB, allowing you to pull in BizToc's news stream in their Terminal.""",
required_credentials=["api_key"],
credentials=["api_key"],
fetcher_dict={
"GlobalNews": BiztocGlobalNewsFetcher,
},

View File

@@ -26,7 +26,7 @@ cboe_provider = Provider(
description="""Cboe is the world's go-to derivatives and exchange network,
delivering cutting-edge trading, clearing and investment solutions to people
around the world.""",
required_credentials=None,
credentials=None,
fetcher_dict={
"EquitySearch": CboeEquitySearchFetcher,
"OptionsChains": CboeOptionsChainsFetcher,

View File

@@ -7,7 +7,7 @@ finra_provider = Provider(
name="finra",
website="https://finra.org",
description="Financial Industry Regulatory Authority.",
required_credentials=None,
credentials=None,
fetcher_dict={
"OTCAggregate": FinraOTCAggregateFetcher,
"EquityShortInterest": FinraShortInterestFetcher,

View File

@@ -66,7 +66,7 @@ fmp_provider = Provider(
website="https://financialmodelingprep.com/",
description="""Financial Modeling Prep is a new concept that informs you about
stock market information (news, currencies, and stock prices).""",
required_credentials=["api_key"],
credentials=["api_key"],
fetcher_dict={
"AnalystEstimates": FMPAnalystEstimatesFetcher,
"AvailableIndices": FMPAvailableIndicesFetcher,

View File

@@ -28,7 +28,7 @@ fred_provider = Provider(
description="""Federal Reserve Economic Data is a database maintained by the
Research division of the Federal Reserve Bank of St. Louis that has more than
816,000 economic time series from various sources.""",
required_credentials=["api_key"],
credentials=["api_key"],
fetcher_dict={
"ConsumerPriceIndex": FREDConsumerPriceIndexFetcher,
"USYieldCurve": FREDYieldCurveFetcher,

View File

@@ -25,7 +25,7 @@ intrinio_provider = Provider(
website="https://intrinio.com/",
description="""Intrinio is a financial data platform that provides real-time and
historical financial market data to businesses and developers through an API.""",
required_credentials=["api_key"],
credentials=["api_key"],
fetcher_dict={
"BalanceSheet": IntrinioBalanceSheetFetcher,
"CalendarIpo": IntrinioCalendarIpoFetcher,

View File

@@ -15,7 +15,7 @@ nasdaq_provider = Provider(
description="""Positioned at the nexus of technology and the capital markets, Nasdaq
provides premier platforms and services for global capital markets and beyond with
unmatched technology, insights and markets expertise.""",
required_credentials=["api_key"],
credentials=["api_key"],
fetcher_dict={
"CalendarDividend": NasdaqDividendCalendarFetcher,
"CalendarEarnings": NasdaqCalendarEarningsFetcher,

View File

@@ -60,6 +60,8 @@ class NasdaqDividendCalendarFetcher(
):
"""Transform the query, extract and transform the data from the Nasdaq endpoints."""
require_credentials = False
@staticmethod
def transform_query(params: Dict[str, Any]) -> NasdaqDividendCalendarQueryParams:
"""Transform the query params."""

View File

@@ -116,6 +116,8 @@ class NasdaqCalendarIpoFetcher(
):
"""Transform the query, extract and transform the data from the Nasdaq endpoints."""
require_credentials = False
@staticmethod
def transform_query(params: Dict[str, Any]) -> NasdaqCalendarIpoQueryParams:
"""Transform the query params."""

View File

@@ -22,9 +22,13 @@ class NasdaqCotSearchData(CotSearchData):
"""Nasdaq CFTC Commitment of Traders Reports Search data."""
class NasdaqCotSearchFetcher(Fetcher[CotSearchQueryParams, List[NasdaqCotSearchData]]):
class NasdaqCotSearchFetcher(
Fetcher[NasdaqCotSearchQueryParams, List[NasdaqCotSearchData]]
):
"""Nasdaq CFTC Commitment of Traders Reports Search Fetcher."""
require_credentials = False
@staticmethod
def transform_query(params: Dict[str, Any]) -> NasdaqCotSearchQueryParams:
return NasdaqCotSearchQueryParams(**params)

View File

@@ -76,6 +76,8 @@ class NasdaqEconomicCalendarFetcher(
):
"""Transform the query, extract and transform the data from the Nasdaq endpoints."""
require_credentials = False
@staticmethod
def transform_query(params: Dict[str, Any]) -> NasdaqEconomicCalendarQueryParams:
"""Transform the query params."""

View File

@@ -28,7 +28,9 @@ class NasdaqTopRetailData(TopRetailData):
return datetime.strptime(v, "%Y-%m-%d").date()
class NasdaqTopRetailFetcher(Fetcher[TopRetailQueryParams, List[NasdaqTopRetailData]]):
class NasdaqTopRetailFetcher(
Fetcher[NasdaqTopRetailQueryParams, List[NasdaqTopRetailData]]
):
"""Nasdaq Top Retail Fetcher."""
@staticmethod

View File

@@ -20,7 +20,7 @@ polygon_provider = Provider(
description="""The Polygon.io Stocks API provides REST endpoints that let you query
the latest market data from all US stock exchanges. You can also find data on
company financials, stock market holidays, corporate actions, and more.""",
required_credentials=["api_key"],
credentials=["api_key"],
fetcher_dict={
"BalanceSheet": PolygonBalanceSheetFetcher,
"CashFlowStatement": PolygonCashFlowStatementFetcher,

View File

@@ -14,7 +14,7 @@ sec_provider = Provider(
name="sec",
website="https://sec.gov",
description="SEC is the public listings regulatory body for the United States.",
required_credentials=None,
credentials=None,
fetcher_dict={
"CikMap": SecCikMapFetcher,
"CompanyFilings": SecCompanyFilingsFetcher,

View File

@@ -14,7 +14,7 @@ tiingo_provider = Provider(
name="tiingo",
website="https://tiingo.com/",
description="""""",
required_credentials=["token"],
credentials=["token"],
fetcher_dict={
"EquityHistorical": TiingoEquityHistoricalFetcher,
"CompanyNews": TiingoCompanyNewsFetcher,

View File

@@ -6,6 +6,6 @@ tradingeconomics_provider = Provider(
name="tradingeconomics",
website="https://tradingeconomics.com/",
description="""Trading Economics""",
required_credentials=["api_key"],
credentials=["api_key"],
fetcher_dict={"EconomicCalendar": TEEconomicCalendarFetcher},
)

View File

@@ -6,7 +6,7 @@ ultima_provider = Provider(
name="ultima",
website="https://www.ultimainsights.ai/openbb",
description="""Ultima harnesses the power of LLMs to deliver news before it hits the frontpage of Bloomberg.""",
required_credentials=["api_key"],
credentials=["api_key"],
fetcher_dict={
"CompanyNews": UltimaCompanyNewsFetcher,
},

View File

@@ -133,13 +133,13 @@ from openbb_<provider_name>.models.equity_historical import <ProviderName>Equity
name="<provider_name>",
website="<URL to the provider website>",
description="Provider description goes here",
required_credentials=["api_key"],
credentials=["api_key"],
fetcher_dict={
"EquityHistorical": <ProviderName>EquityHistoricalFetcher,
},
)
```
If the provider does not require any credentials, you can remove that parameter. On the other hand, if it requires more than 2 items to authenticate, you can add a list of all the required items to the `required_credentials` list.
If the provider does not require any credentials, you can remove that parameter. On the other hand, if it requires more than 2 items to authenticate, you can add a list of all the required items to the `credentials` list.
After running `pip install .` on `openbb_platform/providers/<provider_name>` your provider should be ready for usage, both from the Python interface and the API.