jerry/OpenBB
1
0
Fork 0
mirror of https://github.com/OpenBB-finance/OpenBB.git synced 2026-11-05 00:42:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

[BugFix] Fix World News (#6064)

Browse Source 
* fix fmp

* fix intrinio

* fix biztoc

* actually commit intrinio fix

* fix tiingo

* unused import

* fix tests

* black

* ruff

* benzinga empty strings

* codespell

* fix test?

* more empty strings

---------

Co-authored-by: Igor Radovanovic <74266147+IgorWounds@users.noreply.github.com>
This commit is contained in:
Danglewood
2024-02-12 05:23:30 -08:00
committed by GitHub
parent ed0446e038
commit d31eec2354
12 changed files with 80 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
openbb_platform/core/openbb_core/provider/standard_models/world_news.py
View File

@@ -21,9 +21,9 @@ class WorldNewsQueryParams(QueryParams):
"""World News Query."""
limit: NonNegativeInt = Field(
default=20,
default=2500,
description=QUERY_DESCRIPTIONS.get("limit", "")
+ " Here its the no. of articles to return.",
+ " The number of articles to return.",
)
start_date: Optional[dateType] = Field(
default=None, description=QUERY_DESCRIPTIONS.get("start_date", "")
@@ -38,7 +38,7 @@ class WorldNewsQueryParams(QueryParams):
"""Populate start date if empty."""
if not v:
now = datetime.now().date()
v = now - relativedelta(years=1)
v = now - relativedelta(weeks=2)
return v
@field_validator("end_date", mode="before")
@@ -55,11 +55,11 @@ class WorldNewsData(Data):
date: datetime = Field(
description=DATA_DESCRIPTIONS.get("date", "")
+ " Here it is the published date of the news."
+ " The published date of the article."
)
title: str = Field(description="Title of the news.")
title: str = Field(description="Title of the article.")
images: Optional[List[Dict[str, str]]] = Field(
default=None, description="Images associated with the news."
default=None, description="Images associated with the article."
)
text: Optional[str] = Field(default=None, description="Text/body of the news.")
url: Optional[str] = Field(default=None, description="URL of the news.")
text: Optional[str] = Field(default=None, description="Text/body of the article.")
url: Optional[str] = Field(default=None, description="URL to the article.")

1
openbb_platform/extensions/news/integration/test_news_api.py
View File

@@ -76,6 +76,7 @@ def headers():
"source": "bloomberg.com",
"start_date": None,
"end_date": None,
"offset": 0,
}
),
],

1
openbb_platform/extensions/news/integration/test_news_python.py
View File

@@ -75,6 +75,7 @@ def obb(pytestconfig): # pylint: disable=inconsistent-return-statements
"source": "bloomberg.com",
"start_date": None,
"end_date": None,
"offset": 0,
}
),
],

35
openbb_platform/providers/benzinga/openbb_benzinga/models/world_news.py
View File

@@ -1,5 +1,7 @@
"""Benzinga World News Model."""
# pylint: disable=unused-argument
import math
from datetime import (
date as dateType,
@@ -97,21 +99,31 @@ class BenzingaWorldNewsData(WorldNewsData):
)
@field_validator("date", "updated", mode="before", check_fields=False)
def date_validate(cls, v): # pylint: disable=E0213
@classmethod
def date_validate(cls, v):
"""Return the date as a datetime object."""
return datetime.strptime(v, "%a, %d %b %Y %H:%M:%S %z")
@field_validator("stocks", "channels", "tags", mode="before", check_fields=False)
def list_validate(cls, v): # pylint: disable=E0213
@classmethod
def list_validate(cls, v):
"""Return the list as a string."""
return ",".join(
[item.get("name", None) for item in v if item.get("name", None)]
)
v = ",".join([item.get("name", None) for item in v if item.get("name", None)])
return v if v != "" else None
@field_validator("id", mode="before", check_fields=False)
def id_validate(cls, v): # pylint: disable=E0213
"""Return the id as a string."""
return str(v)
@field_validator(
"id", "text", "teaser", "title", "author", mode="before", check_fields=False
)
@classmethod
def id_validate(cls, v):
"""Return the a string if the field is not empty."""
return str(v) if v else None
@field_validator("images", mode="before", check_fields=False)
@classmethod
def empty_list(cls, v):
"""Return None instead of []"""
return None if v == [] else v
class BenzingaWorldNewsFetcher(
@@ -132,7 +144,7 @@ class BenzingaWorldNewsFetcher(
query: BenzingaWorldNewsQueryParams,
credentials: Optional[Dict[str, str]],
**kwargs: Any,
) -> List[dict]:
) -> List[Dict]:
"""Extract the data."""
token = credentials.get("benzinga_api_key") if credentials else ""
base_url = "https://api.benzinga.com/api/v2/news"
@@ -151,11 +163,10 @@ class BenzingaWorldNewsFetcher(
return data[: query.limit]
# pylint: disable=unused-argument
@staticmethod
def transform_data(
query: BenzingaWorldNewsQueryParams,
data: List[dict],
data: List[Dict],
**kwargs: Any,
) -> List[BenzingaWorldNewsData]:
"""Transform the data."""

2
openbb_platform/providers/benzinga/tests/test_benzinga_fetchers.py
View File

@@ -21,7 +21,7 @@ def vcr_config():
@pytest.mark.record_http
def test_benzinga_world_news_fetcher(credentials=test_credentials):
params = {}
params = {"limit": 20}
fetcher = BenzingaWorldNewsFetcher()
result = fetcher.test(params, credentials)

18
openbb_platform/providers/biztoc/openbb_biztoc/models/world_news.py
View File

@@ -1,5 +1,6 @@
"""Biztoc World News Model."""
import warnings
from datetime import datetime, timedelta
from typing import Any, Dict, List, Literal, Optional
@@ -9,10 +10,11 @@ from openbb_core.provider.standard_models.world_news import (
WorldNewsData,
WorldNewsQueryParams,
)
from openbb_core.provider.utils.helpers import filter_by_dates
from pandas import to_datetime
from pydantic import Field, field_validator
_warn = warnings.warn
class BiztocWorldNewsQueryParams(WorldNewsQueryParams):
"""Biztoc World News Query."""
@@ -37,7 +39,12 @@ class BiztocWorldNewsQueryParams(WorldNewsQueryParams):
class BiztocWorldNewsData(WorldNewsData):
"""Biztoc World News Data."""
__alias_dict__ = {"date": "created", "text": "body", "site": "domain"}
__alias_dict__ = {
"date": "created",
"text": "body",
"site": "domain",
"images": "img",
}
images: Optional[Dict[str, str]] = Field(
description="Images for the article.", alias="images", default=None
@@ -69,6 +76,8 @@ class BiztocWorldNewsFetcher(
@staticmethod
def transform_query(params: Dict[str, Any]) -> BiztocWorldNewsQueryParams:
"""Transform the query."""
if params.get("start_date") or params.get("end_date"):
_warn("start_date and end_date are not supported for this endpoint.")
return BiztocWorldNewsQueryParams(**params)
@staticmethod
@@ -96,7 +105,7 @@ class BiztocWorldNewsFetcher(
hours=times[item["created"]]
)
return data
return sorted(data, key=lambda x: x["created"], reverse=True)
# pylint: disable=unused-argument
@staticmethod
@@ -104,5 +113,4 @@ class BiztocWorldNewsFetcher(
query: BiztocWorldNewsQueryParams, data: List[Dict], **kwargs: Any
) -> List[BiztocWorldNewsData]:
"""Transform the data to the standard format."""
modeled_data = [BiztocWorldNewsData.model_validate(d) for d in data]
return filter_by_dates(modeled_data, query.start_date, query.end_date)
return [BiztocWorldNewsData.model_validate(d) for d in data[: query.limit]]

22
openbb_platform/providers/fmp/openbb_fmp/models/world_news.py
View File

@@ -1,6 +1,7 @@
"""FMP World News Model."""
import math
import warnings
from datetime import datetime
from typing import Any, Dict, List, Optional
@@ -9,9 +10,11 @@ from openbb_core.provider.standard_models.world_news import (
WorldNewsData,
WorldNewsQueryParams,
)
from openbb_core.provider.utils.helpers import amake_requests, filter_by_dates
from openbb_core.provider.utils.helpers import amake_requests
from pydantic import Field, field_validator
_warn = warnings.warn
class FMPWorldNewsQueryParams(WorldNewsQueryParams):
"""FMP World News Query.
@@ -32,6 +35,13 @@ class FMPWorldNewsData(WorldNewsData):
"""Return the date as a datetime object."""
return datetime.strptime(v, "%Y-%m-%dT%H:%M:%S.%fZ")
@field_validator("images", mode="before", check_fields=False)
def images_validate(cls, v): # pylint: disable=E0213
"""Conform the response to a list."""
if isinstance(v, str):
return [{"o": v}]
return v if isinstance(v, List[Dict]) else None
class FMPWorldNewsFetcher(
Fetcher[
@@ -44,6 +54,8 @@ class FMPWorldNewsFetcher(
@staticmethod
def transform_query(params: Dict[str, Any]) -> FMPWorldNewsQueryParams:
"""Transform the query params."""
if params.get("start_date") or params.get("end_date"):
_warn("start_date and end_date are not supported for this endpoint.")
return FMPWorldNewsQueryParams(**params)
@staticmethod
@@ -62,7 +74,6 @@ class FMPWorldNewsFetcher(
f"{base_url}/general_news?page={page}&apikey={api_key}"
for page in range(pages)
]
response = await amake_requests(urls, **kwargs)
data = sorted(response, key=lambda x: x["publishedDate"], reverse=True)
@@ -74,9 +85,4 @@ class FMPWorldNewsFetcher(
query: FMPWorldNewsQueryParams, data: List[Dict], **kwargs: Any
) -> List[FMPWorldNewsData]:
"""Return the transformed data."""
for d in data:
if isinstance(d["image"], str):
d["image"] = [{"url": d["image"]}]
modeled_data = [FMPWorldNewsData.model_validate(d) for d in data]
return filter_by_dates(modeled_data, query.start_date, query.end_date)
return [FMPWorldNewsData.model_validate(d) for d in data]

9
openbb_platform/providers/intrinio/openbb_intrinio/models/world_news.py
View File

@@ -1,5 +1,6 @@
"""Intrinio World News Model."""
import warnings
from datetime import datetime
from typing import Any, Dict, List, Optional
@@ -8,10 +9,11 @@ from openbb_core.provider.standard_models.world_news import (
WorldNewsData,
WorldNewsQueryParams,
)
from openbb_core.provider.utils.helpers import filter_by_dates
from openbb_intrinio.utils.helpers import get_data_many
from pydantic import Field, field_validator
_warn = warnings.warn
class IntrinioWorldNewsQueryParams(WorldNewsQueryParams):
"""Intrinio World News Query.
@@ -47,6 +49,8 @@ class IntrinioWorldNewsFetcher(
@staticmethod
def transform_query(params: Dict[str, Any]) -> IntrinioWorldNewsQueryParams:
"""Transform the query params."""
if params.get("start_date") or params.get("end_date"):
_warn("start_date and end_date are not supported for this endpoint.")
return IntrinioWorldNewsQueryParams(**params)
@staticmethod
@@ -69,5 +73,4 @@ class IntrinioWorldNewsFetcher(
query: IntrinioWorldNewsQueryParams, data: List[Dict], **kwargs: Any
) -> List[IntrinioWorldNewsData]:
"""Return the transformed data."""
modeled_data = [IntrinioWorldNewsData.model_validate(d) for d in data]
return filter_by_dates(modeled_data, query.start_date, query.end_date)
return [IntrinioWorldNewsData.model_validate(d) for d in data]

2
openbb_platform/providers/intrinio/tests/record/http/test_intrinio_fetchers/test_intrinio_world_news_fetcher.yaml vendored
View File

@@ -9,7 +9,7 @@ interactions:
Connection:
- keep-alive
method: GET
uri: https://api-v2.intrinio.com/companies/news?api_key=MOCK_API_KEY&page_size=20
uri: https://api-v2.intrinio.com/companies/news?page_size=2500&api_key=MOCK_API_KEY
response:
body:
string: !!binary |

15
openbb_platform/providers/tiingo/openbb_tiingo/models/world_news.py
View File

@@ -9,11 +9,10 @@ from openbb_core.provider.standard_models.world_news import (
WorldNewsData,
WorldNewsQueryParams,
)
from openbb_core.provider.utils.helpers import filter_by_dates, get_querystring
from openbb_core.provider.utils.helpers import get_querystring
from openbb_tiingo.utils.helpers import get_data_many
from pydantic import Field, field_validator
from ..utils.helpers import get_data_many
class TiingoWorldNewsQueryParams(WorldNewsQueryParams):
"""Tiingo World News Query.
@@ -21,6 +20,13 @@ class TiingoWorldNewsQueryParams(WorldNewsQueryParams):
Source: https://www.tiingo.com/documentation/news
"""
__alias_dict__ = {
"start_date": "startDate",
"end_date": "endDate",
}
offset: Optional[int] = Field(
default=0, description="Page offset, used in conjunction with limit."
)
source: Optional[str] = Field(
default=None, description="A comma-separated list of the domains requested."
)
@@ -95,5 +101,4 @@ class TiingoWorldNewsFetcher(
query: TiingoWorldNewsQueryParams, data: List[Dict], **kwargs: Any
) -> List[TiingoWorldNewsData]:
"""Return the transformed data."""
modeled_data = [TiingoWorldNewsData.model_validate(d) for d in data]
return filter_by_dates(modeled_data, query.start_date, query.end_date)
return [TiingoWorldNewsData.model_validate(d) for d in data]

2
openbb_platform/providers/tiingo/tests/record/http/test_tiingo_fetchers/test_tiingo_world_news_fetcher.yaml vendored
View File

@@ -9,7 +9,7 @@ interactions:
Connection:
- keep-alive
method: GET
uri: https://api.tiingo.com/tiingo/news?limit=20&token=MOCK_TOKEN
uri: https://api.tiingo.com/tiingo/news?limit=20&offset=0&token=MOCK_TOKEN
response:
body:
string: "[{\"id\":59591461,\"publishedDate\":\"2023-11-16T08:38:56Z\",\"title\":\"How

2
openbb_platform/providers/tiingo/tests/test_tiingo_fetchers.py
View File

@@ -48,7 +48,7 @@ def test_tiingo_company_news_fetcher(credentials=test_credentials):
@pytest.mark.record_http
def test_tiingo_world_news_fetcher(credentials=test_credentials):
params = {}
params = {"limit": 20}
fetcher = TiingoWorldNewsFetcher()
result = fetcher.test(params, credentials)