From f7196e86617683c4a42cc59ed124262eaabc40ff Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Fri, 6 Sep 2024 15:29:23 -0700 Subject: [PATCH] [BugFix] Fix Tests & Related Bugs (#6657) * fix tests and related bugs * docstring * add trust_env to ClientSession initializations * pylint * other pylint --- .../equity/integration/test_equity_api.py | 6 +-- .../equity/integration/test_equity_python.py | 6 +-- .../integration/test_regulators_api.py | 4 +- .../integration/test_regulators_python.py | 4 +- .../charting/openbb_charting/charting.py | 28 +++++++++++ .../providers/bls/tests/test_bls_fetchers.py | 2 +- .../test_cftc_cot_fetcher_urllib3_v1.yaml | 17 +++---- .../test_cftc_cot_fetcher_urllib3_v2.yaml | 17 +++---- .../cftc/tests/test_cftc_fetchers.py | 15 ++++++ .../openbb_intrinio/models/calendar_ipo.py | 5 ++ .../models/calendar_earnings.py | 47 +++++++++++-------- .../models/forward_eps_estimates.py | 32 +++++++++---- .../models/forward_sales_estimates.py | 31 ++++++++---- .../openbb_seeking_alpha/utils/helpers.py | 17 ++++--- 14 files changed, 158 insertions(+), 73 deletions(-) diff --git a/openbb_platform/extensions/equity/integration/test_equity_api.py b/openbb_platform/extensions/equity/integration/test_equity_api.py index a90d83a1f0b..652b9495eef 100644 --- a/openbb_platform/extensions/equity/integration/test_equity_api.py +++ b/openbb_platform/extensions/equity/integration/test_equity_api.py @@ -631,7 +631,7 @@ def test_equity_ownership_institutional(params, headers): [ ( { - "symbol": "", + "symbol": None, "start_date": "2023-01-01", "end_date": "2023-06-06", "limit": 100, @@ -1131,8 +1131,8 @@ def test_equity_compare_groups(params, headers): "adjustment": "splits_only", "provider": "yfinance", "symbol": "AAPL", - "start_date": "2023-06-01", - "end_date": "2023-06-03", + "start_date": None, + "end_date": None, "interval": "1h", } ), diff --git a/openbb_platform/extensions/equity/integration/test_equity_python.py b/openbb_platform/extensions/equity/integration/test_equity_python.py index d6b079e10a9..7e361d424b6 100644 --- a/openbb_platform/extensions/equity/integration/test_equity_python.py +++ b/openbb_platform/extensions/equity/integration/test_equity_python.py @@ -514,7 +514,7 @@ def test_equity_ownership_institutional(params, obb): [ ( { - "symbol": "", + "symbol": None, "start_date": "2023-01-01", "end_date": "2023-06-06", "limit": 100, @@ -1078,8 +1078,8 @@ def test_equity_compare_groups(params, obb): "adjustment": "splits_and_dividends", "provider": "yfinance", "symbol": "AAPL", - "start_date": "2023-06-01", - "end_date": "2023-06-03", + "start_date": None, + "end_date": None, "interval": "1h", } ), diff --git a/openbb_platform/extensions/regulators/integration/test_regulators_api.py b/openbb_platform/extensions/regulators/integration/test_regulators_api.py index 34e32cb0f72..c062c3b7aa8 100644 --- a/openbb_platform/extensions/regulators/integration/test_regulators_api.py +++ b/openbb_platform/extensions/regulators/integration/test_regulators_api.py @@ -160,7 +160,7 @@ def test_regulators_cftc_cot_search(params, headers): [ ( { - "id": "13874P", + "id": "045601", "data_type": "FO", "legacy_format": True, "report_type": "ALL", @@ -174,7 +174,7 @@ def test_regulators_cftc_cot_search(params, headers): ), ( { - "id": "13874P", + "id": "045601", "report_type": "legacy", "start_date": "2023-01-01", "end_date": "2023-06-06", diff --git a/openbb_platform/extensions/regulators/integration/test_regulators_python.py b/openbb_platform/extensions/regulators/integration/test_regulators_python.py index 70c00d54557..b5da950f16b 100644 --- a/openbb_platform/extensions/regulators/integration/test_regulators_python.py +++ b/openbb_platform/extensions/regulators/integration/test_regulators_python.py @@ -145,7 +145,7 @@ def test_regulators_cftc_cot_search(params, obb): [ ( { - "id": "13874P", + "id": "045601", "data_type": "FO", "legacy_format": True, "report_type": "ALL", @@ -159,7 +159,7 @@ def test_regulators_cftc_cot_search(params, obb): ), ( { - "id": "13874P", + "id": "045601", "report_type": "legacy", "start_date": "2023-01-01", "end_date": "2023-06-06", diff --git a/openbb_platform/obbject_extensions/charting/openbb_charting/charting.py b/openbb_platform/obbject_extensions/charting/openbb_charting/charting.py index 4888e99b0f9..edf6f2caf92 100644 --- a/openbb_platform/obbject_extensions/charting/openbb_charting/charting.py +++ b/openbb_platform/obbject_extensions/charting/openbb_charting/charting.py @@ -546,6 +546,32 @@ class Charting: external=True ).to_plotly_json() + @staticmethod + def _convert_to_string(x): + """Sanitize the data for the table.""" + # pylint: disable=import-outside-toplevel + from numpy import isnan + + if isinstance(x, (float, int)) and not isnan(x): + return x + if isinstance(x, dict): + return ", ".join([str(v) for v in x.values()]) + if isinstance(x, list): + if all(isinstance(i, dict) for i in x): + return ", ".join( + str(", ".join([str(v) for v in i.values()])) for i in x + ) + return ", ".join([str(i) for i in x]) + + return ( + str(x) + .replace("[", "") + .replace("]", "") + .replace("'{", "") + .replace("}'", "") + .replace("nan", "") + ) + def table( self, data: Optional[Union["DataFrame", "Series"]] = None, @@ -569,6 +595,8 @@ class Charting: data_as_df.reset_index(inplace=True, drop=True) else: data_as_df.reset_index(inplace=True) + for col in data_as_df.columns: + data_as_df[col] = data_as_df[col].apply(self._convert_to_string) if self._backend.isatty: try: self._backend.send_table( diff --git a/openbb_platform/providers/bls/tests/test_bls_fetchers.py b/openbb_platform/providers/bls/tests/test_bls_fetchers.py index 334cee2d4d7..c208ff108cf 100644 --- a/openbb_platform/providers/bls/tests/test_bls_fetchers.py +++ b/openbb_platform/providers/bls/tests/test_bls_fetchers.py @@ -3,8 +3,8 @@ from datetime import date import pytest -from openbb_bls.models.series import BlsSeriesFetcher from openbb_bls.models.search import BlsSearchFetcher +from openbb_bls.models.series import BlsSeriesFetcher from openbb_core.app.service.user_service import UserService test_credentials = UserService().default_user_settings.credentials.model_dump( diff --git a/openbb_platform/providers/cftc/tests/record/http/test_cftc_fetchers/test_cftc_cot_fetcher_urllib3_v1.yaml b/openbb_platform/providers/cftc/tests/record/http/test_cftc_fetchers/test_cftc_cot_fetcher_urllib3_v1.yaml index e140d0b236c..24e5e174851 100644 --- a/openbb_platform/providers/cftc/tests/record/http/test_cftc_fetchers/test_cftc_cot_fetcher_urllib3_v1.yaml +++ b/openbb_platform/providers/cftc/tests/record/http/test_cftc_fetchers/test_cftc_cot_fetcher_urllib3_v1.yaml @@ -43,11 +43,10 @@ interactions: Content-Type: - application/json;charset=utf-8 Date: - - Tue, 27 Aug 2024 06:28:40 GMT - Etag: - - '"aG90ZWwuNTEzMDVfM18yODJtVzlSN0o5WVlrTFhDbnYyakxHb3Z1NTJCcHc---gzipn8hfNknVynA7XV4li5cBTkJ7u_Q--gzip--gzip"' + - Thu, 05 Sep 2024 18:51:48 GMT + Etag: null Last-Modified: - - Fri, 23 Aug 2024 19:30:46 GMT + - Fri, 30 Aug 2024 19:30:46 GMT Server: - nginx Strict-Transport-Security: @@ -61,15 +60,13 @@ interactions: X-SODA2-Fields: - '["id","market_and_exchange_names","report_date_as_yyyy_mm_dd","yyyy_report_week_ww","contract_market_name","cftc_contract_market_code","cftc_market_code","cftc_region_code","cftc_commodity_code","commodity_name","open_interest_all","noncomm_positions_long_all","noncomm_positions_short_all","noncomm_postions_spread_all","comm_positions_long_all","comm_positions_short_all","tot_rept_positions_long_all","tot_rept_positions_short","nonrept_positions_long_all","nonrept_positions_short_all","open_interest_old","noncomm_positions_long_old","noncomm_positions_short_old","noncomm_positions_spread","comm_positions_long_old","comm_positions_short_old","tot_rept_positions_long_old","tot_rept_positions_short_1","nonrept_positions_long_old","nonrept_positions_short_old","open_interest_other","noncomm_positions_long_other","noncomm_positions_short_other","noncomm_positions_spread_1","comm_positions_long_other","comm_positions_short_other","tot_rept_positions_long_other","tot_rept_positions_short_2","nonrept_positions_long_other","nonrept_positions_short_other","change_in_open_interest_all","change_in_noncomm_long_all","change_in_noncomm_short_all","change_in_noncomm_spead_all","change_in_comm_long_all","change_in_comm_short_all","change_in_tot_rept_long_all","change_in_tot_rept_short","change_in_nonrept_long_all","change_in_nonrept_short_all","pct_of_open_interest_all","pct_of_oi_noncomm_long_all","pct_of_oi_noncomm_short_all","pct_of_oi_noncomm_spread","pct_of_oi_comm_long_all","pct_of_oi_comm_short_all","pct_of_oi_tot_rept_long_all","pct_of_oi_tot_rept_short","pct_of_oi_nonrept_long_all","pct_of_oi_nonrept_short_all","pct_of_open_interest_old","pct_of_oi_noncomm_long_old","pct_of_oi_noncomm_short_old","pct_of_oi_noncomm_spread_1","pct_of_oi_comm_long_old","pct_of_oi_comm_short_old","pct_of_oi_tot_rept_long_old","pct_of_oi_tot_rept_short_1","pct_of_oi_nonrept_long_old","pct_of_oi_nonrept_short_old","pct_of_open_interest_other","pct_of_oi_noncomm_long_other","pct_of_oi_noncomm_short_other","pct_of_oi_noncomm_spread_2","pct_of_oi_comm_long_other","pct_of_oi_comm_short_other","pct_of_oi_tot_rept_long_other","pct_of_oi_tot_rept_short_2","pct_of_oi_nonrept_long_other","pct_of_oi_nonrept_short_other","traders_tot_all","traders_noncomm_long_all","traders_noncomm_short_all","traders_noncomm_spread_all","traders_comm_long_all","traders_comm_short_all","traders_tot_rept_long_all","traders_tot_rept_short_all","traders_tot_old","traders_noncomm_long_old","traders_noncomm_short_old","traders_noncomm_spead_old","traders_comm_long_old","traders_comm_short_old","traders_tot_rept_long_old","traders_tot_rept_short_old","traders_tot_other","traders_noncomm_long_other","traders_noncomm_short_other","traders_noncomm_spread_other","traders_comm_long_other","traders_comm_short_other","traders_tot_rept_long_other","traders_tot_rept_short_other","conc_gross_le_4_tdr_long","conc_gross_le_4_tdr_short","conc_gross_le_8_tdr_long","conc_gross_le_8_tdr_short","conc_net_le_4_tdr_long_all","conc_net_le_4_tdr_short_all","conc_net_le_8_tdr_long_all","conc_net_le_8_tdr_short_all","conc_gross_le_4_tdr_long_1","conc_gross_le_4_tdr_short_1","conc_gross_le_8_tdr_long_1","conc_gross_le_8_tdr_short_1","conc_net_le_4_tdr_long_old","conc_net_le_4_tdr_short_old","conc_net_le_8_tdr_long_old","conc_net_le_8_tdr_short_old","conc_gross_le_4_tdr_long_2","conc_gross_le_4_tdr_short_2","conc_gross_le_8_tdr_long_2","conc_gross_le_8_tdr_short_2","conc_net_le_4_tdr_long_other","conc_net_le_4_tdr_short_other","conc_net_le_8_tdr_long_other","conc_net_le_8_tdr_short_other","contract_units","commodity","commodity_subgroup_name","commodity_group_name","futonly_or_combined"]' X-SODA2-Secondary-Last-Modified: - - Fri, 23 Aug 2024 19:30:46 GMT + - Fri, 30 Aug 2024 19:30:46 GMT X-SODA2-Truth-Last-Modified: - - Fri, 23 Aug 2024 19:30:46 GMT + - Fri, 30 Aug 2024 19:30:46 GMT X-SODA2-Types: - '["text","text","floating_timestamp","text","text","text","text","text","text","text","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","text","text","text","text","text"]' - X-Socrata-Region: - - aws-us-east-1-fedramp-prod - X-Socrata-RequestId: - - 0abb185a1eec56a4fa1748718eb9ef7d + X-Socrata-Region: null + X-Socrata-RequestId: null status: code: 200 message: OK diff --git a/openbb_platform/providers/cftc/tests/record/http/test_cftc_fetchers/test_cftc_cot_fetcher_urllib3_v2.yaml b/openbb_platform/providers/cftc/tests/record/http/test_cftc_fetchers/test_cftc_cot_fetcher_urllib3_v2.yaml index a1e30c75b3e..b9b71f4524f 100644 --- a/openbb_platform/providers/cftc/tests/record/http/test_cftc_fetchers/test_cftc_cot_fetcher_urllib3_v2.yaml +++ b/openbb_platform/providers/cftc/tests/record/http/test_cftc_fetchers/test_cftc_cot_fetcher_urllib3_v2.yaml @@ -43,11 +43,10 @@ interactions: Content-Type: - application/json;charset=utf-8 Date: - - Tue, 27 Aug 2024 06:27:59 GMT - Etag: - - '"aG90ZWwuNTEzMDVfM18yODJtVzlSN0o5WVlrTFhDbnYyakxHb3Z1NTJCcHc---gzipn8hfNknVynA7XV4li5cBTkJ7u_Q--gzip--gzip"' + - Thu, 05 Sep 2024 18:48:26 GMT + Etag: null Last-Modified: - - Fri, 23 Aug 2024 19:30:46 GMT + - Fri, 30 Aug 2024 19:30:46 GMT Server: - nginx Strict-Transport-Security: @@ -61,15 +60,13 @@ interactions: X-SODA2-Fields: - '["id","market_and_exchange_names","report_date_as_yyyy_mm_dd","yyyy_report_week_ww","contract_market_name","cftc_contract_market_code","cftc_market_code","cftc_region_code","cftc_commodity_code","commodity_name","open_interest_all","noncomm_positions_long_all","noncomm_positions_short_all","noncomm_postions_spread_all","comm_positions_long_all","comm_positions_short_all","tot_rept_positions_long_all","tot_rept_positions_short","nonrept_positions_long_all","nonrept_positions_short_all","open_interest_old","noncomm_positions_long_old","noncomm_positions_short_old","noncomm_positions_spread","comm_positions_long_old","comm_positions_short_old","tot_rept_positions_long_old","tot_rept_positions_short_1","nonrept_positions_long_old","nonrept_positions_short_old","open_interest_other","noncomm_positions_long_other","noncomm_positions_short_other","noncomm_positions_spread_1","comm_positions_long_other","comm_positions_short_other","tot_rept_positions_long_other","tot_rept_positions_short_2","nonrept_positions_long_other","nonrept_positions_short_other","change_in_open_interest_all","change_in_noncomm_long_all","change_in_noncomm_short_all","change_in_noncomm_spead_all","change_in_comm_long_all","change_in_comm_short_all","change_in_tot_rept_long_all","change_in_tot_rept_short","change_in_nonrept_long_all","change_in_nonrept_short_all","pct_of_open_interest_all","pct_of_oi_noncomm_long_all","pct_of_oi_noncomm_short_all","pct_of_oi_noncomm_spread","pct_of_oi_comm_long_all","pct_of_oi_comm_short_all","pct_of_oi_tot_rept_long_all","pct_of_oi_tot_rept_short","pct_of_oi_nonrept_long_all","pct_of_oi_nonrept_short_all","pct_of_open_interest_old","pct_of_oi_noncomm_long_old","pct_of_oi_noncomm_short_old","pct_of_oi_noncomm_spread_1","pct_of_oi_comm_long_old","pct_of_oi_comm_short_old","pct_of_oi_tot_rept_long_old","pct_of_oi_tot_rept_short_1","pct_of_oi_nonrept_long_old","pct_of_oi_nonrept_short_old","pct_of_open_interest_other","pct_of_oi_noncomm_long_other","pct_of_oi_noncomm_short_other","pct_of_oi_noncomm_spread_2","pct_of_oi_comm_long_other","pct_of_oi_comm_short_other","pct_of_oi_tot_rept_long_other","pct_of_oi_tot_rept_short_2","pct_of_oi_nonrept_long_other","pct_of_oi_nonrept_short_other","traders_tot_all","traders_noncomm_long_all","traders_noncomm_short_all","traders_noncomm_spread_all","traders_comm_long_all","traders_comm_short_all","traders_tot_rept_long_all","traders_tot_rept_short_all","traders_tot_old","traders_noncomm_long_old","traders_noncomm_short_old","traders_noncomm_spead_old","traders_comm_long_old","traders_comm_short_old","traders_tot_rept_long_old","traders_tot_rept_short_old","traders_tot_other","traders_noncomm_long_other","traders_noncomm_short_other","traders_noncomm_spread_other","traders_comm_long_other","traders_comm_short_other","traders_tot_rept_long_other","traders_tot_rept_short_other","conc_gross_le_4_tdr_long","conc_gross_le_4_tdr_short","conc_gross_le_8_tdr_long","conc_gross_le_8_tdr_short","conc_net_le_4_tdr_long_all","conc_net_le_4_tdr_short_all","conc_net_le_8_tdr_long_all","conc_net_le_8_tdr_short_all","conc_gross_le_4_tdr_long_1","conc_gross_le_4_tdr_short_1","conc_gross_le_8_tdr_long_1","conc_gross_le_8_tdr_short_1","conc_net_le_4_tdr_long_old","conc_net_le_4_tdr_short_old","conc_net_le_8_tdr_long_old","conc_net_le_8_tdr_short_old","conc_gross_le_4_tdr_long_2","conc_gross_le_4_tdr_short_2","conc_gross_le_8_tdr_long_2","conc_gross_le_8_tdr_short_2","conc_net_le_4_tdr_long_other","conc_net_le_4_tdr_short_other","conc_net_le_8_tdr_long_other","conc_net_le_8_tdr_short_other","contract_units","commodity","commodity_subgroup_name","commodity_group_name","futonly_or_combined"]' X-SODA2-Secondary-Last-Modified: - - Fri, 23 Aug 2024 19:30:46 GMT + - Fri, 30 Aug 2024 19:30:46 GMT X-SODA2-Truth-Last-Modified: - - Fri, 23 Aug 2024 19:30:46 GMT + - Fri, 30 Aug 2024 19:30:46 GMT X-SODA2-Types: - '["text","text","floating_timestamp","text","text","text","text","text","text","text","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","number","text","text","text","text","text"]' - X-Socrata-Region: - - aws-us-east-1-fedramp-prod - X-Socrata-RequestId: - - 4223a5a28d93ad5780819953fa087052 + X-Socrata-Region: null + X-Socrata-RequestId: null status: code: 200 message: OK diff --git a/openbb_platform/providers/cftc/tests/test_cftc_fetchers.py b/openbb_platform/providers/cftc/tests/test_cftc_fetchers.py index 1c4360bc71e..41f8eb22276 100644 --- a/openbb_platform/providers/cftc/tests/test_cftc_fetchers.py +++ b/openbb_platform/providers/cftc/tests/test_cftc_fetchers.py @@ -12,6 +12,16 @@ test_credentials = UserService().default_user_settings.credentials.model_dump( ) +def scrub_string(key): + """Scrub a string from the response.""" + + def before_record_response(response): + response["headers"][key] = response["headers"].update({key: "MOCK_VALUE"}) + return response + + return before_record_response + + @pytest.fixture(scope="module") def vcr_config(): """VCR configuration.""" @@ -23,6 +33,11 @@ def vcr_config(): ("$order", "MOCK_ORDER"), ("$where", "MOCK_WHERE"), ], + "before_record_response": [ + scrub_string("Etag"), + scrub_string("X-Socrata-RequestId"), + scrub_string("X-Socrata-Region"), + ], } diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/models/calendar_ipo.py b/openbb_platform/providers/intrinio/openbb_intrinio/models/calendar_ipo.py index 81cc49b5821..aafcfa43b24 100644 --- a/openbb_platform/providers/intrinio/openbb_intrinio/models/calendar_ipo.py +++ b/openbb_platform/providers/intrinio/openbb_intrinio/models/calendar_ipo.py @@ -1,5 +1,7 @@ """Intrinio IPO Calendar Model.""" +# pylint: disable=unused-argument + from typing import Any, Dict, List, Literal, Optional from openbb_core.provider.abstract.fetcher import Fetcher @@ -7,6 +9,7 @@ from openbb_core.provider.standard_models.calendar_ipo import ( CalendarIpoData, CalendarIpoQueryParams, ) +from openbb_core.provider.utils.errors import EmptyDataError from openbb_core.provider.utils.helpers import get_querystring from openbb_intrinio.utils.helpers import get_data_one from openbb_intrinio.utils.references import IntrinioCompany, IntrinioSecurity @@ -185,4 +188,6 @@ class IntrinioCalendarIpoFetcher( query: IntrinioCalendarIpoQueryParams, data: List[Dict], **kwargs: Any ) -> List[IntrinioCalendarIpoData]: """Return the transformed data.""" + if not data: + raise EmptyDataError("The request was returned empty.") return [IntrinioCalendarIpoData.model_validate(d) for d in data] diff --git a/openbb_platform/providers/seeking_alpha/openbb_seeking_alpha/models/calendar_earnings.py b/openbb_platform/providers/seeking_alpha/openbb_seeking_alpha/models/calendar_earnings.py index ebb83d0289a..fd6601981e0 100644 --- a/openbb_platform/providers/seeking_alpha/openbb_seeking_alpha/models/calendar_earnings.py +++ b/openbb_platform/providers/seeking_alpha/openbb_seeking_alpha/models/calendar_earnings.py @@ -14,7 +14,6 @@ from openbb_core.provider.standard_models.calendar_earnings import ( CalendarEarningsData, CalendarEarningsQueryParams, ) -from openbb_core.provider.utils.helpers import amake_request from openbb_seeking_alpha.utils.helpers import HEADERS, date_range from pydantic import Field, field_validator @@ -86,6 +85,10 @@ class SACalendarEarningsFetcher( **kwargs: Any, ) -> List[Dict]: """Return the raw data from the Seeking Alpha endpoint.""" + # pylint: disable=import-outside-toplevel + from openbb_core.provider.utils.client import ClientSession + from openbb_core.provider.utils.helpers import amake_request + results: List[Dict] = [] dates = [ date.strftime("%Y-%m-%d") @@ -94,25 +97,31 @@ class SACalendarEarningsFetcher( currency = "USD" if query.country == "us" else "CAD" messages: List = [] - async def get_date(date, currency): - """Get date for one date.""" - url = ( - f"https://seekingalpha.com/api/v3/earnings_calendar/tickers?" - f"filter%5Bselected_date%5D={date}" - f"&filter%5Bwith_rating%5D=false&filter%5Bcurrency%5D={currency}" - ) - response = await amake_request(url=url, headers=HEADERS) - # Try again if the response is blocked. - if "blockScript" in response: - response = await amake_request(url=url, headers=HEADERS) - if "blockScript" in response: - message = json.dumps(response) - messages.append(message) - warn(message) - if "data" in response: - results.extend(response.get("data")) + async with ClientSession() as session: - await asyncio.gather(*[get_date(date, currency) for date in dates]) + async def get_date(date, currency): + """Get date for one date.""" + url = ( + f"https://seekingalpha.com/api/v3/earnings_calendar/tickers?" + f"filter%5Bselected_date%5D={date}" + f"&filter%5Bwith_rating%5D=false&filter%5Bcurrency%5D={currency}" + ) + response = await amake_request( + url=url, headers=HEADERS, session=session + ) + # Try again if the response is blocked. + if "blockScript" in response: + response = await amake_request( + url=url, headers=HEADERS, session=session + ) + if "blockScript" in response: + message = json.dumps(response) + messages.append(message) + warn(message) + if "data" in response: + results.extend(response.get("data")) + + await asyncio.gather(*[get_date(date, currency) for date in dates]) if not results: raise OpenBBError(f"Error with the Seeking Alpha request -> {messages}") diff --git a/openbb_platform/providers/seeking_alpha/openbb_seeking_alpha/models/forward_eps_estimates.py b/openbb_platform/providers/seeking_alpha/openbb_seeking_alpha/models/forward_eps_estimates.py index 52cf3a02fac..68fce535ce0 100644 --- a/openbb_platform/providers/seeking_alpha/openbb_seeking_alpha/models/forward_eps_estimates.py +++ b/openbb_platform/providers/seeking_alpha/openbb_seeking_alpha/models/forward_eps_estimates.py @@ -89,6 +89,9 @@ class SAForwardEpsEstimatesFetcher( **kwargs: Any, ) -> Dict: """Return the raw data from the Seeking Alpha endpoint.""" + # pylint: disable=import-outside-toplevel + from openbb_core.provider.utils.client import ClientSession + tickers = query.symbol.split(",") fp = query.period if query.period == "annual" else "quarterly" url = "https://seekingalpha.com/api/v3/symbol_data/estimates" @@ -97,14 +100,19 @@ class SAForwardEpsEstimatesFetcher( "eps_normalized_consensus_high,eps_normalized_num_of_estimates," "eps_gaap_actual,eps_gaap_consensus_low,eps_gaap_consensus_mean,eps_gaap_consensus_high,", "period_type": fp, - "relative_periods": "-4,-3,-2,-1,0,1,2,3,4,5,6,7,8,9,10,11,12", + "relative_periods": "-3,-2,-1,0,1,2,3,4,5,6,7,8,9,10,11", } - ids: Dict = {ticker: await get_seekingalpha_id(ticker) for ticker in tickers} - querystring["ticker_ids"] = (",").join(list(ids.values())) - payload: str = "" - response = await amake_request( - url, data=payload, headers=HEADERS, params=querystring - ) + async with ClientSession(trust_env=True) as session: + kwargs = {"session": session} + ids = { + ticker: await get_seekingalpha_id(ticker, **kwargs) + for ticker in tickers + } + querystring["ticker_ids"] = ("%2C").join(list(ids.values())) + response = await amake_request( + url, headers=HEADERS, params=querystring, session=session + ) + estimates: Dict = response.get("estimates", {}) # type: ignore if not estimates: raise OpenBBError(f"No estimates data was returned for: {query.symbol}") @@ -128,8 +136,16 @@ class SAForwardEpsEstimatesFetcher( sa_id = str(ids.get(ticker, "")) if sa_id == "" or sa_id not in estimates: warn(f"Symbol Error: No data found for, {ticker}") + continue + seek_object = estimates.get(sa_id, {}) - items = len(seek_object["eps_normalized_num_of_estimates"]) + if not seek_object: + warn(f"No data found for {ticker}") + continue + items = len(seek_object.get("eps_normalized_num_of_estimates")) + if not items: + warn(f"No data found for {ticker}") + continue for i in range(0, items - 4): eps_estimates: Dict = {} eps_estimates["symbol"] = ticker diff --git a/openbb_platform/providers/seeking_alpha/openbb_seeking_alpha/models/forward_sales_estimates.py b/openbb_platform/providers/seeking_alpha/openbb_seeking_alpha/models/forward_sales_estimates.py index bb5db6a970c..d893fb4adfe 100644 --- a/openbb_platform/providers/seeking_alpha/openbb_seeking_alpha/models/forward_sales_estimates.py +++ b/openbb_platform/providers/seeking_alpha/openbb_seeking_alpha/models/forward_sales_estimates.py @@ -73,22 +73,31 @@ class SAForwardSalesEstimatesFetcher( **kwargs: Any, ) -> Dict: """Return the raw data from the Seeking Alpha endpoint.""" + # pylint: disable=import-outside-toplevel + from openbb_core.provider.utils.client import ClientSession + tickers = query.symbol.split(",") fp = query.period if query.period == "annual" else "quarterly" url = "https://seekingalpha.com/api/v3/symbol_data/estimates" - payload = "" querystring = { "estimates_data_items": "revenue_actual,revenue_consensus_low,revenue_consensus_mean," "revenue_consensus_high,revenue_num_of_estimates", "period_type": fp, "relative_periods": "-4,-3,-2,-1,0,1,2,3,4,5,6,7,8,9,10,11,12", } - ids = {ticker: await get_seekingalpha_id(ticker) for ticker in tickers} - querystring["ticker_ids"] = (",").join(list(ids.values())) - response = await amake_request( - url, data=payload, headers=HEADERS, params=querystring - ) - estimates = response.get("estimates", {}) # type: ignore + + async with ClientSession(trust_env=True) as session: + kwargs = {"session": session} + ids = { + ticker: await get_seekingalpha_id(ticker, **kwargs) + for ticker in tickers + } + querystring["ticker_ids"] = ("%2C").join(list(ids.values())) + response = await amake_request( + url, headers=HEADERS, params=querystring, session=session + ) + estimates = response.get("estimates", {}) # type: ignore + if not estimates: raise OpenBBError(f"No estimates data was returned for: {query.symbol}") output: Dict = {"ids": ids, "estimates": estimates} @@ -111,7 +120,13 @@ class SAForwardSalesEstimatesFetcher( if sa_id == "" or sa_id not in estimates: warn(f"Symbol Error: No data found for, {ticker}") seek_object = estimates.get(sa_id, {}) - items = len(seek_object["revenue_num_of_estimates"]) + if not seek_object: + warn(f"No data found for {ticker}") + continue + items = len(seek_object.get("revenue_num_of_estimates")) + if not items: + warn(f"No data found for {ticker}") + continue for i in range(0, items - 4): rev_estimates: Dict = {} rev_estimates["symbol"] = ticker diff --git a/openbb_platform/providers/seeking_alpha/openbb_seeking_alpha/utils/helpers.py b/openbb_platform/providers/seeking_alpha/openbb_seeking_alpha/utils/helpers.py index 656b4d0c4fe..fb3b4d8b87a 100644 --- a/openbb_platform/providers/seeking_alpha/openbb_seeking_alpha/utils/helpers.py +++ b/openbb_platform/providers/seeking_alpha/openbb_seeking_alpha/utils/helpers.py @@ -7,8 +7,8 @@ from openbb_core.provider.utils.helpers import amake_request HEADERS = { "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:106.0) Gecko/20100101 Firefox/106.0", "Accept": "*/*", - "Accept-Language": "de,en-US;q=0.7,en;q=0.3", - "Accept-Encoding": "gzip, deflate, br", + "Accept-Language": "en-US,en;q=0.9", + "Accept-Encoding": "gzip, deflate, br, zstd", "Sec-Fetch-Dest": "empty", "Sec-Fetch-Mode": "cors", "Sec-Fetch-Site": "same-origin", @@ -22,8 +22,9 @@ def date_range(start_date, end_date): yield start_date + timedelta(n) -async def get_seekingalpha_id(symbol: str) -> str: +async def get_seekingalpha_id(symbol: str, **kwargs) -> str: """Map a ticker symbol to its Seeking Alpha ID.""" + session = kwargs.get("session") url = "https://seekingalpha.com/api/v3/searches" querystring = { "filter[type]": "symbols", @@ -31,10 +32,12 @@ async def get_seekingalpha_id(symbol: str) -> str: "page[size]": "100", } querystring["filter[query]"] = symbol - payload = "" - response = await amake_request( - url, data=payload, headers=HEADERS, params=querystring - ) + if session: + response = await amake_request( + url, headers=HEADERS, params=querystring, session=session + ) + else: + response = await amake_request(url, headers=HEADERS, params=querystring) ids = response.get("symbols") # type: ignore return str(ids[0].get("id", ""))