From 305b8656378dff73f4d3268f96c95797b447b80f Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Thu, 25 Jan 2024 12:41:53 -0800 Subject: [PATCH 1/2] hotfix/intrinio-index: Drop None Rows From intrinio Historical Index (#5987) * drop None rows from intrinio historical indexes * add empty data error * static assets --- .../standard_models/index_historical.py | 12 +++-- .../index/integration/test_index_api.py | 4 +- .../index/integration/test_index_python.py | 4 +- .../openbb/package/extension_map.json | 2 +- .../openbb/package/module_map.json | 2 +- .../models/index_historical.py | 41 ++++++++++------ ...est_intrinio_index_historical_fetcher.yaml | 48 +++++++++---------- .../intrinio/tests/test_intrinio_fetchers.py | 3 +- 8 files changed, 65 insertions(+), 51 deletions(-) diff --git a/openbb_platform/core/openbb_core/provider/standard_models/index_historical.py b/openbb_platform/core/openbb_core/provider/standard_models/index_historical.py index 2e19d327260..1443562936b 100644 --- a/openbb_platform/core/openbb_core/provider/standard_models/index_historical.py +++ b/openbb_platform/core/openbb_core/provider/standard_models/index_historical.py @@ -40,7 +40,9 @@ class IndexHistoricalQueryParams(QueryParams): class IndexHistoricalData(Data): """Index Historical Data.""" - date: datetime = Field(description=DATA_DESCRIPTIONS.get("date", "")) + date: Union[dateType, datetime] = Field( + description=DATA_DESCRIPTIONS.get("date", "") + ) open: Optional[StrictFloat] = Field( default=None, description=DATA_DESCRIPTIONS.get("open", "") ) @@ -58,6 +60,10 @@ class IndexHistoricalData(Data): ) @field_validator("date", mode="before", check_fields=False) - def date_validate(cls, v): # pylint: disable=E0213 + @classmethod + def date_validate(cls, v): """Return formatted datetime.""" - return parser.isoparse(str(v)) + v = parser.isoparse(str(v)) + if v.hour == 0 and v.minute == 0: + return v.date() + return v diff --git a/openbb_platform/extensions/index/integration/test_index_api.py b/openbb_platform/extensions/index/integration/test_index_api.py index d8094ad8f9a..3b7fa9c7be4 100644 --- a/openbb_platform/extensions/index/integration/test_index_api.py +++ b/openbb_platform/extensions/index/integration/test_index_api.py @@ -134,11 +134,9 @@ def test_index_constituents(params, headers): "provider": "intrinio", "start_date": "2023-01-01", "end_date": "2023-06-06", - "symbol": "$DJI", - "tag": "level", + "symbol": "DJI", "sort": "desc", "limit": 100, - "type": None, } ), ], diff --git a/openbb_platform/extensions/index/integration/test_index_python.py b/openbb_platform/extensions/index/integration/test_index_python.py index b8073621a7c..7e5787518c7 100644 --- a/openbb_platform/extensions/index/integration/test_index_python.py +++ b/openbb_platform/extensions/index/integration/test_index_python.py @@ -121,11 +121,9 @@ def test_index_constituents(params, obb): "provider": "intrinio", "start_date": "2023-01-01", "end_date": "2023-06-06", - "symbol": "$DJI", - "tag": "level", + "symbol": "DJI", "sort": "desc", "limit": 100, - "type": None, } ), ], diff --git a/openbb_platform/openbb/package/extension_map.json b/openbb_platform/openbb/package/extension_map.json index d2deb1107cc..cb31119c7f8 100644 --- a/openbb_platform/openbb/package/extension_map.json +++ b/openbb_platform/openbb/package/extension_map.json @@ -24,4 +24,4 @@ "tradingeconomics@1.1.1", "yfinance@1.1.1" ] -} \ No newline at end of file +} diff --git a/openbb_platform/openbb/package/module_map.json b/openbb_platform/openbb/package/module_map.json index a14f4bc4032..3c43bc00321 100644 --- a/openbb_platform/openbb/package/module_map.json +++ b/openbb_platform/openbb/package/module_map.json @@ -140,4 +140,4 @@ "regulators_sec_schema_files": "/regulators/sec/schema_files", "regulators_sec_sic_search": "/regulators/sec/sic_search", "regulators_sec_symbol_map": "/regulators/sec/symbol_map" -} \ No newline at end of file +} diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/models/index_historical.py b/openbb_platform/providers/intrinio/openbb_intrinio/models/index_historical.py index 9acd91c68a1..40977203414 100644 --- a/openbb_platform/providers/intrinio/openbb_intrinio/models/index_historical.py +++ b/openbb_platform/providers/intrinio/openbb_intrinio/models/index_historical.py @@ -10,29 +10,25 @@ from openbb_core.provider.standard_models.index_historical import ( IndexHistoricalQueryParams, ) from openbb_core.provider.utils.descriptions import QUERY_DESCRIPTIONS -from openbb_core.provider.utils.helpers import get_querystring -from openbb_intrinio.utils.helpers import get_data_many +from openbb_core.provider.utils.errors import EmptyDataError +from openbb_core.provider.utils.helpers import amake_requests, get_querystring from pydantic import Field, NonNegativeInt class IntrinioIndexHistoricalQueryParams(IndexHistoricalQueryParams): """Intrinio Index Historical Query. - Source: https://docs.intrinio.com/documentation/web_api/get_stock_market_index_historical_data_v2 + Source: + https://docs.intrinio.com/documentation/web_api/get_stock_market_index_historical_data_v2 """ - tag: Optional[str] = Field(default="level", description="Index tag.") - type: Optional[str] = Field( - default=None, - description="Index type.", - ) sort: Literal["asc", "desc"] = Field( - default="desc", + default="asc", description="Sort order.", alias="sort_order", ) limit: NonNegativeInt = Field( - default=1000, + default=10000, description=QUERY_DESCRIPTIONS.get("limit", ""), alias="page_size", ) @@ -73,13 +69,30 @@ class IntrinioIndexHistoricalFetcher( **kwargs: Any, ) -> List[Dict]: """Return the raw data from the Intrinio endpoint.""" + results = [] api_key = credentials.get("intrinio_api_key") if credentials else "" - + symbols = query.symbol.replace("$", "").replace("^", "").split(",") base_url = "https://api-v2.intrinio.com/indices/stock_market" - query_str = get_querystring(query.model_dump(by_alias=True), ["symbol", "tag"]) - url = f"{base_url}/{query.symbol}/historical_data/{query.tag}?{query_str}&api_key={api_key}" + query_str = get_querystring(query.model_dump(by_alias=True), ["symbol"]) + urls = [ + f"{base_url}/${symbol}/historical_data/level?{query_str}&api_key={api_key}" + for symbol in symbols + ] - return await get_data_many(url, "historical_data", **kwargs) + async def callback(response, _) -> List[Dict]: + """Response callback.""" + _response = await response.json() + data = _response.get("historical_data") + symbol = _response["index"].get("symbol").replace("$", "") + data = [d for d in data if d.get("value") is not None] + data = [{"symbol": symbol, **d} for d in data] if len(symbols) > 1 else data + return results.extend(data) if len(data) > 0 else results # type: ignore + + await amake_requests(urls, callback, **kwargs) + + if len(results) == 0: + raise EmptyDataError() + return results @staticmethod def transform_data( diff --git a/openbb_platform/providers/intrinio/tests/record/http/test_intrinio_fetchers/test_intrinio_index_historical_fetcher.yaml b/openbb_platform/providers/intrinio/tests/record/http/test_intrinio_fetchers/test_intrinio_index_historical_fetcher.yaml index 4327fe36106..33b72674f1d 100644 --- a/openbb_platform/providers/intrinio/tests/record/http/test_intrinio_fetchers/test_intrinio_index_historical_fetcher.yaml +++ b/openbb_platform/providers/intrinio/tests/record/http/test_intrinio_fetchers/test_intrinio_index_historical_fetcher.yaml @@ -3,36 +3,36 @@ interactions: body: null headers: Accept: - - '*/*' + - application/json Accept-Encoding: - - gzip, deflate, br + - gzip, deflate Connection: - keep-alive method: GET - uri: https://api-v2.intrinio.com/indices/stock_market/$DJI/historical_data/level?api_key=MOCK_API_KEY&end_date=2023-06-06&page_size=1000&sort_order=desc&start_date=2023-01-01 + uri: https://api-v2.intrinio.com/indices/stock_market/$DJI/historical_data/level?api_key=MOCK_API_KEY&end_date=2023-06-06&page_size=10000&sort_order=asc&start_date=2023-01-01 response: body: string: !!binary | - H4sIAGQlZWUAA3RYTW8bRwz9K4LQoz0YkvPpm4FckkMubS8tCmFjbRoB8iqVVm4cI/+93KRFlzPP - gA6W9oEzfOQj3/pl++lwmU/nw8Nw3O2Hedje/f6y1T/G7d2WPcutT/rZ3myfhuNVfxSJWRyXbzc9 - LBpYYlcSgvEalhWWIYzWMJ/IxdzB4q2sYVx9cQxh3kQLemifQrzl+j9suh6PCGLI8FWcBARbk8E5 - BZciggUDq9VVRjAxh8boIiGYoZZLchFlSXUNC5xcEgQrpp4SXUWHUrbRvMuoAmR5I3aEeCPTRBKK - 65tDUSZR8d4lRBuRhVVMG3mbKDlBfPhq+5tcQdG8oS2RZlARzCSacnCCauXXDSLE2WVEmzcNEig4 - hjDDWyrBRZjpmrfgI7m+ouGWi0HV4qivlcJMfxTtNgxLtlbKLjzUDhnxrvQZBKsrKTm6ng5Fmf4o - OjwqvJrpj6yyAt0WGlmVmp3v+yM0sqoZqi80sqolO+r7Q2E2Ub1bQLSRmFpxRU0ZGl2loAMEoajt - ogyDWVkt0wjR5leJwpkbmgUUSnSMDrSSCkWHPDzQUKZkIOWFRlJJG5J6MsQuIB3h4RXYmgwusTrf - V13MAtJglMH4E6s8lqpqR2ca5XEQdr5PVJodxJLRUBC7g5h8fOXQdQuxV4FSrwKx0uOYPE7UsMYh - oIUmVitUdDJXCEs2mjYkysCsICpaUeA8xCqPKaIRI1Z4VKiitSdWK1R1UyUEMyuIOQZkscSuIHUV - BV7NZ9uSOjpgMCs+3Y4BRjNykepdRazZDeS9YHLNBuK07Nr+UG6EkDSFPhg3OihFtdePD250UCgh - L8aNF6MoyBZx48V8iK8casYH6WCOELbqDzgkuVkYy54FI54bH5ZqcqUXAVsRBOLiPIStOQteqQX2 - m5v1w0pG7UcMNxujpFeiWSOW1DSXXu7cGLEaKtrHbGUQKL7Cm5VBqeQ86g4rg6pVwIey4W1pItTf - jRGrjAwK2R2kVdBD+2jUvAQtywW0JDWGreaC1gY1hm2hN/SZUmPYchCkZGoMWxbBmVrxpUUuEGYy - lcX+oUytY/PLekHRrGNjlQy8W+PYyCNlkREglDI1chF1KAlRa9xaIBUfGLnU2LWqr++gJakRX/Zq - ZACqeQlaDApK0qolqQuAGRjHxlVhsNWsY2OdC+AVkxrtkSQnEMZNAf642R6m/fhle/eyPewVqd92 - /PXp/dd7RV6eHz+cjvrrT2/evdXv0/C4RHtz+nvz7jSNl83baX+9zOfDcNzcP43n4c9RUQ+naT5M - 4zQr9P3pPH/a3D+Oy79Zvj+7TvP5WZ/8Oh3mcb/5edYbXjanjyvQ9fNy7d3H8/jXdZweFvR+OByf - 9dFxuMy7H8/3/+VFi9J+8XS3fLTC3v+myNOHy3h+GubDadpd5uE8f4fTv/AGME4mWth+02THL/Pu - 85LSD6r+AQAA//8DAAVlFaAwEgAA + H4sIANIhsGUAA4SYT28bNxDFv4qw6NEhyOEM//hmIJfkkEvbS4tCUKxNI0BepdLKjWPku3c2Btod + 8hm9WdIDyRm+38zQz8Pnw2U+nQ/3u+N2v5t3w+3vz4P+MQ63A3mKb3x442m4GR53x6t+OV2Px+83 + vST+J4kxxORiRjJeyyhVl6FMVjKq0TtfkCytV0sqSwHJ6lomIbskQBb8WpY9O7RYCGtVzdF5KFul + LHIo1VUUaFinjaMnHEFI/3sBIZtzBe8KDLKYC6jJ1YRkJmWe2QmSkUlZzOK4IpkxR6L6ymrGHDlG + fDYSI+PoCiOZMUfl6hillmzecsFWi9Yc6iGPIo1rd7Avyfn+bKRKI6uEIiXDncpEEwJXM+mtlJAl + qaGl1OA8IVk2zpXkUh+pykqbXrypMVKq1ZU+vdSwV7QuUI8LNbgQi6sRydjewmuriYmU9Op7ZMjA + t4SQEFnUAFjolbytkYEoq8SUmLDgAlciy6ggS1IDX1hMhC7KwEclaGFA7jC4UNHU4k3X7qCkJuov + IFoMKKWASI4Wg+h9dAJXM4HG6lHJjQ0GHINjuKkNVCPgHtFoMaBci4OLrSkgEnYFLWYoCNVXl9Bq + hoJQQnUBysyNBhF4NANBKJlxcg0ERJyc9BBEC0EoeqO1xz3avqGr6aZQZvwhSfs7Uq3tQV6ngNAb + PFoOKHhxhCKwHFDMTvoSExsOOBJqG7HhQMvkK5safyzdBQVqepBaUtlDRzM9SFfTOabfkxtckg+v + yExTZp1QYh8o24ktciGX+/LBLXxFL6EvH2zgg0WSm44h2mkFbWjHtVTYZbShGddiWuyNVKb9eC3M + oMSzBS8WPRr3SHE7sZXsAsqsndhq1r7S3zo3E1upGfVjbia2rGdLKG22BRVf0IDCzcRWMiox3Axs + ooAWFIEd2KIasqC0UddnAzxaMXelhRnIpJ3EJIBGJU0HWlwEioI0SHFQ2vt8SINUoOwylJm0JeUY + kCfNIJZCQaaU9hG09NreH9Jipd0xokgtVlE7lcDVyMo8cpvYHrSUBVD+pJnDfCDU9qTBismjJ6Y0 + WEkUNBZJgxUTpE+aWYyWaoTuys5iXlsyTJvtQVmHZjCLieWKcmL0rpWGK1+jiyhv6x4Ea640jyDP + WuRRlKYBUdXiAcbvZNGLXh0JJo/UsJeTbtq7IzW0iMrAiJWaBiT6fKfy/Y+b4TDtx6/D7fNw2Kta + P23p2+OHb3eqvjw9fDwd9duf3r5/p5+n3cOy4tvT35v3p2m8bN5N++tlPh92x83d43je/Tmq6v40 + zYdpnGaVfjid58+bu4dx+TfLj9+u03x+0l9+nQ7zuN/8POspL5vTp5Xo+mU5+vbTefzrOk73i3q/ + Oxyf9Kfj7jJvX37fv8TGL8/sX3y4DeF2eYx5/5sqTx8v4/lxNx9O0/Yy787zD3ngf1/la8E4mdVo + +K7Bjl/n7ZclpBdT/AMAAP//AwCMqInFMBIAAA== headers: Connection: - keep-alive @@ -41,7 +41,7 @@ interactions: Content-Type: - application/json Date: - - Mon, 27 Nov 2023 23:25:24 GMT + - Tue, 23 Jan 2024 20:30:10 GMT Transfer-Encoding: - chunked Vary: diff --git a/openbb_platform/providers/intrinio/tests/test_intrinio_fetchers.py b/openbb_platform/providers/intrinio/tests/test_intrinio_fetchers.py index 4fead500911..866e2537150 100644 --- a/openbb_platform/providers/intrinio/tests/test_intrinio_fetchers.py +++ b/openbb_platform/providers/intrinio/tests/test_intrinio_fetchers.py @@ -246,8 +246,7 @@ def test_intrinio_market_indices_fetcher(credentials=test_credentials): @pytest.mark.record_http def test_intrinio_index_historical_fetcher(credentials=test_credentials): params = { - "symbol": "$DJI", - "tag": "level", + "symbol": "DJI", "start_date": date(2023, 1, 1), "end_date": date(2023, 6, 6), } From f7c9865de766445f8dad630400c5125b12550dac Mon Sep 17 00:00:00 2001 From: Danglewood <85772166+deeleeramone@users.noreply.github.com> Date: Thu, 25 Jan 2024 12:55:46 -0800 Subject: [PATCH 2/2] hotfix/insider-trading: Fix #5988 (#5990) * fix validation errors * empty strings * validator --------- Co-authored-by: Igor Radovanovic <74266147+IgorWounds@users.noreply.github.com> Co-authored-by: teh_coderer --- .../standard_models/insider_trading.py | 71 ++++++++++--------- .../openbb_intrinio/models/insider_trading.py | 46 +++++++----- 2 files changed, 68 insertions(+), 49 deletions(-) diff --git a/openbb_platform/core/openbb_core/provider/standard_models/insider_trading.py b/openbb_platform/core/openbb_core/provider/standard_models/insider_trading.py index 950527c0fec..94a893421f6 100644 --- a/openbb_platform/core/openbb_core/provider/standard_models/insider_trading.py +++ b/openbb_platform/core/openbb_core/provider/standard_models/insider_trading.py @@ -1,6 +1,10 @@ """Insider Trading Standard Model.""" -from datetime import date, datetime, time +from datetime import ( + date as dateType, + datetime, + time, +) from typing import List, Optional, Set, Union from dateutil import parser @@ -36,52 +40,55 @@ class InsiderTradingData(Data): """Insider Trading Data.""" symbol: str = Field(description=DATA_DESCRIPTIONS.get("symbol", "")) - company_cik: int = Field(description="Company CIK of the insider trading.") - filing_date: datetime = Field(description="Filing date of the insider trading.") - transaction_date: Optional[date] = Field( - default=None, description="Transaction date of the insider trading." + company_cik: Optional[Union[int, str]] = Field( + default=None, description="CIK number of the company." + ) + filing_date: Optional[Union[dateType, datetime]] = Field( + default=None, description="Filing date of the trade." + ) + transaction_date: Optional[dateType] = Field( + default=None, description="Date of the transaction." + ) + owner_cik: Optional[Union[int, str]] = Field( + default=None, description="Reporting individual's CIK." + ) + owner_name: Optional[str] = Field( + default=None, description="Name of the reporting individual." ) - owner_cik: int = Field(description="Reporting CIK of the insider trading.") - owner_name: str = Field(description="Reporting name of the insider trading.") owner_title: Optional[str] = Field( - default=None, description="Designation of owner of the insider trading." + default=None, description="The title held by the reporting individual." ) - transaction_type: str = Field( - description="Transaction type of the insider trading." + transaction_type: Optional[str] = Field( + default=None, description="Type of transaction being reported." ) acquisition_or_disposition: Optional[str] = Field( - default=None, - description="Acquisition or disposition of the insider trading.", + default=None, description="Acquisition or disposition of the shares." ) security_type: Optional[str] = Field( - default=None, description="Security type of the insider trading." + default=None, description="The type of security transacted." ) securities_owned: Optional[float] = Field( - default=None, description="Number of securities owned in the insider trading." + default=None, + description="Number of securities owned by the reporting individual.", ) securities_transacted: Optional[float] = Field( - default=None, description="Securities transacted of the insider trading." + default=None, + description="Number of securities transacted by the reporting individual.", ) transaction_price: Optional[float] = Field( - default=None, - description="Price of the insider trading.", + default=None, description="The price of the transaction." ) - filing_url: str = Field(description="Link of the insider trading.") + filing_url: Optional[str] = Field(default=None, description="Link to the filing.") - @field_validator("symbol", mode="before", check_fields=False) - @classmethod - def upper_symbol(cls, v: Union[str, List[str], Set[str]]): - """Convert symbol to uppercase.""" - if isinstance(v, str): - return v.upper() - return ",".join([symbol.upper() for symbol in list(v)]) - - @field_validator("filing_date", mode="before", check_fields=False) + @field_validator( + "filing_date", "transaction_date", mode="before", check_fields=False + ) @classmethod def date_validate(cls, v): # pylint: disable=E0213 """Return formatted datetime.""" - filing_date = parser.isoparse(str(v)) - - if filing_date.time() == time(0, 0): - return datetime.combine(filing_date.date(), time(0, 0, 0)) - return filing_date + if v: + filing_date = parser.isoparse(str(v)) + if filing_date.time() == time(0, 0): + return filing_date.date() + return filing_date + return None diff --git a/openbb_platform/providers/intrinio/openbb_intrinio/models/insider_trading.py b/openbb_platform/providers/intrinio/openbb_intrinio/models/insider_trading.py index a371818ba36..eed2a18b442 100644 --- a/openbb_platform/providers/intrinio/openbb_intrinio/models/insider_trading.py +++ b/openbb_platform/providers/intrinio/openbb_intrinio/models/insider_trading.py @@ -4,7 +4,7 @@ from datetime import ( date as dateType, datetime, ) -from typing import Any, Dict, List, Literal, Optional +from typing import Any, Dict, List, Literal, Optional, Union from dateutil.relativedelta import relativedelta from openbb_core.provider.abstract.fetcher import Fetcher @@ -15,7 +15,7 @@ from openbb_core.provider.standard_models.insider_trading import ( from openbb_core.provider.utils.descriptions import QUERY_DESCRIPTIONS from openbb_core.provider.utils.helpers import get_querystring from openbb_intrinio.utils.helpers import get_data_many -from pydantic import Field +from pydantic import Field, model_validator class IntrinioInsiderTradingQueryParams(InsiderTradingQueryParams): @@ -57,25 +57,25 @@ class IntrinioInsiderTradingData(InsiderTradingData): company_name: str = Field(description="Name of the company.") conversion_exercise_price: Optional[float] = Field( default=None, - description="Conversion/Exercise price of the insider trading.", + description="Conversion/Exercise price of the shares.", ) deemed_execution_date: Optional[dateType] = Field( default=None, - description="Deemed execution date of the insider trading.", + description="Deemed execution date of the trade.", ) exercise_date: Optional[dateType] = Field( default=None, - description="Exercise date of the insider trading.", + description="Exercise date of the trade.", ) expiration_date: Optional[dateType] = Field( default=None, - description="Expiration date of the insider trading.", + description="Expiration date of the derivative.", ) underlying_security_title: Optional[str] = Field( default=None, description="Name of the underlying non-derivative security related to this derivative transaction.", ) - underlying_shares: Optional[int] = Field( + underlying_shares: Optional[Union[int, float]] = Field( default=None, description="Number of underlying shares related to this derivative transaction.", ) @@ -83,20 +83,32 @@ class IntrinioInsiderTradingData(InsiderTradingData): default=None, description="Nature of ownership of the insider trading.", ) - director: bool = Field(description="Whether the owner is a director.") - officer: bool = Field(description="Whether the owner is an officer.") - ten_percent_owner: bool = Field(description="Whether the owner is a 10% owner.") - other_relation: bool = Field( - description="Whether the owner is having another relation." + director: Optional[bool] = Field( + default=None, description="Whether the owner is a director." ) - derivative_transaction: bool = Field( - description="Whether the owner is having a derivative transaction." + officer: Optional[bool] = Field( + default=None, description="Whether the owner is an officer." + ) + ten_percent_owner: Optional[bool] = Field( + default=None, description="Whether the owner is a 10% owner." + ) + other_relation: Optional[bool] = Field( + default=None, description="Whether the owner is having another relation." + ) + derivative_transaction: Optional[bool] = Field( + default=None, + description="Whether the owner is having a derivative transaction.", ) report_line_number: Optional[int] = Field( - default=None, - description="Report line number of the insider trading.", + default=None, description="Report line number of the insider trading." ) - filing_url: str = Field(description="URL of the filing.") + filing_url: Optional[str] = Field(default=None, description="URL of the filing.") + + @model_validator(mode="before") + @classmethod + def empty_strings(cls, values): # pylint: disable=no-self-argument + """Check for empty strings and replace with None.""" + return {k: None if v == "" else v for k, v in values.items()} class IntrinioInsiderTradingFetcher(