[BugFix] - Improve Standardization - CotSearch (#6031)

* format w new black

* Revert "format w new black"

This reverts commit 74b9619cc6.

* small adjustment on the test

* index historical

* adjust tests accordingly

* keep the test skipped

* match tests w python

* use cache as std field

* fix tests

* fix tests

* keep the skip

* fix test

* adding missing param

* fix tests accordingly

openbb_platform/core/openbb_core/provider/standard_models/cot_search.py

@@ -13,6 +13,10 @@ class CotSearchQueryParams(QueryParams):
     """Commitment of Traders Reports Search Query."""
 
     query: str = Field(description="Search query.", default="")
+    use_cache: Optional[bool] = Field(
+        default=True,
+        description="Whether or not to use cache. If True, cache will store for seven days.",
+    )
 
 
 class CotSearchData(Data):

openbb_platform/extensions/regulators/integration/test_regulators_api.py

@@ -40,7 +40,7 @@ def test_regulators_sec_cik_map(params, headers):
 @parametrize(
     "params",
     [
-        ({"query": "berkshire hathaway", "provider": "sec"}),
+        ({"query": "berkshire hathaway", "provider": "sec", "use_cache": None}),
     ],
 )
 @pytest.mark.integration
@@ -57,12 +57,13 @@ def test_regulators_sec_institutions_search(params, headers):
 @parametrize(
     "params",
     [
-        ({"query": "2022", "provider": "sec", "url": ""}),
+        ({"query": "2022", "provider": "sec", "url": "", "use_cache": None}),
         (
             {
                 "query": "",
                 "provider": "sec",
                 "url": "https://xbrl.fasb.org/us-gaap/2014/entire/",
+                "use_cache": None,
             }
         ),
     ],
@@ -81,8 +82,8 @@ def test_regulators_sec_schema_files(params, headers):
 @parametrize(
     "params",
     [
-        ({"query": "0000909832", "provider": "sec"}),
-        ({"query": "0001067983", "provider": "sec"}),
+        ({"query": "0000909832", "provider": "sec", "use_cache": None}),
+        ({"query": "0001067983", "provider": "sec", "use_cache": None}),
     ],
 )
 @pytest.mark.integration
@@ -129,7 +130,7 @@ def test_regulators_sec_sic_search(params, headers):
 @parametrize(
     "params",
     [
-        ({"query": "grain", "provider": "nasdaq"}),
+        ({"query": "grain", "provider": "nasdaq", "use_cache": None}),
     ],
 )
 @pytest.mark.integration

openbb_platform/extensions/regulators/integration/test_regulators_python.py

@@ -38,7 +38,7 @@ def test_regulators_sec_cik_map(params, obb):
 @parametrize(
     "params",
     [
-        ({"query": "berkshire hathaway", "provider": "sec"}),
+        ({"query": "berkshire hathaway", "provider": "sec", "use_cache": None}),
     ],
 )
 @pytest.mark.integration
@@ -52,12 +52,20 @@ def test_regulators_sec_institutions_search(params, obb):
 @parametrize(
     "params",
     [
-        ({"query": "2022", "provider": "sec", "url": None}),
+        (
+            {
+                "query": "2022",
+                "provider": "sec",
+                "url": None,
+                "use_cache": None,
+            }
+        ),
         (
             {
                 "query": "",
                 "provider": "sec",
                 "url": "https://xbrl.fasb.org/us-gaap/2014/entire/",
+                "use_cache": None,
             }
         ),
     ],
@@ -73,8 +81,8 @@ def test_regulators_sec_schema_files(params, obb):
 @parametrize(
     "params",
     [
-        ({"query": "0000909832", "provider": "sec"}),
-        ({"query": "0001067983", "provider": "sec"}),
+        ({"query": "0000909832", "provider": "sec", "use_cache": None}),
+        ({"query": "0001067983", "provider": "sec", "use_cache": None}),
     ],
 )
 @pytest.mark.integration
@@ -113,7 +121,7 @@ def test_regulators_sec_sic_search(params, obb):
 @parametrize(
     "params",
     [
-        ({"query": "grain", "provider": "nasdaq"}),
+        ({"query": "grain", "provider": "nasdaq", "use_cache": None}),
     ],
 )
 @pytest.mark.integration

openbb_platform/providers/sec/openbb_sec/models/institutions_search.py

@@ -15,11 +15,6 @@ class SecInstitutionsSearchQueryParams(CotSearchQueryParams):
     Source: https://sec.gov/
     """
 
-    use_cache: bool = Field(
-        default=True,
-        description="Whether or not to use cache. If True, cache will store for seven days.",
-    )
-
 
 class SecInstitutionsSearchData(Data):
     """SEC Institutions Search Data."""
@@ -45,6 +40,7 @@ class SecInstitutionsSearchFetcher(
         """Transform the query."""
         return SecInstitutionsSearchQueryParams(**params)
 
+    # pylint: disable=unused-argument
     @staticmethod
     def extract_data(
         query: SecInstitutionsSearchQueryParams,
@@ -56,9 +52,10 @@ class SecInstitutionsSearchFetcher(
         hp = institutions["Institution"].str.contains(query.query, case=False)
         return institutions[hp].astype(str).to_dict("records")
 
+    # pylint: disable=unused-argument
     @staticmethod
     def transform_data(
-        data: List[Dict], **kwargs: Any
+        query: SecInstitutionsSearchQueryParams, data: List[Dict], **kwargs: Any
     ) -> List[SecInstitutionsSearchData]:
         """Transform the data to the standard format."""
         return [SecInstitutionsSearchData.model_validate(d) for d in data]

openbb_platform/providers/sec/openbb_sec/models/sic_search.py

@@ -17,11 +17,6 @@ class SecSicSearchQueryParams(CotSearchQueryParams):
     Source: https://sec.gov/
     """
 
-    use_cache: bool = Field(
-        default=True,
-        description="Whether to use the cache or not. The full list will be cached for seven days if True.",
-    )
-
 
 class SecSicSearchData(Data):
     """SEC Standard Industrial Classification Code (SIC) Data."""
@@ -42,6 +37,7 @@ class SecSicSearchFetcher(
 ):
     """Transform the query, extract and transform the data from the SEC endpoints."""
 
+    # pylint: disable=unused-argument
     @staticmethod
     def transform_query(
         params: Dict[str, Any], **kwargs: Any
@@ -49,6 +45,7 @@ class SecSicSearchFetcher(
         """Transform the query."""
         return SecSicSearchQueryParams(**params)
 
+    # pylint: disable=unused-argument
     @staticmethod
     def extract_data(
         query: SecSicSearchQueryParams,
@@ -83,7 +80,10 @@ class SecSicSearchFetcher(
 
         return results
 
+    # pylint: disable=unused-argument
     @staticmethod
-    def transform_data(data: List[Dict], **kwargs: Any) -> List[SecSicSearchData]:
+    def transform_data(
+        query: SecSicSearchQueryParams, data: List[Dict], **kwargs: Any
+    ) -> List[SecSicSearchData]:
         """Transform the data."""
         return [SecSicSearchData.model_validate(d) for d in data]