[Feature] - Platform V4 Markdown Generator V2 (#6094)

* add recent `openapi.json`

* fix data types in models

* removed `openapi.json`

* set default date value as None in PolygonCurrencyPairs class

* add function to generate reference.json file

* reworked function to improve readability

* reword function to add seo metadata

* add functions for creating markdown sections

* add code to extract data card markdown title

* add additional functions for generating index and data models file

* extract first sentence of the description for cards

* code cleanup and documentation

* linting

* linting polygon models

* add openbb import statement in create_reference_markdown_examples function

* add POST method functions

* cleanup; reworked generate_reference_index_files function

* moved development section to 7th position in the sidebar

* fix POST function params default value

add standard flag to QueryParams and Data fields

cleanup

* add type expansion from package_builder.MethodDefinition

* sort data models cards alphabetically

make printing less verbose

* make MAX_CARDS global

display less content in cards in Commannds section

* cleanup

* Remove '_' from the cards under Commands section

* " to ' in econometrics/causality

* replace ' with " in ReferenceCard for reference dir index files

* remove extra . from the quantile function description

* shoutout to @deeleeramone for finding POST method description bug!

* set correct value for standard field

* handle BaseModel types in provider data fields

* unit tests for the platform markdown generator v2

* yeet 'Default' and 'Optional' columns in the 'Data' section

* last minute bug fix

* add info for multiple symbols

* make multiple items info same as platform static

* organize sections properly

* sort reference sub-directories alphabetically

* extra space in 'OBBject extra' description

* add type expansion for fields with multiple items

POST method cleanup

---------

Co-authored-by: Danglewood <85772166+deeleeramone@users.noreply.github.com>
Co-authored-by: Igor Radovanovic <74266147+IgorWounds@users.noreply.github.com>

openbb_platform/core/openbb_core/app/static/package_builder.py

@@ -843,7 +843,7 @@ class DocstringGenerator:
             "            List of warnings.\n"
             "        chart : Optional[Chart]\n"
             "            Chart object.\n"
-            "        extra: Dict[str, Any]\n"
+            "        extra : Dict[str, Any]\n"
             "            Extra info.\n"
         )
         obbject_description = obbject_description.replace("NoneType", "None")

openbb_platform/extensions/econometrics/openbb_econometrics/econometrics_router.py

@@ -353,10 +353,10 @@ def causality(
     x_column: str,
     lag: PositiveInt = 3,
 ) -> OBBject[Data]:
-    """Perform Granger causality test to determine if X "causes" y.
+    """Perform Granger causality test to determine if X 'causes' y.
 
     The Granger causality test is a statistical hypothesis test to determine if one time series is useful in
-    forecasting another. While "causality" in this context does not imply a cause-and-effect relationship in
+    forecasting another. While 'causality' in this context does not imply a cause-and-effect relationship in
     the philosophical sense, it does test whether changes in one variable are systematically followed by changes
     in another variable, suggesting a predictive relationship. By specifying a lag, you set the number of periods to
     look back in the time series to assess this relationship. This test is particularly useful in economic and

openbb_platform/extensions/quantitative/openbb_quantitative/rolling/rolling_router.py

@@ -230,7 +230,7 @@ def quantile(
 
     Quantiles are points dividing the range of a probability distribution into  intervals with equal probabilities,
     or dividing the  sample in the same way. This function is useful for understanding the distribution of data
-    within a specified window, allowing for analysis of trends, identification of outliers, and assessment of risk..
+    within a specified window, allowing for analysis of trends, identification of outliers, and assessment of risk.
 
     Parameters:
         data: List[Data]

openbb_platform/providers/polygon/openbb_polygon/models/currency_pairs.py

@@ -26,7 +26,7 @@ class PolygonCurrencyPairsQueryParams(CurrencyPairsQueryParams):
         default=None, description="Symbol of the pair to search."
     )
     date: Optional[dateType] = Field(
-        default=datetime.now().date(), description=QUERY_DESCRIPTIONS.get("date", "")
+        default=None, description=QUERY_DESCRIPTIONS.get("date", "")
     )
     search: Optional[str] = Field(
         default="",
@@ -159,6 +159,7 @@ class PolygonCurrencyPairsFetcher(
 
         return all_data
 
+    # pylint: disable=unused-argument
     @staticmethod
     def transform_data(
         query: PolygonCurrencyPairsQueryParams,

openbb_platform/providers/polygon/openbb_polygon/models/equity_nbbo.py

@@ -92,7 +92,7 @@ class PolygonEquityNBBOData(EquityNBBOData):
     conditions: Optional[Union[str, List[int], List[str]]] = Field(
         default=None, description="A list of condition codes.", alias="conditions"
     )
-    indicators: Optional[List] = Field(
+    indicators: Optional[List[int]] = Field(
         default=None, description="A list of indicator codes.", alias="indicators"
     )
     sequence_num: Optional[int] = Field(
@@ -208,6 +208,7 @@ class PolygonEquityNBBOFetcher(
 
         return data
 
+    # pylint: disable=unused-argument
     @staticmethod
     def transform_data(
         query: PolygonEquityNBBOQueryParams,

openbb_platform/providers/sec/openbb_sec/models/schema_files.py

@@ -23,7 +23,7 @@ class SecSchemaFilesQueryParams(CotSearchQueryParams):
 class SecSchemaFilesData(Data):
     """SEC Schema Files List Data."""
 
-    files: List = Field(description="Dictionary of URLs to SEC Schema Files")
+    files: List[str] = Field(description="Dictionary of URLs to SEC Schema Files")
 
 
 class SecSchemaFilesFetcher(Fetcher[SecSchemaFilesQueryParams, SecSchemaFilesData]):

website/content/platform/development/_category_.json

@@ -1,4 +1,4 @@
 {
   "label": "Development",
-  "position": 6
-}
+  "position": 7
+}

website/tests/test_platform_docs.py

@@ -0,0 +1,308 @@
+import datetime
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+from typing import List, Literal, Optional, Union
+
+import pytest
+from pydantic import BaseModel, Field
+
+sys.path.append(str(Path(__file__).resolve().parent.parent))
+
+# pylint: disable=unused-argument,redefined-outer-name,wrong-import-position
+
+from generate_platform_v4_markdown import (  # noqa: E402
+    get_field_data_type,
+    get_function_params_default_value,
+    get_post_method_parameters_info,
+    get_post_method_returns_info,
+    get_provider_field_params,
+    get_provider_parameter_info,
+)
+
+
+# Mock Classes
+class MockField:
+    def __init__(self, annotation):
+        self.annotation = annotation
+
+
+class MockReturnType:
+    def __init__(self):
+        self.model_fields = {"results": MockField(annotation="List[str]")}
+
+
+@dataclass
+class MockProvider(BaseModel):
+    provider: Literal["provider1", "provider2"] = "provider1"
+
+
+class MockModel(BaseModel):
+    provider: Optional[MockProvider] = Field(default=None)
+
+
+# Mock Pydantic models to simulate your data structure
+class OpenBBQueryParams(BaseModel):
+    field1: Optional[int] = Field(default=None, description="A sample integer field")
+    date: datetime.date = Field(
+        default=None, description="A datetime.date type field (expansion type)"
+    )
+
+
+class OpenBBData(BaseModel):
+    field1: Literal["a", "b"] = Field(
+        default="a", description="A sample Union field with Literal['a', 'b']"
+    )
+    field2: Optional[int] = Field(
+        default=10,
+        description="A field with annotated type",
+        gt=0,
+    )
+
+
+class Provider1QueryParams(BaseModel):
+    field1: Optional[List[str]] = Field(
+        default=None, description="A sample List[str] field"
+    )
+
+
+class EmptyModel(BaseModel):
+    pass
+
+
+@pytest.fixture
+def mock_data():
+    return {
+        "openbb": {
+            "QueryParams": {"fields": OpenBBQueryParams().model_fields},
+            "Data": {"fields": OpenBBData().model_fields},
+        },
+        "provider1": {
+            "QueryParams": {"fields": Provider1QueryParams().model_fields},
+            "Data": {"fields": EmptyModel().model_fields},
+        },
+    }
+
+
+@pytest.fixture
+def mock_endpoint():
+    def endpoint(param1: int, param2: Optional[str] = "default"):
+        pass  # Mock endpoint function
+
+    return endpoint
+
+
+class MockEndpointSetup:
+    def __init__(self):
+        self.mock_endpoint = self._create_mock_endpoint()
+
+    def _create_mock_endpoint(self):
+        def mock_endpoint(provider_choices: Optional[MockProvider] = None):
+            return ["result1", "result2"]
+
+        # Simulate annotations with a dataclass model
+        mock_endpoint.__annotations__ = {
+            "provider_choices": Optional[MockProvider],
+            "return": List[str],
+        }
+
+        return mock_endpoint
+
+
+@pytest.fixture
+def mock_endpoint_setup():
+    return MockEndpointSetup()
+
+
+# Test Classes
+class TestFieldDataType:
+    @pytest.mark.parametrize(
+        "input_type,expected_output",
+        [
+            (int, "int"),
+            (str, "str"),
+            (Optional[int], "int"),
+            (Union[int, None], "int"),
+        ],
+    )
+    def test_get_field_data_type(self, input_type, expected_output):
+        assert get_field_data_type(input_type) == expected_output
+
+
+class TestProviderFieldParams:
+    def test_get_provider_field_params(self, mock_data, monkeypatch):
+        monkeypatch.setattr(
+            "generate_platform_v4_markdown.get_field_data_type",
+            get_field_data_type,
+        )
+        # Define expected_result based on mock_data and logic in get_provider_field_params
+        expected_result_1 = [
+            {
+                "name": "field1",
+                "type": "int",
+                "description": "A sample integer field",
+                "default": "None",
+                "optional": True,
+                "standard": True,
+            },
+            {
+                "name": "date",
+                "type": "Union[str, date]",
+                "description": "A datetime.date type field (expansion type)",
+                "default": "None",
+                "optional": True,
+                "standard": True,
+            },
+        ]
+
+        expected_result_2 = [
+            {
+                "name": "field1",
+                "type": "List[str]",
+                "description": "A sample List[str] field",
+                "default": "None",
+                "optional": True,
+                "standard": False,
+            },
+        ]
+
+        expected_result_3 = [
+            {
+                "name": "field1",
+                "type": "Literal['a', 'b']",
+                "description": "A sample Union field with Literal['a', 'b']",
+                "default": "a",
+                "optional": True,
+                "standard": True,
+            },
+            {
+                "name": "field2",
+                "type": "int",
+                "description": "A field with annotated type",
+                "default": "10",
+                "optional": True,
+                "standard": True,
+            },
+        ]
+
+        result = get_provider_field_params(mock_data, "QueryParams")
+        assert result == expected_result_1
+
+        result = get_provider_field_params(mock_data, "QueryParams", "provider1")
+        assert result == expected_result_2
+
+        result = get_provider_field_params(mock_data, "Data")
+        assert result == expected_result_3
+
+
+class TestProviderParameterInfo:
+    def test_get_provider_parameter_info(self, mock_endpoint_setup):
+        # Access the mock_endpoint from the fixture
+        mock_endpoint = mock_endpoint_setup.mock_endpoint
+
+        expected = {
+            "name": "provider",
+            "type": "Literal['provider1', 'provider2']",
+            "description": "The provider to use for the query, by default None. If None, the provider specified in defaults is selected or 'provider1' if there is no default.",  # noqa: E501
+            "default": "provider1",
+            "optional": True,
+            "standard": True,
+        }
+
+        result = get_provider_parameter_info(mock_endpoint)
+        assert result == expected
+
+
+class TestFunctionParamsDefaultValue:
+    def test_get_function_params_default_value(self, mock_endpoint):
+        expected_result = {
+            "param1": "",
+            "param2": "default",
+        }
+
+        result = get_function_params_default_value(mock_endpoint)
+        assert (
+            result == expected_result
+        ), "Failed to correctly identify default parameter values."
+
+
+class TestPostMethodParametersInfo:
+    def test_get_post_method_parameters_info(self, monkeypatch):
+        monkeypatch.setattr(
+            "generate_platform_v4_markdown.get_field_data_type",
+            get_field_data_type,
+        )
+
+        def mock_endpoint(param1: int, param2: Optional[str] = "default"):
+            """
+            A mock endpoint function.
+
+            Parameters
+            ----------
+            param1 : int
+                The first parameter.
+            param2 : Optional[str]
+                The second parameter, optional.
+            """
+            return
+
+        expected_result = [
+            {
+                "name": "param1",
+                "type": "int",
+                "description": "The first parameter.",
+                "default": "",
+                "optional": False,
+            },
+            {
+                "name": "param2",
+                "type": "str",
+                "description": "The second parameter, optional.",
+                "default": "default",
+                "optional": True,
+            },
+        ]
+
+        result = get_post_method_parameters_info(mock_endpoint)
+        assert (
+            result == expected_result
+        ), "Failed to correctly extract POST method parameters info."
+
+
+class TestPostMethodReturnsInfo:
+    def test_get_post_method_returns_info(self, monkeypatch):
+        monkeypatch.setattr(
+            "generate_platform_v4_markdown.get_field_data_type",
+            get_field_data_type,
+        )
+
+        class MockReturnModel(BaseModel):
+            results: List[str]
+
+        def mock_endpoint() -> MockReturnModel:
+            """
+            A mock endpoint function.
+
+            Parameters
+            ----------
+            None
+
+            Returns
+            -------
+            MockReturnModel
+                A list of strings as results.
+            """
+            return
+
+        expected_result = [
+            {
+                "name": "results",
+                "type": "List[str]",
+                "description": "A list of strings as results.",
+            }
+        ]
+
+        result = get_post_method_returns_info(mock_endpoint)
+        assert (
+            result == expected_result
+        ), "Failed to correctly extract POST method returns info."