[Feature] SEC - Management Discussion & Analysis Sections (#7006)

* open draft

* some table smoothing..

* fix embedded image links

* bullet list cleanup

* some updates..coming back to it later

* saving here as a clear marker for progress

* some linting

* param description

* some edge case...

* cch cch changes

* missing docstring

* walk back..

* f-string

* some improvements, hopefully

* some improvements, hopefully

* lint

* use generator instead

* fixes some table parsing

* find the section in banks like JPM and C

* fixes some cell splitting behaviour

* test stuff

* lock file

* lock file

* test things

* remove commented out lines

* startswith for bulletpoints..

* some more edges

* O

* try not to be tricked into starting at the TOC

* font difference causing start not to be found.

* fix wfc that calls the section something else entirely

* fix target header not being all on the same line

* fix target section missing Item 2 or Item 7 in text

* fix for BKNG

* fix different wording order in target section title

* fix some error with BX

* fix some nested table bullet points

* fix out of range

* fix to find intel's section title

* fix APH issue finding start

* some edge cases

* Help out a lil

* add 'strategy' param to select parser

* some patches

* allow entering CIK for symbol

* update parsing bool on failed attempt

* integration params

* add warning when trafilatura fails, and don't insert cell dividers when (a) follows number

* fix section headers

* format fixes

* more formatting

* linter

* add widget config for markdown

* calendar period as Q1-Q4 enum

* test param

---------

Co-authored-by: James Maslek <jmaslek11@gmail.com>

openbb_platform/core/openbb_core/provider/standard_models/management_discussion_analysis.py

@@ -0,0 +1,48 @@
+"""Management Discussion & Analysis Standard Model."""
+
+from datetime import date as dateType
+from typing import Literal, Optional
+
+from openbb_core.provider.abstract.data import Data
+from openbb_core.provider.abstract.query_params import QueryParams
+from openbb_core.provider.utils.descriptions import (
+    DATA_DESCRIPTIONS,
+    QUERY_DESCRIPTIONS,
+)
+from pydantic import Field, field_validator
+
+
+class ManagementDiscussionAnalysisQueryParams(QueryParams):
+    """Management Discussion & Analysis Query Parameters."""
+
+    symbol: str = Field(description=QUERY_DESCRIPTIONS.get("symbol", ""))
+    calendar_year: Optional[int] = Field(
+        default=None,
+        description="Calendar year of the report. By default, is the current year."
+        + " If the calendar period is not provided, but the calendar year is, it will return the annual report.",
+    )
+    calendar_period: Optional[Literal["Q1", "Q2", "Q3", "Q4"]] = Field(
+        default=None,
+        description="Calendar period of the report. By default, is the most recent report available for the symbol."
+        + " If no calendar year and no calendar period are provided, it will return the most recent report.",
+    )
+
+    @field_validator("symbol", mode="before", check_fields=False)
+    @classmethod
+    def to_upper(cls, v: str):
+        """Convert field to uppercase."""
+        return v.upper()
+
+
+class ManagementDiscussionAnalysisData(Data):
+    """Management Discussion & Analysis Data."""
+
+    symbol: str = Field(description=DATA_DESCRIPTIONS.get("symbol", ""))
+    calendar_year: int = Field(description="The calendar year of the report.")
+    calendar_period: int = Field(description="The calendar period of the report.")
+    period_ending: Optional[dateType] = Field(
+        description="The end date of the reporting period.", default=None
+    )
+    content: str = Field(
+        description="The content of the management discussion and analysis."
+    )

openbb_platform/extensions/equity/integration/test_equity_api.py

@@ -2189,3 +2189,33 @@ def test_equity_ownership_government_trades(params, headers):
     result = requests.get(url, headers=headers, timeout=10)
     assert isinstance(result, requests.Response)
     assert result.status_code == 200
+
+
+@parametrize(
+    "params",
+    [
+        (
+            {
+                "symbol": "AAPL",
+                "calendar_year": 2024,
+                "calendar_period": "Q2",
+                "wrap_length": 120,
+                "include_tables": False,
+                "use_cache": True,
+                "raw_html": False,
+                "strategy": "trafilatura",
+                "provider": "sec",
+            }
+        ),
+    ],
+)
+@pytest.mark.integration
+def test_equity_fundamental_management_discussion_analysis(params, headers):
+    """Test the equity fundamental management discussion analysis endpoint."""
+    params = {p: v for p, v in params.items() if v}
+
+    query_str = get_querystring(params, [])
+    url = f"http://0.0.0.0:8000/api/v1/equity/fundamental/management_discussion_analysis?{query_str}"
+    result = requests.get(url, headers=headers, timeout=10)
+    assert isinstance(result, requests.Response)
+    assert result.status_code == 200

openbb_platform/extensions/equity/integration/test_equity_python.py

@@ -2049,3 +2049,30 @@ def test_equity_ownership_government_trades(params, obb):
     assert result
     assert isinstance(result, OBBject)
     assert len(result.results) > 0
+
+
+@parametrize(
+    "params",
+    [
+        (
+            {
+                "symbol": "AAPL",
+                "calendar_year": 2024,
+                "calendar_period": "Q2",
+                "wrap_length": 120,
+                "include_tables": False,
+                "use_cache": True,
+                "raw_html": False,
+                "strategy": "trafilatura",
+                "provider": "sec",
+            }
+        ),
+    ],
+)
+@pytest.mark.integration
+def test_equity_fundamental_management_discussion_analysis(params, obb):
+    """Test the equity fundamental management discussion analysis endpoint."""
+    result = obb.equity.fundamental.management_discussion_analysis(**params)
+    assert result
+    assert isinstance(result, OBBject)
+    assert len(result.results.content) > 0

openbb_platform/extensions/equity/openbb_equity/fundamental/fundamental_router.py

@@ -477,3 +477,58 @@ async def trailing_dividend_yield(
 ) -> OBBject:
     """Get the 1 year trailing dividend yield for a given company over time."""
     return await OBBject.from_query(Query(**locals()))
+
+
+@router.command(
+    model="ManagementDiscussionAnalysis",
+    examples=[
+        APIEx(parameters={"symbol": "AAPL", "provider": "sec"}),
+        APIEx(
+            description="Get the Management Discussion & Analysis section by calendar year and period.",
+            parameters={
+                "symbol": "AAPL",
+                "calendar_year": 2020,
+                "calendar_period": "Q4",
+                "provider": "sec",
+            },
+        ),
+        APIEx(
+            description="Setting 'include_tables' to True will attempt to extract all tables in valid Markdown.",
+            parameters={
+                "symbol": "AAPL",
+                "calendar_year": 2020,
+                "calendar_period": "Q4",
+                "provider": "sec",
+                "include_tables": True,
+            },
+        ),
+        APIEx(
+            description="Setting 'raw_html' to True will bypass extraction and return the raw HTML file, as is."
+            + " Use this for custom parsing or to access the entire HTML filing.",
+            parameters={
+                "symbol": "AAPL",
+                "calendar_year": 2020,
+                "calendar_period": "Q4",
+                "provider": "sec",
+                "raw_html": True,
+            },
+        ),
+    ],
+    openapi_extra={
+        "widget_config": {
+            "type": "markdown",
+            "data": {"dataKey": "results.content", "columnsDefs": []},
+            "staleTime": 86400000,
+            "refetchInterval": 86400000,
+            "source": "SEC",
+        }
+    },
+)
+async def management_discussion_analysis(
+    cc: CommandContext,
+    provider_choices: ProviderChoices,
+    standard_params: StandardParams,
+    extra_params: ExtraParams,
+) -> OBBject:
+    """Get the Management Discussion & Analysis section from the financial statements for a given company."""
+    return await OBBject.from_query(Query(**locals()))

openbb_platform/openbb/assets/reference.json

@@ -26646,6 +26646,160 @@
             },
             "model": "TrailingDividendYield"
         },
+        "/equity/fundamental/management_discussion_analysis": {
+            "deprecated": {
+                "flag": null,
+                "message": null
+            },
+            "description": "Get the Management Discussion & Analysis section from the financial statements for a given company.",
+            "examples": "\nExamples\n--------\n\n```python\nfrom openbb import obb\nobb.equity.fundamental.management_discussion_analysis(symbol='AAPL', provider='sec')\n# Get the Management Discussion & Analysis section by calendar year and period.\nobb.equity.fundamental.management_discussion_analysis(symbol='AAPL', calendar_year=2020, calendar_period=4, provider='sec')\n# Setting 'include_tables' to True will attempt to extract all tables in valid Markdown.\nobb.equity.fundamental.management_discussion_analysis(symbol='AAPL', calendar_year=2020, calendar_period=4, provider='sec', include_tables=True)\n# Setting 'raw_html' to True will bypass extraction and return the raw HTML file, as is. Use this for custom parsing or to access the entire HTML filing.\nobb.equity.fundamental.management_discussion_analysis(symbol='AAPL', calendar_year=2020, calendar_period=4, provider='sec', raw_html=True)\n```\n\n",
+            "parameters": {
+                "standard": [
+                    {
+                        "name": "symbol",
+                        "type": "str",
+                        "description": "Symbol to get data for.",
+                        "default": "",
+                        "optional": false,
+                        "choices": null
+                    },
+                    {
+                        "name": "calendar_year",
+                        "type": "int",
+                        "description": "Calendar year of the report. By default, is the current year. If the calendar period is not provided, but the calendar year is, it will return the annual report.",
+                        "default": null,
+                        "optional": true,
+                        "choices": null
+                    },
+                    {
+                        "name": "calendar_period",
+                        "type": "int",
+                        "description": "Calendar period of the report. By default, is the most recent report available for the symbol. If no calendar year and no calendar period are provided, it will return the most recent report.",
+                        "default": null,
+                        "optional": true,
+                        "choices": null
+                    }
+                ],
+                "sec": [
+                    {
+                        "name": "wrap_length",
+                        "type": "int",
+                        "description": "The length to wrap the extracted text, excluding tables. Default is 120.",
+                        "default": 120,
+                        "optional": true,
+                        "choices": null
+                    },
+                    {
+                        "name": "include_tables",
+                        "type": "bool",
+                        "description": "Return tables formatted as markdown in the text. Default is False. Tables may reveal 'missing' content, but will likely need some level of manual cleaning, post-request, to display properly. In some cases, tables may not be recoverable due to the nature of the document.",
+                        "default": false,
+                        "optional": true,
+                        "choices": null
+                    },
+                    {
+                        "name": "use_cache",
+                        "type": "bool",
+                        "description": "When True, the file will be cached for use later. Default is True.",
+                        "default": true,
+                        "optional": true,
+                        "choices": null
+                    },
+                    {
+                        "name": "raw_html",
+                        "type": "bool",
+                        "description": "When True, the raw HTML content of the entire filing will be returned. Default is False. Use this option to parse the document manually.",
+                        "default": false,
+                        "optional": true,
+                        "choices": null
+                    }
+                ]
+            },
+            "returns": {
+                "OBBject": [
+                    {
+                        "name": "results",
+                        "type": "List[ManagementDiscussionAnalysis]",
+                        "description": "Serializable results."
+                    },
+                    {
+                        "name": "provider",
+                        "type": "Optional[Literal['sec']]",
+                        "description": "Provider name."
+                    },
+                    {
+                        "name": "warnings",
+                        "type": "Optional[List[Warning_]]",
+                        "description": "List of warnings."
+                    },
+                    {
+                        "name": "chart",
+                        "type": "Optional[Chart]",
+                        "description": "Chart object."
+                    },
+                    {
+                        "name": "extra",
+                        "type": "Dict[str, Any]",
+                        "description": "Extra info."
+                    }
+                ]
+            },
+            "data": {
+                "standard": [
+                    {
+                        "name": "symbol",
+                        "type": "str",
+                        "description": "Symbol representing the entity requested in the data.",
+                        "default": "",
+                        "optional": false,
+                        "choices": null
+                    },
+                    {
+                        "name": "calendar_year",
+                        "type": "int",
+                        "description": "The calendar year of the report.",
+                        "default": "",
+                        "optional": false,
+                        "choices": null
+                    },
+                    {
+                        "name": "calendar_period",
+                        "type": "int",
+                        "description": "The calendar period of the report.",
+                        "default": "",
+                        "optional": false,
+                        "choices": null
+                    },
+                    {
+                        "name": "period_ending",
+                        "type": "date",
+                        "description": "The end date of the reporting period.",
+                        "default": null,
+                        "optional": true,
+                        "choices": null
+                    },
+                    {
+                        "name": "content",
+                        "type": "str",
+                        "description": "The content of the management discussion and analysis.",
+                        "default": "",
+                        "optional": false,
+                        "choices": null
+                    }
+                ],
+                "sec": [
+                    {
+                        "name": "url",
+                        "type": "str",
+                        "description": "The URL of the filing from which the data was extracted.",
+                        "default": "",
+                        "optional": false,
+                        "choices": null
+                    }
+                ]
+            },
+            "model": "ManagementDiscussionAnalysis"
+        },
         "/equity/ownership/major_holders": {
             "deprecated": {
                 "flag": null,

openbb_platform/openbb/package/equity_fundamental.py

@@ -29,6 +29,7 @@ class ROUTER_equity_fundamental(Container):
     latest_attributes
     management
     management_compensation
+    management_discussion_analysis
     metrics
     multiples
     ratios
@@ -2782,6 +2783,112 @@ class ROUTER_equity_fundamental(Container):
             )
         )
 
+    @exception_handler
+    @validate
+    def management_discussion_analysis(
+        self,
+        symbol: Annotated[str, OpenBBField(description="Symbol to get data for.")],
+        calendar_year: Annotated[
+            Optional[int],
+            OpenBBField(
+                description="Calendar year of the report. By default, is the current year. If the calendar period is not provided, but the calendar year is, it will return the annual report."
+            ),
+        ] = None,
+        calendar_period: Annotated[
+            Optional[int],
+            OpenBBField(
+                description="Calendar period of the report. By default, is the most recent report available for the symbol. If no calendar year and no calendar period are provided, it will return the most recent report."
+            ),
+        ] = None,
+        provider: Annotated[
+            Optional[Literal["sec"]],
+            OpenBBField(
+                description="The provider to use, by default None. If None, the priority list configured in the settings is used. Default priority: sec."
+            ),
+        ] = None,
+        **kwargs
+    ) -> OBBject:
+        """Get the Management Discussion & Analysis section from the financial statements for a given company.
+
+        Parameters
+        ----------
+        symbol : str
+            Symbol to get data for.
+        calendar_year : Optional[int]
+            Calendar year of the report. By default, is the current year. If the calendar period is not provided, but the calendar year is, it will return the annual report.
+        calendar_period : Optional[int]
+            Calendar period of the report. By default, is the most recent report available for the symbol. If no calendar year and no calendar period are provided, it will return the most recent report.
+        provider : Optional[Literal['sec']]
+            The provider to use, by default None. If None, the priority list configured in the settings is used. Default priority: sec.
+        wrap_length : int
+            The length to wrap the extracted text, excluding tables. Default is 120. (provider: sec)
+        include_tables : bool
+            Return tables formatted as markdown in the text. Default is False. Tables may reveal 'missing' content, but will likely need some level of manual cleaning, post-request, to display properly. In some cases, tables may not be recoverable due to the nature of the document. (provider: sec)
+        use_cache : bool
+            When True, the file will be cached for use later. Default is True. (provider: sec)
+        raw_html : bool
+            When True, the raw HTML content of the entire filing will be returned. Default is False. Use this option to parse the document manually. (provider: sec)
+
+        Returns
+        -------
+        OBBject
+            results : ManagementDiscussionAnalysis
+                Serializable results.
+            provider : Optional[Literal['sec']]
+                Provider name.
+            warnings : Optional[List[Warning_]]
+                List of warnings.
+            chart : Optional[Chart]
+                Chart object.
+            extra : Dict[str, Any]
+                Extra info.
+
+        ManagementDiscussionAnalysis
+        ----------------------------
+        symbol : str
+            Symbol representing the entity requested in the data.
+        calendar_year : int
+            The calendar year of the report.
+        calendar_period : int
+            The calendar period of the report.
+        period_ending : Optional[date]
+            The end date of the reporting period.
+        content : str
+            The content of the management discussion and analysis.
+        url : Optional[str]
+            The URL of the filing from which the data was extracted. (provider: sec)
+
+        Examples
+        --------
+        >>> from openbb import obb
+        >>> obb.equity.fundamental.management_discussion_analysis(symbol='AAPL', provider='sec')
+        >>> # Get the Management Discussion & Analysis section by calendar year and period.
+        >>> obb.equity.fundamental.management_discussion_analysis(symbol='AAPL', calendar_year=2020, calendar_period=4, provider='sec')
+        >>> # Setting 'include_tables' to True will attempt to extract all tables in valid Markdown.
+        >>> obb.equity.fundamental.management_discussion_analysis(symbol='AAPL', calendar_year=2020, calendar_period=4, provider='sec', include_tables=True)
+        >>> # Setting 'raw_html' to True will bypass extraction and return the raw HTML file, as is. Use this for custom parsing or to access the entire HTML filing.
+        >>> obb.equity.fundamental.management_discussion_analysis(symbol='AAPL', calendar_year=2020, calendar_period=4, provider='sec', raw_html=True)
+        """  # noqa: E501
+
+        return self._run(
+            "/equity/fundamental/management_discussion_analysis",
+            **filter_inputs(
+                provider_choices={
+                    "provider": self._get_provider(
+                        provider,
+                        "equity.fundamental.management_discussion_analysis",
+                        ("sec",),
+                    )
+                },
+                standard_params={
+                    "symbol": symbol,
+                    "calendar_year": calendar_year,
+                    "calendar_period": calendar_period,
+                },
+                extra_params=kwargs,
+            )
+        )
+
     @exception_handler
     @validate
     def metrics(

openbb_platform/providers/sec/openbb_sec/__init__.py

@@ -11,6 +11,9 @@ from openbb_sec.models.form_13FHR import SecForm13FHRFetcher
 from openbb_sec.models.insider_trading import SecInsiderTradingFetcher
 from openbb_sec.models.institutions_search import SecInstitutionsSearchFetcher
 from openbb_sec.models.latest_financial_reports import SecLatestFinancialReportsFetcher
+from openbb_sec.models.management_discussion_analysis import (
+    SecManagementDiscussionAnalysisFetcher,
+)
 from openbb_sec.models.rss_litigation import SecRssLitigationFetcher
 from openbb_sec.models.schema_files import SecSchemaFilesFetcher
 from openbb_sec.models.sic_search import SecSicSearchFetcher
@@ -33,6 +36,7 @@ sec_provider = Provider(
         "InsiderTrading": SecInsiderTradingFetcher,
         "InstitutionsSearch": SecInstitutionsSearchFetcher,
         "LatestFinancialReports": SecLatestFinancialReportsFetcher,
+        "ManagementDiscussionAnalysis": SecManagementDiscussionAnalysisFetcher,
         "RssLitigation": SecRssLitigationFetcher,
         "SchemaFiles": SecSchemaFilesFetcher,
         "SicSearch": SecSicSearchFetcher,

openbb_platform/providers/sec/pyproject.toml

@@ -9,12 +9,14 @@ packages = [{ include = "openbb_sec" }]
 
 [tool.poetry.dependencies]
 python = ">=3.9,<3.13"
-openbb-core = "^1.3.6"
+openbb-core = "^1.3.7"
 aiohttp-client-cache = "^0.11.0"
 aiosqlite = "^0.20.0"
 xmltodict = "^0.13.0"
 beautifulsoup4 = "^4.12"
 lxml = "^5.2.1"
+trafilatura = "^2.0"
+inscriptis = "^2.5.3"
 
 [build-system]
 requires = ["poetry-core"]

openbb_platform/providers/sec/tests/test_sec_fetchers.py

@@ -14,6 +14,9 @@ from openbb_sec.models.form_13FHR import SecForm13FHRFetcher
 from openbb_sec.models.insider_trading import SecInsiderTradingFetcher
 from openbb_sec.models.institutions_search import SecInstitutionsSearchFetcher
 from openbb_sec.models.latest_financial_reports import SecLatestFinancialReportsFetcher
+from openbb_sec.models.management_discussion_analysis import (
+    SecManagementDiscussionAnalysisFetcher,
+)
 from openbb_sec.models.rss_litigation import SecRssLitigationFetcher
 from openbb_sec.models.schema_files import SecSchemaFilesFetcher
 from openbb_sec.models.sic_search import SecSicSearchFetcher
@@ -186,3 +189,21 @@ def test_sec_insider_trading_fetcher(credentials=test_credentials):
     fetcher = SecInsiderTradingFetcher()
     result = fetcher.test(params, credentials)
     assert result is None
+
+
+@pytest.mark.record_http
+def test_sec_management_discussion_analysis_fetcher(credentials=test_credentials):
+    """Test the SEC Management Discussion Analysis fetcher."""
+    params = {
+        "symbol": "AAPL",
+        "calendar_year": 2024,
+        "calendar_period": "Q2",
+        "wrap_length": 120,
+        "include_tables": False,
+        "use_cache": False,
+        "raw_html": False,
+    }
+
+    fetcher = SecManagementDiscussionAnalysisFetcher()
+    result = fetcher.test(params, credentials)
+    assert result is None